open-source-mirrors
/
Readarr
mirror of https://github.com/Readarr/Readarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

New: Limit filenames to a maximum of 255 characters

Browse Source
pull/3996/head
Kyle Butler 2 months ago
parent bc74456944
commit 690de685c7
3 changed files with 326 additions and 25 deletions
Show Stats Download Patch File Download Diff File

214
src/NzbDrone.Core.Test/OrganizerTests/FileNameBuilderTests/TruncatedBookTitlesFixture.cs
Unescape View File

@ -0,0 +1,214 @@
using System.Linq;
using FizzWare.NBuilder;
using FluentAssertions;
using NUnit.Framework;
using NzbDrone.Core.Books;
using NzbDrone.Core.MediaFiles;
using NzbDrone.Core.Organizer;
using NzbDrone.Core.Qualities;
using NzbDrone.Core.Test.Framework;
namespace NzbDrone.Core.Test.OrganizerTests.FileNameBuilderTests
{
[TestFixture]
public class TruncatedBookTitlesFixture : CoreTest<FileNameBuilder>
{
private BookFile _bookFile;
private NamingConfig _namingConfig;
[SetUp]
public void Setup()
{
_namingConfig = NamingConfig.Default;
_namingConfig.RenameBooks = true;
Mocker.GetMock<INamingConfigService>()
.Setup(c => c.GetConfig()).Returns(_namingConfig);
_bookFile = new BookFile { Quality = new QualityModel(Quality.EPUB), ReleaseGroup = "ReadarrTest" };
Mocker.GetMock<IQualityDefinitionService>()
.Setup(v => v.Get(Moq.It.IsAny<Quality>()))
.Returns<Quality>(v => Quality.DefaultQualityDefinitions.First(c => c.Quality == v));
}
private (Author, Edition) BuildTestInputs(string authorName, string bookTitle, string seriesName, string seriesNumber)
{
var author = Builder<Author>
.CreateNew()
.With(s => s.Name = authorName)
.Build();
var series = Builder<Series>
.CreateNew()
.With(x => x.Title = seriesName)
.Build();
var seriesLink = Builder<SeriesBookLink>
.CreateListOfSize(1)
.All()
.With(s => s.Position = seriesNumber)
.With(s => s.Series = series)
.BuildListOfNew();
var book = Builder<Book>
.CreateNew()
.With(s => s.Title = bookTitle)
.With(s => s.AuthorMetadata = author.Metadata.Value)
.With(s => s.SeriesLinks = seriesLink)
.Build();
var edition = Builder<Edition>
.CreateNew()
.With(s => s.Title = book.Title)
.With(s => s.Book = book)
.Build();
return (author, edition);
}
[Test]
public void should_not_truncate_filename_if_length_is_less_than_max_length_limit_long_book_name()
{
var authorName = "Brandon Sanderson";
var bookTitle = "Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var seriesName = "The Stormlight Archive";
var seriesNumber = "5";
var expected = "Brandon Sanderson - The Stormlight Archive #5 - Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().BeLessThan(255);
}
[Test]
public void should_not_truncate_filename_if_length_is_less_than_max_length_limit_long_author_name()
{
var authorName = "Brandon Sanderson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var bookTitle = "Knights of Wind and Truth";
var seriesName = "The Stormlight Archive";
var seriesNumber = "5";
var expected = "Brandon Sanderson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum - The Stormlight Archive #5 - Knights of Wind and Truth";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().BeLessThan(255);
}
[Test]
public void should_not_truncate_filename_if_length_is_less_than_max_length_limit_long_series_name()
{
var authorName = "Brandon Sanderson";
var bookTitle = "Knights of Wind and Truth";
var seriesName = "The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var seriesNumber = "5";
var expected = "Brandon Sanderson - The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum #5 - Knights of Wind and Truth";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().BeLessThan(255);
}
[Test]
public void should_not_truncate_filename_if_length_is_equal_to_than_max_length_limit_long_book_name()
{
var authorName = "Brandon Sanderson";
var bookTitle = "Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var seriesName = "The Stormlight Archive";
var seriesNumber = "5-1";
var expected = "Brandon Sanderson - The Stormlight Archive #5-1 - Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().Be(255);
}
[Test]
public void should_not_truncate_filename_if_length_is_equal_to_than_max_length_limit_long_author_name()
{
var authorName = "Brandon Sanderson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var bookTitle = "Knights of Wind and Truth";
var seriesName = "The Stormlight Archive";
var seriesNumber = "5-1";
var expected = "Brandon Sanderson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum - The Stormlight Archive #5-1 - Knights of Wind and Truth";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().Be(255);
}
[Test]
public void should_not_truncate_filename_if_length_is_equal_to_than_max_length_limit_long_series_name()
{
var authorName = "Brandon Sanderson";
var bookTitle = "Knights of Wind and Truth";
var seriesName = "The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var seriesNumber = "5-1";
var expected = "Brandon Sanderson - The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum #5-1 - Knights of Wind and Truth";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().Be(255);
}
[Test]
public void should_truncate_filename_if_length_is_greater_than_max_length_limit_long_book_name()
{
var authorName = "Brandon Sanderson and Janci Patterson";
var bookTitle = "Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var seriesName = "The Stormlight Archive";
var seriesNumber = "5";
var expected = "Brandon Sanderson and Janci Patterson - The Stormlight Archive #5 - Knights of Wind and Truth and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Belo...";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().Be(255);
}
[Test]
public void should_truncate_filename_if_length_is_greater_than_than_max_length_limit_long_author_name()
{
var authorName = "Brandon Sanderson and Janci Patterson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var bookTitle = "Knights of Wind and Truth";
var seriesName = "The Stormlight Archive";
var seriesNumber = "5";
var expected = "Brandon Sanderson and Janci Patterson and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum - The Stormlight Archive #5 - Knig...";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().Be(255);
}
[Test]
public void should_truncate_filename_if_length_is_greater_than_than_max_length_limit_long_series_name()
{
var authorName = "Brandon Sanderson and Janci Patterson";
var bookTitle = "Knights of Wind and Truth";
var seriesName = "The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum";
var seriesNumber = "5";
var expected = "Brandon Sanderson and Janci Patterson - The Stormlight Archive and Several Extra Words to Get The Length Close to 255 Characters in Total So That We Can Test Whether Or Not The Filename Is Truncated But The Length Is Below the Allowed Maximum #5 - Knig...";
_namingConfig.StandardBookFormat = "{Author Name} - {Book SeriesTitle} - {Book Title}";
var (author, edition) = BuildTestInputs(authorName, bookTitle, seriesName, seriesNumber);
var result = Subject.BuildBookFileName(author, edition, _bookFile);
result.Should().Be(expected);
result.Length.Should().Be(255);
}
}
}

5
src/NzbDrone.Core/Fluent.cs
Unescape View File

@ -97,6 +97,11 @@ namespace NzbDrone.Core
return intList.Max();
}
public static int GetByteCount(this string input)
{
return Encoding.UTF8.GetByteCount(input);
}
public static string Truncate(this string s, int maxLength)
{
if (Encoding.UTF8.GetByteCount(s) <= maxLength)

132
src/NzbDrone.Core/Organizer/FileNameBuilder.cs
Unescape View File

@ -6,6 +6,7 @@ using System.Linq;
using System.Text.RegularExpressions;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Disk;
using NzbDrone.Common.EnsureThat;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Books;
@ -30,7 +31,7 @@ namespace NzbDrone.Core.Organizer
private readonly INamingConfigService _namingConfigService;
private readonly IQualityDefinitionService _qualityDefinitionService;
private readonly ICustomFormatCalculationService _formatCalculator;
private readonly ICached<BookFormat[]> _trackFormatCache;
private readonly ICached<BookFormat[]> _bookFormatCache;
private readonly Logger _logger;
private static readonly Regex TitleRegex = new Regex(@"\{(?<prefix>[- ._\[(]*)(?<token>(?:[a-z0-9]+)(?:(?<separator>[- ._]+)(?:[a-z0-9]+))?)(?::(?<customFormat>[a-z0-9]+))?(?<suffix>[- ._)\]]*)\}",
@ -56,6 +57,8 @@ namespace NzbDrone.Core.Organizer
private static readonly Regex TitlePrefixRegex = new Regex(@"^(The|An|A) (.*?)((?: *\([^)]+\))*)$", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly char[] BookTitleTrimCharacters = new[] { ' ', '.', '?' };
public FileNameBuilder(INamingConfigService namingConfigService,
IQualityDefinitionService qualityDefinitionService,
ICacheManager cacheManager,
@ -65,11 +68,11 @@ namespace NzbDrone.Core.Organizer
_namingConfigService = namingConfigService;
_qualityDefinitionService = qualityDefinitionService;
_formatCalculator = formatCalculator;
_trackFormatCache = cacheManager.GetCache<BookFormat[]>(GetType(), "bookFormat");
_bookFormatCache = cacheManager.GetCache<BookFormat[]>(GetType(), "bookFormat");
_logger = logger;
}
public string BuildBookFileName(Author author, Edition edition, BookFile bookFile, NamingConfig namingConfig = null, List<CustomFormat> customFormats = null)
private string BuildBookFileName(Author author, Edition edition, BookFile bookFile, int maxPath, NamingConfig namingConfig = null, List<CustomFormat> customFormats = null)
{
if (namingConfig == null)
{
@ -88,15 +91,6 @@ namespace NzbDrone.Core.Organizer
var pattern = namingConfig.StandardBookFormat;
var tokenHandlers = new Dictionary<string, Func<TokenMatch, string>>(FileNameBuilderTokenEqualityComparer.Instance);
AddAuthorTokens(tokenHandlers, author);
AddBookTokens(tokenHandlers, edition);
AddBookFileTokens(tokenHandlers, bookFile);
AddQualityTokens(tokenHandlers, author, bookFile);
AddMediaInfoTokens(tokenHandlers, bookFile);
AddCustomFormats(tokenHandlers, author, bookFile, customFormats);
var splitPatterns = pattern.Split(new char[] { '\\', '/' }, StringSplitOptions.RemoveEmptyEntries);
var components = new List<string>();
@ -104,11 +98,29 @@ namespace NzbDrone.Core.Organizer
{
var splitPattern = s;
var tokenHandlers = new Dictionary<string, Func<TokenMatch, string>>(FileNameBuilderTokenEqualityComparer.Instance);
/* Replace all tokens excluding the book title */
AddAuthorTokens(tokenHandlers, author);
AddBookTokens(tokenHandlers, edition);
AddBookTitlePlaceholderTokens(tokenHandlers);
AddBookFileTokens(tokenHandlers, bookFile);
AddQualityTokens(tokenHandlers, author, bookFile);
AddMediaInfoTokens(tokenHandlers, bookFile);
AddCustomFormats(tokenHandlers, author, bookFile, customFormats);
var component = ReplacePartTokens(splitPattern, tokenHandlers, namingConfig).Trim();
component = ReplaceTokens(component, tokenHandlers, namingConfig).Trim();
/* Determine how long the name is and compute the max book title length based on what is left below the max */
var maxPathSegmentLength = Math.Min(LongPathSupport.MaxFileNameLength, maxPath);
var maxBookTitleLength = maxPathSegmentLength - GetLengthWithoutBookTitle(component, namingConfig);
/* Add the book title, truncating the length as necessary */
AddBookTitleTokens(tokenHandlers, edition, maxBookTitleLength);
component = ReplaceTokens(component, tokenHandlers, namingConfig).Trim();
component = FileNameCleanupRegex.Replace(component, match => match.Captures[0].Value[0].ToString());
component = TrimSeparatorsRegex.Replace(component, string.Empty);
component = component.Replace("{ellipsis}", "...");
if (component.IsNotNullOrWhiteSpace())
{
@ -119,6 +131,11 @@ namespace NzbDrone.Core.Organizer
return Path.Combine(components.ToArray());
}
public string BuildBookFileName(Author author, Edition edition, BookFile bookFile, NamingConfig namingConfig = null, List<CustomFormat> customFormats = null)
{
return BuildBookFileName(author, edition, bookFile, LongPathSupport.MaxFilePathLength, namingConfig, customFormats);
}
public string BuildBookFilePath(Author author, Edition edition, string fileName, string extension)
{
Ensure.That(extension, () => extension).IsNotNullOrWhiteSpace();
@ -135,16 +152,16 @@ namespace NzbDrone.Core.Organizer
public BasicNamingConfig GetBasicNamingConfig(NamingConfig nameSpec)
{
var trackFormat = GetTrackFormat(nameSpec.StandardBookFormat).LastOrDefault();
var bookFormat = GetBookFormat(nameSpec.StandardBookFormat).LastOrDefault();
if (trackFormat == null)
if (bookFormat == null)
{
return new BasicNamingConfig();
}
var basicNamingConfig = new BasicNamingConfig
{
Separator = trackFormat.Separator
Separator = bookFormat.Separator
};
var titleTokens = TitleRegex.Matches(nameSpec.StandardBookFormat);
@ -251,10 +268,6 @@ namespace NzbDrone.Core.Organizer
private void AddBookTokens(Dictionary<string, Func<TokenMatch, string>> tokenHandlers, Edition edition)
{
tokenHandlers["{Book Title}"] = m => edition.Title;
tokenHandlers["{Book CleanTitle}"] = m => CleanTitle(edition.Title);
tokenHandlers["{Book TitleThe}"] = m => TitleThe(edition.Title);
var (titleNoSub, subtitle) = edition.Title.SplitBookTitle(edition.Book.Value.AuthorMetadata.Value.Name);
tokenHandlers["{Book TitleNoSub}"] = m => titleNoSub;
@ -313,6 +326,20 @@ namespace NzbDrone.Core.Organizer
}
}
private void AddBookTitlePlaceholderTokens(Dictionary<string, Func<TokenMatch, string>> tokenHandlers)
{
tokenHandlers["{Book Title}"] = m => null;
tokenHandlers["{Book CleanTitle}"] = m => null;
tokenHandlers["{Book TitleThe}"] = m => null;
}
private void AddBookTitleTokens(Dictionary<string, Func<TokenMatch, string>> tokenHandlers, Edition edition, int maxLength)
{
tokenHandlers["{Book Title}"] = m => GetBookTitle(edition.Title, maxLength);
tokenHandlers["{Book CleanTitle}"] = m => GetBookTitle(CleanTitle(edition.Title), maxLength);
tokenHandlers["{Book TitleThe}"] = m => GetBookTitle(TitleThe(edition.Title), maxLength);
}
private void AddBookFileTokens(Dictionary<string, Func<TokenMatch, string>> tokenHandlers, BookFile bookFile)
{
tokenHandlers["{Original Title}"] = m => GetOriginalTitle(bookFile);
@ -372,13 +399,29 @@ namespace NzbDrone.Core.Organizer
tokenHandlers["{Custom Formats}"] = m => string.Join(" ", customFormats.Where(x => x.IncludeCustomFormatWhenRenaming));
}
private string ReplaceTokens(string pattern, Dictionary<string, Func<TokenMatch, string>> tokenHandlers, NamingConfig namingConfig)
private string ReplaceTokens(string pattern, Dictionary<string, Func<TokenMatch, string>> tokenHandlers, NamingConfig namingConfig, bool escape = false)
{
return TitleRegex.Replace(pattern, match => ReplaceToken(match, tokenHandlers, namingConfig));
return TitleRegex.Replace(pattern, match => ReplaceToken(match, tokenHandlers, namingConfig, escape));
}
private string ReplaceToken(Match match, Dictionary<string, Func<TokenMatch, string>> tokenHandlers, NamingConfig namingConfig)
private string ReplaceToken(Match match, Dictionary<string, Func<TokenMatch, string>> tokenHandlers, NamingConfig namingConfig, bool escape = false)
{
if (match.Groups["escaped"].Success)
{
if (escape)
{
return match.Value;
}
else if (match.Value == "{{")
{
return "{";
}
else if (match.Value == "}}")
{
return "}";
}
}
var tokenMatch = new TokenMatch
{
RegexMatch = match,
@ -396,7 +439,14 @@ namespace NzbDrone.Core.Organizer
var tokenHandler = tokenHandlers.GetValueOrDefault(tokenMatch.Token, m => string.Empty);
var replacementText = tokenHandler(tokenMatch).Trim();
var replacementText = tokenHandler(tokenMatch);
if (replacementText == null)
{
// Preserve original token if handler returned null
return match.Value;
}
replacementText = replacementText.Trim();
if (tokenMatch.Token.All(t => !char.IsLetter(t) || char.IsLower(t)))
{
@ -419,6 +469,11 @@ namespace NzbDrone.Core.Organizer
replacementText = tokenMatch.Prefix + replacementText + tokenMatch.Suffix;
}
if (escape)
{
replacementText = replacementText.Replace("{", "{{").Replace("}", "}}");
}
return replacementText;
}
@ -456,9 +511,9 @@ namespace NzbDrone.Core.Organizer
return $"{prefix}{tokenText1}{separator}{tokenText2}{suffix}";
}
private BookFormat[] GetTrackFormat(string pattern)
private BookFormat[] GetBookFormat(string pattern)
{
return _trackFormatCache.Get(pattern, () => SeasonEpisodePatternRegex.Matches(pattern).OfType<Match>()
return _bookFormatCache.Get(pattern, () => SeasonEpisodePatternRegex.Matches(pattern).OfType<Match>()
.Select(match => new BookFormat
{
BookSeparator = match.Groups["episodeSeparator"].Value,
@ -467,6 +522,23 @@ namespace NzbDrone.Core.Organizer
}).ToArray());
}
private string GetBookTitle(string title, int maxLength)
{
if (title.GetByteCount() <= maxLength)
{
return title;
}
var titleLength = title.GetByteCount();
if (titleLength + 3 <= maxLength)
{
return $"{title.TrimEnd(' ', '.')}{{ellipsis}}";
}
return $"{title.Truncate(maxLength - 3).TrimEnd(' ', '.')}{{ellipsis}}";
}
private string GetQualityProper(QualityModel quality)
{
if (quality.Revision.Version > 1)
@ -497,6 +569,16 @@ namespace NzbDrone.Core.Organizer
return Path.GetFileNameWithoutExtension(bookFile.Path);
}
private int GetLengthWithoutBookTitle(string pattern, NamingConfig namingConfig)
{
var tokenHandlers = new Dictionary<string, Func<TokenMatch, string>>(FileNameBuilderTokenEqualityComparer.Instance);
tokenHandlers["{Book Title}"] = m => string.Empty;
tokenHandlers["{Book CleanTitle}"] = m => string.Empty;
tokenHandlers["{Book TitleThe}"] = m => string.Empty;
var result = ReplaceTokens(pattern, tokenHandlers, namingConfig);
return result.GetByteCount();
}
private static string CleanFileName(string name, NamingConfig namingConfig)
{
var result = name;

Write Preview
Loading…
Cancel
Save