Fixed: Contain memory usage during library import

src/Directory.Packages.props

@@ -13,8 +13,10 @@
     <PackageVersion Include="FluentValidation" Version="8.6.2" />
     <PackageVersion Include="Ical.Net" Version="4.2.0" />
     <PackageVersion Include="ImpromptuInterface" Version="7.0.1" />
+    <PackageVersion Include="LazyCache" Version="2.4.0" />
     <PackageVersion Include="Mailkit" Version="3.1.0" />
     <PackageVersion Include="Microsoft.AspNetCore.SignalR.Client" Version="6.0.1" />
+    <PackageVersion Include="Microsoft.Extensions.Caching.Memory" Version="6.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Configuration" Version="6.0.0" />
     <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="6.0.0" />
     <PackageVersion Include="Microsoft.Extensions.Hosting.WindowsServices" Version="6.0.0" />

src/NzbDrone.Core/Books/Model/Author.cs

@@ -77,7 +77,9 @@ namespace NzbDrone.Core.Books
             RootFolderPath = other.RootFolderPath;
             Added = other.Added;
             QualityProfileId = other.QualityProfileId;
+            QualityProfile = other.QualityProfile;
             MetadataProfileId = other.MetadataProfileId;
+            MetadataProfile = other.MetadataProfile;
             Tags = other.Tags;
             AddOptions = other.AddOptions;
         }

src/NzbDrone.Core/MediaFiles/BookImport/Identification/IdentificationService.cs

@@ -74,7 +74,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
             // 3 find best candidate
             var watch = System.Diagnostics.Stopwatch.StartNew();
 
-            _logger.Debug("Starting track identification");
+            _logger.Debug("Starting book identification");
 
             var releases = GetLocalBookReleases(localTracks, config.SingleRelease);
 
@@ -183,7 +183,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Identification
 
             _logger.Debug($"Best release found in {watch.ElapsedMilliseconds}ms");
 
-            localBookRelease.PopulateMatch();
+            localBookRelease.PopulateMatch(config.KeepAllEditions);
 
             _logger.Debug($"IdentifyRelease done in {watch.ElapsedMilliseconds}ms");
         }

src/NzbDrone.Core/MediaFiles/BookImport/ImportDecisionMaker.cs

@@ -42,6 +42,7 @@ namespace NzbDrone.Core.MediaFiles.BookImport
         public bool SingleRelease { get; set; }
         public bool IncludeExisting { get; set; }
         public bool AddNewAuthors { get; set; }
+        public bool KeepAllEditions { get; set; }
     }
 
     public class ImportDecisionMaker : IMakeImportDecision

src/NzbDrone.Core/MediaFiles/BookImport/Manual/ManualImportService.cs

@@ -117,7 +117,8 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Manual
                     NewDownload = true,
                     SingleRelease = false,
                     IncludeExisting = !replaceExistingFiles,
-                    AddNewAuthors = false
+                    AddNewAuthors = false,
+                    KeepAllEditions = true
                 };
 
                 var decision = _importDecisionMaker.GetImportDecisions(files, null, null, config);
@@ -162,7 +163,8 @@ namespace NzbDrone.Core.MediaFiles.BookImport.Manual
                 NewDownload = true,
                 SingleRelease = false,
                 IncludeExisting = !replaceExistingFiles,
-                AddNewAuthors = false
+                AddNewAuthors = false,
+                KeepAllEditions = true
             };
 
             var decisions = _importDecisionMaker.GetImportDecisions(authorFiles, idOverrides, itemInfo, config);

src/NzbDrone.Core/MetadataSource/BookInfo/BookInfoProxy.cs

@@ -5,6 +5,9 @@ using System.Linq;
 using System.Net;
 using System.Text.Json;
 using System.Threading;
+using LazyCache;
+using LazyCache.Providers;
+using Microsoft.Extensions.Caching.Memory;
 using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Extensions;
@@ -35,7 +38,7 @@ namespace NzbDrone.Core.MetadataSource.BookInfo
         private readonly Logger _logger;
         private readonly IMetadataRequestBuilder _requestBuilder;
         private readonly ICached<HashSet<string>> _cache;
-        private readonly ICached<Author> _authorCache;
+        private readonly CachingService _authorCache;
 
         public BookInfoProxy(IHttpClient httpClient,
                              ICachedHttpResponseService cachedHttpClient,
@@ -55,8 +58,13 @@ namespace NzbDrone.Core.MetadataSource.BookInfo
             _editionService = editionService;
             _requestBuilder = requestBuilder;
             _cache = cacheManager.GetCache<HashSet<string>>(GetType());
-            _authorCache = cacheManager.GetRollingCache<Author>(GetType(), "authorCache", TimeSpan.FromMinutes(5));
             _logger = logger;
+
+            _authorCache = new CachingService(new MemoryCacheProvider(new MemoryCache(new MemoryCacheOptions { SizeLimit = 10 })));
+            _authorCache.DefaultCachePolicy = new CacheDefaults
+            {
+                DefaultCacheDurationSeconds = 60
+            };
         }
 
         public HashSet<string> GetChangedAuthors(DateTime startTime)
@@ -532,7 +540,16 @@ namespace NzbDrone.Core.MetadataSource.BookInfo
 
         private Author PollAuthor(string foreignAuthorId)
         {
-            return _authorCache.Get(foreignAuthorId, () => PollAuthorUncached(foreignAuthorId));
+            return _authorCache.GetOrAdd(foreignAuthorId,
+                () => PollAuthorUncached(foreignAuthorId),
+                new LazyCacheEntryOptions
+                {
+                    AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(10),
+                    ImmediateAbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(10),
+                    Size = 1,
+                    SlidingExpiration = TimeSpan.FromMinutes(1),
+                    ExpirationMode = ExpirationMode.ImmediateEviction
+                }.RegisterPostEvictionCallback((key, value, reason, state) => _logger.Debug($"Clearing cache for {key} due to {reason}")));
         }
 
         private Author PollAuthorUncached(string foreignAuthorId)

src/NzbDrone.Core/Parser/Model/LocalEdition.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Books;
 using NzbDrone.Core.MediaFiles.BookImport.Identification;
+using SixLabors.ImageSharp.Processing;
 
 namespace NzbDrone.Core.Parser.Model
 {
@@ -35,17 +36,77 @@ namespace NzbDrone.Core.Parser.Model
         public List<LocalBook> ExistingTracks { get; set; }
         public bool NewDownload { get; set; }
 
-        public void PopulateMatch()
+        public void PopulateMatch(bool keepAllEditions)
         {
             if (Edition != null)
             {
                 LocalBooks = LocalBooks.Concat(ExistingTracks).DistinctBy(x => x.Path).ToList();
-                foreach (var localTrack in LocalBooks)
+
+                if (!keepAllEditions)
+                {
+                    // Manually clone the edition / book to avoid holding references to *every* edition we have
+                    // seen during the matching process
+                    var edition = new Edition();
+                    edition.UseMetadataFrom(Edition);
+                    edition.UseDbFieldsFrom(Edition);
+
+                    var fullBook = Edition.Book.Value;
+
+                    var book = new Book();
+                    book.UseMetadataFrom(fullBook);
+                    book.UseDbFieldsFrom(fullBook);
+                    book.Author.Value.UseMetadataFrom(fullBook.Author.Value);
+                    book.Author.Value.UseDbFieldsFrom(fullBook.Author.Value);
+                    book.Author.Value.Metadata = fullBook.AuthorMetadata.Value;
+                    book.AuthorMetadata = fullBook.AuthorMetadata.Value;
+                    book.BookFiles = fullBook.BookFiles;
+                    book.Editions = new List<Edition> { edition };
+
+                    if (fullBook.SeriesLinks.IsLoaded)
+                    {
+                        book.SeriesLinks = fullBook.SeriesLinks.Value.Select(l => new SeriesBookLink
+                        {
+                            Book = book,
+                            Series = new Series
+                            {
+                                ForeignSeriesId = l.Series.Value.ForeignSeriesId,
+                                Title = l.Series.Value.Title,
+                                Description = l.Series.Value.Description,
+                                Numbered = l.Series.Value.Numbered,
+                                WorkCount = l.Series.Value.WorkCount,
+                                PrimaryWorkCount = l.Series.Value.PrimaryWorkCount
+                            },
+                            IsPrimary = l.IsPrimary,
+                            Position = l.Position,
+                            SeriesPosition = l.SeriesPosition
+                        }).ToList();
+                    }
+                    else
+                    {
+                        book.SeriesLinks = fullBook.SeriesLinks;
+                    }
+
+                    edition.Book = book;
+
+                    Edition = edition;
+
+                    foreach (var localTrack in LocalBooks)
+                    {
+                        localTrack.Edition = edition;
+                        localTrack.Book = book;
+                        localTrack.Author = book.Author.Value;
+                        localTrack.PartCount = LocalBooks.Count;
+                    }
+                }
+                else
                 {
-                    localTrack.Edition = Edition;
-                    localTrack.Book = Edition.Book.Value;
-                    localTrack.Author = Edition.Book.Value.Author.Value;
-                    localTrack.PartCount = LocalBooks.Count;
+                    foreach (var localTrack in LocalBooks)
+                    {
+                        localTrack.Edition = Edition;
+                        localTrack.Book = Edition.Book.Value;
+                        localTrack.Author = Edition.Book.Value.Author.Value;
+                        localTrack.PartCount = LocalBooks.Count;
+                    }
                 }
             }
         }

src/NzbDrone.Core/Readarr.Core.csproj

@@ -4,9 +4,11 @@
   </PropertyGroup>
   <ItemGroup>
     <PackageReference Include="Dapper" />
+    <PackageReference Include="LazyCache" />
     <PackageReference Include="System.Text.Json" />
     <PackageReference Include="System.Text.Encoding.CodePages" />
     <PackageReference Include="System.Memory" />
+    <PackageReference Include="Microsoft.Extensions.Caching.Memory" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
     <PackageReference Include="Microsoft.Extensions.Logging" />
     <PackageReference Include="Microsoft.Extensions.Configuration" />