From 3f6f4fc65fa91368126a866a15955f8bebdb09c1 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 8 Oct 2023 11:59:17 -0500 Subject: [PATCH 1/3] Remove db calls from list threads --- .../ImportListSyncServiceFixture.cs | 4 +- .../FetchAndParseImportListService.cs | 74 +++++-------------- .../ImportLists/ImportListBase.cs | 5 +- .../ImportLists/ImportListSyncService.cs | 62 ++++++++++++++-- 4 files changed, 81 insertions(+), 64 deletions(-) diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs index 95283449c..a58aa25e5 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs @@ -60,7 +60,7 @@ namespace NzbDrone.Core.Test.ImportList { Movies = _list1Movies, AnyFailure = false, - SyncedLists = 1 + SyncedLists = new List { 1 } }; _commandAll = new ImportListSyncCommand @@ -100,7 +100,7 @@ namespace NzbDrone.Core.Test.ImportList private void GivenNoListSync() { - _importListFetch.SyncedLists = 0; + _importListFetch.SyncedLists = new List(); } private void GivenCleanLevel(string cleanLevel) diff --git a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs index d47aeb71f..828c3b428 100644 --- a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs +++ b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs @@ -5,9 +5,6 @@ using System.Threading.Tasks; using NLog; using NzbDrone.Common.Instrumentation.Extensions; using NzbDrone.Common.TPL; -using NzbDrone.Core.ImportLists.ImportListMovies; -using NzbDrone.Core.MetadataSource; -using NzbDrone.Core.Movies; namespace NzbDrone.Core.ImportLists { @@ -21,26 +18,14 @@ namespace NzbDrone.Core.ImportLists { private readonly IImportListFactory _importListFactory; private readonly IImportListStatusService _importListStatusService; - private readonly IImportListMovieService _listMovieService; - private readonly ISearchForNewMovie _movieSearch; - private readonly IProvideMovieInfo _movieInfoService; - private readonly IMovieMetadataService _movieMetadataService; private readonly Logger _logger; public FetchAndParseImportListService(IImportListFactory importListFactory, IImportListStatusService importListStatusService, - IImportListMovieService listMovieService, - ISearchForNewMovie movieSearch, - IProvideMovieInfo movieInfoService, - IMovieMetadataService movieMetadataService, Logger logger) { _importListFactory = importListFactory; _importListStatusService = importListStatusService; - _listMovieService = listMovieService; - _movieSearch = movieSearch; - _movieInfoService = movieInfoService; - _movieMetadataService = movieMetadataService; _logger = logger; } @@ -101,21 +86,17 @@ namespace NzbDrone.Core.ImportLists if (!importListReports.AnyFailure) { - var alreadyMapped = result.Movies.Where(x => importListReports.Movies.Any(r => r.TmdbId == x.TmdbId)); - var listMovies = MapMovieReports(importListReports.Movies.Where(x => result.Movies.All(r => r.TmdbId != x.TmdbId))).Where(x => x.TmdbId > 0).ToList(); + var listMovies = importListReports.Movies; - listMovies.AddRange(alreadyMapped); - listMovies = listMovies.DistinctBy(x => x.TmdbId).ToList(); listMovies.ForEach(m => m.ListId = importList.Definition.Id); result.Movies.AddRange(listMovies); - _listMovieService.SyncMoviesForList(listMovies, importList.Definition.Id); + + result.SyncedWithoutFailure.Add(importList.Definition.Id); } result.AnyFailure |= importListReports.AnyFailure; - result.SyncedLists++; - - _importListStatusService.UpdateListSyncStatus(importList.Definition.Id); + result.SyncedLists.Add(importList.Definition.Id); } } catch (Exception e) @@ -129,9 +110,17 @@ namespace NzbDrone.Core.ImportLists Task.WaitAll(taskList.ToArray()); + foreach (var list in importLists) + { + if (result.SyncedLists.Contains(list.Definition.Id)) + { + _importListStatusService.UpdateListSyncStatus(list.Definition.Id); + } + } + result.Movies = result.Movies.DistinctBy(r => new { r.TmdbId, r.ImdbId, r.Title }).ToList(); - _logger.Debug("Found {0} total reports from {1} lists", result.Movies.Count, result.SyncedLists); + _logger.Debug("Found {0} total reports from {1} lists", result.Movies.Count, result.SyncedLists.Count); return result; } @@ -160,19 +149,19 @@ namespace NzbDrone.Core.ImportLists if (!importListReports.AnyFailure) { - var listMovies = MapMovieReports(importListReports.Movies) - .Where(x => x.TmdbId > 0) - .DistinctBy(x => x.TmdbId) - .ToList(); + var listMovies = importListReports.Movies; listMovies.ForEach(m => m.ListId = importList.Definition.Id); result.Movies.AddRange(listMovies); - _listMovieService.SyncMoviesForList(listMovies, importList.Definition.Id); + + result.SyncedWithoutFailure.Add(importList.Definition.Id); } result.AnyFailure |= importListReports.AnyFailure; + result.SyncedLists.Add(importList.Definition.Id); + _importListStatusService.UpdateListSyncStatus(importList.Definition.Id); } } @@ -187,32 +176,5 @@ namespace NzbDrone.Core.ImportLists return result; } - - private List MapMovieReports(IEnumerable reports) - { - var mappedMovies = reports.Select(m => _movieSearch.MapMovieToTmdbMovie(new MovieMetadata { Title = m.Title, TmdbId = m.TmdbId, ImdbId = m.ImdbId, Year = m.Year })) - .Where(x => x != null) - .DistinctBy(x => x.TmdbId) - .ToList(); - - _movieMetadataService.UpsertMany(mappedMovies); - - var mappedListMovies = new List(); - - foreach (var movieMeta in mappedMovies) - { - var mappedListMovie = new ImportListMovie(); - - if (movieMeta != null) - { - mappedListMovie.MovieMetadata = movieMeta; - mappedListMovie.MovieMetadataId = movieMeta.Id; - } - - mappedListMovies.Add(mappedListMovie); - } - - return mappedListMovies; - } } } diff --git a/src/NzbDrone.Core/ImportLists/ImportListBase.cs b/src/NzbDrone.Core/ImportLists/ImportListBase.cs index e8c962317..247d8d611 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListBase.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListBase.cs @@ -15,11 +15,14 @@ namespace NzbDrone.Core.ImportLists public ImportListFetchResult() { Movies = new List(); + SyncedLists = new List(); + SyncedWithoutFailure = new List(); } public List Movies { get; set; } public bool AnyFailure { get; set; } - public int SyncedLists { get; set; } + public List SyncedLists { get; set; } + public List SyncedWithoutFailure { get; set; } } public abstract class ImportListBase : IImportList diff --git a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs index f73d0b7dd..4a031b47e 100644 --- a/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs +++ b/src/NzbDrone.Core/ImportLists/ImportListSyncService.cs @@ -7,6 +7,7 @@ using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists.ImportExclusions; using NzbDrone.Core.ImportLists.ImportListMovies; using NzbDrone.Core.Messaging.Commands; +using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Movies; namespace NzbDrone.Core.ImportLists @@ -17,6 +18,8 @@ namespace NzbDrone.Core.ImportLists private readonly IImportListFactory _importListFactory; private readonly IFetchAndParseImportList _listFetcherAndParser; private readonly IMovieService _movieService; + private readonly IMovieMetadataService _movieMetadataService; + private readonly ISearchForNewMovie _movieSearch; private readonly IAddMovieService _addMovieService; private readonly IConfigService _configService; private readonly IImportExclusionsService _exclusionService; @@ -25,6 +28,8 @@ namespace NzbDrone.Core.ImportLists public ImportListSyncService(IImportListFactory importListFactory, IFetchAndParseImportList listFetcherAndParser, IMovieService movieService, + IMovieMetadataService movieMetadataService, + ISearchForNewMovie movieSearch, IAddMovieService addMovieService, IConfigService configService, IImportExclusionsService exclusionService, @@ -34,6 +39,8 @@ namespace NzbDrone.Core.ImportLists _importListFactory = importListFactory; _listFetcherAndParser = listFetcherAndParser; _movieService = movieService; + _movieMetadataService = movieMetadataService; + _movieSearch = movieSearch; _addMovieService = addMovieService; _exclusionService = exclusionService; _listMovieService = listMovieService; @@ -52,17 +59,17 @@ namespace NzbDrone.Core.ImportLists var listItemsResult = _listFetcherAndParser.Fetch(); - if (listItemsResult.SyncedLists == 0) + if (listItemsResult.SyncedLists.Count == 0) { return; } + ProcessListItems(listItemsResult); + if (!listItemsResult.AnyFailure) { CleanLibrary(); } - - ProcessListItems(listItemsResult); } private void SyncList(ImportListDefinition definition) @@ -125,7 +132,25 @@ namespace NzbDrone.Core.ImportLists private void ProcessListItems(ImportListFetchResult listFetchResult) { - listFetchResult.Movies = listFetchResult.Movies.DistinctBy(x => + var allMappedMovies = new List(); + + // Sync ListMovies table for Discovery view and Cleaning task + foreach (var listId in listFetchResult.SyncedWithoutFailure) + { + var listMovies = listFetchResult.Movies.Where(x => x.ListId == listId); + var alreadyMapped = allMappedMovies.Where(x => listMovies.Any(r => r.TmdbId == x.TmdbId)); + var mappedListMovies = MapMovieReports(listMovies.Where(x => allMappedMovies.All(r => r.TmdbId != x.TmdbId)).ToList()).Where(x => x.TmdbId > 0).ToList(); + + mappedListMovies.AddRange(alreadyMapped); + mappedListMovies = mappedListMovies.DistinctBy(x => x.TmdbId).ToList(); + mappedListMovies.ForEach(m => m.ListId = listId); + + allMappedMovies.AddRange(mappedListMovies); + + _listMovieService.SyncMoviesForList(mappedListMovies, listId); + } + + allMappedMovies = allMappedMovies.DistinctBy(x => { if (x.TmdbId != 0) { @@ -140,7 +165,7 @@ namespace NzbDrone.Core.ImportLists return x.Title; }).ToList(); - var listedMovies = listFetchResult.Movies.ToList(); + var listedMovies = allMappedMovies; var importExclusions = _exclusionService.GetAllExclusions(); var dbMovies = _movieService.AllMovieTmdbIds(); @@ -168,6 +193,33 @@ namespace NzbDrone.Core.ImportLists } } + private List MapMovieReports(IEnumerable reports) + { + var mappedMovies = reports.Select(m => _movieSearch.MapMovieToTmdbMovie(new MovieMetadata { Title = m.Title, TmdbId = m.TmdbId, ImdbId = m.ImdbId, Year = m.Year })) + .Where(x => x != null) + .DistinctBy(x => x.TmdbId) + .ToList(); + + _movieMetadataService.UpsertMany(mappedMovies); + + var mappedListMovies = new List(); + + foreach (var movieMeta in mappedMovies) + { + var mappedListMovie = new ImportListMovie(); + + if (movieMeta != null) + { + mappedListMovie.MovieMetadata = movieMeta; + mappedListMovie.MovieMetadataId = movieMeta.Id; + } + + mappedListMovies.Add(mappedListMovie); + } + + return mappedListMovies; + } + public void Execute(ImportListSyncCommand message) { if (message.DefinitionId.HasValue) From b279984bd7e58514e92dabbe7b19a93249f3ac0b Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 8 Oct 2023 12:37:21 -0500 Subject: [PATCH 2/3] fixup! Remove db calls from list threads --- .../ImportListSyncServiceFixture.cs | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs index a58aa25e5..d3c50b7a0 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/ImportListSyncServiceFixture.cs @@ -7,6 +7,7 @@ using NzbDrone.Core.Configuration; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.ImportExclusions; using NzbDrone.Core.ImportLists.ImportListMovies; +using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Movies; using NzbDrone.Core.Test.Framework; @@ -59,8 +60,7 @@ namespace NzbDrone.Core.Test.ImportList _importListFetch = new ImportListFetchResult { Movies = _list1Movies, - AnyFailure = false, - SyncedLists = new List { 1 } + AnyFailure = false }; _commandAll = new ImportListSyncCommand @@ -84,6 +84,10 @@ namespace NzbDrone.Core.Test.ImportList .Setup(v => v.MovieExists(It.IsAny())) .Returns(false); + Mocker.GetMock() + .Setup(v => v.MovieExists(It.IsAny())) + .Returns(false); + Mocker.GetMock() .Setup(v => v.AllMovieTmdbIds()) .Returns(new List()); @@ -91,6 +95,10 @@ namespace NzbDrone.Core.Test.ImportList Mocker.GetMock() .Setup(v => v.Fetch()) .Returns(_importListFetch); + + Mocker.GetMock() + .Setup(v => v.MapMovieToTmdbMovie(It.IsAny())) + .Returns(m => new MovieMetadata { TmdbId = m.TmdbId }); } private void GivenListFailure() @@ -101,6 +109,7 @@ namespace NzbDrone.Core.Test.ImportList private void GivenNoListSync() { _importListFetch.SyncedLists = new List(); + _importListFetch.SyncedWithoutFailure = new List(); } private void GivenCleanLevel(string cleanLevel) @@ -114,6 +123,9 @@ namespace NzbDrone.Core.Test.ImportList { var importListDefinition = new ImportListDefinition { Id = id, EnableAuto = enabledAuto }; + _importListFetch.SyncedLists.Add(id); + _importListFetch.SyncedWithoutFailure.Add(id); + Mocker.GetMock() .Setup(v => v.Get(id)) .Returns(importListDefinition); From 200be6451ab6ef011c9e272ed3bfe0947d41a9e8 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 8 Oct 2023 12:40:04 -0500 Subject: [PATCH 3/3] fixup! Remove db calls from list threads --- .../FetchAndParseImportListServiceFixture.cs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListServiceFixture.cs b/src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListServiceFixture.cs index e89ed32d7..e6504abb7 100644 --- a/src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListServiceFixture.cs +++ b/src/NzbDrone.Core.Test/ImportListTests/FetchAndParseImportListServiceFixture.cs @@ -7,8 +7,6 @@ using Moq; using NUnit.Framework; using NzbDrone.Core.ImportLists; using NzbDrone.Core.ImportLists.ImportListMovies; -using NzbDrone.Core.MetadataSource; -using NzbDrone.Core.Movies; using NzbDrone.Core.Test.Framework; namespace NzbDrone.Core.Test.ImportListTests @@ -36,10 +34,6 @@ namespace NzbDrone.Core.Test.ImportListTests _listMovies = Builder.CreateListOfSize(5) .Build().ToList(); - - Mocker.GetMock() - .Setup(v => v.MapMovieToTmdbMovie(It.IsAny())) - .Returns(m => new MovieMetadata { TmdbId = m.TmdbId }); } private void GivenList(int id, bool enabled, bool enabledAuto, ImportListFetchResult fetchResult) @@ -135,9 +129,6 @@ namespace NzbDrone.Core.Test.ImportListTests var listResult = Subject.Fetch(); listResult.AnyFailure.Should().BeFalse(); - - Mocker.GetMock() - .Verify(v => v.SyncMoviesForList(It.IsAny>(), listId), Times.Once()); } [Test] @@ -149,9 +140,6 @@ namespace NzbDrone.Core.Test.ImportListTests var listResult = Subject.Fetch(); listResult.AnyFailure.Should().BeTrue(); - - Mocker.GetMock() - .Verify(v => v.SyncMoviesForList(It.IsAny>(), listId), Times.Never()); } [Test] @@ -166,9 +154,6 @@ namespace NzbDrone.Core.Test.ImportListTests var listResult = Subject.Fetch(); listResult.AnyFailure.Should().BeTrue(); - - Mocker.GetMock() - .Verify(v => v.SyncMoviesForList(It.IsAny>(), passedListId), Times.Once()); } [Test]