From 0f93e04186f24abdb0cf0b3ba6a3505fda834e06 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 20 Aug 2023 16:46:55 -0500 Subject: [PATCH] Switch to Parallel.ForEach for list processing with MaxParallelism --- .../FetchAndParseImportListService.cs | 66 ++++++++----------- 1 file changed, 26 insertions(+), 40 deletions(-) diff --git a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs index 40dc132b3..06fdd1371 100644 --- a/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs +++ b/src/NzbDrone.Core/ImportLists/FetchAndParseImportListService.cs @@ -4,7 +4,6 @@ using System.Linq; using System.Threading.Tasks; using NLog; using NzbDrone.Common.Instrumentation.Extensions; -using NzbDrone.Common.TPL; using NzbDrone.Core.ImportLists.ImportListMovies; using NzbDrone.Core.MetadataSource; using NzbDrone.Core.Movies; @@ -23,7 +22,6 @@ namespace NzbDrone.Core.ImportLists private readonly IImportListStatusService _importListStatusService; private readonly IImportListMovieService _listMovieService; private readonly ISearchForNewMovie _movieSearch; - private readonly IProvideMovieInfo _movieInfoService; private readonly IMovieMetadataService _movieMetadataService; private readonly Logger _logger; @@ -31,7 +29,6 @@ namespace NzbDrone.Core.ImportLists IImportListStatusService importListStatusService, IImportListMovieService listMovieService, ISearchForNewMovie movieSearch, - IProvideMovieInfo movieInfoService, IMovieMetadataService movieMetadataService, Logger logger) { @@ -39,7 +36,6 @@ namespace NzbDrone.Core.ImportLists _importListStatusService = importListStatusService; _listMovieService = listMovieService; _movieSearch = movieSearch; - _movieInfoService = movieInfoService; _movieMetadataService = movieMetadataService; _logger = logger; } @@ -58,10 +54,7 @@ namespace NzbDrone.Core.ImportLists _logger.Debug("Available import lists {0}", importLists.Count); - var taskList = new List(); - var taskFactory = new TaskFactory(TaskCreationOptions.LongRunning, TaskContinuationOptions.None); - - foreach (var importList in importLists) + Parallel.ForEach(importLists, new ParallelOptions { MaxDegreeOfParallelism = 5 }, importList => { var importListLocal = importList; var importListStatus = _importListStatusService.GetLastSyncListInfo(importListLocal.Definition.Id); @@ -74,7 +67,7 @@ namespace NzbDrone.Core.ImportLists { _logger.Trace("Skipping refresh of Import List {0} ({1}) due to minimum refresh interval. Next Sync after {2}", importList.Name, importListLocal.Definition.Name, importListNextSync); - continue; + return; } } @@ -86,48 +79,41 @@ namespace NzbDrone.Core.ImportLists { _logger.Debug("Temporarily ignoring Import List {0} ({1}) till {2} due to recent failures.", importList.Name, importListLocal.Definition.Name, blockedListStatus.DisabledTill.Value.ToLocalTime()); result.AnyFailure |= true; // Ensure we don't clean if a list is down - continue; + return; } - var task = taskFactory.StartNew(() => + try { - try + var importListReports = importListLocal.Fetch(); + + lock (result) { - var importListReports = importListLocal.Fetch(); + _logger.Debug("Found {0} from Import List {1} ({2})", importListReports.Movies.Count, importList.Name, importListLocal.Definition.Name); - lock (result) + if (!importListReports.AnyFailure) { - _logger.Debug("Found {0} from Import List {1} ({2})", importListReports.Movies.Count, importList.Name, importListLocal.Definition.Name); - - if (!importListReports.AnyFailure) - { - var alreadyMapped = result.Movies.Where(x => importListReports.Movies.Any(r => r.TmdbId == x.TmdbId)); - var listMovies = MapMovieReports(importListReports.Movies.Where(x => result.Movies.All(r => r.TmdbId != x.TmdbId)).ToList()).Where(x => x.TmdbId > 0).ToList(); - - listMovies.AddRange(alreadyMapped); - listMovies = listMovies.DistinctBy(x => x.TmdbId).ToList(); - listMovies.ForEach(m => m.ListId = importList.Definition.Id); + var alreadyMapped = result.Movies.Where(x => importListReports.Movies.Any(r => r.TmdbId == x.TmdbId)); + var listMovies = MapMovieReports(importListReports.Movies.Where(x => result.Movies.All(r => r.TmdbId != x.TmdbId)).ToList()).Where(x => x.TmdbId > 0).ToList(); - result.Movies.AddRange(listMovies); - _listMovieService.SyncMoviesForList(listMovies, importList.Definition.Id); - } + listMovies.AddRange(alreadyMapped); + listMovies = listMovies.DistinctBy(x => x.TmdbId).ToList(); + listMovies.ForEach(m => m.ListId = importList.Definition.Id); - result.AnyFailure |= importListReports.AnyFailure; - result.SyncedLists++; - - _importListStatusService.UpdateListSyncStatus(importList.Definition.Id); + result.Movies.AddRange(listMovies); + _listMovieService.SyncMoviesForList(listMovies, importList.Definition.Id); } - } - catch (Exception e) - { - _logger.Error(e, "Error during Import List Sync of {0} ({1})", importList.Name, importListLocal.Definition.Name); - } - }).LogExceptions(); - taskList.Add(task); - } + result.AnyFailure |= importListReports.AnyFailure; + result.SyncedLists++; - Task.WaitAll(taskList.ToArray()); + _importListStatusService.UpdateListSyncStatus(importList.Definition.Id); + } + } + catch (Exception e) + { + _logger.Error(e, "Error during Import List Sync of {0} ({1})", importList.Name, importListLocal.Definition.Name); + } + }); result.Movies = result.Movies.DistinctBy(r => new { r.TmdbId, r.ImdbId, r.Title }).ToList();