From a621f0d49bf75e6cf6f4e96fd7883ceb32022526 Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Sun, 18 Sep 2016 11:04:56 -0700 Subject: [PATCH] Fixed: Prevent duplicate parsing of extra files --- .../Extras/ExistingExtraFileService.cs | 13 +++++----- .../ImportExistingExtraFileFilterResult.cs | 18 +++++++++++++ .../Extras/ImportExistingExtraFilesBase.cs | 16 +++++++----- .../MetaData/ExistingMetadataImporter.cs | 17 ++++++++++--- .../Others/ExistingOtherExtraImporter.cs | 9 ++++--- .../Subtitles/ExistingSubtitleImporter.cs | 9 ++++--- .../MediaFiles/DiskScanService.cs | 25 ++++++++++++++++--- src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + 8 files changed, 82 insertions(+), 26 deletions(-) create mode 100644 src/NzbDrone.Core/Extras/ImportExistingExtraFileFilterResult.cs diff --git a/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs b/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs index c4621d1a3..354d2b1c7 100644 --- a/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs +++ b/src/NzbDrone.Core/Extras/ExistingExtraFileService.cs @@ -14,18 +14,18 @@ namespace NzbDrone.Core.Extras public class ExistingExtraFileService : IHandle { private readonly IDiskProvider _diskProvider; + private readonly IDiskScanService _diskScanService; private readonly List _existingExtraFileImporters; - private readonly List _extraFileManagers; private readonly Logger _logger; public ExistingExtraFileService(IDiskProvider diskProvider, + IDiskScanService diskScanService, List existingExtraFileImporters, - List extraFileManagers, Logger logger) { _diskProvider = diskProvider; + _diskScanService = diskScanService; _existingExtraFileImporters = existingExtraFileImporters.OrderBy(e => e.Order).ToList(); - _extraFileManagers = extraFileManagers.OrderBy(e => e.Order).ToList(); _logger = logger; } @@ -41,9 +41,8 @@ namespace NzbDrone.Core.Extras _logger.Debug("Looking for existing extra files in {0}", series.Path); - var filesOnDisk = _diskProvider.GetFiles(series.Path, SearchOption.AllDirectories); - var possibleExtraFiles = filesOnDisk.Where(c => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(c).ToLower()) && - !c.StartsWith(Path.Combine(series.Path, "EXTRAS"))).ToList(); + var filesOnDisk = _diskScanService.GetNonVideoFiles(series.Path); + var possibleExtraFiles = _diskScanService.FilterFiles(series, filesOnDisk); var filteredFiles = possibleExtraFiles; var importedFiles = new List(); @@ -55,7 +54,7 @@ namespace NzbDrone.Core.Extras importedFiles.AddRange(imported.Select(f => Path.Combine(series.Path, f.RelativePath))); } - _logger.Info("Found {0} extra files", extraFiles); + _logger.Info("Found {0} extra files", extraFiles.Count); } } } diff --git a/src/NzbDrone.Core/Extras/ImportExistingExtraFileFilterResult.cs b/src/NzbDrone.Core/Extras/ImportExistingExtraFileFilterResult.cs new file mode 100644 index 000000000..910f338dd --- /dev/null +++ b/src/NzbDrone.Core/Extras/ImportExistingExtraFileFilterResult.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using NzbDrone.Core.Extras.Files; + +namespace NzbDrone.Core.Extras +{ + public class ImportExistingExtraFileFilterResult + where TExtraFile : ExtraFile, new() + { + public ImportExistingExtraFileFilterResult(List previouslyImported, List filesOnDisk) + { + PreviouslyImported = previouslyImported; + FilesOnDisk = filesOnDisk; + } + + public List PreviouslyImported { get; set; } + public List FilesOnDisk { get; set; } + } +} diff --git a/src/NzbDrone.Core/Extras/ImportExistingExtraFilesBase.cs b/src/NzbDrone.Core/Extras/ImportExistingExtraFilesBase.cs index 2c94038c6..a2dddaa69 100644 --- a/src/NzbDrone.Core/Extras/ImportExistingExtraFilesBase.cs +++ b/src/NzbDrone.Core/Extras/ImportExistingExtraFilesBase.cs @@ -21,7 +21,7 @@ namespace NzbDrone.Core.Extras public abstract int Order { get; } public abstract IEnumerable ProcessFiles(Series series, List filesOnDisk, List importedFiles); - public virtual List FilterAndClean(Series series, List filesOnDisk, List importedFiles) + public virtual ImportExistingExtraFileFilterResult FilterAndClean(Series series, List filesOnDisk, List importedFiles) { var seriesFiles = _extraFileService.GetFilesBySeries(series.Id); @@ -30,12 +30,16 @@ namespace NzbDrone.Core.Extras return Filter(series, filesOnDisk, importedFiles, seriesFiles); } - private List Filter(Series series, List filesOnDisk, List importedFiles, List seriesFiles) + private ImportExistingExtraFileFilterResult Filter(Series series, List filesOnDisk, List importedFiles, List seriesFiles) { - var filteredFiles = filesOnDisk; - - filteredFiles = filteredFiles.Except(seriesFiles.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance).ToList(); - return filteredFiles.Except(importedFiles, PathEqualityComparer.Instance).ToList(); + var previouslyImported = seriesFiles.IntersectBy(s => Path.Combine(series.Path, s.RelativePath), filesOnDisk, f => f, PathEqualityComparer.Instance).ToList(); + var filteredFiles = filesOnDisk.Except(previouslyImported.Select(f => Path.Combine(series.Path, f.RelativePath)).ToList(), PathEqualityComparer.Instance) + .Except(importedFiles, PathEqualityComparer.Instance) + .ToList(); + + // Return files that are already imported so they aren't imported again by other importers. + // Filter out files that were previously imported and as well as ones imported by other importers. + return new ImportExistingExtraFileFilterResult(previouslyImported, filteredFiles); } private void Clean(Series series, List filesOnDisk, List importedFiles, List seriesFiles) diff --git a/src/NzbDrone.Core/Extras/MetaData/ExistingMetadataImporter.cs b/src/NzbDrone.Core/Extras/MetaData/ExistingMetadataImporter.cs index eb4f8ee31..059be3e5f 100644 --- a/src/NzbDrone.Core/Extras/MetaData/ExistingMetadataImporter.cs +++ b/src/NzbDrone.Core/Extras/MetaData/ExistingMetadataImporter.cs @@ -5,6 +5,7 @@ using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Core.Extras.Files; using NzbDrone.Core.Extras.Metadata.Files; +using NzbDrone.Core.Extras.Subtitles; using NzbDrone.Core.Parser; using NzbDrone.Core.Tv; @@ -42,10 +43,17 @@ namespace NzbDrone.Core.Extras.Metadata _logger.Debug("Looking for existing metadata in {0}", series.Path); var metadataFiles = new List(); - var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles); + var filterResult = FilterAndClean(series, filesOnDisk, importedFiles); - foreach (var possibleMetadataFile in filteredFiles) + foreach (var possibleMetadataFile in filterResult.FilesOnDisk) { + // Don't process files that have known Subtitle file extensions (saves a bit of unecessary processing) + + if (SubtitleFileExtensions.Extensions.Contains(Path.GetExtension(possibleMetadataFile))) + { + continue; + } + foreach (var consumer in _consumers) { var metadata = consumer.FindMetadataFile(series, possibleMetadataFile); @@ -90,7 +98,10 @@ namespace NzbDrone.Core.Extras.Metadata _logger.Info("Found {0} existing metadata files", metadataFiles.Count); _metadataFileService.Upsert(metadataFiles); - return metadataFiles; + // Return files that were just imported along with files that were + // previously imported so previously imported files aren't imported twice + + return metadataFiles.Concat(filterResult.PreviouslyImported); } } } diff --git a/src/NzbDrone.Core/Extras/Others/ExistingOtherExtraImporter.cs b/src/NzbDrone.Core/Extras/Others/ExistingOtherExtraImporter.cs index d744c2259..6b3d3682b 100644 --- a/src/NzbDrone.Core/Extras/Others/ExistingOtherExtraImporter.cs +++ b/src/NzbDrone.Core/Extras/Others/ExistingOtherExtraImporter.cs @@ -38,9 +38,9 @@ namespace NzbDrone.Core.Extras.Others _logger.Debug("Looking for existing extra files in {0}", series.Path); var extraFiles = new List(); - var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles); + var filterResult = FilterAndClean(series, filesOnDisk, importedFiles); - foreach (var possibleExtraFile in filteredFiles) + foreach (var possibleExtraFile in filterResult.FilesOnDisk) { var localEpisode = _parsingService.GetLocalEpisode(possibleExtraFile, series); @@ -77,7 +77,10 @@ namespace NzbDrone.Core.Extras.Others _logger.Info("Found {0} existing other extra files", extraFiles.Count); _otherExtraFileService.Upsert(extraFiles); - return extraFiles; + // Return files that were just imported along with files that were + // previously imported so previously imported files aren't imported twice + + return extraFiles.Concat(filterResult.PreviouslyImported); } } } diff --git a/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs b/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs index 11e1b7742..eb095123e 100644 --- a/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs +++ b/src/NzbDrone.Core/Extras/Subtitles/ExistingSubtitleImporter.cs @@ -38,9 +38,9 @@ namespace NzbDrone.Core.Extras.Subtitles _logger.Debug("Looking for existing subtitle files in {0}", series.Path); var subtitleFiles = new List(); - var filteredFiles = FilterAndClean(series, filesOnDisk, importedFiles); + var filterResult = FilterAndClean(series, filesOnDisk, importedFiles); - foreach (var possibleSubtitleFile in filteredFiles) + foreach (var possibleSubtitleFile in filterResult.FilesOnDisk) { var extension = Path.GetExtension(possibleSubtitleFile); @@ -83,7 +83,10 @@ namespace NzbDrone.Core.Extras.Subtitles _logger.Info("Found {0} existing subtitle files", subtitleFiles.Count); _subtitleFileService.Upsert(subtitleFiles); - return subtitleFiles; + // Return files that were just imported along with files that were + // previously imported so previously imported files aren't imported twice + + return subtitleFiles.Concat(filterResult.PreviouslyImported); } } } diff --git a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs index a1e8eb1a4..a7a6d8ace 100644 --- a/src/NzbDrone.Core/MediaFiles/DiskScanService.cs +++ b/src/NzbDrone.Core/MediaFiles/DiskScanService.cs @@ -23,6 +23,8 @@ namespace NzbDrone.Core.MediaFiles { void Scan(Series series); string[] GetVideoFiles(string path, bool allDirectories = true); + string[] GetNonVideoFiles(string path, bool allDirectories = true); + List FilterFiles(Series series, IEnumerable files); } public class DiskScanService : @@ -59,7 +61,7 @@ namespace NzbDrone.Core.MediaFiles } private static readonly Regex ExcludedSubFoldersRegex = new Regex(@"(?:\\|\/|^)(extras|@eadir|\..+)(?:\\|\/)", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex ExcludedFilesRegex = new Regex(@"^\._|Thumbs\.db", RegexOptions.Compiled | RegexOptions.IgnoreCase); public void Scan(Series series) { @@ -133,10 +135,25 @@ namespace NzbDrone.Core.MediaFiles return mediaFileList.ToArray(); } - private IEnumerable FilterFiles(Series series, IEnumerable videoFiles) + public string[] GetNonVideoFiles(string path, bool allDirectories = true) { - return videoFiles.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file))) - .Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file))); + _logger.Debug("Scanning '{0}' for non-video files", path); + + var searchOption = allDirectories ? SearchOption.AllDirectories : SearchOption.TopDirectoryOnly; + var filesOnDisk = _diskProvider.GetFiles(path, searchOption); + + var mediaFileList = filesOnDisk.Where(file => !MediaFileExtensions.Extensions.Contains(Path.GetExtension(file).ToLower())) + .ToList(); + + _logger.Debug("{0} non-video files were found in {1}", mediaFileList.Count, path); + return mediaFileList.ToArray(); + } + + public List FilterFiles(Series series, IEnumerable files) + { + return files.Where(file => !ExcludedSubFoldersRegex.IsMatch(series.Path.GetRelativePath(file))) + .Where(file => !ExcludedFilesRegex.IsMatch(Path.GetFileName(file))) + .ToList(); } private void SetPermissions(string path) diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 00b0c0df6..6605dd5a5 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -495,6 +495,7 @@ +