From b676f868ceaa0a22ac3f4414a81e92e8b17d5715 Mon Sep 17 00:00:00 2001 From: "kay.one" Date: Thu, 18 Jul 2013 22:05:07 -0700 Subject: [PATCH] Disk scan is much much much much faster. --- .../Messaging/MessageAggregator.cs | 6 +- .../NotAlreadyImportedSpecificationFixture.cs | 44 ------------- .../NotSampleSpecificationFixture.cs | 13 ++++ .../MediaFileTests/MediaFileServiceTest.cs | 65 ++++++++++++++++++- NzbDrone.Core.Test/NzbDrone.Core.Test.csproj | 1 - .../EpisodeImport/ImportDecisionMaker.cs | 11 +++- .../NotAlreadyImportedSpecification.cs | 30 --------- .../Specifications/NotSampleSpecification.cs | 9 ++- NzbDrone.Core/MediaFiles/MediaFileService.cs | 13 ++++ NzbDrone.Core/NzbDrone.Core.csproj | 1 - 10 files changed, 112 insertions(+), 81 deletions(-) delete mode 100644 NzbDrone.Core.Test/MediaFileTests/EpisodeImportTests/NotAlreadyImportedSpecificationFixture.cs delete mode 100644 NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotAlreadyImportedSpecification.cs diff --git a/NzbDrone.Common/Messaging/MessageAggregator.cs b/NzbDrone.Common/Messaging/MessageAggregator.cs index 5e73c5a4b..725f84ccf 100644 --- a/NzbDrone.Common/Messaging/MessageAggregator.cs +++ b/NzbDrone.Common/Messaging/MessageAggregator.cs @@ -1,4 +1,5 @@ using System; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using NLog; @@ -83,9 +84,12 @@ namespace NzbDrone.Common.Messaging _logger.Debug("{0} -> {1}", command.GetType().Name, handler.GetType().Name); + var sw = Stopwatch.StartNew(); + try { handler.Execute(command); + sw.Stop(); PublishEvent(new CommandCompletedEvent(command)); } catch (Exception e) @@ -98,7 +102,7 @@ namespace NzbDrone.Common.Messaging PublishEvent(new CommandExecutedEvent(command)); } - _logger.Debug("{0} <- {1}", command.GetType().Name, handler.GetType().Name); + _logger.Debug("{0} <- {1} [{2}]", command.GetType().Name, handler.GetType().Name, sw.Elapsed.ToString("")); } public void PublishCommand(string commandTypeName) diff --git a/NzbDrone.Core.Test/MediaFileTests/EpisodeImportTests/NotAlreadyImportedSpecificationFixture.cs b/NzbDrone.Core.Test/MediaFileTests/EpisodeImportTests/NotAlreadyImportedSpecificationFixture.cs deleted file mode 100644 index 28dd6a87a..000000000 --- a/NzbDrone.Core.Test/MediaFileTests/EpisodeImportTests/NotAlreadyImportedSpecificationFixture.cs +++ /dev/null @@ -1,44 +0,0 @@ -using FluentAssertions; -using NUnit.Framework; -using NzbDrone.Core.MediaFiles; -using NzbDrone.Core.MediaFiles.EpisodeImport.Specifications; -using NzbDrone.Core.Parser.Model; -using NzbDrone.Core.Test.Framework; - -namespace NzbDrone.Core.Test.MediaFileTests.EpisodeImportTests -{ - [TestFixture] - public class NotAlreadyImportedSpecificationFixture : CoreTest - { - private LocalEpisode _localEpisode; - - [SetUp] - public void Setup() - { - _localEpisode = new LocalEpisode - { - Path = @"C:\Test\30 Rock\30.rock.s01e01.avi" - }; - } - - [Test] - public void should_return_false_if_path_is_already_in_episodeFiles() - { - Mocker.GetMock() - .Setup(s => s.Exists(_localEpisode.Path)) - .Returns(true); - - Subject.IsSatisfiedBy(_localEpisode).Should().BeFalse(); - } - - [Test] - public void should_return_true_if_new_file() - { - Mocker.GetMock() - .Setup(s => s.Exists(_localEpisode.Path)) - .Returns(false); - - Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue(); - } - } -} diff --git a/NzbDrone.Core.Test/MediaFileTests/EpisodeImportTests/NotSampleSpecificationFixture.cs b/NzbDrone.Core.Test/MediaFileTests/EpisodeImportTests/NotSampleSpecificationFixture.cs index dd4f5a6e4..8d7afcd1f 100644 --- a/NzbDrone.Core.Test/MediaFileTests/EpisodeImportTests/NotSampleSpecificationFixture.cs +++ b/NzbDrone.Core.Test/MediaFileTests/EpisodeImportTests/NotSampleSpecificationFixture.cs @@ -112,5 +112,18 @@ namespace NzbDrone.Core.Test.MediaFileTests.EpisodeImportTests Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue(); } + + [Test] + public void should_not_check_lenght_if_file_is_large_enough() + { + WithFileSize(100.Megabytes()); + + Subject.IsSatisfiedBy(_localEpisode).Should().BeTrue(); + + + Mocker.GetMock().Verify(c => c.GetRunTime(It.IsAny()), Times.Never()); + } + + } } diff --git a/NzbDrone.Core.Test/MediaFileTests/MediaFileServiceTest.cs b/NzbDrone.Core.Test/MediaFileTests/MediaFileServiceTest.cs index 0d753a399..e9b876e27 100644 --- a/NzbDrone.Core.Test/MediaFileTests/MediaFileServiceTest.cs +++ b/NzbDrone.Core.Test/MediaFileTests/MediaFileServiceTest.cs @@ -1,4 +1,7 @@ +using System.Collections.Generic; +using System.Linq; using FluentAssertions; +using Moq; using NUnit.Framework; using NzbDrone.Core.MediaFiles; using NzbDrone.Core.Organizer; @@ -11,10 +14,70 @@ namespace NzbDrone.Core.Test.MediaFileTests { [Test] - [TestCase("Law & Order: Criminal Intent - S10E07 - Icarus [HDTV-720p]", "Law & Order- Criminal Intent - S10E07 - Icarus [HDTV-720p]")] + [TestCase("Law & Order: Criminal Intent - S10E07 - Icarus [HDTV-720p]", + "Law & Order- Criminal Intent - S10E07 - Icarus [HDTV-720p]")] public void CleanFileName(string name, string expectedName) { FileNameBuilder.CleanFilename(name).Should().Be(expectedName); } + + [Test] + public void filter_should_return_all_files_if_no_existing_files() + { + var files = new List() + { + "c:\\file1.avi", + "c:\\file2.avi", + "c:\\file3.avi", + }; + + Mocker.GetMock() + .Setup(c => c.GetFilesBySeries(It.IsAny())) + .Returns(new List()); + + + Subject.FilterExistingFiles(files, 10).Should().BeEquivalentTo(files); + } + + + [Test] + public void filter_should_return_none_if_all_files_exist() + { + var files = new List() + { + "c:\\file1.avi", + "c:\\file2.avi", + "c:\\file3.avi", + }; + + Mocker.GetMock() + .Setup(c => c.GetFilesBySeries(It.IsAny())) + .Returns(files.Select(f => new EpisodeFile { Path = f }).ToList()); + + + Subject.FilterExistingFiles(files, 10).Should().BeEmpty(); + } + + [Test] + public void filter_should_return_none_existing_files() + { + var files = new List() + { + "c:\\file1.avi", + "c:\\file2.avi", + "c:\\file3.avi", + }; + + Mocker.GetMock() + .Setup(c => c.GetFilesBySeries(It.IsAny())) + .Returns(new List + { + new EpisodeFile{Path = "c:\\file2.avi"} + }); + + + Subject.FilterExistingFiles(files, 10).Should().HaveCount(2); + Subject.FilterExistingFiles(files, 10).Should().NotContain("c:\\file2.avi"); + } } } \ No newline at end of file diff --git a/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index b521911c2..89e607835 100644 --- a/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -153,7 +153,6 @@ - diff --git a/NzbDrone.Core/MediaFiles/EpisodeImport/ImportDecisionMaker.cs b/NzbDrone.Core/MediaFiles/EpisodeImport/ImportDecisionMaker.cs index a157d8a6c..e748787b0 100644 --- a/NzbDrone.Core/MediaFiles/EpisodeImport/ImportDecisionMaker.cs +++ b/NzbDrone.Core/MediaFiles/EpisodeImport/ImportDecisionMaker.cs @@ -4,6 +4,7 @@ using System.Linq; using NLog; using NzbDrone.Common; using NzbDrone.Core.DecisionEngine; +using NzbDrone.Core.MediaFiles.EpisodeImport.Specifications; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Tv; @@ -20,23 +21,31 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport { private readonly IEnumerable _specifications; private readonly IParsingService _parsingService; + private readonly IMediaFileService _mediaFileService; private readonly IDiskProvider _diskProvider; private readonly Logger _logger; public ImportDecisionMaker(IEnumerable specifications, IParsingService parsingService, + IMediaFileService mediaFileService, IDiskProvider diskProvider, + Logger logger) { _specifications = specifications; _parsingService = parsingService; + _mediaFileService = mediaFileService; _diskProvider = diskProvider; _logger = logger; } public List GetImportDecisions(IEnumerable videoFiles, Series series) { - return GetDecisions(videoFiles, series).ToList(); + var newFiles = _mediaFileService.FilterExistingFiles(videoFiles.ToList(), series.Id); + + _logger.Debug("Analysing {0}/{1} files.", newFiles.Count, videoFiles.Count()); + + return GetDecisions(newFiles, series).ToList(); } private IEnumerable GetDecisions(IEnumerable videoFiles, Series series) diff --git a/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotAlreadyImportedSpecification.cs b/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotAlreadyImportedSpecification.cs deleted file mode 100644 index 8fe699877..000000000 --- a/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotAlreadyImportedSpecification.cs +++ /dev/null @@ -1,30 +0,0 @@ -using NLog; -using NzbDrone.Core.Parser.Model; - -namespace NzbDrone.Core.MediaFiles.EpisodeImport.Specifications -{ - public class NotAlreadyImportedSpecification : IImportDecisionEngineSpecification - { - private readonly IMediaFileService _mediaFileService; - private readonly Logger _logger; - - public NotAlreadyImportedSpecification(IMediaFileService mediaFileService, Logger logger) - { - _mediaFileService = mediaFileService; - _logger = logger; - } - - public string RejectionReason { get { return "Is Sample"; } } - - public bool IsSatisfiedBy(LocalEpisode localEpisode) - { - if (_mediaFileService.Exists(localEpisode.Path)) - { - _logger.Trace("[{0}] already exists in the database. skipping.", localEpisode.Path); - return false; - } - - return true; - } - } -} diff --git a/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotSampleSpecification.cs b/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotSampleSpecification.cs index aba58791a..746d3dc1c 100644 --- a/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotSampleSpecification.cs +++ b/NzbDrone.Core/MediaFiles/EpisodeImport/Specifications/NotSampleSpecification.cs @@ -41,11 +41,16 @@ namespace NzbDrone.Core.MediaFiles.EpisodeImport.Specifications return true; } + if (localEpisode.Size > SampleSizeLimit) + { + return true; + } + var runTime = _videoFileInfoReader.GetRunTime(localEpisode.Path); - if (localEpisode.Size < SampleSizeLimit && runTime.TotalMinutes < 3) + if (runTime.TotalMinutes < 3) { - _logger.Trace("[{0}] appears to be a sample.", localEpisode.Path); + _logger.Trace("[{0}] appears to be a sample. Size: {1} Runtime: {2}", localEpisode.Path, localEpisode.Size, runTime); return false; } diff --git a/NzbDrone.Core/MediaFiles/MediaFileService.cs b/NzbDrone.Core/MediaFiles/MediaFileService.cs index 20752ed6c..c6532b3e8 100644 --- a/NzbDrone.Core/MediaFiles/MediaFileService.cs +++ b/NzbDrone.Core/MediaFiles/MediaFileService.cs @@ -1,4 +1,6 @@ using System.Collections.Generic; +using System.IO; +using System.Linq; using NLog; using NzbDrone.Common.Messaging; using NzbDrone.Core.Configuration; @@ -15,6 +17,8 @@ namespace NzbDrone.Core.MediaFiles bool Exists(string path); EpisodeFile GetFileByPath(string path); List GetFilesBySeries(int seriesId); + + List FilterExistingFiles(List files, int seriesId); } public class MediaFileService : IMediaFileService, IHandleAsync @@ -65,6 +69,15 @@ namespace NzbDrone.Core.MediaFiles return _mediaFileRepository.GetFilesBySeries(seriesId); } + public List FilterExistingFiles(List files, int seriesId) + { + var seriesFiles = GetFilesBySeries(seriesId); + + if (!seriesFiles.Any()) return files; + + return files.Select(f => f.Normalize()).Except(seriesFiles.Select(c => c.Path)).ToList(); + } + public void HandleAsync(SeriesDeletedEvent message) { var files = GetFilesBySeries(message.Series.Id); diff --git a/NzbDrone.Core/NzbDrone.Core.csproj b/NzbDrone.Core/NzbDrone.Core.csproj index e61968fbf..5e6918d2d 100644 --- a/NzbDrone.Core/NzbDrone.Core.csproj +++ b/NzbDrone.Core/NzbDrone.Core.csproj @@ -276,7 +276,6 @@ -