From 8cac84b4ad72192420b5da6a4a02a1aeea5550bc Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Fri, 14 Oct 2011 08:37:19 -0700 Subject: [PATCH] PostDownloadProvider broken down further. Will try to reprocess _NzbDrone_ directories each pass, but will mark with an error when possible. Attempt to process _UNPACK_ and _FAILED_ directories 30 minutes after first detected by NzbDrone (to give SAB time to unpack properly before processing). --- NzbDrone.Core.Test/EpisodeProviderTest.cs | 12 +- NzbDrone.Core.Test/NzbDrone.Core.Test.csproj | 1 + .../PostDownloadProviderTest.cs | 190 ++++++++++++++++++ NzbDrone.Core/Model/PostDownloadInfoModel.cs | 15 ++ NzbDrone.Core/Model/PostDownloadStatusType.cs | 12 +- NzbDrone.Core/NzbDrone.Core.csproj | 1 + NzbDrone.Core/Providers/EpisodeProvider.cs | 6 +- .../Providers/Jobs/PostDownloadScanJob.cs | 2 +- .../Providers/PostDownloadProvider.cs | 160 ++++++++++++--- 9 files changed, 359 insertions(+), 40 deletions(-) create mode 100644 NzbDrone.Core.Test/PostDownloadProviderTest.cs create mode 100644 NzbDrone.Core/Model/PostDownloadInfoModel.cs diff --git a/NzbDrone.Core.Test/EpisodeProviderTest.cs b/NzbDrone.Core.Test/EpisodeProviderTest.cs index 3d3112305..5a15aa203 100644 --- a/NzbDrone.Core.Test/EpisodeProviderTest.cs +++ b/NzbDrone.Core.Test/EpisodeProviderTest.cs @@ -1466,10 +1466,10 @@ namespace NzbDrone.Core.Test result.Should().BeFalse(); } - [TestCase("The Office (US) - S01E05 - Episode Title", PostDownloadStatusType.Unpacking, 1)] - [TestCase("The Office (US) - S01E05 - Episode Title", PostDownloadStatusType.Failed, 1)] - [TestCase("The Office (US) - S01E05E06 - Episode Title", PostDownloadStatusType.Unpacking, 2)] - [TestCase("The Office (US) - S01E05E06 - Episode Title", PostDownloadStatusType.Failed, 2)] + [TestCase("The Office (US) - S01E01 - Episode Title", PostDownloadStatusType.Unpacking, 1)] + [TestCase("The Office (US) - S01E01 - Episode Title", PostDownloadStatusType.Failed, 1)] + [TestCase("The Office (US) - S01E01E02 - Episode Title", PostDownloadStatusType.Unpacking, 2)] + [TestCase("The Office (US) - S01E01E02 - Episode Title", PostDownloadStatusType.Failed, 2)] [TestCase("The Office (US) - Season 01 - Episode Title", PostDownloadStatusType.Unpacking, 10)] [TestCase("The Office (US) - Season 01 - Episode Title", PostDownloadStatusType.Failed, 10)] public void SetPostDownloadStatus(string folderName, PostDownloadStatusType postDownloadStatus, int episodeCount) @@ -1483,7 +1483,7 @@ namespace NzbDrone.Core.Test .With(s => s.CleanTitle = "officeus") .Build(); - var fakeEpisodes = Builder.CreateListOfSize(10) + var fakeEpisodes = Builder.CreateListOfSize(episodeCount) .WhereAll() .Have(c => c.SeriesId = 12345) .Have(c => c.SeasonNumber = 1) @@ -1496,7 +1496,7 @@ namespace NzbDrone.Core.Test mocker.GetMock().Setup(s => s.FindSeries("officeus")).Returns(fakeSeries); //Act - mocker.Resolve().SetPostDownloadStatus(folderName, postDownloadStatus); + mocker.Resolve().SetPostDownloadStatus(fakeEpisodes.Select(e => e.EpisodeId), postDownloadStatus); //Assert var result = db.Fetch(); diff --git a/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index dc45e4208..86e4fcfdd 100644 --- a/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -91,6 +91,7 @@ + diff --git a/NzbDrone.Core.Test/PostDownloadProviderTest.cs b/NzbDrone.Core.Test/PostDownloadProviderTest.cs new file mode 100644 index 000000000..5ad8fdd66 --- /dev/null +++ b/NzbDrone.Core.Test/PostDownloadProviderTest.cs @@ -0,0 +1,190 @@ +// ReSharper disable RedundantUsingDirective +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using AutoMoq; +using FizzWare.NBuilder; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Model; +using NzbDrone.Core.Providers; +using NzbDrone.Core.Providers.Core; +using NzbDrone.Core.Repository; +using NzbDrone.Core.Repository.Quality; +using NzbDrone.Core.Test.Framework; +using PetaPoco; +using TvdbLib.Data; + +namespace NzbDrone.Core.Test +{ + [TestFixture] + // ReSharper disable InconsistentNaming + public class PostDownloadProviderTest : TestBase + { + [TestCase("The Office (US) - S01E05 - Episode Title", PostDownloadStatusType.Unpacking, 1)] + [TestCase("The Office (US) - S01E05 - Episode Title", PostDownloadStatusType.Failed, 1)] + [TestCase("The Office (US) - S01E05E06 - Episode Title", PostDownloadStatusType.Unpacking, 2)] + [TestCase("The Office (US) - S01E05E06 - Episode Title", PostDownloadStatusType.Failed, 2)] + [TestCase("The Office (US) - Season 01 - Episode Title", PostDownloadStatusType.Unpacking, 10)] + [TestCase("The Office (US) - Season 01 - Episode Title", PostDownloadStatusType.Failed, 10)] + public void SetPostDownloadStatus(string folderName, PostDownloadStatusType postDownloadStatus, int episodeCount) + { + var db = MockLib.GetEmptyDatabase(); + var mocker = new AutoMoqer(); + mocker.SetConstant(db); + + var fakeSeries = Builder.CreateNew() + .With(s => s.SeriesId = 12345) + .With(s => s.CleanTitle = "officeus") + .Build(); + + var fakeEpisodes = Builder.CreateListOfSize(10) + .WhereAll() + .Have(c => c.SeriesId = 12345) + .Have(c => c.SeasonNumber = 1) + .Have(c => c.PostDownloadStatus = PostDownloadStatusType.Unknown) + .Build(); + + db.Insert(fakeSeries); + db.InsertMany(fakeEpisodes); + + mocker.GetMock().Setup(s => s.FindSeries("officeus")).Returns(fakeSeries); + + //Act + //mocker.Resolve().SetPostDownloadStatus(folderName, postDownloadStatus); + + //Assert + var result = db.Fetch(); + result.Where(e => e.PostDownloadStatus == postDownloadStatus).Count().Should().Be(episodeCount); + } + + [TestCase(PostDownloadStatusType.Unpacking, 8)] + [TestCase(PostDownloadStatusType.Failed, 8)] + [TestCase(PostDownloadStatusType.InvalidSeries, 24)] + [TestCase(PostDownloadStatusType.ParseError, 21)] + [TestCase(PostDownloadStatusType.Unknown, 10)] + [TestCase(PostDownloadStatusType.Processed, 0)] + public void GetPrefixLength(PostDownloadStatusType postDownloadStatus, int expected) + { + //Setup + var mocker = new AutoMoqer(); + + //Act + var result = mocker.Resolve().GetPrefixLength(postDownloadStatus); + + //Assert + result.Should().Be(expected); + } + + [Test] + public void ProcessDownload_InvalidSeries() + { + //Setup + var mocker = new AutoMoqer(MockBehavior.Strict); + var di = new DirectoryInfo(@"C:\Test\Unsorted TV\The Office - S01E01 - Episode Title"); + + var newFolder = @"C:\Test\Unsorted TV\_NzbDrone_InvalidSeries_The Office - S01E01 - Episode Title"; + Series nullSeries = null; + + //Act + mocker.GetMock().Setup(s => s.FindSeries("office")).Returns(nullSeries); + mocker.GetMock().Setup(s => s.MoveDirectory(di.FullName, newFolder)); + + mocker.Resolve().ProcessDownload(di); + + //Assert + mocker.VerifyAllMocks(); + ExceptionVerification.ExcpectedWarns(1); + } + + [Test] + public void ProcessDownload_ParseError() + { + //Setup + var mocker = new AutoMoqer(MockBehavior.Strict); + var di = new DirectoryInfo(@"C:\Test\Unsorted TV\The Office - S01E01 - Episode Title"); + + var newFolder = @"C:\Test\Unsorted TV\_NzbDrone_ParseError_The Office - S01E01 - Episode Title"; + + var fakeSeries = Builder.CreateNew() + .With(s => s.Title = "The Office") + .Build(); + + //Act + mocker.GetMock().Setup(s => s.FindSeries("office")).Returns(fakeSeries); + mocker.GetMock().Setup(s => s.MoveDirectory(di.FullName, newFolder)); + mocker.GetMock().Setup(s => s.GetDirectorySize(di.FullName)).Returns(100.Megabytes()); + mocker.GetMock().Setup(s => s.Scan(fakeSeries, di.FullName)).Returns( + new List()); + + mocker.Resolve().ProcessDownload(di); + + //Assert + mocker.VerifyAllMocks(); + ExceptionVerification.ExcpectedWarns(1); + } + + [Test] + public void ProcessDownload_Unknown_Error() + { + //Setup + var mocker = new AutoMoqer(MockBehavior.Strict); + var di = new DirectoryInfo(@"C:\Test\Unsorted TV\The Office - Season 01"); + + var newFolder = @"C:\Test\Unsorted TV\_NzbDrone_The Office - Season 01"; + + var fakeSeries = Builder.CreateNew() + .With(s => s.Title = "The Office") + .Build(); + + var fakeEpisodeFiles = Builder.CreateListOfSize(2) + .WhereAll() + .Have(f => f.SeriesId = fakeSeries.SeriesId) + .Build().ToList(); + + //Act + mocker.GetMock().Setup(s => s.FindSeries("office")).Returns(fakeSeries); + mocker.GetMock().Setup(s => s.MoveDirectory(di.FullName, newFolder)); + mocker.GetMock().Setup(s => s.GetDirectorySize(di.FullName)).Returns(100.Megabytes()); + mocker.GetMock().Setup(s => s.Scan(fakeSeries, di.FullName)).Returns(fakeEpisodeFiles); + mocker.GetMock().Setup(s => s.MoveEpisodeFile(It.IsAny(), true)).Returns(true); + + mocker.Resolve().ProcessDownload(di); + + //Assert + mocker.VerifyAllMocks(); + ExceptionVerification.ExcpectedWarns(1); + } + + [Test] + public void ProcessDownload_Success() + { + //Setup + var mocker = new AutoMoqer(MockBehavior.Strict); + var di = new DirectoryInfo(@"C:\Test\Unsorted TV\The Office - Season 01"); + + var fakeSeries = Builder.CreateNew() + .With(s => s.Title = "The Office") + .Build(); + + var fakeEpisodeFiles = Builder.CreateListOfSize(2) + .WhereAll() + .Have(f => f.SeriesId = fakeSeries.SeriesId) + .Build().ToList(); + + //Act + mocker.GetMock().Setup(s => s.FindSeries("office")).Returns(fakeSeries); + mocker.GetMock().Setup(s => s.DeleteFolder(di.FullName, true)); + mocker.GetMock().Setup(s => s.GetDirectorySize(di.FullName)).Returns(1.Megabytes()); + mocker.GetMock().Setup(s => s.Scan(fakeSeries, di.FullName)).Returns(fakeEpisodeFiles); + mocker.GetMock().Setup(s => s.MoveEpisodeFile(It.IsAny(), true)).Returns(true); + + mocker.Resolve().ProcessDownload(di); + + //Assert + mocker.VerifyAllMocks(); + } + } +} \ No newline at end of file diff --git a/NzbDrone.Core/Model/PostDownloadInfoModel.cs b/NzbDrone.Core/Model/PostDownloadInfoModel.cs new file mode 100644 index 000000000..572ef529e --- /dev/null +++ b/NzbDrone.Core/Model/PostDownloadInfoModel.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.Model +{ + public class PostDownloadInfoModel + { + public string Name { get; set; } + public DateTime Added { get; set; } + public PostDownloadStatusType Status { get; set; } + } +} diff --git a/NzbDrone.Core/Model/PostDownloadStatusType.cs b/NzbDrone.Core/Model/PostDownloadStatusType.cs index a330c8954..ce5a5d4ca 100644 --- a/NzbDrone.Core/Model/PostDownloadStatusType.cs +++ b/NzbDrone.Core/Model/PostDownloadStatusType.cs @@ -20,6 +20,16 @@ /// /// Processed /// - Processed = 3 + Processed = 3, + + /// + /// InvalidSeries + /// + InvalidSeries = 4, + + /// + /// ParseError + /// + ParseError = 5 } } \ No newline at end of file diff --git a/NzbDrone.Core/NzbDrone.Core.csproj b/NzbDrone.Core/NzbDrone.Core.csproj index f5a464d3b..ff2e269f4 100644 --- a/NzbDrone.Core/NzbDrone.Core.csproj +++ b/NzbDrone.Core/NzbDrone.Core.csproj @@ -193,6 +193,7 @@ + diff --git a/NzbDrone.Core/Providers/EpisodeProvider.cs b/NzbDrone.Core/Providers/EpisodeProvider.cs index 676f1c144..547c0aa64 100644 --- a/NzbDrone.Core/Providers/EpisodeProvider.cs +++ b/NzbDrone.Core/Providers/EpisodeProvider.cs @@ -405,12 +405,8 @@ namespace NzbDrone.Core.Providers Logger.Trace("Finished deleting invalid episodes for {0}", series.SeriesId); } - public virtual void SetPostDownloadStatus(string folderName, PostDownloadStatusType postDownloadStatus) + public virtual void SetPostDownloadStatus(IEnumerable episodeIds, PostDownloadStatusType postDownloadStatus) { - var parseResult = Parser.ParseTitle(folderName); - parseResult.Series = _seriesProvider.FindSeries(parseResult.CleanTitle); - - var episodeIds = GetEpisodesByParseResult(parseResult).Select(e => e.EpisodeId); var episodeIdString = String.Join(", ", episodeIds); var episodeIdQuery = String.Format(@"UPDATE Episodes SET PostDownloadStatus = {0} diff --git a/NzbDrone.Core/Providers/Jobs/PostDownloadScanJob.cs b/NzbDrone.Core/Providers/Jobs/PostDownloadScanJob.cs index a2a67aa93..741b79e89 100644 --- a/NzbDrone.Core/Providers/Jobs/PostDownloadScanJob.cs +++ b/NzbDrone.Core/Providers/Jobs/PostDownloadScanJob.cs @@ -34,7 +34,7 @@ namespace NzbDrone.Core.Providers.Jobs public virtual void Start(ProgressNotification notification, int targetId, int secondaryTargetId) { - _postDownloadProvider.Start(notification); + _postDownloadProvider.ScanDropFolder(notification); } } } \ No newline at end of file diff --git a/NzbDrone.Core/Providers/PostDownloadProvider.cs b/NzbDrone.Core/Providers/PostDownloadProvider.cs index 06d9053b4..f592f9d83 100644 --- a/NzbDrone.Core/Providers/PostDownloadProvider.cs +++ b/NzbDrone.Core/Providers/PostDownloadProvider.cs @@ -22,6 +22,8 @@ namespace NzbDrone.Core.Providers private readonly EpisodeProvider _episodeProvider; private static readonly Logger Logger = LogManager.GetCurrentClassLogger(); + private static readonly List InfoList = new List(); + [Inject] public PostDownloadProvider(ConfigProvider configProvider, DiskProvider diskProvider, DiskScanProvider diskScanProvider, SeriesProvider seriesProvider, @@ -39,7 +41,7 @@ namespace NzbDrone.Core.Providers } - public virtual void Start(ProgressNotification notification) + public virtual void ScanDropFolder(ProgressNotification notification) { var dropFolder = _configProvider.SabDropDirectory; @@ -55,55 +57,47 @@ namespace NzbDrone.Core.Providers return; } + ProcessDropFolder(dropFolder); + } + + public virtual void ProcessDropFolder(string dropFolder) + { foreach (var subfolder in _diskProvider.GetDirectories(dropFolder)) { try { var subfolderInfo = new DirectoryInfo(subfolder); - if (subfolderInfo.Name.StartsWith("_UNPACK_", StringComparison.CurrentCultureIgnoreCase)) + if (subfolderInfo.Name.StartsWith("_UNPACK_")) { - _episodeProvider.SetPostDownloadStatus(subfolderInfo.Name.Substring(8), PostDownloadStatusType.Unpacking); + ProcessFailedOrUnpackingDownload(subfolderInfo, PostDownloadStatusType.Unpacking); Logger.Debug("Folder [{0}] is still being unpacked. skipping.", subfolder); continue; } - if (subfolderInfo.Name.StartsWith("_FAILED_", StringComparison.CurrentCultureIgnoreCase)) + if (subfolderInfo.Name.StartsWith("_FAILED_")) { - _episodeProvider.SetPostDownloadStatus(subfolderInfo.Name.Substring(8), PostDownloadStatusType.Failed); + ProcessFailedOrUnpackingDownload(subfolderInfo, PostDownloadStatusType.Failed); Logger.Debug("Folder [{0}] is marked as failed. skipping.", subfolder); continue; } - if (subfolderInfo.Name.StartsWith("_NzbDrone_", StringComparison.CurrentCultureIgnoreCase)) + if (subfolderInfo.Name.StartsWith("_NzbDrone_")) { - Logger.Debug("Folder [{0}] is marked as already processedby NzbDrone. skipping.", subfolder); - continue; - } + if (subfolderInfo.Name.StartsWith("_NzbDrone_InvalidSeries_")) + ReProcessDownload(new PostDownloadInfoModel{ Name = subfolderInfo.FullName, Status = PostDownloadStatusType.InvalidSeries }); - //Parse the Folder name - var seriesName = Parser.ParseSeriesName(subfolderInfo.Name); - var series = _seriesProvider.FindSeries(seriesName); + else if (subfolderInfo.Name.StartsWith("_NzbDrone_ParseError_")) + ReProcessDownload(new PostDownloadInfoModel { Name = subfolderInfo.FullName, Status = PostDownloadStatusType.ParseError }); - if (series == null) - { - Logger.Warn("Unable to Import new download [{0}], series doesn't exist in database.", subfolder); + else + ReProcessDownload(new PostDownloadInfoModel { Name = subfolderInfo.FullName, Status = PostDownloadStatusType.Unknown }); - //Rename the Directory so it's not processed again. - _diskProvider.MoveDirectory(subfolderInfo.FullName, Path.Combine(subfolderInfo.Parent.FullName, "_NzbDrone_" + subfolderInfo.Name)); continue; } - var importedFiles = _diskScanProvider.Scan(series, subfolder); - importedFiles.ForEach(file => _diskScanProvider.MoveEpisodeFile(file, true)); - - //Delete the folder only if folder is small enough - if (_diskProvider.GetDirectorySize(subfolder) < 10.Megabytes()) - _diskProvider.DeleteFolder(subfolder, true); - - //Otherwise rename the folder to say it was already processed once by NzbDrone so it will not be continually processed - else - _diskProvider.MoveDirectory(subfolderInfo.FullName, Path.Combine(subfolderInfo.Parent.FullName, "_NzbDrone_" + subfolderInfo.Name)); + //Process a successful download + ProcessDownload(subfolderInfo); } catch (Exception e) @@ -112,5 +106,117 @@ namespace NzbDrone.Core.Providers } } } + + public virtual void ProcessDownload(DirectoryInfo subfolderInfo) + { + //Parse the Folder name + var seriesName = Parser.ParseSeriesName(subfolderInfo.Name); + var series = _seriesProvider.FindSeries(seriesName); + + if (series == null) + { + Logger.Warn("Unable to Import new download [{0}], series doesn't exist in database.", subfolderInfo.FullName); + + //Rename the Directory so it's not processed again. + _diskProvider.MoveDirectory(subfolderInfo.FullName, + Path.Combine(subfolderInfo.Parent.FullName, + "_NzbDrone_InvalidSeries_" + subfolderInfo.Name)); + return; + } + + var importedFiles = _diskScanProvider.Scan(series, subfolderInfo.FullName); + importedFiles.ForEach(file => _diskScanProvider.MoveEpisodeFile(file, true)); + + //Delete the folder only if folder is small enough + if (_diskProvider.GetDirectorySize(subfolderInfo.FullName) < 10.Megabytes()) + _diskProvider.DeleteFolder(subfolderInfo.FullName, true); + + //Otherwise rename the folder to say it was already processed once by NzbDrone + else + { + if (importedFiles.Count == 0) + { + Logger.Warn("Unable to Import new download [{0}], unable to parse episode file(s).", subfolderInfo.FullName); + _diskProvider.MoveDirectory(subfolderInfo.FullName, + Path.Combine(subfolderInfo.Parent.FullName, + "_NzbDrone_ParseError_" + subfolderInfo.Name)); + } + + //Unknown Error Importing (Possibly a lesser quality than episode currently on disk) + else + { + Logger.Warn("Unable to Import new download [{0}].", subfolderInfo.FullName); + + _diskProvider.MoveDirectory(subfolderInfo.FullName, + Path.Combine(subfolderInfo.Parent.FullName, + "_NzbDrone_" + subfolderInfo.Name)); + } + } + } + + public virtual void ProcessFailedOrUnpackingDownload(DirectoryInfo directoryInfo, PostDownloadStatusType postDownloadStatus) + { + //Check to see if its already in InfoList, if it is, check if enough time has passed to process + if (InfoList.Any(i => i.Name == directoryInfo.FullName)) + { + var model = InfoList.Single(i => i.Name == directoryInfo.FullName); + + //Process if 30 minutes has passed + if (model.Added > DateTime.Now.AddMinutes(30)) + ReProcessDownload(model); + + //If everything processed successfully up until now, remove it from InfoList + InfoList.Remove(model); + return; + } + + //Add to InfoList for possible later processing + InfoList.Add(new PostDownloadInfoModel{ Name = directoryInfo.FullName, + Added = DateTime.Now, + Status = postDownloadStatus + }); + + //Remove the first 8 characters of the folder name (removes _UNPACK_ or _FAILED_) before processing + var parseResult = Parser.ParseTitle(directoryInfo.Name.Substring(8)); + parseResult.Series = _seriesProvider.FindSeries(parseResult.CleanTitle); + + var episodeIds = _episodeProvider.GetEpisodesByParseResult(parseResult).Select(e => e.EpisodeId); + + _episodeProvider.SetPostDownloadStatus(episodeIds, postDownloadStatus); + } + + public virtual void ReProcessDownload(PostDownloadInfoModel model) + { + var directoryInfo = new DirectoryInfo(model.Name); + var newName = Path.Combine(directoryInfo.Parent.FullName, directoryInfo.Name.Substring(GetPrefixLength(model.Status))); + + _diskProvider.MoveDirectory(directoryInfo.FullName, newName); + + directoryInfo = new DirectoryInfo(newName); + + ProcessDownload(directoryInfo); + } + + public int GetPrefixLength(PostDownloadStatusType postDownloadStatus) + { + //_UNPACK_ & _FAILED_ have a length of 8 + if (postDownloadStatus == PostDownloadStatusType.Unpacking || postDownloadStatus == PostDownloadStatusType.Failed) + return 8; + + //_NzbDrone_InvalidSeries_ - Length = 24 + if (postDownloadStatus == PostDownloadStatusType.InvalidSeries) + return 24; + + //_NzbDrone_ParseError_ - Length = + if (postDownloadStatus == PostDownloadStatusType.ParseError) + return 21; + + //_NzbDrone_ - Length = 10 + if (postDownloadStatus == PostDownloadStatusType.Unknown) + return 10; + + //Default to zero + return 0; + } } }