open-source-mirrors
/
Lidarr
mirror of https://github.com/Lidarr/Lidarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

PostDownloadProvider broken down further.

Browse Source 
Will try to reprocess _NzbDrone_ directories each pass, but will mark with an error when possible.
Attempt to process _UNPACK_ and _FAILED_ directories 30 minutes after first detected by NzbDrone (to give SAB time to unpack properly before processing).
pull/4/head
Mark McDowall 13 years ago
parent 04ed22db55
commit 8cac84b4ad
9 changed files with 359 additions and 40 deletions
Show Stats Download Patch File Download Diff File

12
NzbDrone.Core.Test/EpisodeProviderTest.cs
Unescape View File

@ -1466,10 +1466,10 @@ namespace NzbDrone.Core.Test
result.Should().BeFalse();
}
[TestCase("The Office (US) - S01E05 - Episode Title", PostDownloadStatusType.Unpacking, 1)]
[TestCase("The Office (US) - S01E05 - Episode Title", PostDownloadStatusType.Failed, 1)]
[TestCase("The Office (US) - S01E05E06 - Episode Title", PostDownloadStatusType.Unpacking, 2)]
[TestCase("The Office (US) - S01E05E06 - Episode Title", PostDownloadStatusType.Failed, 2)]
[TestCase("The Office (US) - S01E01 - Episode Title", PostDownloadStatusType.Unpacking, 1)]
[TestCase("The Office (US) - S01E01 - Episode Title", PostDownloadStatusType.Failed, 1)]
[TestCase("The Office (US) - S01E01E02 - Episode Title", PostDownloadStatusType.Unpacking, 2)]
[TestCase("The Office (US) - S01E01E02 - Episode Title", PostDownloadStatusType.Failed, 2)]
[TestCase("The Office (US) - Season 01 - Episode Title", PostDownloadStatusType.Unpacking, 10)]
[TestCase("The Office (US) - Season 01 - Episode Title", PostDownloadStatusType.Failed, 10)]
public void SetPostDownloadStatus(string folderName, PostDownloadStatusType postDownloadStatus, int episodeCount)
@ -1483,7 +1483,7 @@ namespace NzbDrone.Core.Test
.With(s => s.CleanTitle = "officeus")
.Build();
var fakeEpisodes = Builder<Episode>.CreateListOfSize(10)
var fakeEpisodes = Builder<Episode>.CreateListOfSize(episodeCount)
.WhereAll()
.Have(c => c.SeriesId = 12345)
.Have(c => c.SeasonNumber = 1)
@ -1496,7 +1496,7 @@ namespace NzbDrone.Core.Test
mocker.GetMock<SeriesProvider>().Setup(s => s.FindSeries("officeus")).Returns(fakeSeries);
//Act
mocker.Resolve<EpisodeProvider>().SetPostDownloadStatus(folderName, postDownloadStatus);
mocker.Resolve<EpisodeProvider>().SetPostDownloadStatus(fakeEpisodes.Select(e => e.EpisodeId), postDownloadStatus);
//Assert
var result = db.Fetch<Episode>();

1
NzbDrone.Core.Test/NzbDrone.Core.Test.csproj
Unescape View File

@ -91,6 +91,7 @@
<Compile Include="BacklogSearchJobTest.cs" />
<Compile Include="BannerDownloadJobTest.cs" />
<Compile Include="ConfigFileProviderTest.cs" />
<Compile Include="PostDownloadProviderTest.cs" />
<Compile Include="SortHelperTest.cs" />
<Compile Include="EpisodeProviderTest_DeleteInvalidEpisodes.cs" />
<Compile Include="InventoryProvider_IsAcceptableSizeTest.cs" />

190
NzbDrone.Core.Test/PostDownloadProviderTest.cs
Unescape View File

@ -0,0 +1,190 @@
// ReSharper disable RedundantUsingDirective
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using AutoMoq;
using FizzWare.NBuilder;
using FluentAssertions;
using Moq;
using NUnit.Framework;
using NzbDrone.Core.Model;
using NzbDrone.Core.Providers;
using NzbDrone.Core.Providers.Core;
using NzbDrone.Core.Repository;
using NzbDrone.Core.Repository.Quality;
using NzbDrone.Core.Test.Framework;
using PetaPoco;
using TvdbLib.Data;
namespace NzbDrone.Core.Test
{
[TestFixture]
// ReSharper disable InconsistentNaming
public class PostDownloadProviderTest : TestBase
{
[TestCase("The Office (US) - S01E05 - Episode Title", PostDownloadStatusType.Unpacking, 1)]
[TestCase("The Office (US) - S01E05 - Episode Title", PostDownloadStatusType.Failed, 1)]
[TestCase("The Office (US) - S01E05E06 - Episode Title", PostDownloadStatusType.Unpacking, 2)]
[TestCase("The Office (US) - S01E05E06 - Episode Title", PostDownloadStatusType.Failed, 2)]
[TestCase("The Office (US) - Season 01 - Episode Title", PostDownloadStatusType.Unpacking, 10)]
[TestCase("The Office (US) - Season 01 - Episode Title", PostDownloadStatusType.Failed, 10)]
public void SetPostDownloadStatus(string folderName, PostDownloadStatusType postDownloadStatus, int episodeCount)
{
var db = MockLib.GetEmptyDatabase();
var mocker = new AutoMoqer();
mocker.SetConstant(db);
var fakeSeries = Builder<Series>.CreateNew()
.With(s => s.SeriesId = 12345)
.With(s => s.CleanTitle = "officeus")
.Build();
var fakeEpisodes = Builder<Episode>.CreateListOfSize(10)
.WhereAll()
.Have(c => c.SeriesId = 12345)
.Have(c => c.SeasonNumber = 1)
.Have(c => c.PostDownloadStatus = PostDownloadStatusType.Unknown)
.Build();
db.Insert(fakeSeries);
db.InsertMany(fakeEpisodes);
mocker.GetMock<SeriesProvider>().Setup(s => s.FindSeries("officeus")).Returns(fakeSeries);
//Act
//mocker.Resolve<EpisodeProvider>().SetPostDownloadStatus(folderName, postDownloadStatus);
//Assert
var result = db.Fetch<Episode>();
result.Where(e => e.PostDownloadStatus == postDownloadStatus).Count().Should().Be(episodeCount);
}
[TestCase(PostDownloadStatusType.Unpacking, 8)]
[TestCase(PostDownloadStatusType.Failed, 8)]
[TestCase(PostDownloadStatusType.InvalidSeries, 24)]
[TestCase(PostDownloadStatusType.ParseError, 21)]
[TestCase(PostDownloadStatusType.Unknown, 10)]
[TestCase(PostDownloadStatusType.Processed, 0)]
public void GetPrefixLength(PostDownloadStatusType postDownloadStatus, int expected)
{
//Setup
var mocker = new AutoMoqer();
//Act
var result = mocker.Resolve<PostDownloadProvider>().GetPrefixLength(postDownloadStatus);
//Assert
result.Should().Be(expected);
}
[Test]
public void ProcessDownload_InvalidSeries()
{
//Setup
var mocker = new AutoMoqer(MockBehavior.Strict);
var di = new DirectoryInfo(@"C:\Test\Unsorted TV\The Office - S01E01 - Episode Title");
var newFolder = @"C:\Test\Unsorted TV\_NzbDrone_InvalidSeries_The Office - S01E01 - Episode Title";
Series nullSeries = null;
//Act
mocker.GetMock<SeriesProvider>().Setup(s => s.FindSeries("office")).Returns(nullSeries);
mocker.GetMock<DiskProvider>().Setup(s => s.MoveDirectory(di.FullName, newFolder));
mocker.Resolve<PostDownloadProvider>().ProcessDownload(di);
//Assert
mocker.VerifyAllMocks();
ExceptionVerification.ExcpectedWarns(1);
}
[Test]
public void ProcessDownload_ParseError()
{
//Setup
var mocker = new AutoMoqer(MockBehavior.Strict);
var di = new DirectoryInfo(@"C:\Test\Unsorted TV\The Office - S01E01 - Episode Title");
var newFolder = @"C:\Test\Unsorted TV\_NzbDrone_ParseError_The Office - S01E01 - Episode Title";
var fakeSeries = Builder<Series>.CreateNew()
.With(s => s.Title = "The Office")
.Build();
//Act
mocker.GetMock<SeriesProvider>().Setup(s => s.FindSeries("office")).Returns(fakeSeries);
mocker.GetMock<DiskProvider>().Setup(s => s.MoveDirectory(di.FullName, newFolder));
mocker.GetMock<DiskProvider>().Setup(s => s.GetDirectorySize(di.FullName)).Returns(100.Megabytes());
mocker.GetMock<DiskScanProvider>().Setup(s => s.Scan(fakeSeries, di.FullName)).Returns(
new List<EpisodeFile>());
mocker.Resolve<PostDownloadProvider>().ProcessDownload(di);
//Assert
mocker.VerifyAllMocks();
ExceptionVerification.ExcpectedWarns(1);
}
[Test]
public void ProcessDownload_Unknown_Error()
{
//Setup
var mocker = new AutoMoqer(MockBehavior.Strict);
var di = new DirectoryInfo(@"C:\Test\Unsorted TV\The Office - Season 01");
var newFolder = @"C:\Test\Unsorted TV\_NzbDrone_The Office - Season 01";
var fakeSeries = Builder<Series>.CreateNew()
.With(s => s.Title = "The Office")
.Build();
var fakeEpisodeFiles = Builder<EpisodeFile>.CreateListOfSize(2)
.WhereAll()
.Have(f => f.SeriesId = fakeSeries.SeriesId)
.Build().ToList();
//Act
mocker.GetMock<SeriesProvider>().Setup(s => s.FindSeries("office")).Returns(fakeSeries);
mocker.GetMock<DiskProvider>().Setup(s => s.MoveDirectory(di.FullName, newFolder));
mocker.GetMock<DiskProvider>().Setup(s => s.GetDirectorySize(di.FullName)).Returns(100.Megabytes());
mocker.GetMock<DiskScanProvider>().Setup(s => s.Scan(fakeSeries, di.FullName)).Returns(fakeEpisodeFiles);
mocker.GetMock<DiskScanProvider>().Setup(s => s.MoveEpisodeFile(It.IsAny<EpisodeFile>(), true)).Returns(true);
mocker.Resolve<PostDownloadProvider>().ProcessDownload(di);
//Assert
mocker.VerifyAllMocks();
ExceptionVerification.ExcpectedWarns(1);
}
[Test]
public void ProcessDownload_Success()
{
//Setup
var mocker = new AutoMoqer(MockBehavior.Strict);
var di = new DirectoryInfo(@"C:\Test\Unsorted TV\The Office - Season 01");
var fakeSeries = Builder<Series>.CreateNew()
.With(s => s.Title = "The Office")
.Build();
var fakeEpisodeFiles = Builder<EpisodeFile>.CreateListOfSize(2)
.WhereAll()
.Have(f => f.SeriesId = fakeSeries.SeriesId)
.Build().ToList();
//Act
mocker.GetMock<SeriesProvider>().Setup(s => s.FindSeries("office")).Returns(fakeSeries);
mocker.GetMock<DiskProvider>().Setup(s => s.DeleteFolder(di.FullName, true));
mocker.GetMock<DiskProvider>().Setup(s => s.GetDirectorySize(di.FullName)).Returns(1.Megabytes());
mocker.GetMock<DiskScanProvider>().Setup(s => s.Scan(fakeSeries, di.FullName)).Returns(fakeEpisodeFiles);
mocker.GetMock<DiskScanProvider>().Setup(s => s.MoveEpisodeFile(It.IsAny<EpisodeFile>(), true)).Returns(true);
mocker.Resolve<PostDownloadProvider>().ProcessDownload(di);
//Assert
mocker.VerifyAllMocks();
}
}
}

15
NzbDrone.Core/Model/PostDownloadInfoModel.cs
Unescape View File

@ -0,0 +1,15 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
namespace NzbDrone.Core.Model
{
public class PostDownloadInfoModel
{
public string Name { get; set; }
public DateTime Added { get; set; }
public PostDownloadStatusType Status { get; set; }
}
}

12
NzbDrone.Core/Model/PostDownloadStatusType.cs
Unescape View File

@ -20,6 +20,16 @@
/// <summary>
/// Processed
/// </summary>
Processed = 3
Processed = 3,
/// <summary>
/// InvalidSeries
/// </summary>
InvalidSeries = 4,
/// <summary>
/// ParseError
/// </summary>
ParseError = 5
}
}

1
NzbDrone.Core/NzbDrone.Core.csproj
Unescape View File

@ -193,6 +193,7 @@
<Compile Include="Model\AtomicParsleyTitleType.cs" />
<Compile Include="Model\AuthenticationType.cs" />
<Compile Include="Model\ConnectionInfoModel.cs" />
<Compile Include="Model\PostDownloadInfoModel.cs" />
<Compile Include="Model\PostDownloadStatusType.cs" />
<Compile Include="Model\ExternalNotificationType.cs" />
<Compile Include="Model\JobQueueItem.cs" />

6
NzbDrone.Core/Providers/EpisodeProvider.cs
Unescape View File

@ -405,12 +405,8 @@ namespace NzbDrone.Core.Providers
Logger.Trace("Finished deleting invalid episodes for {0}", series.SeriesId);
}
public virtual void SetPostDownloadStatus(string folderName, PostDownloadStatusType postDownloadStatus)
public virtual void SetPostDownloadStatus(IEnumerable<int> episodeIds, PostDownloadStatusType postDownloadStatus)
{
var parseResult = Parser.ParseTitle(folderName);
parseResult.Series = _seriesProvider.FindSeries(parseResult.CleanTitle);
var episodeIds = GetEpisodesByParseResult(parseResult).Select(e => e.EpisodeId);
var episodeIdString = String.Join(", ", episodeIds);
var episodeIdQuery = String.Format(@"UPDATE Episodes SET PostDownloadStatus = {0}

2
NzbDrone.Core/Providers/Jobs/PostDownloadScanJob.cs
Unescape View File

@ -34,7 +34,7 @@ namespace NzbDrone.Core.Providers.Jobs
public virtual void Start(ProgressNotification notification, int targetId, int secondaryTargetId)
{
_postDownloadProvider.Start(notification);
_postDownloadProvider.ScanDropFolder(notification);
}
}
}

160
NzbDrone.Core/Providers/PostDownloadProvider.cs
Unescape View File

@ -22,6 +22,8 @@ namespace NzbDrone.Core.Providers
private readonly EpisodeProvider _episodeProvider;
private static readonly Logger Logger = LogManager.GetCurrentClassLogger();
private static readonly List<PostDownloadInfoModel> InfoList = new List<PostDownloadInfoModel>();
[Inject]
public PostDownloadProvider(ConfigProvider configProvider, DiskProvider diskProvider,
DiskScanProvider diskScanProvider, SeriesProvider seriesProvider,
@ -39,7 +41,7 @@ namespace NzbDrone.Core.Providers
}
public virtual void Start(ProgressNotification notification)
public virtual void ScanDropFolder(ProgressNotification notification)
{
var dropFolder = _configProvider.SabDropDirectory;
@ -55,55 +57,47 @@ namespace NzbDrone.Core.Providers
return;
}
ProcessDropFolder(dropFolder);
}
public virtual void ProcessDropFolder(string dropFolder)
{
foreach (var subfolder in _diskProvider.GetDirectories(dropFolder))
{
try
{
var subfolderInfo = new DirectoryInfo(subfolder);
if (subfolderInfo.Name.StartsWith("_UNPACK_", StringComparison.CurrentCultureIgnoreCase))
if (subfolderInfo.Name.StartsWith("_UNPACK_"))
{
_episodeProvider.SetPostDownloadStatus(subfolderInfo.Name.Substring(8), PostDownloadStatusType.Unpacking);
ProcessFailedOrUnpackingDownload(subfolderInfo, PostDownloadStatusType.Unpacking);
Logger.Debug("Folder [{0}] is still being unpacked. skipping.", subfolder);
continue;
}
if (subfolderInfo.Name.StartsWith("_FAILED_", StringComparison.CurrentCultureIgnoreCase))
if (subfolderInfo.Name.StartsWith("_FAILED_"))
{
_episodeProvider.SetPostDownloadStatus(subfolderInfo.Name.Substring(8), PostDownloadStatusType.Failed);
ProcessFailedOrUnpackingDownload(subfolderInfo, PostDownloadStatusType.Failed);
Logger.Debug("Folder [{0}] is marked as failed. skipping.", subfolder);
continue;
}
if (subfolderInfo.Name.StartsWith("_NzbDrone_", StringComparison.CurrentCultureIgnoreCase))
if (subfolderInfo.Name.StartsWith("_NzbDrone_"))
{
Logger.Debug("Folder [{0}] is marked as already processedby NzbDrone. skipping.", subfolder);
continue;
}
if (subfolderInfo.Name.StartsWith("_NzbDrone_InvalidSeries_"))
ReProcessDownload(new PostDownloadInfoModel{ Name = subfolderInfo.FullName, Status = PostDownloadStatusType.InvalidSeries });
//Parse the Folder name
var seriesName = Parser.ParseSeriesName(subfolderInfo.Name);
var series = _seriesProvider.FindSeries(seriesName);
else if (subfolderInfo.Name.StartsWith("_NzbDrone_ParseError_"))
ReProcessDownload(new PostDownloadInfoModel { Name = subfolderInfo.FullName, Status = PostDownloadStatusType.ParseError });
if (series == null)
{
Logger.Warn("Unable to Import new download [{0}], series doesn't exist in database.", subfolder);
else
ReProcessDownload(new PostDownloadInfoModel { Name = subfolderInfo.FullName, Status = PostDownloadStatusType.Unknown });
//Rename the Directory so it's not processed again.
_diskProvider.MoveDirectory(subfolderInfo.FullName, Path.Combine(subfolderInfo.Parent.FullName, "_NzbDrone_" + subfolderInfo.Name));
continue;
}
var importedFiles = _diskScanProvider.Scan(series, subfolder);
importedFiles.ForEach(file => _diskScanProvider.MoveEpisodeFile(file, true));
//Delete the folder only if folder is small enough
if (_diskProvider.GetDirectorySize(subfolder) < 10.Megabytes())
_diskProvider.DeleteFolder(subfolder, true);
//Otherwise rename the folder to say it was already processed once by NzbDrone so it will not be continually processed
else
_diskProvider.MoveDirectory(subfolderInfo.FullName, Path.Combine(subfolderInfo.Parent.FullName, "_NzbDrone_" + subfolderInfo.Name));
//Process a successful download
ProcessDownload(subfolderInfo);
}
catch (Exception e)
@ -112,5 +106,117 @@ namespace NzbDrone.Core.Providers
}
}
}
public virtual void ProcessDownload(DirectoryInfo subfolderInfo)
{
//Parse the Folder name
var seriesName = Parser.ParseSeriesName(subfolderInfo.Name);
var series = _seriesProvider.FindSeries(seriesName);
if (series == null)
{
Logger.Warn("Unable to Import new download [{0}], series doesn't exist in database.", subfolderInfo.FullName);
//Rename the Directory so it's not processed again.
_diskProvider.MoveDirectory(subfolderInfo.FullName,
Path.Combine(subfolderInfo.Parent.FullName,
"_NzbDrone_InvalidSeries_" + subfolderInfo.Name));
return;
}
var importedFiles = _diskScanProvider.Scan(series, subfolderInfo.FullName);
importedFiles.ForEach(file => _diskScanProvider.MoveEpisodeFile(file, true));
//Delete the folder only if folder is small enough
if (_diskProvider.GetDirectorySize(subfolderInfo.FullName) < 10.Megabytes())
_diskProvider.DeleteFolder(subfolderInfo.FullName, true);
//Otherwise rename the folder to say it was already processed once by NzbDrone
else
{
if (importedFiles.Count == 0)
{
Logger.Warn("Unable to Import new download [{0}], unable to parse episode file(s).", subfolderInfo.FullName);
_diskProvider.MoveDirectory(subfolderInfo.FullName,
Path.Combine(subfolderInfo.Parent.FullName,
"_NzbDrone_ParseError_" + subfolderInfo.Name));
}
//Unknown Error Importing (Possibly a lesser quality than episode currently on disk)
else
{
Logger.Warn("Unable to Import new download [{0}].", subfolderInfo.FullName);
_diskProvider.MoveDirectory(subfolderInfo.FullName,
Path.Combine(subfolderInfo.Parent.FullName,
"_NzbDrone_" + subfolderInfo.Name));
}
}
}
public virtual void ProcessFailedOrUnpackingDownload(DirectoryInfo directoryInfo, PostDownloadStatusType postDownloadStatus)
{
//Check to see if its already in InfoList, if it is, check if enough time has passed to process
if (InfoList.Any(i => i.Name == directoryInfo.FullName))
{
var model = InfoList.Single(i => i.Name == directoryInfo.FullName);
//Process if 30 minutes has passed
if (model.Added > DateTime.Now.AddMinutes(30))
ReProcessDownload(model);
//If everything processed successfully up until now, remove it from InfoList
InfoList.Remove(model);
return;
}
//Add to InfoList for possible later processing
InfoList.Add(new PostDownloadInfoModel{ Name = directoryInfo.FullName,
Added = DateTime.Now,
Status = postDownloadStatus
});
//Remove the first 8 characters of the folder name (removes _UNPACK_ or _FAILED_) before processing
var parseResult = Parser.ParseTitle(directoryInfo.Name.Substring(8));
parseResult.Series = _seriesProvider.FindSeries(parseResult.CleanTitle);
var episodeIds = _episodeProvider.GetEpisodesByParseResult(parseResult).Select(e => e.EpisodeId);
_episodeProvider.SetPostDownloadStatus(episodeIds, postDownloadStatus);
}
public virtual void ReProcessDownload(PostDownloadInfoModel model)
{
var directoryInfo = new DirectoryInfo(model.Name);
var newName = Path.Combine(directoryInfo.Parent.FullName, directoryInfo.Name.Substring(GetPrefixLength(model.Status)));
_diskProvider.MoveDirectory(directoryInfo.FullName, newName);
directoryInfo = new DirectoryInfo(newName);
ProcessDownload(directoryInfo);
}
public int GetPrefixLength(PostDownloadStatusType postDownloadStatus)
{
//_UNPACK_ & _FAILED_ have a length of 8
if (postDownloadStatus == PostDownloadStatusType.Unpacking || postDownloadStatus == PostDownloadStatusType.Failed)
return 8;
//_NzbDrone_InvalidSeries_ - Length = 24
if (postDownloadStatus == PostDownloadStatusType.InvalidSeries)
return 24;
//_NzbDrone_ParseError_ - Length =
if (postDownloadStatus == PostDownloadStatusType.ParseError)
return 21;
//_NzbDrone_ - Length = 10
if (postDownloadStatus == PostDownloadStatusType.Unknown)
return 10;
//Default to zero
return 0;
}
}
}

Write Preview
Loading…
Cancel
Save