From d4bc835b1c1f00020aec47852f6f655c7ae0d9df Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sun, 14 Feb 2016 17:55:32 +0100 Subject: [PATCH] New: Delaying Blackhole imports while they're still being updated. --- src/NzbDrone.Common/Crypto/HashConverter.cs | 10 +- .../Extensions/StringExtensions.cs | 5 + .../Blackhole/ScanWatchFolderFixture.cs | 94 +++++++++ .../Blackhole/TorrentBlackholeFixture.cs | 34 +-- .../Blackhole/UsenetBlackholeFixture.cs | 33 +-- .../DownloadClientFixtureBase.cs | 9 + .../NzbDrone.Core.Test.csproj | 1 + .../Clients/Blackhole/ScanWatchFolder.cs | 196 ++++++++++++++++++ .../TorrentBlackhole.cs | 77 ++----- .../TorrentBlackholeSettings.cs | 2 +- .../UsenetBlackhole.cs | 71 ++----- .../UsenetBlackholeSettings.cs | 2 +- .../Clients/Blackhole/WatchFolderItem.cs | 22 ++ src/NzbDrone.Core/NzbDrone.Core.csproj | 10 +- 14 files changed, 411 insertions(+), 155 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs create mode 100644 src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs rename src/NzbDrone.Core/Download/Clients/{TorrentBlackhole => Blackhole}/TorrentBlackhole.cs (58%) rename src/NzbDrone.Core/Download/Clients/{TorrentBlackhole => Blackhole}/TorrentBlackholeSettings.cs (96%) rename src/NzbDrone.Core/Download/Clients/{UsenetBlackhole => Blackhole}/UsenetBlackhole.cs (53%) rename src/NzbDrone.Core/Download/Clients/{UsenetBlackhole => Blackhole}/UsenetBlackholeSettings.cs (95%) create mode 100644 src/NzbDrone.Core/Download/Clients/Blackhole/WatchFolderItem.cs diff --git a/src/NzbDrone.Common/Crypto/HashConverter.cs b/src/NzbDrone.Common/Crypto/HashConverter.cs index 2645e2b09..60d71dc15 100644 --- a/src/NzbDrone.Common/Crypto/HashConverter.cs +++ b/src/NzbDrone.Common/Crypto/HashConverter.cs @@ -10,12 +10,16 @@ namespace NzbDrone.Common.Crypto public static int GetHashInt31(string target) { - byte[] hash; + var hash = GetHash(target); + return BitConverter.ToInt32(hash, 0) & 0x7fffffff; + } + + public static byte[] GetHash(string target) + { lock (Sha1) { - hash = Sha1.ComputeHash(Encoding.Default.GetBytes(target)); + return Sha1.ComputeHash(Encoding.Default.GetBytes(target)); } - return BitConverter.ToInt32(hash, 0) & 0x7fffffff; } } } diff --git a/src/NzbDrone.Common/Extensions/StringExtensions.cs b/src/NzbDrone.Common/Extensions/StringExtensions.cs index 4396f863c..e5bd84902 100644 --- a/src/NzbDrone.Common/Extensions/StringExtensions.cs +++ b/src/NzbDrone.Common/Extensions/StringExtensions.cs @@ -100,5 +100,10 @@ namespace NzbDrone.Common.Extensions .Select(x => Convert.ToByte(input.Substring(x, 2), 16)) .ToArray(); } + + public static string ToHexString(this byte[] input) + { + return string.Concat(Array.ConvertAll(input, x => x.ToString("X2"))); + } } } \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs new file mode 100644 index 000000000..199b206e2 --- /dev/null +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/ScanWatchFolderFixture.cs @@ -0,0 +1,94 @@ +using System; +using System.IO; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Disk; +using NzbDrone.Core.Download; +using NzbDrone.Test.Common; +using System.Threading; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.Download.Clients.Blackhole; + +namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole +{ + [TestFixture] + public class ScanWatchFolderFixture : CoreTest + { + protected readonly string _title = "Droned.S01E01.Pilot.1080p.WEB-DL-DRONE"; + protected string _completedDownloadFolder = @"c:\blackhole\completed".AsOsAgnostic(); + + protected void GivenCompletedItem() + { + var targetDir = Path.Combine(_completedDownloadFolder, _title); + Mocker.GetMock() + .Setup(c => c.GetDirectories(_completedDownloadFolder)) + .Returns(new[] { targetDir }); + + Mocker.GetMock() + .Setup(c => c.GetFiles(targetDir, SearchOption.AllDirectories)) + .Returns(new[] { Path.Combine(targetDir, "somefile.mkv") }); + + Mocker.GetMock() + .Setup(c => c.GetFileSize(It.IsAny())) + .Returns(1000000); + } + + protected void GivenChangedItem() + { + var currentSize = Mocker.GetMock().Object.GetFileSize("abc"); + + Mocker.GetMock() + .Setup(c => c.GetFileSize(It.IsAny())) + .Returns(currentSize + 1); + } + + private void VerifySingleItem(DownloadItemStatus status) + { + var items = Subject.GetItems(_completedDownloadFolder, TimeSpan.FromMilliseconds(50)).ToList(); + + items.Count.Should().Be(1); + items.First().Status.Should().Be(status); + } + + [Test] + public void GetItems_should_considered_locked_files_queued() + { + GivenCompletedItem(); + + Mocker.GetMock() + .Setup(c => c.IsFileLocked(It.IsAny())) + .Returns(true); + + Thread.Sleep(60); + + VerifySingleItem(DownloadItemStatus.Downloading); + } + + [Test] + public void GetItems_should_considered_changing_files_queued() + { + GivenCompletedItem(); + + VerifySingleItem(DownloadItemStatus.Downloading); + + // If we keep changing the file every 20ms we should stay Downloading. + for (int i = 0; i < 10; i++) + { + TestLogger.Info("Iteration {0}", i); + + GivenChangedItem(); + + VerifySingleItem(DownloadItemStatus.Downloading); + + Thread.Sleep(10); + } + + // Until it remains unchanged for >=50ms. + Thread.Sleep(60); + + VerifySingleItem(DownloadItemStatus.Completed); + } + } +} diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs index 9356a8e39..6665ca980 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/TorrentBlackholeFixture.cs @@ -8,7 +8,7 @@ using NUnit.Framework; using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Download; -using NzbDrone.Core.Download.Clients.TorrentBlackhole; +using NzbDrone.Core.Download.Clients.Blackhole; using NzbDrone.Core.Exceptions; using NzbDrone.Core.MediaFiles.TorrentInfo; using NzbDrone.Core.Parser.Model; @@ -30,6 +30,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole _blackholeFolder = @"c:\blackhole\torrent".AsOsAgnostic(); _filePath = (@"c:\blackhole\torrent\" + _title + ".torrent").AsOsAgnostic(); + Mocker.SetConstant(Mocker.Resolve()); + Subject.Definition = new DownloadClientDefinition(); Subject.Definition.Settings = new TorrentBlackholeSettings { @@ -56,13 +58,14 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole protected void GivenCompletedItem() { var targetDir = Path.Combine(_completedDownloadFolder, _title); + Mocker.GetMock() .Setup(c => c.GetDirectories(_completedDownloadFolder)) .Returns(new[] { targetDir }); Mocker.GetMock() .Setup(c => c.GetFiles(targetDir, SearchOption.AllDirectories)) - .Returns(new[] { Path.Combine(_completedDownloadFolder, "somefile.mkv") }); + .Returns(new[] { Path.Combine(targetDir, "somefile.mkv") }); Mocker.GetMock() .Setup(c => c.GetFileSize(It.IsAny())) @@ -87,6 +90,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole [Test] public void completed_download_should_have_required_properties() { + Subject.ScanGracePeriod = TimeSpan.Zero; + GivenCompletedItem(); var result = Subject.GetItems().Single(); @@ -94,6 +99,16 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole VerifyCompleted(result); } + [Test] + public void partial_download_should_have_required_properties() + { + GivenCompletedItem(); + + var result = Subject.GetItems().Single(); + + VerifyPostprocessing(result); + } + [Test] public void should_return_category() { @@ -142,21 +157,6 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Assert.Throws(() => Subject.Download(remoteEpisode)); } - [Test] - public void GetItems_should_considered_locked_files_queued() - { - GivenCompletedItem(); - - Mocker.GetMock() - .Setup(c => c.IsFileLocked(It.IsAny())) - .Returns(true); - - var items = Subject.GetItems().ToList(); - - items.Count.Should().Be(1); - items.First().Status.Should().Be(DownloadItemStatus.Downloading); - } - [Test] public void RemoveItem_should_delete_file() { diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs index 3dc4a93f9..f8aab95bb 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/Blackhole/UsenetBlackholeFixture.cs @@ -9,7 +9,7 @@ using NUnit.Framework; using NzbDrone.Common.Disk; using NzbDrone.Common.Http; using NzbDrone.Core.Download; -using NzbDrone.Core.Download.Clients.UsenetBlackhole; +using NzbDrone.Core.Download.Clients.Blackhole; using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole @@ -29,6 +29,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole _blackholeFolder = @"c:\blackhole\nzb".AsOsAgnostic(); _filePath = (@"c:\blackhole\nzb\" + _title + ".nzb").AsOsAgnostic(); + Mocker.SetConstant(Mocker.Resolve()); + Subject.Definition = new DownloadClientDefinition(); Subject.Definition.Settings = new UsenetBlackholeSettings { @@ -58,7 +60,7 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Mocker.GetMock() .Setup(c => c.GetFiles(targetDir, SearchOption.AllDirectories)) - .Returns(new[] { Path.Combine(_completedDownloadFolder, "somefile.mkv") }); + .Returns(new[] { Path.Combine(targetDir, "somefile.mkv") }); Mocker.GetMock() .Setup(c => c.GetFileSize(It.IsAny())) @@ -68,6 +70,8 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole [Test] public void completed_download_should_have_required_properties() { + Subject.ScanGracePeriod = TimeSpan.Zero; + GivenCompletedItem(); var result = Subject.GetItems().Single(); @@ -75,6 +79,17 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole VerifyCompleted(result); } + [Test] + public void partial_download_should_have_required_properties() + { + GivenCompletedItem(); + + var result = Subject.GetItems().Single(); + + VerifyPostprocessing(result); + } + + [Test] public void should_return_category() { @@ -114,20 +129,6 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests.Blackhole Mocker.GetMock().Verify(c => c.DownloadFile(It.IsAny(), It.IsAny()), Times.Never()); } - [Test] - public void GetItems_should_considered_locked_files_downloading() - { - GivenCompletedItem(); - - Mocker.GetMock() - .Setup(c => c.IsFileLocked(It.IsAny())) - .Returns(true); - - var result = Subject.GetItems().Single(); - - result.Status.Should().Be(DownloadItemStatus.Downloading); - } - [Test] public void RemoveItem_should_delete_file() { diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientTests/DownloadClientFixtureBase.cs b/src/NzbDrone.Core.Test/Download/DownloadClientTests/DownloadClientFixtureBase.cs index c0eacf7bc..762137861 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadClientTests/DownloadClientFixtureBase.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadClientTests/DownloadClientFixtureBase.cs @@ -96,6 +96,15 @@ namespace NzbDrone.Core.Test.Download.DownloadClientTests downloadClientItem.Status.Should().Be(DownloadItemStatus.Downloading); } + protected void VerifyPostprocessing(DownloadClientItem downloadClientItem) + { + VerifyIdentifiable(downloadClientItem); + + //downloadClientItem.RemainingTime.Should().NotBe(TimeSpan.Zero); + //downloadClientItem.OutputPath.Should().NotBeNullOrEmpty(); + downloadClientItem.Status.Should().Be(DownloadItemStatus.Downloading); + } + protected void VerifyCompleted(DownloadClientItem downloadClientItem) { VerifyIdentifiable(downloadClientItem); diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index aa5cbd9ba..f0653da74 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -158,6 +158,7 @@ + diff --git a/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs b/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs new file mode 100644 index 000000000..433a12147 --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Blackhole/ScanWatchFolder.cs @@ -0,0 +1,196 @@ +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Crypto; +using NzbDrone.Common.Disk; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.MediaFiles; +using NzbDrone.Core.Organizer; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.Download.Clients.Blackhole +{ + public interface IScanWatchFolder + { + IEnumerable GetItems(string watchFolder, TimeSpan waitPeriod); + } + + public class ScanWatchFolder : IScanWatchFolder + { + private readonly Logger _logger; + private readonly IDiskProvider _diskProvider; + private readonly IDiskScanService _diskScanService; + private readonly ICached> _watchFolderItemCache; + + public ScanWatchFolder(ICacheManager cacheManager, IDiskScanService diskScanService, IDiskProvider diskProvider, Logger logger) + { + _logger = logger; + _diskProvider = diskProvider; + _diskScanService = diskScanService; + _watchFolderItemCache = cacheManager.GetCache>(GetType()); + } + + public IEnumerable GetItems(string watchFolder, TimeSpan waitPeriod) + { + var newWatchItems = new Dictionary(); + var lastWatchItems = _watchFolderItemCache.Get(watchFolder, () => newWatchItems); + + foreach (var newWatchItem in GetDownloadItems(watchFolder, lastWatchItems, waitPeriod)) + { + newWatchItems[newWatchItem.DownloadId] = newWatchItem; + } + + _watchFolderItemCache.Set(watchFolder, newWatchItems, TimeSpan.FromMinutes(5)); + + return newWatchItems.Values; + } + + private IEnumerable GetDownloadItems(string watchFolder, Dictionary lastWatchItems, TimeSpan waitPeriod) + { + foreach (var folder in _diskProvider.GetDirectories(watchFolder)) + { + var title = FileNameBuilder.CleanFileName(Path.GetFileName(folder)); + + var newWatchItem = new WatchFolderItem + { + DownloadId = Path.GetFileName(folder) + "_" + _diskProvider.FolderGetCreationTime(folder).Ticks, + Title = title, + + OutputPath = new OsPath(folder), + + Status = DownloadItemStatus.Completed, + RemainingTime = TimeSpan.Zero + }; + + var oldWatchItem = lastWatchItems.GetValueOrDefault(newWatchItem.DownloadId); + + if (PreCheckWatchItemExpiry(newWatchItem, oldWatchItem)) + { + var files = _diskProvider.GetFiles(folder, SearchOption.AllDirectories); + + newWatchItem.TotalSize = files.Select(_diskProvider.GetFileSize).Sum(); + newWatchItem.Hash = GetHash(folder, files); + + if (files.Any(_diskProvider.IsFileLocked)) + { + newWatchItem.Status = DownloadItemStatus.Downloading; + newWatchItem.RemainingTime = null; + } + + UpdateWatchItemExpiry(newWatchItem, oldWatchItem, waitPeriod); + } + + yield return newWatchItem; + } + + foreach (var videoFile in _diskScanService.GetVideoFiles(watchFolder, false)) + { + var title = FileNameBuilder.CleanFileName(Path.GetFileName(videoFile)); + + var newWatchItem = new WatchFolderItem + { + DownloadId = Path.GetFileName(videoFile) + "_" + _diskProvider.FileGetLastWrite(videoFile).Ticks, + Title = title, + + OutputPath = new OsPath(videoFile), + + Status = DownloadItemStatus.Completed, + RemainingTime = TimeSpan.Zero + }; + + var oldWatchItem = lastWatchItems.GetValueOrDefault(newWatchItem.DownloadId); + + if (PreCheckWatchItemExpiry(oldWatchItem, newWatchItem)) + { + newWatchItem.TotalSize = _diskProvider.GetFileSize(videoFile); + newWatchItem.Hash = GetHash(videoFile); + + if (_diskProvider.IsFileLocked(videoFile)) + { + newWatchItem.Status = DownloadItemStatus.Downloading; + } + + UpdateWatchItemExpiry(newWatchItem, oldWatchItem, waitPeriod); + } + + yield return newWatchItem; + } + } + + private static bool PreCheckWatchItemExpiry(WatchFolderItem newWatchItem, WatchFolderItem oldWatchItem) + { + if (oldWatchItem == null || oldWatchItem.LastChanged.AddHours(1) > DateTime.UtcNow) + { + return true; + } + + newWatchItem.TotalSize = oldWatchItem.TotalSize; + newWatchItem.Hash = oldWatchItem.Hash; + + return false; + } + + private static void UpdateWatchItemExpiry(WatchFolderItem newWatchItem, WatchFolderItem oldWatchItem, TimeSpan waitPeriod) + { + if (oldWatchItem != null && newWatchItem.Hash == oldWatchItem.Hash) + { + newWatchItem.LastChanged = oldWatchItem.LastChanged; + } + else + { + newWatchItem.LastChanged = DateTime.UtcNow; + } + + var remainingTime = waitPeriod - (DateTime.UtcNow - newWatchItem.LastChanged); + + if (remainingTime > TimeSpan.Zero) + { + newWatchItem.RemainingTime = remainingTime; + newWatchItem.Status = DownloadItemStatus.Downloading; + } + } + + private string GetHash(string folder, string[] files) + { + var data = new StringBuilder(); + + data.Append(folder); + try + { + data.Append(_diskProvider.FolderGetLastWrite(folder).ToBinary()); + } + catch (Exception ex) + { + _logger.TraceException(string.Format("Ignored hashing error during scan for {0}", folder), ex); + } + + foreach (var file in files.OrderBy(v => v)) + { + data.Append(GetHash(file)); + } + + return HashConverter.GetHash(data.ToString()).ToHexString(); + } + + private string GetHash(string file) + { + var data = new StringBuilder(); + + data.Append(file); + try + { + data.Append(_diskProvider.FileGetLastWrite(file).ToBinary()); + data.Append(_diskProvider.GetFileSize(file)); + } + catch (Exception ex) + { + _logger.TraceException(string.Format("Ignored hashing error during scan for {0}", file), ex); + } + + return HashConverter.GetHash(data.ToString()).ToHexString(); + } + } +} diff --git a/src/NzbDrone.Core/Download/Clients/TorrentBlackhole/TorrentBlackhole.cs b/src/NzbDrone.Core/Download/Clients/Blackhole/TorrentBlackhole.cs similarity index 58% rename from src/NzbDrone.Core/Download/Clients/TorrentBlackhole/TorrentBlackhole.cs rename to src/NzbDrone.Core/Download/Clients/Blackhole/TorrentBlackhole.cs index c0a36caa9..a7f01e81f 100644 --- a/src/NzbDrone.Core/Download/Clients/TorrentBlackhole/TorrentBlackhole.cs +++ b/src/NzbDrone.Core/Download/Clients/Blackhole/TorrentBlackhole.cs @@ -15,13 +15,15 @@ using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; using NzbDrone.Core.ThingiProvider; -namespace NzbDrone.Core.Download.Clients.TorrentBlackhole +namespace NzbDrone.Core.Download.Clients.Blackhole { public class TorrentBlackhole : TorrentClientBase { - private readonly IDiskScanService _diskScanService; + private readonly IScanWatchFolder _scanWatchFolder; - public TorrentBlackhole(IDiskScanService diskScanService, + public TimeSpan ScanGracePeriod { get; set; } + + public TorrentBlackhole(IScanWatchFolder scanWatchFolder, ITorrentFileInfoReader torrentFileInfoReader, IHttpClient httpClient, IConfigService configService, @@ -30,7 +32,9 @@ namespace NzbDrone.Core.Download.Clients.TorrentBlackhole Logger logger) : base(torrentFileInfoReader, httpClient, configService, diskProvider, remotePathMappingService, logger) { - _diskScanService = diskScanService; + _scanWatchFolder = scanWatchFolder; + + ScanGracePeriod = TimeSpan.FromSeconds(30); } protected override string AddFromMagnetLink(RemoteEpisode remoteEpisode, string hash, string magnetLink) @@ -72,71 +76,28 @@ namespace NzbDrone.Core.Download.Clients.TorrentBlackhole } } + + public override IEnumerable GetItems() { - foreach (var folder in _diskProvider.GetDirectories(Settings.WatchFolder)) + foreach (var item in _scanWatchFolder.GetItems(Settings.WatchFolder, ScanGracePeriod)) { - var title = FileNameBuilder.CleanFileName(Path.GetFileName(folder)); - - var files = _diskProvider.GetFiles(folder, SearchOption.AllDirectories); - - var historyItem = new DownloadClientItem + yield return new DownloadClientItem { DownloadClient = Definition.Name, - DownloadId = Definition.Name + "_" + Path.GetFileName(folder) + "_" + _diskProvider.FolderGetCreationTime(folder).Ticks, + DownloadId = Definition.Name + "_" + item.DownloadId, Category = "sonarr", - Title = title, + Title = item.Title, - TotalSize = files.Select(_diskProvider.GetFileSize).Sum(), + TotalSize = item.TotalSize, + RemainingTime = item.RemainingTime, - OutputPath = new OsPath(folder) - }; + OutputPath = item.OutputPath, - if (files.Any(_diskProvider.IsFileLocked)) - { - historyItem.Status = DownloadItemStatus.Downloading; - } - else - { - historyItem.Status = DownloadItemStatus.Completed; - - historyItem.RemainingTime = TimeSpan.Zero; - } - - historyItem.IsReadOnly = Settings.ReadOnly; - - yield return historyItem; - } + Status = item.Status, - foreach (var videoFile in _diskScanService.GetVideoFiles(Settings.WatchFolder, false)) - { - var title = FileNameBuilder.CleanFileName(Path.GetFileName(videoFile)); - - var historyItem = new DownloadClientItem - { - DownloadClient = Definition.Name, - DownloadId = Definition.Name + "_" + Path.GetFileName(videoFile) + "_" + _diskProvider.FileGetLastWrite(videoFile).Ticks, - Category = "sonarr", - Title = title, - - TotalSize = _diskProvider.GetFileSize(videoFile), - - OutputPath = new OsPath(videoFile) + IsReadOnly = Settings.ReadOnly }; - - if (_diskProvider.IsFileLocked(videoFile)) - { - historyItem.Status = DownloadItemStatus.Downloading; - } - else - { - historyItem.Status = DownloadItemStatus.Completed; - historyItem.RemainingTime = TimeSpan.Zero; - } - - historyItem.IsReadOnly = Settings.ReadOnly; - - yield return historyItem; } } diff --git a/src/NzbDrone.Core/Download/Clients/TorrentBlackhole/TorrentBlackholeSettings.cs b/src/NzbDrone.Core/Download/Clients/Blackhole/TorrentBlackholeSettings.cs similarity index 96% rename from src/NzbDrone.Core/Download/Clients/TorrentBlackhole/TorrentBlackholeSettings.cs rename to src/NzbDrone.Core/Download/Clients/Blackhole/TorrentBlackholeSettings.cs index 4e930255f..645607ca4 100644 --- a/src/NzbDrone.Core/Download/Clients/TorrentBlackhole/TorrentBlackholeSettings.cs +++ b/src/NzbDrone.Core/Download/Clients/Blackhole/TorrentBlackholeSettings.cs @@ -6,7 +6,7 @@ using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using NzbDrone.Core.Validation.Paths; -namespace NzbDrone.Core.Download.Clients.TorrentBlackhole +namespace NzbDrone.Core.Download.Clients.Blackhole { public class TorrentBlackholeSettingsValidator : AbstractValidator { diff --git a/src/NzbDrone.Core/Download/Clients/UsenetBlackhole/UsenetBlackhole.cs b/src/NzbDrone.Core/Download/Clients/Blackhole/UsenetBlackhole.cs similarity index 53% rename from src/NzbDrone.Core/Download/Clients/UsenetBlackhole/UsenetBlackhole.cs rename to src/NzbDrone.Core/Download/Clients/Blackhole/UsenetBlackhole.cs index 3d7ac5b0d..52377e339 100644 --- a/src/NzbDrone.Core/Download/Clients/UsenetBlackhole/UsenetBlackhole.cs +++ b/src/NzbDrone.Core/Download/Clients/Blackhole/UsenetBlackhole.cs @@ -13,13 +13,15 @@ using NzbDrone.Core.Organizer; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.RemotePathMappings; -namespace NzbDrone.Core.Download.Clients.UsenetBlackhole +namespace NzbDrone.Core.Download.Clients.Blackhole { public class UsenetBlackhole : UsenetClientBase { - private readonly IDiskScanService _diskScanService; + private readonly IScanWatchFolder _scanWatchFolder; - public UsenetBlackhole(IDiskScanService diskScanService, + public TimeSpan ScanGracePeriod { get; set; } + + public UsenetBlackhole(IScanWatchFolder scanWatchFolder, IHttpClient httpClient, IConfigService configService, IDiskProvider diskProvider, @@ -27,7 +29,9 @@ namespace NzbDrone.Core.Download.Clients.UsenetBlackhole Logger logger) : base(httpClient, configService, diskProvider, remotePathMappingService, logger) { - _diskScanService = diskScanService; + _scanWatchFolder = scanWatchFolder; + + ScanGracePeriod = TimeSpan.FromSeconds(30); } protected override string AddFromNzbFile(RemoteEpisode remoteEpisode, string filename, byte[] fileContent) @@ -58,65 +62,22 @@ namespace NzbDrone.Core.Download.Clients.UsenetBlackhole public override IEnumerable GetItems() { - foreach (var folder in _diskProvider.GetDirectories(Settings.WatchFolder)) + foreach (var item in _scanWatchFolder.GetItems(Settings.WatchFolder, ScanGracePeriod)) { - var title = FileNameBuilder.CleanFileName(Path.GetFileName(folder)); - - var files = _diskProvider.GetFiles(folder, SearchOption.AllDirectories); - - var historyItem = new DownloadClientItem + yield return new DownloadClientItem { DownloadClient = Definition.Name, - DownloadId = Definition.Name + "_" + Path.GetFileName(folder) + "_" + _diskProvider.FolderGetCreationTime(folder).Ticks, + DownloadId = Definition.Name + "_" + item.DownloadId, Category = "sonarr", - Title = title, - - TotalSize = files.Select(_diskProvider.GetFileSize).Sum(), - - OutputPath = new OsPath(folder) - }; - - if (files.Any(_diskProvider.IsFileLocked)) - { - historyItem.Status = DownloadItemStatus.Downloading; - } - else - { - historyItem.Status = DownloadItemStatus.Completed; - - historyItem.RemainingTime = TimeSpan.Zero; - } + Title = item.Title, - yield return historyItem; - } - - foreach (var videoFile in _diskScanService.GetVideoFiles(Settings.WatchFolder, false)) - { - var title = FileNameBuilder.CleanFileName(Path.GetFileName(videoFile)); + TotalSize = item.TotalSize, + RemainingTime = item.RemainingTime, - var historyItem = new DownloadClientItem - { - DownloadClient = Definition.Name, - DownloadId = Definition.Name + "_" + Path.GetFileName(videoFile) + "_" + _diskProvider.FileGetLastWrite(videoFile).Ticks, - Category = "sonarr", - Title = title, + OutputPath = item.OutputPath, - TotalSize = _diskProvider.GetFileSize(videoFile), - - OutputPath = new OsPath(videoFile) + Status = item.Status }; - - if (_diskProvider.IsFileLocked(videoFile)) - { - historyItem.Status = DownloadItemStatus.Downloading; - } - else - { - historyItem.Status = DownloadItemStatus.Completed; - historyItem.RemainingTime = TimeSpan.Zero; - } - - yield return historyItem; } } diff --git a/src/NzbDrone.Core/Download/Clients/UsenetBlackhole/UsenetBlackholeSettings.cs b/src/NzbDrone.Core/Download/Clients/Blackhole/UsenetBlackholeSettings.cs similarity index 95% rename from src/NzbDrone.Core/Download/Clients/UsenetBlackhole/UsenetBlackholeSettings.cs rename to src/NzbDrone.Core/Download/Clients/Blackhole/UsenetBlackholeSettings.cs index 5be32d3e6..0619823c2 100644 --- a/src/NzbDrone.Core/Download/Clients/UsenetBlackhole/UsenetBlackholeSettings.cs +++ b/src/NzbDrone.Core/Download/Clients/Blackhole/UsenetBlackholeSettings.cs @@ -5,7 +5,7 @@ using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; using NzbDrone.Core.Validation.Paths; -namespace NzbDrone.Core.Download.Clients.UsenetBlackhole +namespace NzbDrone.Core.Download.Clients.Blackhole { public class UsenetBlackholeSettingsValidator : AbstractValidator { diff --git a/src/NzbDrone.Core/Download/Clients/Blackhole/WatchFolderItem.cs b/src/NzbDrone.Core/Download/Clients/Blackhole/WatchFolderItem.cs new file mode 100644 index 000000000..daf751d4e --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/Blackhole/WatchFolderItem.cs @@ -0,0 +1,22 @@ +using NzbDrone.Common.Disk; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.Download.Clients.Blackhole +{ + + public class WatchFolderItem + { + public string DownloadId { get; set; } + public string Title { get; set; } + public long TotalSize { get; set; } + public TimeSpan? RemainingTime { get; set; } + public OsPath OutputPath { get; set; } + public DownloadItemStatus Status { get; set; } + + public DateTime LastChanged { get; set; } + public string Hash { get; set; } + } +} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 2a6372013..437bf7d11 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -332,6 +332,8 @@ + + @@ -414,8 +416,8 @@ - - + + @@ -430,8 +432,8 @@ - - + +