From 13bfb73ee9f41e7c5f659e8c595a65befbaf5812 Mon Sep 17 00:00:00 2001 From: Qstick Date: Thu, 26 Oct 2017 23:21:06 -0400 Subject: [PATCH] Upstream Changes to DownloadClients and Indexers --- .../Reflection/ReflectionExtensions.cs | 5 + .../BlockedIndexerSpecificationFixture.cs | 54 ++++++ .../TorrentSeedingSpecificationFixture.cs | 111 +++++++++++++ .../DownloadApprovedFixture.cs | 46 +++++- .../DownloadClientStatusServiceFixture.cs | 156 ++++++++++++++++++ .../PendingReleaseServiceTests/AddFixture.cs | 10 +- .../PendingReleaseServiceFixture.cs | 4 +- .../Checks/DownloadClientCheckFixture.cs | 7 +- .../Checks/IndexerStatusCheckFixture.cs | 9 +- ...ClientUnavailablePendingReleasesFixture.cs | 60 +++++++ .../IndexerStatusServiceFixture.cs | 44 ++--- .../NzbDrone.Core.Test.csproj | 8 +- .../ProviderBaseFixture.cs | 7 +- .../ProviderStatusServiceFixture.cs | 126 ++++++++++++++ .../002_add_release_to_pending_releases.cs | 14 ++ src/NzbDrone.Core/Datastore/TableMapping.cs | 1 + .../BlockedIndexerSpecification.cs | 46 ++++++ .../TorrentSeedingSpecification.cs | 21 ++- .../Download/Clients/Deluge/Deluge.cs | 16 +- .../Download/Clients/Deluge/DelugeProxy.cs | 4 +- .../Clients/DownloadClientException.cs | 5 +- .../DownloadClientUnavailableException.cs | 27 +++ .../Proxies/DiskStationProxyBase.cs | 17 +- .../Download/Clients/Hadouken/Hadouken.cs | 12 +- .../Clients/Hadouken/HadoukenProxy.cs | 20 ++- .../Download/Clients/NzbVortex/NzbVortex.cs | 12 +- .../Clients/NzbVortex/NzbVortexProxy.cs | 4 +- .../Download/Clients/Nzbget/Nzbget.cs | 27 +-- .../Download/Clients/Nzbget/NzbgetProxy.cs | 6 +- .../Clients/QBittorrent/QBittorrent.cs | 15 +- .../Clients/QBittorrent/QBittorrentProxy.cs | 2 +- .../Download/Clients/Sabnzbd/Sabnzbd.cs | 12 +- .../Download/Clients/Sabnzbd/SabnzbdProxy.cs | 4 +- .../Clients/Transmission/TransmissionBase.cs | 24 +-- .../Clients/Transmission/TransmissionProxy.cs | 79 +++++---- .../Download/Clients/rTorrent/RTorrent.cs | 101 ++++++------ .../Clients/rTorrent/RTorrentProxy.cs | 99 ++++++----- .../Download/Clients/uTorrent/UTorrent.cs | 71 ++++---- .../Clients/uTorrent/UTorrentProxy.cs | 4 +- .../Download/DownloadClientFactory.cs | 62 ++++++- .../Download/DownloadClientProvider.cs | 11 +- .../Download/DownloadClientStatus.cs | 9 + .../DownloadClientStatusRepository.cs | 19 +++ .../Download/DownloadClientStatusService.cs | 22 +++ src/NzbDrone.Core/Download/DownloadService.cs | 16 +- .../Download/Pending/PendingRelease.cs | 1 + .../Download/Pending/PendingReleaseReason.cs | 9 + .../Download/Pending/PendingReleaseService.cs | 54 ++++-- .../Download/ProcessDownloadDecisions.cs | 98 +++++++++-- .../DownloadMonitoringService.cs | 28 ++-- .../HealthCheck/CheckOnAttribute.cs | 16 ++ .../Checks/AppDataLocationCheck.cs | 6 +- .../HealthCheck/Checks/DownloadClientCheck.cs | 10 +- .../Checks/DownloadClientStatusCheck.cs | 45 +++++ .../Checks/ImportMechanismCheck.cs | 7 +- .../HealthCheck/Checks/IndexerRssCheck.cs | 6 +- .../HealthCheck/Checks/IndexerSearchCheck.cs | 6 +- .../HealthCheck/Checks/IndexerStatusCheck.cs | 27 +-- .../HealthCheck/Checks/MediaInfoDllCheck.cs | 4 +- .../HealthCheck/Checks/MonoVersionCheck.cs | 2 - .../HealthCheck/Checks/ProxyCheck.cs | 6 +- .../HealthCheck/Checks/RootFolderCheck.cs | 5 +- .../HealthCheck/Checks/UpdateCheck.cs | 6 +- .../HealthCheck/HealthCheckBase.cs | 4 +- .../HealthCheck/HealthCheckService.cs | 79 +++++---- .../HealthCheck/IProvideHealthCheck.cs | 3 +- ...ownloadClientUnavailablePendingReleases.cs | 32 ++++ .../CleanupOrphanedDownloadClientStatus.cs | 26 +++ src/NzbDrone.Core/Indexers/IndexerBase.cs | 5 - src/NzbDrone.Core/Indexers/IndexerFactory.cs | 21 ++- src/NzbDrone.Core/Indexers/IndexerStatus.cs | 17 +- .../Indexers/IndexerStatusRepository.cs | 16 +- .../Indexers/IndexerStatusService.cs | 128 +------------- .../Messaging/Events/EventAggregator.cs | 11 ++ src/NzbDrone.Core/NzbDrone.Core.csproj | 15 ++ .../Events/ProviderStatusChangedEvent.cs | 18 ++ .../ThingiProvider/ProviderFactory.cs | 2 +- .../Status/ProviderStatusBase.cs | 2 +- .../Status/ProviderStatusRepository.cs | 2 +- .../Status/ProviderStatusServiceBase.cs | 19 ++- 80 files changed, 1516 insertions(+), 649 deletions(-) create mode 100644 src/NzbDrone.Core.Test/DecisionEngineTests/BlockedIndexerSpecificationFixture.cs create mode 100644 src/NzbDrone.Core.Test/DecisionEngineTests/Search/TorrentSeedingSpecificationFixture.cs create mode 100644 src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs create mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleasesFixture.cs rename src/NzbDrone.Core.Test/{ThingiProvider => ThingiProviderTests}/ProviderBaseFixture.cs (92%) create mode 100644 src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/002_add_release_to_pending_releases.cs create mode 100644 src/NzbDrone.Core/DecisionEngine/Specifications/BlockedIndexerSpecification.cs create mode 100644 src/NzbDrone.Core/Download/Clients/DownloadClientUnavailableException.cs create mode 100644 src/NzbDrone.Core/Download/DownloadClientStatus.cs create mode 100644 src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs create mode 100644 src/NzbDrone.Core/Download/DownloadClientStatusService.cs create mode 100644 src/NzbDrone.Core/Download/Pending/PendingReleaseReason.cs create mode 100644 src/NzbDrone.Core/HealthCheck/CheckOnAttribute.cs create mode 100644 src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs create mode 100644 src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs diff --git a/src/NzbDrone.Common/Reflection/ReflectionExtensions.cs b/src/NzbDrone.Common/Reflection/ReflectionExtensions.cs index 2f54aa5a2..1e778b843 100644 --- a/src/NzbDrone.Common/Reflection/ReflectionExtensions.cs +++ b/src/NzbDrone.Common/Reflection/ReflectionExtensions.cs @@ -60,6 +60,11 @@ namespace NzbDrone.Common.Reflection return (T)attribute; } + public static T[] GetAttributes(this MemberInfo member) where T : Attribute + { + return member.GetCustomAttributes(typeof(T), false).OfType().ToArray(); + } + public static Type FindTypeByName(this Assembly assembly, string name) { return assembly.GetTypes().SingleOrDefault(c => c.Name.Equals(name, StringComparison.InvariantCultureIgnoreCase)); diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/BlockedIndexerSpecificationFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/BlockedIndexerSpecificationFixture.cs new file mode 100644 index 000000000..e5f872c0d --- /dev/null +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/BlockedIndexerSpecificationFixture.cs @@ -0,0 +1,54 @@ +using System; +using System.Collections.Generic; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.DecisionEngine; +using NzbDrone.Core.DecisionEngine.Specifications; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.DecisionEngineTests +{ + [TestFixture] + + public class BlockedIndexerSpecificationFixture : CoreTest + { + private RemoteAlbum _remoteAlbum; + + [SetUp] + public void Setup() + { + _remoteAlbum = new RemoteAlbum + { + Release = new ReleaseInfo { IndexerId = 1 } + }; + + Mocker.GetMock() + .Setup(v => v.GetBlockedProviders()) + .Returns(new List()); + } + + private void WithBlockedIndexer() + { + Mocker.GetMock() + .Setup(v => v.GetBlockedProviders()) + .Returns(new List { new IndexerStatus { ProviderId = 1, DisabledTill = DateTime.UtcNow } }); + } + + [Test] + public void should_return_true_if_no_blocked_indexer() + { + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_return_false_if_blocked_indexer() + { + WithBlockedIndexer(); + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeFalse(); + Subject.Type.Should().Be(RejectionType.Temporary); + } + } +} diff --git a/src/NzbDrone.Core.Test/DecisionEngineTests/Search/TorrentSeedingSpecificationFixture.cs b/src/NzbDrone.Core.Test/DecisionEngineTests/Search/TorrentSeedingSpecificationFixture.cs new file mode 100644 index 000000000..7a4b86f02 --- /dev/null +++ b/src/NzbDrone.Core.Test/DecisionEngineTests/Search/TorrentSeedingSpecificationFixture.cs @@ -0,0 +1,111 @@ +using FizzWare.NBuilder; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.DecisionEngine.Specifications.Search; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.TorrentRss; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Music; +using NzbDrone.Test.Common; + +namespace NzbDrone.Core.Test.DecisionEngineTests.Search +{ + [TestFixture] + public class TorrentSeedingSpecificationFixture : TestBase + { + private Artist _artist; + private RemoteAlbum _remoteAlbum; + private IndexerDefinition _indexerDefinition; + + [SetUp] + public void Setup() + { + _artist = Builder.CreateNew().With(s => s.Id = 1).Build(); + + _remoteAlbum = new RemoteAlbum + { + Artist = _artist, + Release = new TorrentInfo + { + IndexerId = 1, + Title = "Artist - Album [FLAC-RlsGrp]", + Seeders = 0 + } + }; + + _indexerDefinition = new IndexerDefinition + { + Settings = new TorrentRssIndexerSettings { MinimumSeeders = 5 } + }; + + Mocker.GetMock() + .Setup(v => v.Get(1)) + .Returns(_indexerDefinition); + + } + + private void GivenReleaseSeeders(int? seeders) + { + (_remoteAlbum.Release as TorrentInfo).Seeders = seeders; + } + + [Test] + public void should_return_true_if_not_torrent() + { + _remoteAlbum.Release = new ReleaseInfo + { + IndexerId = 1, + Title = "Artist - Album [FLAC-RlsGrp]" + }; + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_return_true_if_indexer_not_specified() + { + _remoteAlbum.Release.IndexerId = 0; + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_return_true_if_indexer_no_longer_exists() + { + Mocker.GetMock() + .Setup(v => v.Get(It.IsAny())) + .Callback(i => { throw new ModelNotFoundException(typeof(IndexerDefinition), i); }); + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [Test] + public void should_return_true_if_seeds_unknown() + { + GivenReleaseSeeders(null); + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [TestCase(5)] + [TestCase(6)] + public void should_return_true_if_seeds_above_or_equal_to_limit(int seeders) + { + GivenReleaseSeeders(seeders); + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeTrue(); + } + + [TestCase(0)] + [TestCase(4)] + public void should_return_false_if_seeds_belove_limit(int seeders) + { + GivenReleaseSeeders(seeders); + + Subject.IsSatisfiedBy(_remoteAlbum, null).Accepted.Should().BeFalse(); + } + } +} diff --git a/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs index da5230278..cac1446ad 100644 --- a/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs +++ b/src/NzbDrone.Core.Test/Download/DownloadApprovedReportsTests/DownloadApprovedFixture.cs @@ -6,7 +6,9 @@ using Moq; using NUnit.Framework; using NzbDrone.Core.DecisionEngine; using NzbDrone.Core.Download; +using NzbDrone.Core.Download.Clients; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Indexers; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Profiles.Qualities; using NzbDrone.Core.Qualities; @@ -34,7 +36,7 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests .Build(); } - private RemoteAlbum GetRemoteAlbum(List albums, QualityModel quality) + private RemoteAlbum GetRemoteAlbum(List albums, QualityModel quality, DownloadProtocol downloadProtocol = DownloadProtocol.Usenet) { var remoteAlbum = new RemoteAlbum(); remoteAlbum.ParsedAlbumInfo = new ParsedAlbumInfo(); @@ -44,6 +46,7 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests remoteAlbum.Albums.AddRange(albums); remoteAlbum.Release = new ReleaseInfo(); + remoteAlbum.Release.DownloadProtocol = downloadProtocol; remoteAlbum.Release.PublishDate = DateTime.UtcNow; remoteAlbum.Artist = Builder.CreateNew() @@ -191,7 +194,6 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests var decisions = new List(); decisions.Add(new DownloadDecision(remoteAlbum, new Rejection("Failure!", RejectionType.Temporary))); - decisions.Add(new DownloadDecision(remoteAlbum)); Subject.ProcessDecisions(decisions); Mocker.GetMock().Verify(v => v.DownloadReport(It.IsAny()), Times.Never()); @@ -208,7 +210,7 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests decisions.Add(new DownloadDecision(remoteAlbum, new Rejection("Failure!", RejectionType.Temporary))); Subject.ProcessDecisions(decisions); - Mocker.GetMock().Verify(v => v.Add(It.IsAny()), Times.Never()); + Mocker.GetMock().Verify(v => v.Add(It.IsAny(), It.IsAny()), Times.Never()); } [Test] @@ -222,7 +224,43 @@ namespace NzbDrone.Core.Test.Download.DownloadApprovedReportsTests decisions.Add(new DownloadDecision(remoteAlbum, new Rejection("Failure!", RejectionType.Temporary))); Subject.ProcessDecisions(decisions); - Mocker.GetMock().Verify(v => v.Add(It.IsAny()), Times.Exactly(2)); + Mocker.GetMock().Verify(v => v.Add(It.IsAny(), It.IsAny()), Times.Exactly(2)); + } + + [Test] + public void should_add_to_failed_if_already_failed_for_that_protocol() + { + var albums = new List { GetAlbum(1) }; + var remoteAlbum = GetRemoteAlbum(albums, new QualityModel(Quality.MP3_320)); + + var decisions = new List(); + decisions.Add(new DownloadDecision(remoteAlbum)); + decisions.Add(new DownloadDecision(remoteAlbum)); + + Mocker.GetMock().Setup(s => s.DownloadReport(It.IsAny())) + .Throws(new DownloadClientUnavailableException("Download client failed")); + + Subject.ProcessDecisions(decisions); + Mocker.GetMock().Verify(v => v.DownloadReport(It.IsAny()), Times.Once()); + } + + [Test] + public void should_not_add_to_failed_if_failed_for_a_different_protocol() + { + var episodes = new List { GetAlbum(1) }; + var remoteEpisode = GetRemoteAlbum(episodes, new QualityModel(Quality.MP3_320), DownloadProtocol.Usenet); + var remoteEpisode2 = GetRemoteAlbum(episodes, new QualityModel(Quality.MP3_320), DownloadProtocol.Torrent); + + var decisions = new List(); + decisions.Add(new DownloadDecision(remoteEpisode)); + decisions.Add(new DownloadDecision(remoteEpisode2)); + + Mocker.GetMock().Setup(s => s.DownloadReport(It.Is(r => r.Release.DownloadProtocol == DownloadProtocol.Usenet))) + .Throws(new DownloadClientUnavailableException("Download client failed")); + + Subject.ProcessDecisions(decisions); + Mocker.GetMock().Verify(v => v.DownloadReport(It.Is(r => r.Release.DownloadProtocol == DownloadProtocol.Usenet)), Times.Once()); + Mocker.GetMock().Verify(v => v.DownloadReport(It.Is(r => r.Release.DownloadProtocol == DownloadProtocol.Torrent)), Times.Once()); } } } diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs new file mode 100644 index 000000000..c54698f2e --- /dev/null +++ b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs @@ -0,0 +1,156 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Download; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Download +{ + public class DownloadClientStatusServiceFixture : CoreTest + { + private DateTime _epoch; + + [SetUp] + public void SetUp() + { + _epoch = DateTime.UtcNow; + } + + private DownloadClientStatus WithStatus(DownloadClientStatus status) + { + Mocker.GetMock() + .Setup(v => v.FindByProviderId(1)) + .Returns(status); + + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new[] { status }); + + return status; + } + + private void VerifyUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); + } + + private void VerifyNoUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); + } + + [Test] + public void should_not_consider_blocked_within_5_minutes_since_initial_failure() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(4), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + } + + [Test] + public void should_consider_blocked_after_5_minutes_since_initial_failure() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + } + + [Test] + public void should_not_escalate_further_till_after_5_minutes_since_initial_failure() + { + var origStatus = WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(4), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + + origStatus.EscalationLevel.Should().Be(3); + } + + [Test] + public void should_escalate_further_after_5_minutes_since_initial_failure() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + + status.EscalationLevel.Should().BeGreaterThan(3); + } + + [Test] + public void should_not_escalate_beyond_3_hours() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1)); + } + } +} diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs index f6d421cb0..c245ba895 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/AddFixture.cs @@ -102,7 +102,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests [Test] public void should_add() { - Subject.Add(_temporarilyRejected); + Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyInsert(); } @@ -112,7 +112,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests { GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate); - Subject.Add(_temporarilyRejected); + Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyNoInsert(); } @@ -122,7 +122,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests { GivenHeldRelease(_release.Title + "-RP", _release.Indexer, _release.PublishDate); - Subject.Add(_temporarilyRejected); + Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyInsert(); } @@ -132,7 +132,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests { GivenHeldRelease(_release.Title, "AnotherIndexer", _release.PublishDate); - Subject.Add(_temporarilyRejected); + Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyInsert(); } @@ -142,7 +142,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests { GivenHeldRelease(_release.Title, _release.Indexer, _release.PublishDate.AddHours(1)); - Subject.Add(_temporarilyRejected); + Subject.Add(_temporarilyRejected, PendingReleaseReason.Delay); VerifyInsert(); } diff --git a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs index b640003dc..8ff0f0ea0 100644 --- a/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs +++ b/src/NzbDrone.Core.Test/Download/Pending/PendingReleaseServiceTests/PendingReleaseServiceFixture.cs @@ -27,7 +27,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests public void should_not_ignore_pending_items_from_available_indexer() { Mocker.GetMock() - .Setup(v => v.GetBlockedIndexers()) + .Setup(v => v.GetBlockedProviders()) .Returns(new List()); GivenPendingRelease(); @@ -43,7 +43,7 @@ namespace NzbDrone.Core.Test.Download.Pending.PendingReleaseServiceTests public void should_ignore_pending_items_from_unavailable_indexer() { Mocker.GetMock() - .Setup(v => v.GetBlockedIndexers()) + .Setup(v => v.GetBlockedProviders()) .Returns(new List { new IndexerStatus { ProviderId = 1, DisabledTill = DateTime.UtcNow.AddHours(2) } }); GivenPendingRelease(); diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs index dc6986d79..56f83e0a9 100644 --- a/src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs @@ -1,9 +1,8 @@ -using System; +using System; using System.Collections.Generic; using NUnit.Framework; using NzbDrone.Core.Download; using NzbDrone.Core.HealthCheck.Checks; -using NzbDrone.Core.Indexers; using NzbDrone.Core.Test.Framework; using NzbDrone.Test.Common; @@ -26,7 +25,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks public void should_return_error_when_download_client_throws() { var downloadClient = Mocker.GetMock(); - downloadClient.Setup(s => s.Definition).Returns(new IndexerDefinition{Name = "Test"}); + downloadClient.Setup(s => s.Definition).Returns(new DownloadClientDefinition{Name = "Test"}); downloadClient.Setup(s => s.GetItems()) .Throws(); @@ -36,8 +35,6 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks .Returns(new IDownloadClient[] { downloadClient.Object }); Subject.Check().ShouldBeError(); - - ExceptionVerification.ExpectedErrors(1); } [Test] diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs index 246765963..1d71d3a80 100644 --- a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs @@ -22,7 +22,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks .Returns(_indexers); Mocker.GetMock() - .Setup(v => v.GetBlockedIndexers()) + .Setup(v => v.GetBlockedProviders()) .Returns(_blockedIndexers); } @@ -57,13 +57,6 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks { Subject.Check().ShouldBeOk(); } - [Test] - public void should_not_return_error_when_indexer_failed_less_than_an_hour() - { - GivenIndexer(1, 0.1, 0.5); - - Subject.Check().ShouldBeOk(); - } [Test] public void should_return_warning_if_indexer_unavailable() diff --git a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleasesFixture.cs b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleasesFixture.cs new file mode 100644 index 000000000..cda91729d --- /dev/null +++ b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleasesFixture.cs @@ -0,0 +1,60 @@ +using System; +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Housekeeping.Housekeepers; +using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Housekeeping.Housekeepers +{ + [TestFixture] + public class CleanupDownloadClientUnavailablePendingReleasesFixture : DbTest + { + [Test] + public void should_delete_old_DownloadClientUnavailable_pending_items() + { + var pendingRelease = Builder.CreateNew() + .With(h => h.Reason = PendingReleaseReason.DownloadClientUnavailable) + .With(h => h.Added = DateTime.UtcNow.AddDays(-21)) + .With(h => h.ParsedAlbumInfo = new ParsedAlbumInfo()) + .With(h => h.Release = new ReleaseInfo()) + .BuildNew(); + + Db.Insert(pendingRelease); + Subject.Clean(); + AllStoredModels.Should().BeEmpty(); + } + + [Test] + public void should_delete_old_Fallback_pending_items() + { + var pendingRelease = Builder.CreateNew() + .With(h => h.Reason = PendingReleaseReason.Fallback) + .With(h => h.Added = DateTime.UtcNow.AddDays(-21)) + .With(h => h.ParsedAlbumInfo = new ParsedAlbumInfo()) + .With(h => h.Release = new ReleaseInfo()) + .BuildNew(); + + Db.Insert(pendingRelease); + Subject.Clean(); + AllStoredModels.Should().BeEmpty(); + } + + [Test] + public void should_not_delete_old_Delay_pending_items() + { + var pendingRelease = Builder.CreateNew() + .With(h => h.Reason = PendingReleaseReason.Delay) + .With(h => h.Added = DateTime.UtcNow.AddDays(-21)) + .With(h => h.ParsedAlbumInfo = new ParsedAlbumInfo()) + .With(h => h.Release = new ReleaseInfo()) + .BuildNew(); + + Db.Insert(pendingRelease); + Subject.Clean(); + AllStoredModels.Should().HaveCount(1); + } + } +} diff --git a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs index d7bee11f2..3f1699c9f 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Linq; using FluentAssertions; using Moq; @@ -11,7 +11,7 @@ namespace NzbDrone.Core.Test.IndexerTests public class IndexerStatusServiceFixture : CoreTest { private DateTime _epoch; - + [SetUp] public void SetUp() { @@ -21,7 +21,7 @@ namespace NzbDrone.Core.Test.IndexerTests private void WithStatus(IndexerStatus status) { Mocker.GetMock() - .Setup(v => v.FindByIndexerId(1)) + .Setup(v => v.FindByProviderId(1)) .Returns(status); Mocker.GetMock() @@ -29,25 +29,16 @@ namespace NzbDrone.Core.Test.IndexerTests .Returns(new[] { status }); } - private void VerifyUpdate(bool updated = true) + private void VerifyUpdate() { Mocker.GetMock() - .Verify(v => v.Upsert(It.IsAny()), Times.Exactly(updated ? 1 : 0)); + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); } - [Test] - public void should_start_backoff_on_first_failure() + private void VerifyNoUpdate() { - WithStatus(new IndexerStatus()); - - Subject.RecordFailure(1); - - VerifyUpdate(); - - var status = Subject.GetBlockedIndexers().FirstOrDefault(); - status.Should().NotBeNull(); - status.DisabledTill.Should().HaveValue(); - status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500); + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); } [Test] @@ -59,7 +50,7 @@ namespace NzbDrone.Core.Test.IndexerTests VerifyUpdate(); - var status = Subject.GetBlockedIndexers().FirstOrDefault(); + var status = Subject.GetBlockedProviders().FirstOrDefault(); status.Should().BeNull(); } @@ -70,22 +61,7 @@ namespace NzbDrone.Core.Test.IndexerTests Subject.RecordSuccess(1); - VerifyUpdate(false); - } - - [Test] - public void should_preserve_escalation_on_intermittent_success() - { - WithStatus(new IndexerStatus { MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), EscalationLevel = 3 }); - - Subject.RecordSuccess(1); - Subject.RecordSuccess(1); - Subject.RecordFailure(1); - - var status = Subject.GetBlockedIndexers().FirstOrDefault(); - status.Should().NotBeNull(); - status.DisabledTill.Should().HaveValue(); - status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500); + VerifyNoUpdate(); } } } diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index a65554526..f820b351f 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -117,6 +117,7 @@ + @@ -135,10 +136,12 @@ + + @@ -198,6 +201,7 @@ + @@ -324,11 +328,12 @@ - + + @@ -522,6 +527,7 @@ + diff --git a/src/NzbDrone.Core.Test/ThingiProvider/ProviderBaseFixture.cs b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderBaseFixture.cs similarity index 92% rename from src/NzbDrone.Core.Test/ThingiProvider/ProviderBaseFixture.cs rename to src/NzbDrone.Core.Test/ThingiProviderTests/ProviderBaseFixture.cs index db1e21c61..02bbdd8b2 100644 --- a/src/NzbDrone.Core.Test/ThingiProvider/ProviderBaseFixture.cs +++ b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderBaseFixture.cs @@ -1,13 +1,12 @@ -using FizzWare.NBuilder; +using FizzWare.NBuilder; using FluentAssertions; using NUnit.Framework; using NzbDrone.Core.Indexers; using NzbDrone.Core.Indexers.Newznab; using NzbDrone.Core.Test.Framework; -namespace NzbDrone.Core.Test.ThingiProvider +namespace NzbDrone.Core.Test.ThingiProviderTests { - public class ProviderRepositoryFixture : DbTest { [Test] @@ -27,4 +26,4 @@ namespace NzbDrone.Core.Test.ThingiProvider storedSetting.ShouldBeEquivalentTo(newznabSettings, o=>o.IncludingAllRuntimeProperties()); } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs new file mode 100644 index 000000000..ba63a2b6b --- /dev/null +++ b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs @@ -0,0 +1,126 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NLog; +using NUnit.Framework; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Test.ThingiProviderTests +{ + public class MockProviderStatus : ProviderStatusBase + { + } + + public interface IMockProvider : IProvider + { + } + + public interface IMockProviderStatusRepository : IProviderStatusRepository + { + } + + public class MockProviderStatusService : ProviderStatusServiceBase + { + public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + : base(providerStatusRepository, eventAggregator, logger) + { + + } + } + + public class ProviderStatusServiceFixture : CoreTest + { + private DateTime _epoch; + + [SetUp] + public void SetUp() + { + _epoch = DateTime.UtcNow; + } + + private void WithStatus(MockProviderStatus status) + { + Mocker.GetMock() + .Setup(v => v.FindByProviderId(1)) + .Returns(status); + + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new[] { status }); + } + + private void VerifyUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); + } + + private void VerifyNoUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); + } + + [Test] + public void should_start_backoff_on_first_failure() + { + WithStatus(new MockProviderStatus()); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500); + } + + [Test] + public void should_cancel_backoff_on_success() + { + WithStatus(new MockProviderStatus { EscalationLevel = 2 }); + + Subject.RecordSuccess(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + } + + [Test] + public void should_not_store_update_if_already_okay() + { + WithStatus(new MockProviderStatus { EscalationLevel = 0 }); + + Subject.RecordSuccess(1); + + VerifyNoUpdate(); + } + + [Test] + public void should_preserve_escalation_on_intermittent_success() + { + WithStatus(new MockProviderStatus + { + InitialFailure = _epoch - TimeSpan.FromSeconds(20), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordSuccess(1); + Subject.RecordSuccess(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/002_add_release_to_pending_releases.cs b/src/NzbDrone.Core/Datastore/Migration/002_add_release_to_pending_releases.cs new file mode 100644 index 000000000..e4ef749e7 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/002_add_release_to_pending_releases.cs @@ -0,0 +1,14 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(2)] + public class add_reason_to_pending_releases : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("PendingReleases").AddColumn("Reason").AsInt32().WithDefaultValue(0); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 0c4a3bb7a..9d04ba326 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -123,6 +123,7 @@ namespace NzbDrone.Core.Datastore .Ignore(c => c.Message); Mapper.Entity().RegisterModel("IndexerStatus"); + Mapper.Entity().RegisterModel("DownloadClientStatus"); } private static void RegisterMappers() diff --git a/src/NzbDrone.Core/DecisionEngine/Specifications/BlockedIndexerSpecification.cs b/src/NzbDrone.Core/DecisionEngine/Specifications/BlockedIndexerSpecification.cs new file mode 100644 index 000000000..3ec06fe7e --- /dev/null +++ b/src/NzbDrone.Core/DecisionEngine/Specifications/BlockedIndexerSpecification.cs @@ -0,0 +1,46 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.DecisionEngine.Specifications +{ + public class BlockedIndexerSpecification : IDecisionEngineSpecification + { + private readonly IIndexerStatusService _indexerStatusService; + private readonly Logger _logger; + + private readonly ICachedDictionary _blockedIndexerCache; + + public BlockedIndexerSpecification(IIndexerStatusService indexerStatusService, ICacheManager cacheManager, Logger logger) + { + _indexerStatusService = indexerStatusService; + _logger = logger; + + _blockedIndexerCache = cacheManager.GetCacheDictionary(GetType(), "blocked", FetchBlockedIndexer, TimeSpan.FromSeconds(15)); + } + + public SpecificationPriority Priority => SpecificationPriority.Database; + public RejectionType Type => RejectionType.Temporary; + + public virtual Decision IsSatisfiedBy(RemoteAlbum subject, SearchCriteriaBase searchCriteria) + { + var status = _blockedIndexerCache.Find(subject.Release.IndexerId.ToString()); + if (status != null) + { + return Decision.Reject($"Indexer {subject.Release.Indexer} is blocked till {status.DisabledTill} due to failures, cannot grab release."); + } + + return Decision.Accept(); + } + + private IDictionary FetchBlockedIndexer() + { + return _indexerStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId.ToString()); + } + } +} diff --git a/src/NzbDrone.Core/DecisionEngine/Specifications/TorrentSeedingSpecification.cs b/src/NzbDrone.Core/DecisionEngine/Specifications/TorrentSeedingSpecification.cs index fd9400bf6..1a150be94 100644 --- a/src/NzbDrone.Core/DecisionEngine/Specifications/TorrentSeedingSpecification.cs +++ b/src/NzbDrone.Core/DecisionEngine/Specifications/TorrentSeedingSpecification.cs @@ -1,6 +1,5 @@ -using System.Linq; using NLog; -using NzbDrone.Common.Reflection; +using NzbDrone.Core.Datastore; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser.Model; @@ -9,10 +8,10 @@ namespace NzbDrone.Core.DecisionEngine.Specifications.Search { public class TorrentSeedingSpecification : IDecisionEngineSpecification { - private readonly IndexerFactory _indexerFactory; + private readonly IIndexerFactory _indexerFactory; private readonly Logger _logger; - public TorrentSeedingSpecification(IndexerFactory indexerFactory, Logger logger) + public TorrentSeedingSpecification(IIndexerFactory indexerFactory, Logger logger) { _indexerFactory = indexerFactory; _logger = logger; @@ -26,12 +25,22 @@ namespace NzbDrone.Core.DecisionEngine.Specifications.Search { var torrentInfo = remoteAlbum.Release as TorrentInfo; - if (torrentInfo == null) + if (torrentInfo == null || torrentInfo.IndexerId == 0) { return Decision.Accept(); } - var indexer = _indexerFactory.Get(torrentInfo.IndexerId); + IndexerDefinition indexer; + try + { + indexer = _indexerFactory.Get(torrentInfo.IndexerId); + } + catch (ModelNotFoundException) + { + _logger.Debug("Indexer with id {0} does not exist, skipping seeders check", torrentInfo.IndexerId); + return Decision.Accept(); + } + var torrentIndexerSettings = indexer.Settings as ITorrentIndexerSettings; if (torrentIndexerSettings != null) diff --git a/src/NzbDrone.Core/Download/Clients/Deluge/Deluge.cs b/src/NzbDrone.Core/Download/Clients/Deluge/Deluge.cs index 81c5a221f..32e7b2b8b 100644 --- a/src/NzbDrone.Core/Download/Clients/Deluge/Deluge.cs +++ b/src/NzbDrone.Core/Download/Clients/Deluge/Deluge.cs @@ -81,21 +81,13 @@ namespace NzbDrone.Core.Download.Clients.Deluge { IEnumerable torrents; - try + if (!Settings.TvCategory.IsNullOrWhiteSpace()) { - if (!Settings.TvCategory.IsNullOrWhiteSpace()) - { - torrents = _proxy.GetTorrentsByLabel(Settings.TvCategory, Settings); - } - else - { - torrents = _proxy.GetTorrents(Settings); - } + torrents = _proxy.GetTorrentsByLabel(Settings.TvCategory, Settings); } - catch (DownloadClientException ex) + else { - _logger.Error(ex, "Couldn't get list of torrents"); - return Enumerable.Empty(); + torrents = _proxy.GetTorrents(Settings); } var items = new List(); diff --git a/src/NzbDrone.Core/Download/Clients/Deluge/DelugeProxy.cs b/src/NzbDrone.Core/Download/Clients/Deluge/DelugeProxy.cs index 3406685db..62dd60529 100644 --- a/src/NzbDrone.Core/Download/Clients/Deluge/DelugeProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Deluge/DelugeProxy.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Net; @@ -231,7 +231,7 @@ namespace NzbDrone.Core.Download.Clients.Deluge } catch (WebException ex) { - throw new DownloadClientException("Unable to connect to Deluge, please check your settings", ex); + throw new DownloadClientUnavailableException("Unable to connect to Deluge, please check your settings", ex); } } diff --git a/src/NzbDrone.Core/Download/Clients/DownloadClientException.cs b/src/NzbDrone.Core/Download/Clients/DownloadClientException.cs index 9598e04ef..79a93ef1f 100644 --- a/src/NzbDrone.Core/Download/Clients/DownloadClientException.cs +++ b/src/NzbDrone.Core/Download/Clients/DownloadClientException.cs @@ -1,4 +1,4 @@ -using System; +using System; using NzbDrone.Common.Exceptions; namespace NzbDrone.Core.Download.Clients @@ -8,19 +8,16 @@ namespace NzbDrone.Core.Download.Clients public DownloadClientException(string message, params object[] args) : base(string.Format(message, args)) { - } public DownloadClientException(string message) : base(message) { - } public DownloadClientException(string message, Exception innerException, params object[] args) : base(string.Format(message, args), innerException) { - } public DownloadClientException(string message, Exception innerException) diff --git a/src/NzbDrone.Core/Download/Clients/DownloadClientUnavailableException.cs b/src/NzbDrone.Core/Download/Clients/DownloadClientUnavailableException.cs new file mode 100644 index 000000000..923698cef --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/DownloadClientUnavailableException.cs @@ -0,0 +1,27 @@ +using System; + +namespace NzbDrone.Core.Download.Clients +{ + public class DownloadClientUnavailableException : DownloadClientException + { + public DownloadClientUnavailableException(string message, params object[] args) + : base(string.Format(message, args)) + { + } + + public DownloadClientUnavailableException(string message) + : base(message) + { + } + + public DownloadClientUnavailableException(string message, Exception innerException, params object[] args) + : base(string.Format(message, args), innerException) + { + } + + public DownloadClientUnavailableException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/NzbDrone.Core/Download/Clients/DownloadStation/Proxies/DiskStationProxyBase.cs b/src/NzbDrone.Core/Download/Clients/DownloadStation/Proxies/DiskStationProxyBase.cs index 2a8e4b144..2162a3d57 100644 --- a/src/NzbDrone.Core/Download/Clients/DownloadStation/Proxies/DiskStationProxyBase.cs +++ b/src/NzbDrone.Core/Download/Clients/DownloadStation/Proxies/DiskStationProxyBase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Net; @@ -72,7 +72,20 @@ namespace NzbDrone.Core.Download.Clients.DownloadStation.Proxies DownloadStationSettings settings) where T : new() { var request = requestBuilder.Build(); - var response = _httpClient.Execute(request); + HttpResponse response; + + try + { + response = _httpClient.Execute(request); + } + catch (HttpException ex) + { + throw new DownloadClientException("Unable to connect to Diskstation, please check your settings", ex); + } + catch (WebException ex) + { + throw new DownloadClientUnavailableException("Unable to connect to Diskstation, please check your settings", ex); + } _logger.Debug("Trying to {0}", operation); diff --git a/src/NzbDrone.Core/Download/Clients/Hadouken/Hadouken.cs b/src/NzbDrone.Core/Download/Clients/Hadouken/Hadouken.cs index c91833637..4d0ee1049 100644 --- a/src/NzbDrone.Core/Download/Clients/Hadouken/Hadouken.cs +++ b/src/NzbDrone.Core/Download/Clients/Hadouken/Hadouken.cs @@ -35,17 +35,7 @@ namespace NzbDrone.Core.Download.Clients.Hadouken public override IEnumerable GetItems() { - HadoukenTorrent[] torrents; - - try - { - torrents = _proxy.GetTorrents(Settings); - } - catch (DownloadClientException ex) - { - _logger.ErrorException(ex.Message, ex); - return Enumerable.Empty(); - } + var torrents = _proxy.GetTorrents(Settings); var items = new List(); diff --git a/src/NzbDrone.Core/Download/Clients/Hadouken/HadoukenProxy.cs b/src/NzbDrone.Core/Download/Clients/Hadouken/HadoukenProxy.cs index e044dd912..9eee399a3 100644 --- a/src/NzbDrone.Core/Download/Clients/Hadouken/HadoukenProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Hadouken/HadoukenProxy.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Net; using NLog; @@ -77,7 +77,21 @@ namespace NzbDrone.Core.Download.Clients.Hadouken requestBuilder.Headers.Add("Accept-Encoding", "gzip,deflate"); var httpRequest = requestBuilder.Build(); - var response = _httpClient.Execute(httpRequest); + HttpResponse response; + + try + { + response = _httpClient.Execute(httpRequest); + } + catch (HttpException ex) + { + throw new DownloadClientException("Unable to connect to Hadouken, please check your settings", ex); + } + catch (WebException ex) + { + throw new DownloadClientUnavailableException("Unable to connect to Hadouken, please check your settings", ex); + } + var result = Json.Deserialize>(response.Content); if (result.Error != null) @@ -160,4 +174,4 @@ namespace NzbDrone.Core.Download.Clients.Hadouken return HadoukenTorrentState.Unknown; } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs b/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs index 1b156a0ca..f4a8db185 100644 --- a/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs +++ b/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortex.cs @@ -47,17 +47,7 @@ namespace NzbDrone.Core.Download.Clients.NzbVortex public override IEnumerable GetItems() { - List vortexQueue; - - try - { - vortexQueue = _proxy.GetQueue(30, Settings); - } - catch (DownloadClientException ex) - { - _logger.Warn("Couldn't get download queue. {0}", ex.Message); - return Enumerable.Empty(); - } + var vortexQueue = _proxy.GetQueue(30, Settings); var queueItems = new List(); diff --git a/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortexProxy.cs b/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortexProxy.cs index 15450c280..c93d18b57 100644 --- a/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortexProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/NzbVortex/NzbVortexProxy.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Net; using Newtonsoft.Json; @@ -164,7 +164,7 @@ namespace NzbDrone.Core.Download.Clients.NzbVortex } catch (WebException ex) { - throw new DownloadClientException("Unable to connect to NZBVortex, please check your settings", ex); + throw new DownloadClientUnavailableException("Unable to connect to NZBVortex, please check your settings", ex); } } diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs index f2d613192..2a84ab1b4 100644 --- a/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/Nzbget.cs @@ -51,19 +51,8 @@ namespace NzbDrone.Core.Download.Clients.Nzbget private IEnumerable GetQueue() { - NzbgetGlobalStatus globalStatus; - List queue; - - try - { - globalStatus = _proxy.GetGlobalStatus(Settings); - queue = _proxy.GetQueue(Settings); - } - catch (DownloadClientException ex) - { - _logger.Error(ex, ex.Message); - return Enumerable.Empty(); - } + var globalStatus = _proxy.GetGlobalStatus(Settings); + var queue = _proxy.GetQueue(Settings); var queueItems = new List(); @@ -119,17 +108,7 @@ namespace NzbDrone.Core.Download.Clients.Nzbget private IEnumerable GetHistory() { - List history; - - try - { - history = _proxy.GetHistory(Settings).Take(_configService.DownloadClientHistoryLimit).ToList(); - } - catch (DownloadClientException ex) - { - _logger.Error(ex, ex.Message); - return Enumerable.Empty(); - } + var history = _proxy.GetHistory(Settings).Take(_configService.DownloadClientHistoryLimit).ToList(); var historyItems = new List(); diff --git a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs index 366bb9a2c..129d48450 100644 --- a/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Nzbget/NzbgetProxy.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using NLog; @@ -235,14 +235,14 @@ namespace NzbDrone.Core.Download.Clients.Nzbget { if (ex.Response.StatusCode == HttpStatusCode.Unauthorized) { - throw new DownloadClientException("Authentication failed for NzbGet, please check your settings", ex); + throw new DownloadClientAuthenticationException("Authentication failed for NzbGet, please check your settings", ex); } throw new DownloadClientException("Unable to connect to NzbGet. " + ex.Message, ex); } catch (WebException ex) { - throw new DownloadClientException("Unable to connect to NzbGet. " + ex.Message, ex); + throw new DownloadClientUnavailableException("Unable to connect to NzbGet. " + ex.Message, ex); } var result = Json.Deserialize>(response.Content); diff --git a/src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrent.cs b/src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrent.cs index d1fd5f17a..da17970ee 100644 --- a/src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrent.cs +++ b/src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrent.cs @@ -89,19 +89,8 @@ namespace NzbDrone.Core.Download.Clients.QBittorrent public override IEnumerable GetItems() { - QBittorrentPreferences config; - List torrents; - - try - { - config = _proxy.GetConfig(Settings); - torrents = _proxy.GetTorrents(Settings); - } - catch (DownloadClientException ex) - { - _logger.Error(ex, ex.Message); - return Enumerable.Empty(); - } + var config = _proxy.GetConfig(Settings); + var torrents = _proxy.GetTorrents(Settings); var queueItems = new List(); diff --git a/src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrentProxy.cs b/src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrentProxy.cs index 6f5412b85..61c6424eb 100644 --- a/src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrentProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/QBittorrent/QBittorrentProxy.cs @@ -226,7 +226,7 @@ namespace NzbDrone.Core.Download.Clients.QBittorrent } catch (WebException ex) { - throw new DownloadClientException("Failed to connect to qBitTorrent, please check your settings.", ex); + throw new DownloadClientUnavailableException("Failed to connect to qBitTorrent, please check your settings.", ex); } if (response.Content != "Ok.") // returns "Fails." on bad login diff --git a/src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs b/src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs index 26072b33c..5969f5d0c 100644 --- a/src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs +++ b/src/NzbDrone.Core/Download/Clients/Sabnzbd/Sabnzbd.cs @@ -112,17 +112,7 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd private IEnumerable GetHistory() { - SabnzbdHistory sabHistory; - - try - { - sabHistory = _proxy.GetHistory(0, _configService.DownloadClientHistoryLimit, Settings.TvCategory, Settings); - } - catch (DownloadClientException ex) - { - _logger.Warn(ex, "Couldn't get download queue. {0}", ex.Message); - return Enumerable.Empty(); - } + var sabHistory = _proxy.GetHistory(0, _configService.DownloadClientHistoryLimit, Settings.TvCategory, Settings); var historyItems = new List(); diff --git a/src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs b/src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs index 397771ff2..26185cf44 100644 --- a/src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Sabnzbd/SabnzbdProxy.cs @@ -1,4 +1,4 @@ -using System; +using System; using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common.Extensions; @@ -183,7 +183,7 @@ namespace NzbDrone.Core.Download.Clients.Sabnzbd } catch (WebException ex) { - throw new DownloadClientException("Unable to connect to SABnzbd, please check your settings", ex); + throw new DownloadClientUnavailableException("Unable to connect to SABnzbd, please check your settings", ex); } CheckForError(response); diff --git a/src/NzbDrone.Core/Download/Clients/Transmission/TransmissionBase.cs b/src/NzbDrone.Core/Download/Clients/Transmission/TransmissionBase.cs index 4dae1d494..72dfa4913 100644 --- a/src/NzbDrone.Core/Download/Clients/Transmission/TransmissionBase.cs +++ b/src/NzbDrone.Core/Download/Clients/Transmission/TransmissionBase.cs @@ -33,17 +33,7 @@ namespace NzbDrone.Core.Download.Clients.Transmission public override IEnumerable GetItems() { - List torrents; - - try - { - torrents = _proxy.GetTorrents(Settings); - } - catch (DownloadClientException ex) - { - _logger.Error(ex, ex.Message); - return Enumerable.Empty(); - } + var torrents = _proxy.GetTorrents(Settings); var items = new List(); @@ -211,17 +201,13 @@ namespace NzbDrone.Core.Download.Clients.Transmission DetailedDescription = string.Format("Please verify your username and password. Also verify if the host running Lidarr isn't blocked from accessing {0} by WhiteList limitations in the {0} configuration.", Name) }; } - catch (WebException ex) + catch (DownloadClientUnavailableException ex) { _logger.Error(ex, ex.Message); - if (ex.Status == WebExceptionStatus.ConnectFailure) + return new NzbDroneValidationFailure("Host", "Unable to connect") { - return new NzbDroneValidationFailure("Host", "Unable to connect") - { - DetailedDescription = "Please verify the hostname and port." - }; - } - return new NzbDroneValidationFailure(string.Empty, "Unknown exception: " + ex.Message); + DetailedDescription = "Please verify the hostname and port." + }; } catch (Exception ex) { diff --git a/src/NzbDrone.Core/Download/Clients/Transmission/TransmissionProxy.cs b/src/NzbDrone.Core/Download/Clients/Transmission/TransmissionProxy.cs index cada83cae..7c06b9278 100644 --- a/src/NzbDrone.Core/Download/Clients/Transmission/TransmissionProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Transmission/TransmissionProxy.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Net; using System.Collections.Generic; using NzbDrone.Common.Extensions; @@ -238,54 +238,65 @@ namespace NzbDrone.Core.Download.Clients.Transmission public TransmissionResponse ProcessRequest(string action, object arguments, TransmissionSettings settings) { - var requestBuilder = BuildRequest(settings); - requestBuilder.Headers.ContentType = "application/json"; - requestBuilder.SuppressHttpError = true; + try + { + var requestBuilder = BuildRequest(settings); + requestBuilder.Headers.ContentType = "application/json"; + requestBuilder.SuppressHttpError = true; - AuthenticateClient(requestBuilder, settings); + AuthenticateClient(requestBuilder, settings); - var request = requestBuilder.Post().Build(); + var request = requestBuilder.Post().Build(); - var data = new Dictionary(); - data.Add("method", action); + var data = new Dictionary(); + data.Add("method", action); - if (arguments != null) - { - data.Add("arguments", arguments); - } + if (arguments != null) + { + data.Add("arguments", arguments); + } - request.SetContent(data.ToJson()); - request.ContentSummary = string.Format("{0}(...)", action); + request.SetContent(data.ToJson()); + request.ContentSummary = string.Format("{0}(...)", action); - var response = _httpClient.Execute(request); - if (response.StatusCode == HttpStatusCode.Conflict) - { - AuthenticateClient(requestBuilder, settings, true); + var response = _httpClient.Execute(request); - request = requestBuilder.Post().Build(); + if (response.StatusCode == HttpStatusCode.Conflict) + { + AuthenticateClient(requestBuilder, settings, true); - request.SetContent(data.ToJson()); - request.ContentSummary = string.Format("{0}(...)", action); + request = requestBuilder.Post().Build(); - response = _httpClient.Execute(request); - } - else if (response.StatusCode == HttpStatusCode.Unauthorized) - { - throw new DownloadClientAuthenticationException("User authentication failed."); - } + request.SetContent(data.ToJson()); + request.ContentSummary = string.Format("{0}(...)", action); + + response = _httpClient.Execute(request); + } + else if (response.StatusCode == HttpStatusCode.Unauthorized) + { + throw new DownloadClientAuthenticationException("User authentication failed."); + } - var transmissionResponse = Json.Deserialize(response.Content); + var transmissionResponse = Json.Deserialize(response.Content); - if (transmissionResponse == null) + if (transmissionResponse == null) + { + throw new TransmissionException("Unexpected response"); + } + else if (transmissionResponse.Result != "success") + { + throw new TransmissionException(transmissionResponse.Result); + } + return transmissionResponse; + } + catch (HttpException ex) { - throw new TransmissionException("Unexpected response"); + throw new DownloadClientException("Unable to connect to Transmission, please check your settings", ex); } - else if (transmissionResponse.Result != "success") + catch (WebException ex) { - throw new TransmissionException(transmissionResponse.Result); + throw new DownloadClientUnavailableException("Unable to connect to Transmission, please check your settings", ex); } - - return transmissionResponse; } } } diff --git a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs index 24f7bfacc..00ffd0ec6 100644 --- a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs +++ b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrent.cs @@ -81,69 +81,60 @@ namespace NzbDrone.Core.Download.Clients.RTorrent public override IEnumerable GetItems() { - try + var torrents = _proxy.GetTorrents(Settings); + + _logger.Debug("Retrieved metadata of {0} torrents in client", torrents.Count); + + var items = new List(); + foreach (RTorrentTorrent torrent in torrents) { - var torrents = _proxy.GetTorrents(Settings); + // Don't concern ourselves with categories other than specified + if (torrent.Category != Settings.TvCategory) continue; - _logger.Debug("Retrieved metadata of {0} torrents in client", torrents.Count); + if (torrent.Path.StartsWith(".")) + { + throw new DownloadClientException("Download paths paths must be absolute. Please specify variable \"directory\" in rTorrent."); + } - var items = new List(); - foreach (RTorrentTorrent torrent in torrents) + var item = new DownloadClientItem(); + item.DownloadClient = Definition.Name; + item.Title = torrent.Name; + item.DownloadId = torrent.Hash; + item.OutputPath = _remotePathMappingService.RemapRemoteToLocal(Settings.Host, new OsPath(torrent.Path)); + item.TotalSize = torrent.TotalSize; + item.RemainingSize = torrent.RemainingSize; + item.Category = torrent.Category; + + if (torrent.DownRate > 0) + { + var secondsLeft = torrent.RemainingSize / torrent.DownRate; + item.RemainingTime = TimeSpan.FromSeconds(secondsLeft); + } + else { - // Don't concern ourselves with categories other than specified - if (torrent.Category != Settings.TvCategory) continue; - - if (torrent.Path.StartsWith(".")) - { - throw new DownloadClientException("Download paths paths must be absolute. Please specify variable \"directory\" in rTorrent."); - } - - var item = new DownloadClientItem(); - item.DownloadClient = Definition.Name; - item.Title = torrent.Name; - item.DownloadId = torrent.Hash; - item.OutputPath = _remotePathMappingService.RemapRemoteToLocal(Settings.Host, new OsPath(torrent.Path)); - item.TotalSize = torrent.TotalSize; - item.RemainingSize = torrent.RemainingSize; - item.Category = torrent.Category; - - if (torrent.DownRate > 0) - { - var secondsLeft = torrent.RemainingSize / torrent.DownRate; - item.RemainingTime = TimeSpan.FromSeconds(secondsLeft); - } - else - { - item.RemainingTime = TimeSpan.Zero; - } - - if (torrent.IsFinished) - { - item.Status = DownloadItemStatus.Completed; - } - else if (torrent.IsActive) - { - item.Status = DownloadItemStatus.Downloading; - } - else if (!torrent.IsActive) - { - item.Status = DownloadItemStatus.Paused; - } - - // No stop ratio data is present, so do not delete - item.CanMoveFiles = item.CanBeRemoved = false; - - items.Add(item); + item.RemainingTime = TimeSpan.Zero; } - return items; - } - catch (DownloadClientException ex) - { - _logger.Error(ex, ex.Message); - return Enumerable.Empty(); + if (torrent.IsFinished) + { + item.Status = DownloadItemStatus.Completed; + } + else if (torrent.IsActive) + { + item.Status = DownloadItemStatus.Downloading; + } + else if (!torrent.IsActive) + { + item.Status = DownloadItemStatus.Paused; + } + + // No stop ratio data is present, so do not delete + item.CanMoveFiles = item.CanBeRemoved = false; + + items.Add(item); } + return items; } public override void RemoveItem(string downloadId, bool deleteData) diff --git a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs index 749a68d7a..c00df292e 100644 --- a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs @@ -1,7 +1,9 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Net; +using System.Net.Sockets; +using System.Runtime.InteropServices.ComTypes; using NLog; using NzbDrone.Common.Extensions; using CookComputing.XmlRpc; @@ -54,8 +56,7 @@ namespace NzbDrone.Core.Download.Clients.RTorrent _logger.Debug("Executing remote method: system.client_version"); var client = BuildClient(settings); - - var version = client.GetVersion(); + var version = ExecuteRequest(() => client.GetVersion()); return version; } @@ -65,20 +66,22 @@ namespace NzbDrone.Core.Download.Clients.RTorrent _logger.Debug("Executing remote method: d.multicall2"); var client = BuildClient(settings); - var ret = client.TorrentMulticall("", "", - "d.name=", // string - "d.hash=", // string - "d.base_path=", // string - "d.custom1=", // string (label) - "d.size_bytes=", // long - "d.left_bytes=", // long - "d.down.rate=", // long (in bytes / s) - "d.ratio=", // long - "d.is_open=", // long - "d.is_active=", // long - "d.complete="); //long + var ret = ExecuteRequest(() => client.TorrentMulticall("", "", + "d.name=", // string + "d.hash=", // string + "d.base_path=", // string + "d.custom1=", // string (label) + "d.size_bytes=", // long + "d.left_bytes=", // long + "d.down.rate=", // long (in bytes / s) + "d.ratio=", // long + "d.is_open=", // long + "d.is_active=", // long + "d.complete=") //long + ); var items = new List(); + foreach (object[] torrent in ret) { var labelDecoded = System.Web.HttpUtility.UrlDecode((string) torrent[3]); @@ -107,8 +110,8 @@ namespace NzbDrone.Core.Download.Clients.RTorrent _logger.Debug("Executing remote method: load.normal"); var client = BuildClient(settings); + var response = ExecuteRequest(() => client.LoadStart("", torrentUrl, GetCommands(label, priority, directory))); - var response = client.LoadStart("", torrentUrl, GetCommands(label, priority, directory)); if (response != 0) { throw new DownloadClientException("Could not add torrent: {0}.", torrentUrl); @@ -120,8 +123,8 @@ namespace NzbDrone.Core.Download.Clients.RTorrent _logger.Debug("Executing remote method: load.raw"); var client = BuildClient(settings); + var response = ExecuteRequest(() => client.LoadRawStart("", fileContent, GetCommands(label, priority, directory))); - var response = client.LoadRawStart("", fileContent, GetCommands(label, priority, directory)); if (response != 0) { throw new DownloadClientException("Could not add torrent: {0}.", fileName); @@ -133,14 +136,39 @@ namespace NzbDrone.Core.Download.Clients.RTorrent _logger.Debug("Executing remote method: d.erase"); var client = BuildClient(settings); + var response = ExecuteRequest(() => client.Remove(hash)); - var response = client.Remove(hash); if (response != 0) { throw new DownloadClientException("Could not remove torrent: {0}.", hash); } } + public bool HasHashTorrent(string hash, RTorrentSettings settings) + { + _logger.Debug("Executing remote method: d.name"); + + var client = BuildClient(settings); + + try + { + var name = ExecuteRequest(() => client.GetName(hash)); + + if (name.IsNullOrWhiteSpace()) + { + return false; + } + + var metaTorrent = name == (hash + ".meta"); + + return !metaTorrent; + } + catch (Exception) + { + return false; + } + } + private string[] GetCommands(string label, RTorrentPriority priority, string directory) { var result = new List(); @@ -163,25 +191,6 @@ namespace NzbDrone.Core.Download.Clients.RTorrent return result.ToArray(); } - public bool HasHashTorrent(string hash, RTorrentSettings settings) - { - _logger.Debug("Executing remote method: d.name"); - - var client = BuildClient(settings); - - try - { - var name = client.GetName(hash); - if (name.IsNullOrWhiteSpace()) return false; - bool metaTorrent = name == (hash + ".meta"); - return !metaTorrent; - } - catch (Exception) - { - return false; - } - } - private IRTorrent BuildClient(RTorrentSettings settings) { var client = XmlRpcProxyGen.Create(); @@ -201,5 +210,21 @@ namespace NzbDrone.Core.Download.Clients.RTorrent return client; } + + private T ExecuteRequest(Func task) + { + try + { + return task(); + } + catch (XmlRpcServerException ex) + { + throw new DownloadClientException("Unable to connect to rTorrent, please check your settings", ex); + } + catch (WebException ex) + { + throw new DownloadClientUnavailableException("Unable to connect to rTorrent, please check your settings", ex); + } + } } } diff --git a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs index 208f9816a..034ae1e46 100644 --- a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs +++ b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrent.cs @@ -72,42 +72,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent public override IEnumerable GetItems() { - List torrents; - - try - { - var cacheKey = string.Format("{0}:{1}:{2}", Settings.Host, Settings.Port, Settings.TvCategory); - var cache = _torrentCache.Find(cacheKey); - - var response = _proxy.GetTorrents(cache == null ? null : cache.CacheID, Settings); - - if (cache != null && response.Torrents == null) - { - var removedAndUpdated = new HashSet(response.TorrentsChanged.Select(v => v.Hash).Concat(response.TorrentsRemoved)); - - torrents = cache.Torrents - .Where(v => !removedAndUpdated.Contains(v.Hash)) - .Concat(response.TorrentsChanged) - .ToList(); - } - else - { - torrents = response.Torrents; - } - - cache = new UTorrentTorrentCache - { - CacheID = response.CacheNumber, - Torrents = torrents - }; - - _torrentCache.Set(cacheKey, cache, TimeSpan.FromMinutes(15)); - } - catch (DownloadClientException ex) - { - _logger.Error(ex, ex.Message); - return Enumerable.Empty(); - } + var torrents = GetTorrents(); var queueItems = new List(); @@ -173,6 +138,40 @@ namespace NzbDrone.Core.Download.Clients.UTorrent return queueItems; } + private List GetTorrents() + { + List torrents; + + var cacheKey = string.Format("{0}:{1}:{2}", Settings.Host, Settings.Port, Settings.TvCategory); + var cache = _torrentCache.Find(cacheKey); + + var response = _proxy.GetTorrents(cache == null ? null : cache.CacheID, Settings); + + if (cache != null && response.Torrents == null) + { + var removedAndUpdated = new HashSet(response.TorrentsChanged.Select(v => v.Hash).Concat(response.TorrentsRemoved)); + + torrents = cache.Torrents + .Where(v => !removedAndUpdated.Contains(v.Hash)) + .Concat(response.TorrentsChanged) + .ToList(); + } + else + { + torrents = response.Torrents; + } + + cache = new UTorrentTorrentCache + { + CacheID = response.CacheNumber, + Torrents = torrents + }; + + _torrentCache.Set(cacheKey, cache, TimeSpan.FromMinutes(15)); + + return torrents; + } + public override void RemoveItem(string downloadId, bool deleteData) { _proxy.RemoveTorrent(downloadId, deleteData, Settings); diff --git a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs index 64117f328..662b5322c 100644 --- a/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/uTorrent/UTorrentProxy.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Net; using NLog; @@ -244,7 +244,7 @@ namespace NzbDrone.Core.Download.Clients.UTorrent } catch (WebException ex) { - throw new DownloadClientException("Unable to connect to uTorrent, please check your settings", ex); + throw new DownloadClientUnavailableException("Unable to connect to uTorrent, please check your settings", ex); } cookies = response.GetCookies(); diff --git a/src/NzbDrone.Core/Download/DownloadClientFactory.cs b/src/NzbDrone.Core/Download/DownloadClientFactory.cs index dc0f218b5..909b48ed6 100644 --- a/src/NzbDrone.Core/Download/DownloadClientFactory.cs +++ b/src/NzbDrone.Core/Download/DownloadClientFactory.cs @@ -1,5 +1,7 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using System.Linq; +using FluentValidation.Results; using NLog; using NzbDrone.Common.Composition; using NzbDrone.Core.Messaging.Events; @@ -9,17 +11,24 @@ namespace NzbDrone.Core.Download { public interface IDownloadClientFactory : IProviderFactory { - + List DownloadHandlingEnabled(bool filterBlockedClients = true); } public class DownloadClientFactory : ProviderFactory, IDownloadClientFactory { - private readonly IDownloadClientRepository _providerRepository; + private readonly IDownloadClientStatusService _downloadClientStatusService; + private readonly Logger _logger; - public DownloadClientFactory(IDownloadClientRepository providerRepository, IEnumerable providers, IContainer container, IEventAggregator eventAggregator, Logger logger) + public DownloadClientFactory(IDownloadClientStatusService downloadClientStatusService, + IDownloadClientRepository providerRepository, + IEnumerable providers, + IContainer container, + IEventAggregator eventAggregator, + Logger logger) : base(providerRepository, providers, container, eventAggregator, logger) { - _providerRepository = providerRepository; + _downloadClientStatusService = downloadClientStatusService; + _logger = logger; } protected override List Active() @@ -33,5 +42,46 @@ namespace NzbDrone.Core.Download definition.Protocol = provider.Protocol; } + + public List DownloadHandlingEnabled(bool filterBlockedClients = true) + { + var enabledClients = GetAvailableProviders(); + + if (filterBlockedClients) + { + return FilterBlockedClients(enabledClients).ToList(); + } + + return enabledClients.ToList(); + } + + private IEnumerable FilterBlockedClients(IEnumerable clients) + { + var blockedIndexers = _downloadClientStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v); + + foreach (var client in clients) + { + DownloadClientStatus downloadClientStatus; + if (blockedIndexers.TryGetValue(client.Definition.Id, out downloadClientStatus)) + { + _logger.Debug("Temporarily ignoring download client {0} till {1} due to recent failures.", client.Definition.Name, downloadClientStatus.DisabledTill.Value.ToLocalTime()); + continue; + } + + yield return client; + } + } + + public override ValidationResult Test(DownloadClientDefinition definition) + { + var result = base.Test(definition); + + if ((result == null || result.IsValid) && definition.Id != 0) + { + _downloadClientStatusService.RecordSuccess(definition.Id); + } + + return result; + } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Download/DownloadClientProvider.cs b/src/NzbDrone.Core/Download/DownloadClientProvider.cs index 5cb899806..7ed7cd5b9 100644 --- a/src/NzbDrone.Core/Download/DownloadClientProvider.cs +++ b/src/NzbDrone.Core/Download/DownloadClientProvider.cs @@ -1,4 +1,4 @@ -using System.Linq; +using System.Linq; using System.Collections.Generic; using NzbDrone.Core.Indexers; @@ -27,19 +27,12 @@ namespace NzbDrone.Core.Download public IEnumerable GetDownloadClients() { - return _downloadClientFactory.GetAvailableProviders();//.Select(MapDownloadClient); + return _downloadClientFactory.GetAvailableProviders(); } public IDownloadClient Get(int id) { return _downloadClientFactory.GetAvailableProviders().Single(d => d.Definition.Id == id); } - - public IDownloadClient MapDownloadClient(IDownloadClient downloadClient) - { - _downloadClientFactory.SetProviderCharacteristics(downloadClient, (DownloadClientDefinition)downloadClient.Definition); - - return downloadClient; - } } } diff --git a/src/NzbDrone.Core/Download/DownloadClientStatus.cs b/src/NzbDrone.Core/Download/DownloadClientStatus.cs new file mode 100644 index 000000000..f4d819424 --- /dev/null +++ b/src/NzbDrone.Core/Download/DownloadClientStatus.cs @@ -0,0 +1,9 @@ +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Download +{ + public class DownloadClientStatus : ProviderStatusBase + { + + } +} diff --git a/src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs b/src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs new file mode 100644 index 000000000..ac6cfc0b9 --- /dev/null +++ b/src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs @@ -0,0 +1,19 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Download +{ + public interface IDownloadClientStatusRepository : IProviderStatusRepository + { + + } + + public class DownloadClientStatusRepository : ProviderStatusRepository, IDownloadClientStatusRepository + { + public DownloadClientStatusRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/Download/DownloadClientStatusService.cs b/src/NzbDrone.Core/Download/DownloadClientStatusService.cs new file mode 100644 index 000000000..ba3360dcf --- /dev/null +++ b/src/NzbDrone.Core/Download/DownloadClientStatusService.cs @@ -0,0 +1,22 @@ +using System; +using NLog; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Download +{ + public interface IDownloadClientStatusService : IProviderStatusServiceBase + { + + } + + public class DownloadClientStatusService : ProviderStatusServiceBase, IDownloadClientStatusService + { + public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + : base(providerStatusRepository, eventAggregator, logger) + { + MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5); + MaximumEscalationLevel = 5; + } + } +} diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index c4f1e155e..a4f30e918 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -1,4 +1,4 @@ -using System; +using System; using NLog; using NzbDrone.Common.EnsureThat; using NzbDrone.Common.Extensions; @@ -20,18 +20,21 @@ namespace NzbDrone.Core.Download public class DownloadService : IDownloadService { private readonly IProvideDownloadClient _downloadClientProvider; + private readonly IDownloadClientStatusService _downloadClientStatusService; private readonly IIndexerStatusService _indexerStatusService; private readonly IRateLimitService _rateLimitService; private readonly IEventAggregator _eventAggregator; private readonly Logger _logger; public DownloadService(IProvideDownloadClient downloadClientProvider, - IIndexerStatusService indexerStatusService, - IRateLimitService rateLimitService, - IEventAggregator eventAggregator, - Logger logger) + IDownloadClientStatusService downloadClientStatusService, + IIndexerStatusService indexerStatusService, + IRateLimitService rateLimitService, + IEventAggregator eventAggregator, + Logger logger) { _downloadClientProvider = downloadClientProvider; + _downloadClientStatusService = downloadClientStatusService; _indexerStatusService = indexerStatusService; _rateLimitService = rateLimitService; _eventAggregator = eventAggregator; @@ -63,6 +66,7 @@ namespace NzbDrone.Core.Download try { downloadClientId = downloadClient.Download(remoteAlbum); + _downloadClientStatusService.RecordSuccess(downloadClient.Definition.Id); _indexerStatusService.RecordSuccess(remoteAlbum.Release.IndexerId); } catch (ReleaseDownloadException ex) @@ -91,4 +95,4 @@ namespace NzbDrone.Core.Download _eventAggregator.PublishEvent(albumGrabbedEvent); } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Download/Pending/PendingRelease.cs b/src/NzbDrone.Core/Download/Pending/PendingRelease.cs index 087de71e6..a9273ec2e 100644 --- a/src/NzbDrone.Core/Download/Pending/PendingRelease.cs +++ b/src/NzbDrone.Core/Download/Pending/PendingRelease.cs @@ -11,6 +11,7 @@ namespace NzbDrone.Core.Download.Pending public DateTime Added { get; set; } public ParsedAlbumInfo ParsedAlbumInfo { get; set; } public ReleaseInfo Release { get; set; } + public PendingReleaseReason Reason { get; set; } //Not persisted public RemoteAlbum RemoteAlbum { get; set; } diff --git a/src/NzbDrone.Core/Download/Pending/PendingReleaseReason.cs b/src/NzbDrone.Core/Download/Pending/PendingReleaseReason.cs new file mode 100644 index 000000000..ba83714b7 --- /dev/null +++ b/src/NzbDrone.Core/Download/Pending/PendingReleaseReason.cs @@ -0,0 +1,9 @@ +namespace NzbDrone.Core.Download.Pending +{ + public enum PendingReleaseReason + { + Delay = 0, + DownloadClientUnavailable = 1, + Fallback = 2 + } +} diff --git a/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs b/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs index e341e6c0d..0bda1304a 100644 --- a/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs +++ b/src/NzbDrone.Core/Download/Pending/PendingReleaseService.cs @@ -20,8 +20,7 @@ namespace NzbDrone.Core.Download.Pending { public interface IPendingReleaseService { - void Add(DownloadDecision decision); - + void Add(DownloadDecision decision, PendingReleaseReason reason); List GetPending(); List GetPendingRemoteAlbums(int artistId); List GetPendingQueue(); @@ -67,7 +66,7 @@ namespace NzbDrone.Core.Download.Pending } - public void Add(DownloadDecision decision) + public void Add(DownloadDecision decision, PendingReleaseReason reason) { var alreadyPending = GetPendingReleases(); @@ -77,14 +76,32 @@ namespace NzbDrone.Core.Download.Pending .Intersect(albumIds) .Any()); - if (existingReports.Any(MatchingReleasePredicate(decision.RemoteAlbum.Release))) + var matchingReports = existingReports.Where(MatchingReleasePredicate(decision.RemoteAlbum.Release)).ToList(); + + if (matchingReports.Any()) { - _logger.Debug("This release is already pending, not adding again"); - return; + var sameReason = true; + + foreach (var matchingReport in matchingReports) + { + if (matchingReport.Reason != reason) + { + _logger.Debug("This release is already pending with reason {0}, changing to {1}", matchingReport.Reason, reason); + matchingReport.Reason = reason; + _repository.Update(matchingReport); + sameReason = false; + } + } + + if (sameReason) + { + _logger.Debug("This release is already pending with reason {0}, not adding again", reason); + return; + } } - _logger.Debug("Adding release to pending releases"); - Insert(decision); + _logger.Debug("Adding release to pending releases with reason {0}", reason); + Insert(decision, reason); } public List GetPending() @@ -101,7 +118,7 @@ namespace NzbDrone.Core.Download.Pending private List FilterBlockedIndexers(List releases) { - var blockedIndexers = new HashSet(_indexerStatusService.GetBlockedIndexers().Select(v => v.ProviderId)); + var blockedIndexers = new HashSet(_indexerStatusService.GetBlockedProviders().Select(v => v.ProviderId)); return releases.Where(release => !blockedIndexers.Contains(release.IndexerId)).ToList(); } @@ -117,7 +134,7 @@ namespace NzbDrone.Core.Download.Pending var nextRssSync = new Lazy(() => _taskManager.GetNextExecution(typeof(RssSyncCommand))); - foreach (var pendingRelease in GetPendingReleases()) + foreach (var pendingRelease in GetPendingReleases().Where(p => p.Reason != PendingReleaseReason.Fallback)) { foreach (var album in pendingRelease.RemoteAlbum.Albums) { @@ -132,6 +149,13 @@ namespace NzbDrone.Core.Download.Pending ect = ect.AddMinutes(_configService.RssSyncInterval); } + var timeleft = ect.Subtract(DateTime.UtcNow); + + if (timeleft.TotalSeconds < 0) + { + timeleft = TimeSpan.Zero; + } + var queue = new Queue.Queue { Id = GetQueueId(pendingRelease, album), @@ -142,12 +166,13 @@ namespace NzbDrone.Core.Download.Pending Size = pendingRelease.RemoteAlbum.Release.Size, Sizeleft = pendingRelease.RemoteAlbum.Release.Size, RemoteAlbum = pendingRelease.RemoteAlbum, - Timeleft = ect.Subtract(DateTime.UtcNow), + Timeleft = timeleft, EstimatedCompletionTime = ect, - Status = "Pending", + Status = pendingRelease.Reason.ToString(), Protocol = pendingRelease.RemoteAlbum.Release.DownloadProtocol, Indexer = pendingRelease.RemoteAlbum.Release.Indexer }; + queued.Add(queue); } } @@ -230,7 +255,7 @@ namespace NzbDrone.Core.Download.Pending }; } - private void Insert(DownloadDecision decision) + private void Insert(DownloadDecision decision, PendingReleaseReason reason) { _repository.Insert(new PendingRelease { @@ -238,7 +263,8 @@ namespace NzbDrone.Core.Download.Pending ParsedAlbumInfo = decision.RemoteAlbum.ParsedAlbumInfo, Release = decision.RemoteAlbum.Release, Title = decision.RemoteAlbum.Release.Title, - Added = DateTime.UtcNow + Added = DateTime.UtcNow, + Reason = reason }); _eventAggregator.PublishEvent(new PendingReleasesUpdatedEvent()); diff --git a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs index 1c82f3253..6519df2ef 100644 --- a/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs +++ b/src/NzbDrone.Core/Download/ProcessDownloadDecisions.cs @@ -1,9 +1,12 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; +using System.Net; using NLog; using NzbDrone.Core.DecisionEngine; +using NzbDrone.Core.Download.Clients; using NzbDrone.Core.Download.Pending; +using NzbDrone.Core.Indexers; namespace NzbDrone.Core.Download { @@ -36,36 +39,33 @@ namespace NzbDrone.Core.Download var prioritizedDecisions = _prioritizeDownloadDecision.PrioritizeDecisions(qualifiedReports); var grabbed = new List(); var pending = new List(); + var failed = new List(); + + var usenetFailed = false; + var torrentFailed = false; foreach (var report in prioritizedDecisions) { var remoteAlbum = report.RemoteAlbum; - - var albumIds = remoteAlbum.Albums.Select(e => e.Id).ToList(); + var downloadProtocol = report.RemoteAlbum.Release.DownloadProtocol; //Skip if already grabbed - if (grabbed.SelectMany(r => r.RemoteAlbum.Albums) - .Select(e => e.Id) - .ToList() - .Intersect(albumIds) - .Any()) + if (IsAlbumProcessed(grabbed, report)) { continue; } if (report.TemporarilyRejected) { - _pendingReleaseService.Add(report); + _pendingReleaseService.Add(report, PendingReleaseReason.Delay); pending.Add(report); continue; } - if (pending.SelectMany(r => r.RemoteAlbum.Albums) - .Select(e => e.Id) - .ToList() - .Intersect(albumIds) - .Any()) + if (downloadProtocol == DownloadProtocol.Usenet && usenetFailed || + downloadProtocol == DownloadProtocol.Torrent && torrentFailed) { + failed.Add(report); continue; } @@ -74,14 +74,31 @@ namespace NzbDrone.Core.Download _downloadService.DownloadReport(remoteAlbum); grabbed.Add(report); } - catch (Exception e) + catch (Exception ex) { - //TODO: support for store & forward - //We'll need to differentiate between a download client error and an indexer error - _logger.Warn(e, "Couldn't add report to download queue. " + remoteAlbum); + if (ex is DownloadClientUnavailableException || ex is DownloadClientAuthenticationException) + { + _logger.Debug("Failed to send release to download client, storing until later"); + failed.Add(report); + + if (downloadProtocol == DownloadProtocol.Usenet) + { + usenetFailed = true; + } + else if (downloadProtocol == DownloadProtocol.Torrent) + { + torrentFailed = true; + } + } + else + { + _logger.Warn(ex, "Couldn't add report to download queue. " + remoteAlbum); + } } } + pending.AddRange(ProcessFailedGrabs(grabbed, failed)); + return new ProcessedDecisions(grabbed, pending, decisions.Where(d => d.Rejected).ToList()); } @@ -90,5 +107,50 @@ namespace NzbDrone.Core.Download //Process both approved and temporarily rejected return decisions.Where(c => (c.Approved || c.TemporarilyRejected) && c.RemoteAlbum.Albums.Any()).ToList(); } + + private bool IsAlbumProcessed(List decisions, DownloadDecision report) + { + var albumIds = report.RemoteAlbum.Albums.Select(e => e.Id).ToList(); + + return decisions.SelectMany(r => r.RemoteAlbum.Albums) + .Select(e => e.Id) + .ToList() + .Intersect(albumIds) + .Any(); + } + + private List ProcessFailedGrabs(List grabbed, List failed) + { + var pending = new List(); + var stored = new List(); + + foreach (var report in failed) + { + // If a release was already grabbed with matching albums we should store it as a fallback + // and filter it out the next time it is processed incase a higher quality release failed to + // add to the download client, but a lower quality release was sent to another client + // If the release wasn't grabbed already, but was already stored, store it as a fallback, + // otherwise store it as DownloadClientUnavailable. + + if (IsAlbumProcessed(grabbed, report)) + { + _pendingReleaseService.Add(report, PendingReleaseReason.Fallback); + pending.Add(report); + } + else if (IsAlbumProcessed(stored, report)) + { + _pendingReleaseService.Add(report, PendingReleaseReason.Fallback); + pending.Add(report); + } + else + { + _pendingReleaseService.Add(report, PendingReleaseReason.DownloadClientUnavailable); + pending.Add(report); + stored.Add(report); + } + } + + return pending; + } } } diff --git a/src/NzbDrone.Core/Download/TrackedDownloads/DownloadMonitoringService.cs b/src/NzbDrone.Core/Download/TrackedDownloads/DownloadMonitoringService.cs index 345f20353..b1c27d869 100644 --- a/src/NzbDrone.Core/Download/TrackedDownloads/DownloadMonitoringService.cs +++ b/src/NzbDrone.Core/Download/TrackedDownloads/DownloadMonitoringService.cs @@ -17,7 +17,8 @@ namespace NzbDrone.Core.Download.TrackedDownloads IHandle { - private readonly IProvideDownloadClient _downloadClientProvider; + private readonly IDownloadClientStatusService _downloadClientStatusService; + private readonly IDownloadClientFactory _downloadClientFactory; private readonly IEventAggregator _eventAggregator; private readonly IManageCommandQueue _manageCommandQueue; private readonly IConfigService _configService; @@ -27,16 +28,18 @@ namespace NzbDrone.Core.Download.TrackedDownloads private readonly Logger _logger; private readonly Debouncer _refreshDebounce; - public DownloadMonitoringService(IProvideDownloadClient downloadClientProvider, - IEventAggregator eventAggregator, - IManageCommandQueue manageCommandQueue, - IConfigService configService, - IFailedDownloadService failedDownloadService, - ICompletedDownloadService completedDownloadService, - ITrackedDownloadService trackedDownloadService, - Logger logger) + public DownloadMonitoringService(IDownloadClientStatusService downloadClientStatusService, + IDownloadClientFactory downloadClientFactory, + IEventAggregator eventAggregator, + IManageCommandQueue manageCommandQueue, + IConfigService configService, + IFailedDownloadService failedDownloadService, + ICompletedDownloadService completedDownloadService, + ITrackedDownloadService trackedDownloadService, + Logger logger) { - _downloadClientProvider = downloadClientProvider; + _downloadClientStatusService = downloadClientStatusService; + _downloadClientFactory = downloadClientFactory; _eventAggregator = eventAggregator; _manageCommandQueue = manageCommandQueue; _configService = configService; @@ -58,7 +61,7 @@ namespace NzbDrone.Core.Download.TrackedDownloads _refreshDebounce.Pause(); try { - var downloadClients = _downloadClientProvider.GetDownloadClients(); + var downloadClients = _downloadClientFactory.DownloadHandlingEnabled(); var trackedDownloads = new List(); @@ -86,9 +89,12 @@ namespace NzbDrone.Core.Download.TrackedDownloads try { downloadClientHistory = downloadClient.GetItems().ToList(); + + _downloadClientStatusService.RecordSuccess(downloadClient.Definition.Id); } catch (Exception ex) { + _downloadClientStatusService.RecordFailure(downloadClient.Definition.Id); _logger.Warn(ex, "Unable to retrieve queue and history items from " + downloadClient.Definition.Name); } diff --git a/src/NzbDrone.Core/HealthCheck/CheckOnAttribute.cs b/src/NzbDrone.Core/HealthCheck/CheckOnAttribute.cs new file mode 100644 index 000000000..a40aee50e --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/CheckOnAttribute.cs @@ -0,0 +1,16 @@ +using System; +using NzbDrone.Common.Messaging; + +namespace NzbDrone.Core.HealthCheck +{ + [AttributeUsage(AttributeTargets.Class, AllowMultiple = true)] + public class CheckOnAttribute : Attribute + { + public Type EventType { get; set; } + + public CheckOnAttribute(Type eventType) + { + EventType = eventType; + } + } +} diff --git a/src/NzbDrone.Core/HealthCheck/Checks/AppDataLocationCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/AppDataLocationCheck.cs index ad4f2db9e..d4f9a7cc7 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/AppDataLocationCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/AppDataLocationCheck.cs @@ -1,5 +1,6 @@ -using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; +using NzbDrone.Core.Configuration.Events; namespace NzbDrone.Core.HealthCheck.Checks { @@ -22,7 +23,6 @@ namespace NzbDrone.Core.HealthCheck.Checks return new HealthCheck(GetType()); } - - public override bool CheckOnConfigChange => false; + } } diff --git a/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs index d99eed1a3..ebe520960 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs @@ -1,10 +1,13 @@ -using System; +using System; using System.Linq; using NLog; using NzbDrone.Core.Download; +using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.HealthCheck.Checks { + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] public class DownloadClientCheck : HealthCheckBase { private readonly IProvideDownloadClient _downloadClientProvider; @@ -33,11 +36,10 @@ namespace NzbDrone.Core.HealthCheck.Checks } catch (Exception ex) { - - _logger.Error(ex, "Unable to communicate with {0}", downloadClient.Definition.Name); + _logger.Debug(ex, "Unable to communicate with {0}", downloadClient.Definition.Name); var message = $"Unable to communicate with {downloadClient.Definition.Name}."; - return new HealthCheck(GetType(), HealthCheckResult.Error, $"{message} {ex.Message}"); + return new HealthCheck(GetType(), HealthCheckResult.Error, $"{message} {ex.Message}", "#unable-to-communicate-with-download-client"); } } diff --git a/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs new file mode 100644 index 000000000..91b9b122d --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs @@ -0,0 +1,45 @@ +using System; +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Download; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] + public class DownloadClientStatusCheck : HealthCheckBase + { + private readonly IDownloadClientFactory _providerFactory; + private readonly IDownloadClientStatusService _providerStatusService; + + public DownloadClientStatusCheck(IDownloadClientFactory providerFactory, IDownloadClientStatusService providerStatusService) + { + _providerFactory = providerFactory; + _providerStatusService = providerStatusService; + } + + public override HealthCheck Check() + { + var enabledProviders = _providerFactory.GetAvailableProviders(); + var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new { Provider = i, Status = s }) + .ToList(); + + if (backOffProviders.Empty()) + { + return new HealthCheck(GetType()); + } + + if (backOffProviders.Count == enabledProviders.Count) + { + return new HealthCheck(GetType(), HealthCheckResult.Error, "All download clients are unavailable due to failures", "#download-clients-are-unavailable-due-to-failures"); + } + + return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Download clients unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), "#download-clients-are-unavailable-due-to-failures"); + } + } +} diff --git a/src/NzbDrone.Core/HealthCheck/Checks/ImportMechanismCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/ImportMechanismCheck.cs index 24a1fab1e..29172e0d8 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/ImportMechanismCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/ImportMechanismCheck.cs @@ -2,13 +2,18 @@ using System; using System.Collections.Generic; using System.Linq; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Configuration.Events; using NzbDrone.Core.Download; using NzbDrone.Core.Download.Clients; using NzbDrone.Core.Download.Clients.Nzbget; using NzbDrone.Core.Download.Clients.Sabnzbd; +using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.HealthCheck.Checks { + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ConfigSavedEvent))] public class ImportMechanismCheck : HealthCheckBase { private readonly IConfigService _configService; @@ -33,7 +38,7 @@ namespace NzbDrone.Core.HealthCheck.Checks Status = v.GetStatus() }).ToList(); } - catch (DownloadClientException) + catch (Exception) { // One or more download clients failed, assume the health is okay and verify later return new HealthCheck(GetType()); diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerRssCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerRssCheck.cs index 1d9ebf37e..557e07dce 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/IndexerRssCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerRssCheck.cs @@ -1,9 +1,13 @@ -using System.Linq; +using System.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers; +using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.HealthCheck.Checks { + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] public class IndexerRssCheck : HealthCheckBase { private readonly IIndexerFactory _indexerFactory; diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerSearchCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerSearchCheck.cs index f02e5de29..220c08c0e 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/IndexerSearchCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerSearchCheck.cs @@ -1,9 +1,13 @@ -using System.Linq; +using System.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers; +using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.HealthCheck.Checks { + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] public class IndexerSearchCheck : HealthCheckBase { private readonly IIndexerFactory _indexerFactory; diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs index 275c2cb24..d608686a2 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs @@ -1,42 +1,45 @@ -using System; +using System; using System.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers; +using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.HealthCheck.Checks { + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] public class IndexerStatusCheck : HealthCheckBase { - private readonly IIndexerFactory _indexerFactory; - private readonly IIndexerStatusService _indexerStatusService; + private readonly IIndexerFactory _providerFactory; + private readonly IIndexerStatusService _providerStatusService; - public IndexerStatusCheck(IIndexerFactory indexerFactory, IIndexerStatusService indexerStatusService) + public IndexerStatusCheck(IIndexerFactory providerFactory, IIndexerStatusService providerStatusService) { - _indexerFactory = indexerFactory; - _indexerStatusService = indexerStatusService; + _providerFactory = providerFactory; + _providerStatusService = providerStatusService; } public override HealthCheck Check() { - var enabledIndexers = _indexerFactory.GetAvailableProviders(); - var backOffIndexers = enabledIndexers.Join(_indexerStatusService.GetBlockedIndexers(), + var enabledProviders = _providerFactory.GetAvailableProviders(); + var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), i => i.Definition.Id, s => s.ProviderId, (i, s) => new { Indexer = i, Status = s }) - .Where(v => (v.Status.MostRecentFailure - v.Status.InitialFailure) > TimeSpan.FromHours(1)) .ToList(); - if (backOffIndexers.Empty()) + if (backOffProviders.Empty()) { return new HealthCheck(GetType()); } - if (backOffIndexers.Count == enabledIndexers.Count) + if (backOffProviders.Count == enabledProviders.Count) { return new HealthCheck(GetType(), HealthCheckResult.Error, "All indexers are unavailable due to failures", "#indexers-are-unavailable-due-to-failures"); } - return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexers unavailable due to failures: {0}", string.Join(", ", backOffIndexers.Select(v => v.Indexer.Definition.Name))), "#indexers-are-unavailable-due-to-failures"); + return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexers unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Indexer.Definition.Name))), "#indexers-are-unavailable-due-to-failures"); } } } diff --git a/src/NzbDrone.Core/HealthCheck/Checks/MediaInfoDllCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/MediaInfoDllCheck.cs index 5b5a9f3f4..badc522f4 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/MediaInfoDllCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/MediaInfoDllCheck.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.CompilerServices; using NzbDrone.Core.MediaFiles.MediaInfo; @@ -20,7 +20,5 @@ namespace NzbDrone.Core.HealthCheck.Checks return new HealthCheck(GetType()); } - - public override bool CheckOnConfigChange => false; } } diff --git a/src/NzbDrone.Core/HealthCheck/Checks/MonoVersionCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/MonoVersionCheck.cs index f4718c2c2..8a485396c 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/MonoVersionCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/MonoVersionCheck.cs @@ -41,8 +41,6 @@ namespace NzbDrone.Core.HealthCheck.Checks return new HealthCheck(GetType(), HealthCheckResult.Warning, "You are running an old and unsupported version of Mono. Please upgrade Mono for improved stability."); } - public override bool CheckOnConfigChange => false; - public override bool CheckOnSchedule => false; } diff --git a/src/NzbDrone.Core/HealthCheck/Checks/ProxyCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/ProxyCheck.cs index 346a12b1a..71ededd18 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/ProxyCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/ProxyCheck.cs @@ -1,13 +1,15 @@ -using NLog; +using NLog; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using System; using System.Linq; using System.Net; using NzbDrone.Common.Cloud; +using NzbDrone.Core.Configuration.Events; namespace NzbDrone.Core.HealthCheck.Checks { + [CheckOn(typeof(ConfigSavedEvent))] public class ProxyCheck : HealthCheckBase { private readonly Logger _logger; @@ -43,7 +45,7 @@ namespace NzbDrone.Core.HealthCheck.Checks { var response = _client.Execute(request); - // We only care about 400 responses, other error codes can be ignored + // We only care about 400 responses, other error codes can be ignored if (response.StatusCode == HttpStatusCode.BadRequest) { _logger.Error("Proxy Health Check failed: {0}", response.StatusCode); diff --git a/src/NzbDrone.Core/HealthCheck/Checks/RootFolderCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/RootFolderCheck.cs index 317f197fc..86a8fc4ef 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/RootFolderCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/RootFolderCheck.cs @@ -1,9 +1,12 @@ using System.Linq; using NzbDrone.Common.Disk; using NzbDrone.Core.Music; +using NzbDrone.Core.Music.Events; namespace NzbDrone.Core.HealthCheck.Checks { + [CheckOn(typeof(ArtistDeletedEvent))] + [CheckOn(typeof(ArtistMovedEvent))] public class RootFolderCheck : HealthCheckBase { private readonly IArtistService _artistService; @@ -36,7 +39,5 @@ namespace NzbDrone.Core.HealthCheck.Checks return new HealthCheck(GetType()); } - - public override bool CheckOnConfigChange => false; } } diff --git a/src/NzbDrone.Core/HealthCheck/Checks/UpdateCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/UpdateCheck.cs index c0d7a5c31..d474b503c 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/UpdateCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/UpdateCheck.cs @@ -1,13 +1,15 @@ -using System; +using System; using System.IO; using NzbDrone.Common.Disk; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Configuration.Events; using NzbDrone.Core.Update; namespace NzbDrone.Core.HealthCheck.Checks { + [CheckOn(typeof(ConfigFileSavedEvent))] public class UpdateCheck : HealthCheckBase { private readonly IDiskProvider _diskProvider; @@ -66,7 +68,5 @@ namespace NzbDrone.Core.HealthCheck.Checks return new HealthCheck(GetType()); } - - public override bool CheckOnConfigChange => false; } } diff --git a/src/NzbDrone.Core/HealthCheck/HealthCheckBase.cs b/src/NzbDrone.Core/HealthCheck/HealthCheckBase.cs index 5e1700ac6..d9c715ca9 100644 --- a/src/NzbDrone.Core/HealthCheck/HealthCheckBase.cs +++ b/src/NzbDrone.Core/HealthCheck/HealthCheckBase.cs @@ -1,4 +1,4 @@ -namespace NzbDrone.Core.HealthCheck +namespace NzbDrone.Core.HealthCheck { public abstract class HealthCheckBase : IProvideHealthCheck { @@ -6,8 +6,6 @@ public virtual bool CheckOnStartup => true; - public virtual bool CheckOnConfigChange => true; - public virtual bool CheckOnSchedule => true; } } diff --git a/src/NzbDrone.Core/HealthCheck/HealthCheckService.cs b/src/NzbDrone.Core/HealthCheck/HealthCheckService.cs index 56789b8a1..b6487b492 100644 --- a/src/NzbDrone.Core/HealthCheck/HealthCheckService.cs +++ b/src/NzbDrone.Core/HealthCheck/HealthCheckService.cs @@ -1,8 +1,11 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using NLog; using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Messaging; +using NzbDrone.Common.Reflection; using NzbDrone.Core.Configuration.Events; using NzbDrone.Core.Download; using NzbDrone.Core.Indexers; @@ -21,13 +24,12 @@ namespace NzbDrone.Core.HealthCheck public class HealthCheckService : IHealthCheckService, IExecute, IHandleAsync, - IHandleAsync, - IHandleAsync>, - IHandleAsync>, - IHandleAsync>, - IHandleAsync> + IHandleAsync { - private readonly IEnumerable _healthChecks; + private readonly IProvideHealthCheck[] _healthChecks; + private readonly IProvideHealthCheck[] _startupHealthChecks; + private readonly IProvideHealthCheck[] _scheduledHealthChecks; + private readonly Dictionary _eventDrivenHealthChecks; private readonly IEventAggregator _eventAggregator; private readonly ICacheManager _cacheManager; private readonly Logger _logger; @@ -39,12 +41,16 @@ namespace NzbDrone.Core.HealthCheck ICacheManager cacheManager, Logger logger) { - _healthChecks = healthChecks; + _healthChecks = healthChecks.ToArray(); _eventAggregator = eventAggregator; _cacheManager = cacheManager; _logger = logger; _healthCheckResults = _cacheManager.GetCache(GetType()); + + _startupHealthChecks = _healthChecks.Where(v => v.CheckOnStartup).ToArray(); + _scheduledHealthChecks = _healthChecks.Where(v => v.CheckOnSchedule).ToArray(); + _eventDrivenHealthChecks = GetEventDrivenHealthChecks(); } public List Results() @@ -52,11 +58,18 @@ namespace NzbDrone.Core.HealthCheck return _healthCheckResults.Values.ToList(); } - private void PerformHealthCheck(Func predicate) + private Dictionary GetEventDrivenHealthChecks() + { + return _healthChecks + .SelectMany(h => h.GetType().GetAttributes().Select(a => Tuple.Create(a.EventType, h))) + .GroupBy(t => t.Item1, t => t.Item2) + .ToDictionary(g => g.Key, g => g.ToArray()); + } + + private void PerformHealthCheck(IProvideHealthCheck[] healthChecks) { - var results = _healthChecks.Where(predicate) - .Select(c => c.Check()) - .ToList(); + var results = healthChecks.Select(c => c.Check()) + .ToList(); foreach (var result in results) { @@ -76,37 +89,37 @@ namespace NzbDrone.Core.HealthCheck public void Execute(CheckHealthCommand message) { - PerformHealthCheck(c => message.Trigger == CommandTrigger.Manual || c.CheckOnSchedule); + if (message.Trigger == CommandTrigger.Manual) + { + PerformHealthCheck(_healthChecks); + } + else + { + PerformHealthCheck(_scheduledHealthChecks); + } } public void HandleAsync(ApplicationStartedEvent message) { - PerformHealthCheck(c => c.CheckOnStartup); - } - - public void HandleAsync(ConfigSavedEvent message) - { - PerformHealthCheck(c => c.CheckOnConfigChange); + PerformHealthCheck(_startupHealthChecks); } - public void HandleAsync(ProviderUpdatedEvent message) + public void HandleAsync(IEvent message) { - PerformHealthCheck(c => c.CheckOnConfigChange); - } + if (message is HealthCheckCompleteEvent) + { + return; + } - public void HandleAsync(ProviderDeletedEvent message) - { - PerformHealthCheck(c => c.CheckOnConfigChange); - } + IProvideHealthCheck[] checks; + if (!_eventDrivenHealthChecks.TryGetValue(message.GetType(), out checks)) + { + return; + } - public void HandleAsync(ProviderUpdatedEvent message) - { - PerformHealthCheck(c => c.CheckOnConfigChange); - } + // TODO: Add debounce - public void HandleAsync(ProviderDeletedEvent message) - { - PerformHealthCheck(c => c.CheckOnConfigChange); + PerformHealthCheck(checks); } } } diff --git a/src/NzbDrone.Core/HealthCheck/IProvideHealthCheck.cs b/src/NzbDrone.Core/HealthCheck/IProvideHealthCheck.cs index ece0b7952..d71f2653f 100644 --- a/src/NzbDrone.Core/HealthCheck/IProvideHealthCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/IProvideHealthCheck.cs @@ -1,10 +1,9 @@ -namespace NzbDrone.Core.HealthCheck +namespace NzbDrone.Core.HealthCheck { public interface IProvideHealthCheck { HealthCheck Check(); bool CheckOnStartup { get; } - bool CheckOnConfigChange { get; } bool CheckOnSchedule { get; } } } diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs new file mode 100644 index 000000000..eb3073a1f --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupDownloadClientUnavailablePendingReleases.cs @@ -0,0 +1,32 @@ +using System; +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Download.Pending; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class CleanupDownloadClientUnavailablePendingReleases : IHousekeepingTask + { + private readonly IMainDatabase _database; + + public CleanupDownloadClientUnavailablePendingReleases(IMainDatabase database) + { + _database = database; + } + + public void Clean() + { + var mapper = _database.GetDataMapper(); + var twoWeeksAgo = DateTime.UtcNow.AddDays(-14); + + mapper.Delete(p => p.Added < twoWeeksAgo && + (p.Reason == PendingReleaseReason.DownloadClientUnavailable || + p.Reason == PendingReleaseReason.Fallback)); + + // mapper.AddParameter("twoWeeksAgo", $"{DateTime.UtcNow.AddDays(-14).ToString("s")}Z"); + + // mapper.ExecuteNonQuery(@"DELETE FROM PendingReleases + // WHERE Added < @twoWeeksAgo + // AND (Reason = 'DownloadClientUnavailable' OR Reason = 'Fallback')"); + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs new file mode 100644 index 000000000..258a51ab9 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs @@ -0,0 +1,26 @@ +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class CleanupOrphanedDownloadClientStatus : IHousekeepingTask + { + private readonly IMainDatabase _database; + + public CleanupOrphanedDownloadClientStatus(IMainDatabase database) + { + _database = database; + } + + public void Clean() + { + var mapper = _database.GetDataMapper(); + + mapper.ExecuteNonQuery(@"DELETE FROM DownloadClientStatus + WHERE Id IN ( + SELECT DownloadClientStatus.Id FROM DownloadClientStatus + LEFT OUTER JOIN DownloadClients + ON DownloadClientStatus.ProviderId = DownloadClients.Id + WHERE DownloadClients.Id IS NULL)"); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index fd8a278fc..6894b4af3 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -95,11 +95,6 @@ namespace NzbDrone.Core.Indexers failures.Add(new ValidationFailure(string.Empty, "Test was aborted due to an error: " + ex.Message)); } - if (Definition.Id != 0) - { - _indexerStatusService.RecordSuccess(Definition.Id); - } - return new ValidationResult(failures); } diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs index c4ef7ac79..ba1c9a347 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs @@ -1,5 +1,6 @@ -using System.Collections.Generic; +using System.Collections.Generic; using System.Linq; +using FluentValidation.Results; using NLog; using NzbDrone.Common.Composition; using NzbDrone.Core.Messaging.Events; @@ -21,7 +22,7 @@ namespace NzbDrone.Core.Indexers public IndexerFactory(IIndexerStatusService indexerStatusService, IIndexerRepository providerRepository, IEnumerable providers, - IContainer container, + IContainer container, IEventAggregator eventAggregator, Logger logger) : base(providerRepository, providers, container, eventAggregator, logger) @@ -70,7 +71,7 @@ namespace NzbDrone.Core.Indexers private IEnumerable FilterBlockedIndexers(IEnumerable indexers) { - var blockedIndexers = _indexerStatusService.GetBlockedIndexers().ToDictionary(v => v.ProviderId, v => v); + var blockedIndexers = _indexerStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v); foreach (var indexer in indexers) { @@ -84,5 +85,17 @@ namespace NzbDrone.Core.Indexers yield return indexer; } } + + public override ValidationResult Test(IndexerDefinition definition) + { + var result = base.Test(definition); + + if ((result == null || result.IsValid) && definition.Id != 0) + { + _indexerStatusService.RecordSuccess(definition.Id); + } + + return result; + } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Indexers/IndexerStatus.cs b/src/NzbDrone.Core/Indexers/IndexerStatus.cs index 92830490d..72546da7c 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatus.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatus.cs @@ -1,23 +1,10 @@ -using System; -using NzbDrone.Core.Datastore; using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider.Status; namespace NzbDrone.Core.Indexers { - public class IndexerStatus : ModelBase + public class IndexerStatus : ProviderStatusBase { - public int ProviderId { get; set; } - - public DateTime? InitialFailure { get; set; } - public DateTime? MostRecentFailure { get; set; } - public int EscalationLevel { get; set; } - public DateTime? DisabledTill { get; set; } - public ReleaseInfo LastRssSyncReleaseInfo { get; set; } - - public bool IsDisabled() - { - return DisabledTill.HasValue && DisabledTill.Value > DateTime.UtcNow; - } } } diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs b/src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs index 79173ff6b..02a656125 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatusRepository.cs @@ -1,26 +1,20 @@ -using System.Linq; using NzbDrone.Core.Datastore; using NzbDrone.Core.Messaging.Events; -using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.ThingiProvider.Status; namespace NzbDrone.Core.Indexers { - public interface IIndexerStatusRepository : IProviderRepository + public interface IIndexerStatusRepository : IProviderStatusRepository { - IndexerStatus FindByIndexerId(int indexerId); - } + } + + public class IndexerStatusRepository : ProviderStatusRepository, IIndexerStatusRepository - public class IndexerStatusRepository : ProviderRepository, IIndexerStatusRepository { public IndexerStatusRepository(IMainDatabase database, IEventAggregator eventAggregator) : base(database, eventAggregator) { } - - public IndexerStatus FindByIndexerId(int indexerId) - { - return Query.Where(c => c.ProviderId == indexerId).SingleOrDefault(); - } } } diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs index 09be1f00b..b49a3895d 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs @@ -1,131 +1,31 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using NLog; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider.Events; +using NzbDrone.Core.ThingiProvider.Status; namespace NzbDrone.Core.Indexers { - public interface IIndexerStatusService + public interface IIndexerStatusService : IProviderStatusServiceBase { - List GetBlockedIndexers(); ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId); - void RecordSuccess(int indexerId); - void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan)); - void RecordConnectionFailure(int indexerId); void UpdateRssSyncStatus(int indexerId, ReleaseInfo releaseInfo); } - public class IndexerStatusService : IIndexerStatusService, IHandleAsync> + public class IndexerStatusService : ProviderStatusServiceBase, IIndexerStatusService { - private static readonly int[] EscalationBackOffPeriods = { - 0, - 5 * 60, - 15 * 60, - 30 * 60, - 60 * 60, - 3 * 60 * 60, - 6 * 60 * 60, - 12 * 60 * 60, - 24 * 60 * 60 - }; - private static readonly int MaximumEscalationLevel = EscalationBackOffPeriods.Length - 1; - - private static readonly object _syncRoot = new object(); - - private readonly IIndexerStatusRepository _indexerStatusRepository; - private readonly Logger _logger; - - public IndexerStatusService(IIndexerStatusRepository indexerStatusRepository, Logger logger) - { - _indexerStatusRepository = indexerStatusRepository; - _logger = logger; - } - - public List GetBlockedIndexers() + public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + : base(providerStatusRepository, eventAggregator, logger) { - return _indexerStatusRepository.All().Where(v => v.IsDisabled()).ToList(); } public ReleaseInfo GetLastRssSyncReleaseInfo(int indexerId) { - return GetIndexerStatus(indexerId).LastRssSyncReleaseInfo; - } - - private IndexerStatus GetIndexerStatus(int indexerId) - { - return _indexerStatusRepository.FindByIndexerId(indexerId) ?? new IndexerStatus { ProviderId = indexerId }; - } - - private TimeSpan CalculateBackOffPeriod(IndexerStatus status) - { - var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel); - - return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]); - } - - public void RecordSuccess(int indexerId) - { - lock (_syncRoot) - { - var status = GetIndexerStatus(indexerId); - - if (status.EscalationLevel == 0) - { - return; - } - - status.EscalationLevel--; - status.DisabledTill = null; - - _indexerStatusRepository.Upsert(status); - } - } - - protected void RecordFailure(int indexerId, TimeSpan minimumBackOff, bool escalate) - { - lock (_syncRoot) - { - var status = GetIndexerStatus(indexerId); - - var now = DateTime.UtcNow; - - if (status.EscalationLevel == 0) - { - status.InitialFailure = now; - } - - status.MostRecentFailure = now; - if (escalate) - { - status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1); - } - - if (minimumBackOff != TimeSpan.Zero) - { - while (status.EscalationLevel < MaximumEscalationLevel && CalculateBackOffPeriod(status) < minimumBackOff) - { - status.EscalationLevel++; - } - } - - status.DisabledTill = now + CalculateBackOffPeriod(status); - - _indexerStatusRepository.Upsert(status); - } - } - - public void RecordFailure(int indexerId, TimeSpan minimumBackOff = default(TimeSpan)) - { - RecordFailure(indexerId, minimumBackOff, true); - } - - public void RecordConnectionFailure(int indexerId) - { - RecordFailure(indexerId, default(TimeSpan), false); + return GetProviderStatus(indexerId).LastRssSyncReleaseInfo; } @@ -133,21 +33,11 @@ namespace NzbDrone.Core.Indexers { lock (_syncRoot) { - var status = GetIndexerStatus(indexerId); + var status = GetProviderStatus(indexerId); status.LastRssSyncReleaseInfo = releaseInfo; - _indexerStatusRepository.Upsert(status); - } - } - - public void HandleAsync(ProviderDeletedEvent message) - { - var indexerStatus = _indexerStatusRepository.FindByIndexerId(message.ProviderId); - - if (indexerStatus != null) - { - _indexerStatusRepository.Delete(indexerStatus); + _providerStatusRepository.Upsert(status); } } } diff --git a/src/NzbDrone.Core/Messaging/Events/EventAggregator.cs b/src/NzbDrone.Core/Messaging/Events/EventAggregator.cs index a66d22c2c..0111a7342 100644 --- a/src/NzbDrone.Core/Messaging/Events/EventAggregator.cs +++ b/src/NzbDrone.Core/Messaging/Events/EventAggregator.cs @@ -62,6 +62,17 @@ namespace NzbDrone.Core.Messaging.Events } } + foreach (var handler in _serviceFactory.BuildAll>()) + { + var handlerLocal = handler; + + _taskFactory.StartNew(() => + { + handlerLocal.HandleAsync(@event); + }, TaskCreationOptions.PreferFairness) + .LogExceptions(); + } + foreach (var handler in _serviceFactory.BuildAll>()) { var handlerLocal = handler; diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 7eda4001b..a38629225 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -170,6 +170,7 @@ + @@ -200,6 +201,7 @@ + @@ -242,6 +244,7 @@ + @@ -373,7 +376,11 @@ + + + + @@ -435,8 +442,10 @@ + + @@ -459,9 +468,11 @@ + + @@ -994,6 +1005,7 @@ + @@ -1004,6 +1016,9 @@ + + + diff --git a/src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs b/src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs new file mode 100644 index 000000000..8def1f0c7 --- /dev/null +++ b/src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs @@ -0,0 +1,18 @@ +using NzbDrone.Common.Messaging; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.ThingiProvider.Events +{ + public class ProviderStatusChangedEvent : IEvent + { + public int ProviderId { get; private set; } + + public ProviderStatusBase Status { get; private set; } + + public ProviderStatusChangedEvent(int id, ProviderStatusBase status) + { + ProviderId = id; + Status = status; + } + } +} diff --git a/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs b/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs index db98416d3..750f0cf3d 100644 --- a/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs +++ b/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using FluentValidation.Results; diff --git a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusBase.cs b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusBase.cs index cbf101f39..395a43efd 100644 --- a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusBase.cs +++ b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusBase.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; diff --git a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusRepository.cs b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusRepository.cs index 2c1b184f1..c2782b409 100644 --- a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusRepository.cs +++ b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusRepository.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Text; diff --git a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs index b74153947..6931ed626 100644 --- a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs +++ b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs @@ -1,8 +1,9 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using NLog; using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Parser.Model; using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.ThingiProvider.Status @@ -20,13 +21,25 @@ namespace NzbDrone.Core.ThingiProvider.Status where TProvider : IProvider where TModel : ProviderStatusBase, new() { + private static readonly int[] EscalationBackOffPeriods = { + 0, + 5 * 60, + 15 * 60, + 30 * 60, + 60 * 60, + 3 * 60 * 60, + 6 * 60 * 60, + 12 * 60 * 60, + 24 * 60 * 60 + }; + protected readonly object _syncRoot = new object(); protected readonly IProviderStatusRepository _providerStatusRepository; protected readonly IEventAggregator _eventAggregator; protected readonly Logger _logger; - protected int MaximumEscalationLevel { get; set; } = EscalationBackOff.Periods.Length - 1; + protected int MaximumEscalationLevel { get; set; } = EscalationBackOffPeriods.Length - 1; protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero; public ProviderStatusServiceBase(IProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) @@ -50,7 +63,7 @@ namespace NzbDrone.Core.ThingiProvider.Status { var level = Math.Min(MaximumEscalationLevel, status.EscalationLevel); - return TimeSpan.FromSeconds(EscalationBackOff.Periods[level]); + return TimeSpan.FromSeconds(EscalationBackOffPeriods[level]); } public virtual void RecordSuccess(int providerId)