From f335cc1af8ae40d021ed2a408dd9b310d9624833 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Fri, 12 May 2017 21:42:17 +0200 Subject: [PATCH] Fixed: Prevent Download Client from being queried every minute if it failed repeatedly similar to Indexer temporarily disabled logic. --- .../DownloadClientStatusServiceFixture.cs | 106 +++++++++++++++ .../Checks/DownloadClientCheckFixture.cs | 5 +- .../IndexerStatusServiceFixture.cs | 12 +- .../NzbDrone.Core.Test.csproj | 4 +- .../ThingiProviderTests/NullConfigFixture.cs | 2 +- .../ProviderBaseFixture.cs | 5 +- .../ProviderStatusServiceFixture.cs | 126 ++++++++++++++++++ .../115_add_downloadclient_status.cs | 19 +++ src/NzbDrone.Core/Datastore/TableMapping.cs | 7 +- .../Download/DownloadClientStatus.cs | 9 ++ .../DownloadClientStatusRepository.cs | 19 +++ .../Download/DownloadClientStatusService.cs | 22 +++ .../HealthCheck/Checks/DownloadClientCheck.cs | 3 +- .../Checks/DownloadClientStatusCheck.cs | 42 ++++++ .../CleanupOrphanedDownloadClientStatus.cs | 26 ++++ .../Indexers/IndexerStatusService.cs | 4 +- src/NzbDrone.Core/NzbDrone.Core.csproj | 7 + .../Events/ProviderStatusChangedEvent.cs | 18 +++ .../Status/ProviderStatusServiceBase.cs | 18 ++- 19 files changed, 431 insertions(+), 23 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs rename src/NzbDrone.Core.Test/{ThingiProvider => ThingiProviderTests}/ProviderBaseFixture.cs (94%) create mode 100644 src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/115_add_downloadclient_status.cs create mode 100644 src/NzbDrone.Core/Download/DownloadClientStatus.cs create mode 100644 src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs create mode 100644 src/NzbDrone.Core/Download/DownloadClientStatusService.cs create mode 100644 src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs create mode 100644 src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs diff --git a/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs new file mode 100644 index 000000000..0ddae11a2 --- /dev/null +++ b/src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs @@ -0,0 +1,106 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Core.Download; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Download +{ + public class DownloadClientStatusServiceFixture : CoreTest + { + private DateTime _epoch; + + [SetUp] + public void SetUp() + { + _epoch = DateTime.UtcNow; + } + + private void WithStatus(DownloadClientStatus status) + { + Mocker.GetMock() + .Setup(v => v.FindByProviderId(1)) + .Returns(status); + + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new[] { status }); + } + + private void VerifyUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); + } + + private void VerifyNoUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); + } + + [Test] + public void should_not_consider_blocked_within_5_minutes_since_initial_failure() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(4), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + } + + [Test] + public void should_consider_blocked_after_5_minutes_since_initial_failure() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + } + + [Test] + public void should_not_escalate_beyond_3_hours() + { + WithStatus(new DownloadClientStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1)); + } + } +} diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs index dc6986d79..665038e32 100644 --- a/src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/DownloadClientCheckFixture.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using NUnit.Framework; using NzbDrone.Core.Download; using NzbDrone.Core.HealthCheck.Checks; -using NzbDrone.Core.Indexers; using NzbDrone.Core.Test.Framework; using NzbDrone.Test.Common; @@ -26,7 +25,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks public void should_return_error_when_download_client_throws() { var downloadClient = Mocker.GetMock(); - downloadClient.Setup(s => s.Definition).Returns(new IndexerDefinition{Name = "Test"}); + downloadClient.Setup(s => s.Definition).Returns(new DownloadClientDefinition{Name = "Test"}); downloadClient.Setup(s => s.GetItems()) .Throws(); @@ -36,8 +35,6 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks .Returns(new IDownloadClient[] { downloadClient.Object }); Subject.Check().ShouldBeError(); - - ExceptionVerification.ExpectedErrors(1); } [Test] diff --git a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs index 9f2c120d1..17a2eeaa6 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs @@ -29,10 +29,16 @@ namespace NzbDrone.Core.Test.IndexerTests .Returns(new[] { status }); } - private void VerifyUpdate(bool updated = true) + private void VerifyUpdate() { Mocker.GetMock() - .Verify(v => v.Upsert(It.IsAny()), Times.Exactly(updated ? 1 : 0)); + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); + } + + private void VerifyNoUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); } [Test] @@ -70,7 +76,7 @@ namespace NzbDrone.Core.Test.IndexerTests Subject.RecordSuccess(1); - VerifyUpdate(false); + VerifyNoUpdate(); } [Test] diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index ffd18cfd2..4bc77554d 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -178,6 +178,7 @@ + @@ -369,7 +370,8 @@ - + + diff --git a/src/NzbDrone.Core.Test/ThingiProviderTests/NullConfigFixture.cs b/src/NzbDrone.Core.Test/ThingiProviderTests/NullConfigFixture.cs index 0f47d9a91..efc5c58bd 100644 --- a/src/NzbDrone.Core.Test/ThingiProviderTests/NullConfigFixture.cs +++ b/src/NzbDrone.Core.Test/ThingiProviderTests/NullConfigFixture.cs @@ -14,4 +14,4 @@ namespace NzbDrone.Core.Test.ThingiProviderTests Subject.Validate().IsValid.Should().BeTrue(); } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core.Test/ThingiProvider/ProviderBaseFixture.cs b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderBaseFixture.cs similarity index 94% rename from src/NzbDrone.Core.Test/ThingiProvider/ProviderBaseFixture.cs rename to src/NzbDrone.Core.Test/ThingiProviderTests/ProviderBaseFixture.cs index db1e21c61..a3a5a0c51 100644 --- a/src/NzbDrone.Core.Test/ThingiProvider/ProviderBaseFixture.cs +++ b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderBaseFixture.cs @@ -5,9 +5,8 @@ using NzbDrone.Core.Indexers; using NzbDrone.Core.Indexers.Newznab; using NzbDrone.Core.Test.Framework; -namespace NzbDrone.Core.Test.ThingiProvider +namespace NzbDrone.Core.Test.ThingiProviderTests { - public class ProviderRepositoryFixture : DbTest { [Test] @@ -27,4 +26,4 @@ namespace NzbDrone.Core.Test.ThingiProvider storedSetting.ShouldBeEquivalentTo(newznabSettings, o=>o.IncludingAllRuntimeProperties()); } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs new file mode 100644 index 000000000..32a9c4b7a --- /dev/null +++ b/src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs @@ -0,0 +1,126 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NLog; +using NUnit.Framework; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Test.Framework; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Test.ThingiProviderTests +{ + public class MockProviderStatus : ProviderStatusBase + { + } + + public interface IMockProvider : IProvider + { + } + + public interface IMockProviderStatusRepository : IProviderStatusRepository + { + } + + public class MockProviderStatusService : ProviderStatusServiceBase + { + public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + : base(providerStatusRepository, eventAggregator, logger) + { + + } + } + + public class ProviderStatusServiceFixture : CoreTest + { + private DateTime _epoch; + + [SetUp] + public void SetUp() + { + _epoch = DateTime.UtcNow; + } + + private void WithStatus(MockProviderStatus status) + { + Mocker.GetMock() + .Setup(v => v.FindByProviderId(1)) + .Returns(status); + + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new[] { status }); + } + + private void VerifyUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); + } + + private void VerifyNoUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); + } + + [Test] + public void should_start_backoff_on_first_failure() + { + WithStatus(new MockProviderStatus()); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500); + } + + [Test] + public void should_cancel_backoff_on_success() + { + WithStatus(new MockProviderStatus { EscalationLevel = 2 }); + + Subject.RecordSuccess(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + } + + [Test] + public void should_not_store_update_if_already_okay() + { + WithStatus(new MockProviderStatus { EscalationLevel = 0 }); + + Subject.RecordSuccess(1); + + VerifyNoUpdate(); + } + + [Test] + public void should_preserve_escalation_on_intermittent_success() + { + WithStatus(new MockProviderStatus + { + InitialFailure = _epoch - TimeSpan.FromSeconds(20), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordSuccess(1); + Subject.RecordSuccess(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/115_add_downloadclient_status.cs b/src/NzbDrone.Core/Datastore/Migration/115_add_downloadclient_status.cs new file mode 100644 index 000000000..552fd682e --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/115_add_downloadclient_status.cs @@ -0,0 +1,19 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(115)] + public class add_downloadclient_status : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Create.TableForModel("DownloadClientStatus") + .WithColumn("ProviderId").AsInt32().NotNullable().Unique() + .WithColumn("InitialFailure").AsDateTime().Nullable() + .WithColumn("MostRecentFailure").AsDateTime().Nullable() + .WithColumn("EscalationLevel").AsInt32().NotNullable() + .WithColumn("DisabledTill").AsDateTime().Nullable(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 62f6aeb8b..38787b074 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -60,7 +60,7 @@ namespace NzbDrone.Core.Datastore .Ignore(i => i.SupportsOnDownload) .Ignore(i => i.SupportsOnUpgrade) .Ignore(i => i.SupportsOnRename); - + Mapper.Entity().RegisterDefinition("Metadata"); Mapper.Entity().RegisterDefinition("DownloadClients") @@ -80,7 +80,7 @@ namespace NzbDrone.Core.Datastore .Ignore(f => f.Path) .Relationships.AutoMapICollectionOrComplexProperties() .For("Episodes") - .LazyLoad(condition: parent => parent.Id > 0, + .LazyLoad(condition: parent => parent.Id > 0, query: (db, parent) => db.Query().Where(c => c.EpisodeFileId == parent.Id).ToList()) .HasOne(file => file.Series, file => file.SeriesId); @@ -116,6 +116,7 @@ namespace NzbDrone.Core.Datastore .Ignore(c => c.Message); Mapper.Entity().RegisterModel("IndexerStatus"); + Mapper.Entity().RegisterModel("DownloadClientStatus"); } private static void RegisterMappers() @@ -171,4 +172,4 @@ namespace NzbDrone.Core.Datastore } } } -} \ No newline at end of file +} diff --git a/src/NzbDrone.Core/Download/DownloadClientStatus.cs b/src/NzbDrone.Core/Download/DownloadClientStatus.cs new file mode 100644 index 000000000..455196672 --- /dev/null +++ b/src/NzbDrone.Core/Download/DownloadClientStatus.cs @@ -0,0 +1,9 @@ +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Download +{ + public class DownloadClientStatus : ProviderStatusBase + { + + } +} diff --git a/src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs b/src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs new file mode 100644 index 000000000..4f6fd6dfa --- /dev/null +++ b/src/NzbDrone.Core/Download/DownloadClientStatusRepository.cs @@ -0,0 +1,19 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Download +{ + public interface IDownloadClientStatusRepository : IProviderStatusRepository + { + + } + + public class DownloadClientStatusRepository : ProviderStatusRepository, IDownloadClientStatusRepository + { + public DownloadClientStatusRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/Download/DownloadClientStatusService.cs b/src/NzbDrone.Core/Download/DownloadClientStatusService.cs new file mode 100644 index 000000000..b4fd0f83c --- /dev/null +++ b/src/NzbDrone.Core/Download/DownloadClientStatusService.cs @@ -0,0 +1,22 @@ +using System; +using NLog; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Download +{ + public interface IDownloadClientStatusService : IProviderStatusServiceBase + { + + } + + public class DownloadClientStatusService : ProviderStatusServiceBase, IDownloadClientStatusService + { + public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + : base(providerStatusRepository, eventAggregator, logger) + { + MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5); + MaximumEscalationLevel = 5; + } + } +} diff --git a/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs index d99eed1a3..ea3b87586 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientCheck.cs @@ -33,8 +33,7 @@ namespace NzbDrone.Core.HealthCheck.Checks } catch (Exception ex) { - - _logger.Error(ex, "Unable to communicate with {0}", downloadClient.Definition.Name); + _logger.Debug(ex, "Unable to communicate with {0}", downloadClient.Definition.Name); var message = $"Unable to communicate with {downloadClient.Definition.Name}."; return new HealthCheck(GetType(), HealthCheckResult.Error, $"{message} {ex.Message}"); diff --git a/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs new file mode 100644 index 000000000..bf3f268b8 --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/DownloadClientStatusCheck.cs @@ -0,0 +1,42 @@ +using System; +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Download; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + public class DownloadClientStatusCheck : HealthCheckBase + { + private readonly IDownloadClientFactory _providerFactory; + private readonly IDownloadClientStatusService _providerStatusService; + + public DownloadClientStatusCheck(IDownloadClientFactory providerFactory, IDownloadClientStatusService providerStatusService) + { + _providerFactory = providerFactory; + _providerStatusService = providerStatusService; + } + + public override HealthCheck Check() + { + var enabledProviders = _providerFactory.GetAvailableProviders(); + var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new { Provider = i, Status = s }) + .Where(v => (v.Status.MostRecentFailure - v.Status.InitialFailure) > TimeSpan.FromHours(1)) + .ToList(); + + if (backOffProviders.Empty()) + { + return new HealthCheck(GetType()); + } + + if (backOffProviders.Count == enabledProviders.Count) + { + return new HealthCheck(GetType(), HealthCheckResult.Error, "All download clients are unavailable due to failures", "#download-clients-are-unavailable-due-to-failures"); + } + + return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Download clients unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), "#download-clients-are-unavailable-due-to-failures"); + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs new file mode 100644 index 000000000..3bb631eb9 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedDownloadClientStatus.cs @@ -0,0 +1,26 @@ +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class CleanupOrphanedDownloadClientStatus : IHousekeepingTask + { + private readonly IMainDatabase _database; + + public CleanupOrphanedDownloadClientStatus(IMainDatabase database) + { + _database = database; + } + + public void Clean() + { + var mapper = _database.GetDataMapper(); + + mapper.ExecuteNonQuery(@"DELETE FROM DownloadClientStatus + WHERE Id IN ( + SELECT DownloadClientStatus.Id FROM DownloadClientStatus + LEFT OUTER JOIN DownloadClients + ON DownloadClientStatus.ProviderId = DownloadClients.Id + WHERE DownloadClients.Id IS NULL)"); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs index 48a853804..6d3b89545 100644 --- a/src/NzbDrone.Core/Indexers/IndexerStatusService.cs +++ b/src/NzbDrone.Core/Indexers/IndexerStatusService.cs @@ -18,8 +18,8 @@ namespace NzbDrone.Core.Indexers public class IndexerStatusService : ProviderStatusServiceBase, IIndexerStatusService { - public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, Logger logger) - : base(providerStatusRepository, logger) + public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) + : base(providerStatusRepository, eventAggregator, logger) { } diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index c3e3c55d4..15a3a84c3 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -293,6 +293,7 @@ + @@ -494,7 +495,10 @@ + + + @@ -557,6 +561,7 @@ + @@ -582,6 +587,7 @@ + @@ -1073,6 +1079,7 @@ + diff --git a/src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs b/src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs new file mode 100644 index 000000000..8def1f0c7 --- /dev/null +++ b/src/NzbDrone.Core/ThingiProvider/Events/ProviderStatusChangedEvent.cs @@ -0,0 +1,18 @@ +using NzbDrone.Common.Messaging; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.ThingiProvider.Events +{ + public class ProviderStatusChangedEvent : IEvent + { + public int ProviderId { get; private set; } + + public ProviderStatusBase Status { get; private set; } + + public ProviderStatusChangedEvent(int id, ProviderStatusBase status) + { + ProviderId = id; + Status = status; + } + } +} diff --git a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs index f3a57823c..8ea34d47e 100644 --- a/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs +++ b/src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs @@ -33,16 +33,19 @@ namespace NzbDrone.Core.ThingiProvider.Status 24 * 60 * 60 }; - private static readonly int MaximumEscalationLevel = EscalationBackOffPeriods.Length - 1; - protected readonly object _syncRoot = new object(); protected readonly IProviderStatusRepository _providerStatusRepository; + protected readonly IEventAggregator _eventAggregator; protected readonly Logger _logger; - public ProviderStatusServiceBase(IProviderStatusRepository providerStatusRepository, Logger logger) + protected int MaximumEscalationLevel { get; set; } = EscalationBackOffPeriods.Length - 1; + protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero; + + public ProviderStatusServiceBase(IProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger) { _providerStatusRepository = providerStatusRepository; + _eventAggregator = eventAggregator; _logger = logger; } @@ -78,6 +81,8 @@ namespace NzbDrone.Core.ThingiProvider.Status status.DisabledTill = null; _providerStatusRepository.Upsert(status); + + _eventAggregator.PublishEvent(new ProviderStatusChangedEvent(providerId, status)); } } @@ -108,9 +113,14 @@ namespace NzbDrone.Core.ThingiProvider.Status } } - status.DisabledTill = now + CalculateBackOffPeriod(status); + if (status.InitialFailure.Value + MinimumTimeSinceInitialFailure <= now || minimumBackOff != TimeSpan.Zero) + { + status.DisabledTill = now + CalculateBackOffPeriod(status); + } _providerStatusRepository.Upsert(status); + + _eventAggregator.PublishEvent(new ProviderStatusChangedEvent(providerId, status)); } }