From 2adedb97da5ad31b65f0dc2eec5c263efe95731f Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Mon, 12 Oct 2020 11:24:19 -0700 Subject: [PATCH] New: Differentiate between short term and long term (more than 6 hours) indexer failures Closes #3279 --- frontend/src/System/Status/Health/Health.js | 1 + .../IndexerLongTermStatusCheckFixture.cs | 86 +++++++++++++++++++ .../Checks/IndexerStatusCheckFixture.cs | 8 +- .../Checks/IndexerLongTermStatusCheck.cs | 52 +++++++++++ .../HealthCheck/Checks/IndexerStatusCheck.cs | 11 ++- 5 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerLongTermStatusCheckFixture.cs create mode 100644 src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs diff --git a/frontend/src/System/Status/Health/Health.js b/frontend/src/System/Status/Health/Health.js index 2d13b7aa2..0344b2e67 100644 --- a/frontend/src/System/Status/Health/Health.js +++ b/frontend/src/System/Status/Health/Health.js @@ -18,6 +18,7 @@ function getInternalLink(source) { case 'IndexerRssCheck': case 'IndexerSearchCheck': case 'IndexerStatusCheck': + case 'IndexerLongTermStatusCheck': return ( + { + private List _indexers = new List(); + private List _blockedIndexers = new List(); + + [SetUp] + public void SetUp() + { + Mocker.GetMock() + .Setup(v => v.GetAvailableProviders()) + .Returns(_indexers); + + Mocker.GetMock() + .Setup(v => v.GetBlockedProviders()) + .Returns(_blockedIndexers); + } + + private Mock GivenIndexer(int id, double backoffHours, double failureHours) + { + var mockIndexer = new Mock(); + mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = id }); + mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true); + + _indexers.Add(mockIndexer.Object); + + if (backoffHours != 0.0) + { + _blockedIndexers.Add(new IndexerStatus + { + ProviderId = id, + InitialFailure = DateTime.UtcNow.AddHours(-failureHours), + MostRecentFailure = DateTime.UtcNow.AddHours(-0.1), + EscalationLevel = 5, + DisabledTill = DateTime.UtcNow.AddHours(backoffHours) + }); + } + + return mockIndexer; + } + + + [Test] + public void should_not_return_error_when_no_indexers() + { + Subject.Check().ShouldBeOk(); + } + + [Test] + public void should_return_warning_if_indexer_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + GivenIndexer(2, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + + [Test] + public void should_return_error_if_all_indexers_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + + Subject.Check().ShouldBeError(); + } + + [Test] + public void should_return_warning_if_few_indexers_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + GivenIndexer(2, 10.0, 24.0); + GivenIndexer(3, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + } +} diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs index e3deac674..c2e645858 100644 --- a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs @@ -59,7 +59,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks [Test] public void should_return_warning_if_indexer_unavailable() { - GivenIndexer(1, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); GivenIndexer(2, 0.0, 0.0); Subject.Check().ShouldBeWarning(); @@ -68,7 +68,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks [Test] public void should_return_error_if_all_indexers_unavailable() { - GivenIndexer(1, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); Subject.Check().ShouldBeError(); } @@ -76,8 +76,8 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks [Test] public void should_return_warning_if_few_indexers_unavailable() { - GivenIndexer(1, 10.0, 24.0); - GivenIndexer(2, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); + GivenIndexer(2, 2.0, 4.0); GivenIndexer(3, 0.0, 0.0); Subject.Check().ShouldBeWarning(); diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs new file mode 100644 index 000000000..ddffd4f27 --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs @@ -0,0 +1,52 @@ +using System; +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] + public class IndexerLongTermStatusCheck : HealthCheckBase + { + private readonly IIndexerFactory _providerFactory; + private readonly IIndexerStatusService _providerStatusService; + + public IndexerLongTermStatusCheck(IIndexerFactory providerFactory, IIndexerStatusService providerStatusService) + { + _providerFactory = providerFactory; + _providerStatusService = providerStatusService; + } + + public override HealthCheck Check() + { + var enabledProviders = _providerFactory.GetAvailableProviders(); + var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new {Provider = i, Status = s}) + .Where(p => p.Status.InitialFailure.HasValue && + p.Status.InitialFailure.Value.Before( + DateTime.UtcNow.AddHours(-6))) + .ToList(); + + if (backOffProviders.Empty()) + { + return new HealthCheck(GetType()); + } + + if (backOffProviders.Count == enabledProviders.Count) + { + return new HealthCheck(GetType(), HealthCheckResult.Error, + "All indexers are unavailable due to failures for more than 6 hours", "#indexers-are-unavailable-due-to-failures"); + } + + return new HealthCheck(GetType(), HealthCheckResult.Warning, + string.Format("Indexers unavailable due to failures for more than 6 hours: {0}", + string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), + "#indexers-are-unavailable-due-to-failures"); + } + } +} diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs index 5c5f89283..c283b525d 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs @@ -24,10 +24,13 @@ namespace NzbDrone.Core.HealthCheck.Checks { var enabledProviders = _providerFactory.GetAvailableProviders(); var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), - i => i.Definition.Id, - s => s.ProviderId, - (i, s) => new { Provider = i, Status = s }) - .ToList(); + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new {Provider = i, Status = s}) + .Where(p => p.Status.InitialFailure.HasValue && + p.Status.InitialFailure.Value.After( + DateTime.UtcNow.AddHours(-6))) + .ToList(); if (backOffProviders.Empty()) {