From 0c8ad37a8f54fd6af7a84d2a17f3c1ae20d6d300 Mon Sep 17 00:00:00 2001 From: Mark McDowall Date: Mon, 12 Oct 2020 11:24:19 -0700 Subject: [PATCH] New: Differentiate between short term and long term (more than 6 hours) indexer failures Signed-off-by: Robin Dadswell --- frontend/src/System/Status/Health/Health.js | 1 + .../IndexerLongTermStatusCheckFixture.cs | 85 +++++++++++++++++++ .../Checks/IndexerStatusCheckFixture.cs | 8 +- .../Checks/IndexerLongTermStatusCheck.cs | 55 ++++++++++++ .../HealthCheck/Checks/IndexerStatusCheck.cs | 14 +-- 5 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerLongTermStatusCheckFixture.cs create mode 100644 src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs diff --git a/frontend/src/System/Status/Health/Health.js b/frontend/src/System/Status/Health/Health.js index efcd91770..db2f29862 100644 --- a/frontend/src/System/Status/Health/Health.js +++ b/frontend/src/System/Status/Health/Health.js @@ -18,6 +18,7 @@ function getInternalLink(source) { case 'IndexerRssCheck': case 'IndexerSearchCheck': case 'IndexerStatusCheck': + case 'IndexerLongTermStatusCheck': return ( + { + private List _indexers = new List(); + private List _blockedIndexers = new List(); + + [SetUp] + public void SetUp() + { + Mocker.GetMock() + .Setup(v => v.GetAvailableProviders()) + .Returns(_indexers); + + Mocker.GetMock() + .Setup(v => v.GetBlockedProviders()) + .Returns(_blockedIndexers); + } + + private Mock GivenIndexer(int id, double backoffHours, double failureHours) + { + var mockIndexer = new Mock(); + mockIndexer.SetupGet(s => s.Definition).Returns(new IndexerDefinition { Id = id }); + mockIndexer.SetupGet(s => s.SupportsSearch).Returns(true); + + _indexers.Add(mockIndexer.Object); + + if (backoffHours != 0.0) + { + _blockedIndexers.Add(new IndexerStatus + { + ProviderId = id, + InitialFailure = DateTime.UtcNow.AddHours(-failureHours), + MostRecentFailure = DateTime.UtcNow.AddHours(-0.1), + EscalationLevel = 5, + DisabledTill = DateTime.UtcNow.AddHours(backoffHours) + }); + } + + return mockIndexer; + } + + [Test] + public void should_not_return_error_when_no_indexers() + { + Subject.Check().ShouldBeOk(); + } + + [Test] + public void should_return_warning_if_indexer_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + GivenIndexer(2, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + + [Test] + public void should_return_error_if_all_indexers_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + + Subject.Check().ShouldBeError(); + } + + [Test] + public void should_return_warning_if_few_indexers_unavailable() + { + GivenIndexer(1, 10.0, 24.0); + GivenIndexer(2, 10.0, 24.0); + GivenIndexer(3, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + } +} diff --git a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs index a77eecf13..1ffac51c3 100644 --- a/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs +++ b/src/NzbDrone.Core.Test/HealthCheck/Checks/IndexerStatusCheckFixture.cs @@ -60,7 +60,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks [Test] public void should_return_warning_if_indexer_unavailable() { - GivenIndexer(1, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); GivenIndexer(2, 0.0, 0.0); Subject.Check().ShouldBeWarning(); @@ -69,7 +69,7 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks [Test] public void should_return_error_if_all_indexers_unavailable() { - GivenIndexer(1, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); Subject.Check().ShouldBeError(); } @@ -77,8 +77,8 @@ namespace NzbDrone.Core.Test.HealthCheck.Checks [Test] public void should_return_warning_if_few_indexers_unavailable() { - GivenIndexer(1, 10.0, 24.0); - GivenIndexer(2, 10.0, 24.0); + GivenIndexer(1, 2.0, 4.0); + GivenIndexer(2, 2.0, 4.0); GivenIndexer(3, 0.0, 0.0); Subject.Check().ShouldBeWarning(); diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs new file mode 100644 index 000000000..7539a6ffd --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerLongTermStatusCheck.cs @@ -0,0 +1,55 @@ +using System; +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] + public class IndexerLongTermStatusCheck : HealthCheckBase + { + private readonly IIndexerFactory _providerFactory; + private readonly IIndexerStatusService _providerStatusService; + + public IndexerLongTermStatusCheck(IIndexerFactory providerFactory, IIndexerStatusService providerStatusService) + { + _providerFactory = providerFactory; + _providerStatusService = providerStatusService; + } + + public override HealthCheck Check() + { + var enabledProviders = _providerFactory.GetAvailableProviders(); + var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new { Provider = i, Status = s }) + .Where(p => p.Status.InitialFailure.HasValue && + p.Status.InitialFailure.Value.Before( + DateTime.UtcNow.AddHours(-6))) + .ToList(); + + if (backOffProviders.Empty()) + { + return new HealthCheck(GetType()); + } + + if (backOffProviders.Count == enabledProviders.Count) + { + return new HealthCheck(GetType(), + HealthCheckResult.Error, + "All indexers are unavailable due to failures for more than 6 hours", + "#indexers-are-unavailable-due-to-failures"); + } + + return new HealthCheck(GetType(), + HealthCheckResult.Warning, + string.Format("Indexers unavailable due to failures for more than 6 hours: {0}", + string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), + "#indexers-are-unavailable-due-to-failures"); + } + } +} diff --git a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs index b7065f45f..c075eb3e7 100644 --- a/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs +++ b/src/NzbDrone.Core/HealthCheck/Checks/IndexerStatusCheck.cs @@ -1,3 +1,4 @@ +using System; using System.Linq; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers; @@ -23,10 +24,13 @@ namespace NzbDrone.Core.HealthCheck.Checks { var enabledProviders = _providerFactory.GetAvailableProviders(); var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), - i => i.Definition.Id, - s => s.ProviderId, - (i, s) => new { Indexer = i, Status = s }) - .ToList(); + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new { Provider = i, Status = s }) + .Where(p => p.Status.InitialFailure.HasValue && + p.Status.InitialFailure.Value.After( + DateTime.UtcNow.AddHours(-6))) + .ToList(); if (backOffProviders.Empty()) { @@ -38,7 +42,7 @@ namespace NzbDrone.Core.HealthCheck.Checks return new HealthCheck(GetType(), HealthCheckResult.Error, "All indexers are unavailable due to failures", "#indexers-are-unavailable-due-to-failures"); } - return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexers unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Indexer.Definition.Name))), "#indexers-are-unavailable-due-to-failures"); + return new HealthCheck(GetType(), HealthCheckResult.Warning, string.Format("Indexers unavailable due to failures: {0}", string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), "#indexers-are-unavailable-due-to-failures"); } } }