(cherry picked from commit e3545801721e00d4e5cac3fa534e66dcbe9d2d05) (cherry picked from commit cb27b05a6c046ca0a6e4998f7e7ecd6b45add1a2)pull/9020/head
Qstick
1 year ago
committed by
Bogdan
parent
c03453f6f7
commit
fa80e8b7a2
@ -0,0 +1,89 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Core.HealthCheck.Checks;
|
||||
using NzbDrone.Core.Localization;
|
||||
using NzbDrone.Core.Notifications;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Test.HealthCheck.Checks
|
||||
{
|
||||
[TestFixture]
|
||||
public class NotificationStatusCheckFixture : CoreTest<NotificationStatusCheck>
|
||||
{
|
||||
private List<INotification> _notifications = new List<INotification>();
|
||||
private List<NotificationStatus> _blockedNotifications = new List<NotificationStatus>();
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
Mocker.GetMock<INotificationFactory>()
|
||||
.Setup(v => v.GetAvailableProviders())
|
||||
.Returns(_notifications);
|
||||
|
||||
Mocker.GetMock<INotificationStatusService>()
|
||||
.Setup(v => v.GetBlockedProviders())
|
||||
.Returns(_blockedNotifications);
|
||||
|
||||
Mocker.GetMock<ILocalizationService>()
|
||||
.Setup(s => s.GetLocalizedString(It.IsAny<string>()))
|
||||
.Returns("Some Warning Message");
|
||||
}
|
||||
|
||||
private Mock<INotification> GivenNotification(int id, double backoffHours, double failureHours)
|
||||
{
|
||||
var mockNotification = new Mock<INotification>();
|
||||
mockNotification.SetupGet(s => s.Definition).Returns(new NotificationDefinition { Id = id });
|
||||
|
||||
_notifications.Add(mockNotification.Object);
|
||||
|
||||
if (backoffHours != 0.0)
|
||||
{
|
||||
_blockedNotifications.Add(new NotificationStatus
|
||||
{
|
||||
ProviderId = id,
|
||||
InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
|
||||
MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
|
||||
EscalationLevel = 5,
|
||||
DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
|
||||
});
|
||||
}
|
||||
|
||||
return mockNotification;
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_return_error_when_no_notifications()
|
||||
{
|
||||
Subject.Check().ShouldBeOk();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_return_warning_if_notification_unavailable()
|
||||
{
|
||||
GivenNotification(1, 10.0, 24.0);
|
||||
GivenNotification(2, 0.0, 0.0);
|
||||
|
||||
Subject.Check().ShouldBeWarning();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_return_error_if_all_notifications_unavailable()
|
||||
{
|
||||
GivenNotification(1, 10.0, 24.0);
|
||||
|
||||
Subject.Check().ShouldBeError();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_return_warning_if_few_notifications_unavailable()
|
||||
{
|
||||
GivenNotification(1, 10.0, 24.0);
|
||||
GivenNotification(2, 10.0, 24.0);
|
||||
GivenNotification(3, 0.0, 0.0);
|
||||
|
||||
Subject.Check().ShouldBeWarning();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,56 @@
|
||||
using FizzWare.NBuilder;
|
||||
using FluentAssertions;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Core.Housekeeping.Housekeepers;
|
||||
using NzbDrone.Core.Notifications;
|
||||
using NzbDrone.Core.Notifications.Join;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
|
||||
{
|
||||
[TestFixture]
|
||||
public class CleanupOrphanedNotificationStatusFixture : DbTest<CleanupOrphanedNotificationStatus, NotificationStatus>
|
||||
{
|
||||
private NotificationDefinition _notification;
|
||||
|
||||
[SetUp]
|
||||
public void Setup()
|
||||
{
|
||||
_notification = Builder<NotificationDefinition>.CreateNew()
|
||||
.With(s => s.Settings = new JoinSettings { })
|
||||
.BuildNew();
|
||||
}
|
||||
|
||||
private void GivenNotification()
|
||||
{
|
||||
Db.Insert(_notification);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_delete_orphaned_notificationstatus()
|
||||
{
|
||||
var status = Builder<NotificationStatus>.CreateNew()
|
||||
.With(h => h.ProviderId = _notification.Id)
|
||||
.BuildNew();
|
||||
Db.Insert(status);
|
||||
|
||||
Subject.Clean();
|
||||
AllStoredModels.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_delete_unorphaned_notificationstatus()
|
||||
{
|
||||
GivenNotification();
|
||||
|
||||
var status = Builder<NotificationStatus>.CreateNew()
|
||||
.With(h => h.ProviderId = _notification.Id)
|
||||
.BuildNew();
|
||||
Db.Insert(status);
|
||||
|
||||
Subject.Clean();
|
||||
AllStoredModels.Should().HaveCount(1);
|
||||
AllStoredModels.Should().Contain(h => h.ProviderId == _notification.Id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,161 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using FluentAssertions;
|
||||
using Moq;
|
||||
using NUnit.Framework;
|
||||
using NzbDrone.Common.EnvironmentInfo;
|
||||
using NzbDrone.Core.Notifications;
|
||||
using NzbDrone.Core.Test.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Test.NotificationTests
|
||||
{
|
||||
public class NotificationStatusServiceFixture : CoreTest<NotificationStatusService>
|
||||
{
|
||||
private DateTime _epoch;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
_epoch = DateTime.UtcNow;
|
||||
|
||||
Mocker.GetMock<IRuntimeInfo>()
|
||||
.SetupGet(v => v.StartTime)
|
||||
.Returns(_epoch - TimeSpan.FromHours(1));
|
||||
}
|
||||
|
||||
private NotificationStatus WithStatus(NotificationStatus status)
|
||||
{
|
||||
Mocker.GetMock<INotificationStatusRepository>()
|
||||
.Setup(v => v.FindByProviderId(1))
|
||||
.Returns(status);
|
||||
|
||||
Mocker.GetMock<INotificationStatusRepository>()
|
||||
.Setup(v => v.All())
|
||||
.Returns(new[] { status });
|
||||
|
||||
return status;
|
||||
}
|
||||
|
||||
private void VerifyUpdate()
|
||||
{
|
||||
Mocker.GetMock<INotificationStatusRepository>()
|
||||
.Verify(v => v.Upsert(It.IsAny<NotificationStatus>()), Times.Once());
|
||||
}
|
||||
|
||||
private void VerifyNoUpdate()
|
||||
{
|
||||
Mocker.GetMock<INotificationStatusRepository>()
|
||||
.Verify(v => v.Upsert(It.IsAny<NotificationStatus>()), Times.Never());
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_consider_blocked_within_5_minutes_since_initial_failure()
|
||||
{
|
||||
WithStatus(new NotificationStatus
|
||||
{
|
||||
InitialFailure = _epoch - TimeSpan.FromMinutes(4),
|
||||
MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
|
||||
EscalationLevel = 3
|
||||
});
|
||||
|
||||
Subject.RecordFailure(1);
|
||||
|
||||
VerifyUpdate();
|
||||
|
||||
var status = Subject.GetBlockedProviders().FirstOrDefault();
|
||||
status.Should().BeNull();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_consider_blocked_after_5_minutes_since_initial_failure()
|
||||
{
|
||||
WithStatus(new NotificationStatus
|
||||
{
|
||||
InitialFailure = _epoch - TimeSpan.FromMinutes(6),
|
||||
MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
|
||||
EscalationLevel = 3
|
||||
});
|
||||
|
||||
Subject.RecordFailure(1);
|
||||
|
||||
VerifyUpdate();
|
||||
|
||||
var status = Subject.GetBlockedProviders().FirstOrDefault();
|
||||
status.Should().NotBeNull();
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_escalate_further_till_after_5_minutes_since_initial_failure()
|
||||
{
|
||||
var origStatus = WithStatus(new NotificationStatus
|
||||
{
|
||||
InitialFailure = _epoch - TimeSpan.FromMinutes(4),
|
||||
MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
|
||||
EscalationLevel = 3
|
||||
});
|
||||
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
|
||||
var status = Subject.GetBlockedProviders().FirstOrDefault();
|
||||
status.Should().BeNull();
|
||||
|
||||
origStatus.EscalationLevel.Should().Be(3);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_escalate_further_after_5_minutes_since_initial_failure()
|
||||
{
|
||||
WithStatus(new NotificationStatus
|
||||
{
|
||||
InitialFailure = _epoch - TimeSpan.FromMinutes(6),
|
||||
MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
|
||||
EscalationLevel = 3
|
||||
});
|
||||
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
|
||||
var status = Subject.GetBlockedProviders().FirstOrDefault();
|
||||
status.Should().NotBeNull();
|
||||
|
||||
status.EscalationLevel.Should().BeGreaterThan(3);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void should_not_escalate_beyond_3_hours()
|
||||
{
|
||||
WithStatus(new NotificationStatus
|
||||
{
|
||||
InitialFailure = _epoch - TimeSpan.FromMinutes(6),
|
||||
MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
|
||||
EscalationLevel = 3
|
||||
});
|
||||
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
Subject.RecordFailure(1);
|
||||
|
||||
var status = Subject.GetBlockedProviders().FirstOrDefault();
|
||||
status.Should().NotBeNull();
|
||||
status.DisabledTill.Should().HaveValue();
|
||||
status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,19 @@
|
||||
using FluentMigrator;
|
||||
using NzbDrone.Core.Datastore.Migration.Framework;
|
||||
|
||||
namespace NzbDrone.Core.Datastore.Migration
|
||||
{
|
||||
[Migration(232)]
|
||||
public class add_notification_status : NzbDroneMigrationBase
|
||||
{
|
||||
protected override void MainDbUpgrade()
|
||||
{
|
||||
Create.TableForModel("NotificationStatus")
|
||||
.WithColumn("ProviderId").AsInt32().NotNullable().Unique()
|
||||
.WithColumn("InitialFailure").AsDateTimeOffset().Nullable()
|
||||
.WithColumn("MostRecentFailure").AsDateTimeOffset().Nullable()
|
||||
.WithColumn("EscalationLevel").AsInt32().NotNullable()
|
||||
.WithColumn("DisabledTill").AsDateTimeOffset().Nullable();
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,52 @@
|
||||
using System.Linq;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Core.Localization;
|
||||
using NzbDrone.Core.Notifications;
|
||||
using NzbDrone.Core.ThingiProvider.Events;
|
||||
|
||||
namespace NzbDrone.Core.HealthCheck.Checks
|
||||
{
|
||||
[CheckOn(typeof(ProviderUpdatedEvent<INotification>))]
|
||||
[CheckOn(typeof(ProviderDeletedEvent<INotification>))]
|
||||
[CheckOn(typeof(ProviderStatusChangedEvent<INotification>))]
|
||||
public class NotificationStatusCheck : HealthCheckBase
|
||||
{
|
||||
private readonly INotificationFactory _providerFactory;
|
||||
private readonly INotificationStatusService _providerStatusService;
|
||||
|
||||
public NotificationStatusCheck(INotificationFactory providerFactory, INotificationStatusService providerStatusService, ILocalizationService localizationService)
|
||||
: base(localizationService)
|
||||
{
|
||||
_providerFactory = providerFactory;
|
||||
_providerStatusService = providerStatusService;
|
||||
}
|
||||
|
||||
public override HealthCheck Check()
|
||||
{
|
||||
var enabledProviders = _providerFactory.GetAvailableProviders();
|
||||
var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
|
||||
i => i.Definition.Id,
|
||||
s => s.ProviderId,
|
||||
(i, s) => new { Provider = i, Status = s })
|
||||
.ToList();
|
||||
|
||||
if (backOffProviders.Empty())
|
||||
{
|
||||
return new HealthCheck(GetType());
|
||||
}
|
||||
|
||||
if (backOffProviders.Count == enabledProviders.Count)
|
||||
{
|
||||
return new HealthCheck(GetType(),
|
||||
HealthCheckResult.Error,
|
||||
_localizationService.GetLocalizedString("NotificationStatusAllClientHealthCheckMessage"),
|
||||
"#notifications-are-unavailable-due-to-failures");
|
||||
}
|
||||
|
||||
return new HealthCheck(GetType(),
|
||||
HealthCheckResult.Warning,
|
||||
string.Format(_localizationService.GetLocalizedString("NotificationStatusSingleClientHealthCheckMessage"), string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))),
|
||||
"#notifications-are-unavailable-due-to-failures");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,27 @@
|
||||
using Dapper;
|
||||
using NzbDrone.Core.Datastore;
|
||||
|
||||
namespace NzbDrone.Core.Housekeeping.Housekeepers
|
||||
{
|
||||
public class CleanupOrphanedNotificationStatus : IHousekeepingTask
|
||||
{
|
||||
private readonly IMainDatabase _database;
|
||||
|
||||
public CleanupOrphanedNotificationStatus(IMainDatabase database)
|
||||
{
|
||||
_database = database;
|
||||
}
|
||||
|
||||
public void Clean()
|
||||
{
|
||||
using var mapper = _database.OpenConnection();
|
||||
|
||||
mapper.Execute(@"DELETE FROM ""NotificationStatus""
|
||||
WHERE ""Id"" IN (
|
||||
SELECT ""NotificationStatus"".""Id"" FROM ""NotificationStatus""
|
||||
LEFT OUTER JOIN ""Notifications""
|
||||
ON ""NotificationStatus"".""ProviderId"" = ""Notifications"".""Id""
|
||||
WHERE ""Notifications"".""Id"" IS NULL)");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
using NzbDrone.Core.Notifications;
|
||||
|
||||
namespace NzbDrone.Core.Housekeeping.Housekeepers
|
||||
{
|
||||
public class FixFutureNotificationStatusTimes : FixFutureProviderStatusTimes<NotificationStatus>, IHousekeepingTask
|
||||
{
|
||||
public FixFutureNotificationStatusTimes(INotificationStatusRepository notificationStatusRepository)
|
||||
: base(notificationStatusRepository)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,8 @@
|
||||
using NzbDrone.Core.ThingiProvider.Status;
|
||||
|
||||
namespace NzbDrone.Core.Notifications
|
||||
{
|
||||
public class NotificationStatus : ProviderStatusBase
|
||||
{
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
using NzbDrone.Core.Datastore;
|
||||
using NzbDrone.Core.Messaging.Events;
|
||||
using NzbDrone.Core.ThingiProvider.Status;
|
||||
|
||||
namespace NzbDrone.Core.Notifications
|
||||
{
|
||||
public interface INotificationStatusRepository : IProviderStatusRepository<NotificationStatus>
|
||||
{
|
||||
}
|
||||
|
||||
public class NotificationStatusRepository : ProviderStatusRepository<NotificationStatus>, INotificationStatusRepository
|
||||
{
|
||||
public NotificationStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
|
||||
: base(database, eventAggregator)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using NLog;
|
||||
using NzbDrone.Common.EnvironmentInfo;
|
||||
using NzbDrone.Core.Messaging.Events;
|
||||
using NzbDrone.Core.ThingiProvider.Status;
|
||||
|
||||
namespace NzbDrone.Core.Notifications
|
||||
{
|
||||
public interface INotificationStatusService : IProviderStatusServiceBase<NotificationStatus>
|
||||
{
|
||||
}
|
||||
|
||||
public class NotificationStatusService : ProviderStatusServiceBase<INotification, NotificationStatus>, INotificationStatusService
|
||||
{
|
||||
public NotificationStatusService(INotificationStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
|
||||
: base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
|
||||
{
|
||||
MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5);
|
||||
MaximumEscalationLevel = 5;
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue