From 7f4b79f84d4f8ab9f6ba9b11eeb70aedfd571b6e Mon Sep 17 00:00:00 2001 From: Qstick Date: Sat, 12 Aug 2023 14:59:22 -0500 Subject: [PATCH] New: Notifications (Connect) Status (cherry picked from commit e3545801721e00d4e5cac3fa534e66dcbe9d2d05) --- frontend/src/System/Status/Health/Health.js | 8 + .../Checks/NotificationStatusCheckFixture.cs | 89 ++++++++++ ...leanupOrphanedNotificationStatusFixture.cs | 56 ++++++ .../NotificationStatusServiceFixture.cs | 161 ++++++++++++++++++ .../Migration/071_add_notification_status.cs | 19 +++ src/NzbDrone.Core/Datastore/TableMapping.cs | 1 + .../Checks/NotificationStatusCheck.cs | 52 ++++++ .../CleanupOrphanedNotificationStatus.cs | 27 +++ .../FixFutureNotificationStatusTimes.cs | 12 ++ .../Notifications/NotificationFactory.cs | 131 +++++++++++--- .../Notifications/NotificationService.cs | 59 ++++++- .../Notifications/NotificationStatus.cs | 8 + .../NotificationStatusRepository.cs | 18 ++ .../NotificationStatusService.cs | 22 +++ 14 files changed, 631 insertions(+), 32 deletions(-) create mode 100644 src/NzbDrone.Core.Test/HealthCheck/Checks/NotificationStatusCheckFixture.cs create mode 100644 src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatusFixture.cs create mode 100644 src/NzbDrone.Core.Test/NotificationTests/NotificationStatusServiceFixture.cs create mode 100644 src/NzbDrone.Core/Datastore/Migration/071_add_notification_status.cs create mode 100644 src/NzbDrone.Core/HealthCheck/Checks/NotificationStatusCheck.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatus.cs create mode 100644 src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureNotificationStatusTimes.cs create mode 100644 src/NzbDrone.Core/Notifications/NotificationStatus.cs create mode 100644 src/NzbDrone.Core/Notifications/NotificationStatusRepository.cs create mode 100644 src/NzbDrone.Core/Notifications/NotificationStatusService.cs diff --git a/frontend/src/System/Status/Health/Health.js b/frontend/src/System/Status/Health/Health.js index 94ce4bae9..d97bbc9e9 100644 --- a/frontend/src/System/Status/Health/Health.js +++ b/frontend/src/System/Status/Health/Health.js @@ -39,6 +39,14 @@ function getInternalLink(source) { to="/settings/downloadclients" /> ); + case 'NotificationStatusCheck': + return ( + + ); case 'RootFolderCheck': return ( + { + private List _notifications = new List(); + private List _blockedNotifications = new List(); + + [SetUp] + public void SetUp() + { + Mocker.GetMock() + .Setup(v => v.GetAvailableProviders()) + .Returns(_notifications); + + Mocker.GetMock() + .Setup(v => v.GetBlockedProviders()) + .Returns(_blockedNotifications); + + Mocker.GetMock() + .Setup(s => s.GetLocalizedString(It.IsAny())) + .Returns("Some Warning Message"); + } + + private Mock GivenNotification(int id, double backoffHours, double failureHours) + { + var mockNotification = new Mock(); + mockNotification.SetupGet(s => s.Definition).Returns(new NotificationDefinition { Id = id }); + + _notifications.Add(mockNotification.Object); + + if (backoffHours != 0.0) + { + _blockedNotifications.Add(new NotificationStatus + { + ProviderId = id, + InitialFailure = DateTime.UtcNow.AddHours(-failureHours), + MostRecentFailure = DateTime.UtcNow.AddHours(-0.1), + EscalationLevel = 5, + DisabledTill = DateTime.UtcNow.AddHours(backoffHours) + }); + } + + return mockNotification; + } + + [Test] + public void should_not_return_error_when_no_notifications() + { + Subject.Check().ShouldBeOk(); + } + + [Test] + public void should_return_warning_if_notification_unavailable() + { + GivenNotification(1, 10.0, 24.0); + GivenNotification(2, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + + [Test] + public void should_return_error_if_all_notifications_unavailable() + { + GivenNotification(1, 10.0, 24.0); + + Subject.Check().ShouldBeError(); + } + + [Test] + public void should_return_warning_if_few_notifications_unavailable() + { + GivenNotification(1, 10.0, 24.0); + GivenNotification(2, 10.0, 24.0); + GivenNotification(3, 0.0, 0.0); + + Subject.Check().ShouldBeWarning(); + } + } +} diff --git a/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatusFixture.cs b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatusFixture.cs new file mode 100644 index 000000000..20e82ff7f --- /dev/null +++ b/src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatusFixture.cs @@ -0,0 +1,56 @@ +using FizzWare.NBuilder; +using FluentAssertions; +using NUnit.Framework; +using NzbDrone.Core.Housekeeping.Housekeepers; +using NzbDrone.Core.Notifications; +using NzbDrone.Core.Notifications.Join; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.Housekeeping.Housekeepers +{ + [TestFixture] + public class CleanupOrphanedNotificationStatusFixture : DbTest + { + private NotificationDefinition _notification; + + [SetUp] + public void Setup() + { + _notification = Builder.CreateNew() + .With(s => s.Settings = new JoinSettings { }) + .BuildNew(); + } + + private void GivenNotification() + { + Db.Insert(_notification); + } + + [Test] + public void should_delete_orphaned_notificationstatus() + { + var status = Builder.CreateNew() + .With(h => h.ProviderId = _notification.Id) + .BuildNew(); + Db.Insert(status); + + Subject.Clean(); + AllStoredModels.Should().BeEmpty(); + } + + [Test] + public void should_not_delete_unorphaned_notificationstatus() + { + GivenNotification(); + + var status = Builder.CreateNew() + .With(h => h.ProviderId = _notification.Id) + .BuildNew(); + Db.Insert(status); + + Subject.Clean(); + AllStoredModels.Should().HaveCount(1); + AllStoredModels.Should().Contain(h => h.ProviderId == _notification.Id); + } + } +} diff --git a/src/NzbDrone.Core.Test/NotificationTests/NotificationStatusServiceFixture.cs b/src/NzbDrone.Core.Test/NotificationTests/NotificationStatusServiceFixture.cs new file mode 100644 index 000000000..183246313 --- /dev/null +++ b/src/NzbDrone.Core.Test/NotificationTests/NotificationStatusServiceFixture.cs @@ -0,0 +1,161 @@ +using System; +using System.Linq; +using FluentAssertions; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Core.Notifications; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.NotificationTests +{ + public class NotificationStatusServiceFixture : CoreTest + { + private DateTime _epoch; + + [SetUp] + public void SetUp() + { + _epoch = DateTime.UtcNow; + + Mocker.GetMock() + .SetupGet(v => v.StartTime) + .Returns(_epoch - TimeSpan.FromHours(1)); + } + + private NotificationStatus WithStatus(NotificationStatus status) + { + Mocker.GetMock() + .Setup(v => v.FindByProviderId(1)) + .Returns(status); + + Mocker.GetMock() + .Setup(v => v.All()) + .Returns(new[] { status }); + + return status; + } + + private void VerifyUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Once()); + } + + private void VerifyNoUpdate() + { + Mocker.GetMock() + .Verify(v => v.Upsert(It.IsAny()), Times.Never()); + } + + [Test] + public void should_not_consider_blocked_within_5_minutes_since_initial_failure() + { + WithStatus(new NotificationStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(4), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + } + + [Test] + public void should_consider_blocked_after_5_minutes_since_initial_failure() + { + WithStatus(new NotificationStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + + VerifyUpdate(); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + } + + [Test] + public void should_not_escalate_further_till_after_5_minutes_since_initial_failure() + { + var origStatus = WithStatus(new NotificationStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(4), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(4), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().BeNull(); + + origStatus.EscalationLevel.Should().Be(3); + } + + [Test] + public void should_escalate_further_after_5_minutes_since_initial_failure() + { + WithStatus(new NotificationStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + + status.EscalationLevel.Should().BeGreaterThan(3); + } + + [Test] + public void should_not_escalate_beyond_3_hours() + { + WithStatus(new NotificationStatus + { + InitialFailure = _epoch - TimeSpan.FromMinutes(6), + MostRecentFailure = _epoch - TimeSpan.FromSeconds(120), + EscalationLevel = 3 + }); + + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + Subject.RecordFailure(1); + + var status = Subject.GetBlockedProviders().FirstOrDefault(); + status.Should().NotBeNull(); + status.DisabledTill.Should().HaveValue(); + status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1)); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Migration/071_add_notification_status.cs b/src/NzbDrone.Core/Datastore/Migration/071_add_notification_status.cs new file mode 100644 index 000000000..8b0ce7dbd --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/071_add_notification_status.cs @@ -0,0 +1,19 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(071)] + public class add_notification_status : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Create.TableForModel("NotificationStatus") + .WithColumn("ProviderId").AsInt32().NotNullable().Unique() + .WithColumn("InitialFailure").AsDateTimeOffset().Nullable() + .WithColumn("MostRecentFailure").AsDateTimeOffset().Nullable() + .WithColumn("EscalationLevel").AsInt32().NotNullable() + .WithColumn("DisabledTill").AsDateTimeOffset().Nullable(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 369f856ba..4049c3fc0 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -196,6 +196,7 @@ namespace NzbDrone.Core.Datastore Mapper.Entity("IndexerStatus").RegisterModel(); Mapper.Entity("DownloadClientStatus").RegisterModel(); Mapper.Entity("ImportListStatus").RegisterModel(); + Mapper.Entity("NotificationStatus").RegisterModel(); Mapper.Entity("CustomFilters").RegisterModel(); Mapper.Entity("ImportListExclusions").RegisterModel(); diff --git a/src/NzbDrone.Core/HealthCheck/Checks/NotificationStatusCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/NotificationStatusCheck.cs new file mode 100644 index 000000000..c9b5e2561 --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/NotificationStatusCheck.cs @@ -0,0 +1,52 @@ +using System.Linq; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Localization; +using NzbDrone.Core.Notifications; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + [CheckOn(typeof(ProviderUpdatedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderStatusChangedEvent))] + public class NotificationStatusCheck : HealthCheckBase + { + private readonly INotificationFactory _providerFactory; + private readonly INotificationStatusService _providerStatusService; + + public NotificationStatusCheck(INotificationFactory providerFactory, INotificationStatusService providerStatusService, ILocalizationService localizationService) + : base(localizationService) + { + _providerFactory = providerFactory; + _providerStatusService = providerStatusService; + } + + public override HealthCheck Check() + { + var enabledProviders = _providerFactory.GetAvailableProviders(); + var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(), + i => i.Definition.Id, + s => s.ProviderId, + (i, s) => new { Provider = i, Status = s }) + .ToList(); + + if (backOffProviders.Empty()) + { + return new HealthCheck(GetType()); + } + + if (backOffProviders.Count == enabledProviders.Count) + { + return new HealthCheck(GetType(), + HealthCheckResult.Error, + _localizationService.GetLocalizedString("NotificationStatusAllClientHealthCheckMessage"), + "#notifications-are-unavailable-due-to-failures"); + } + + return new HealthCheck(GetType(), + HealthCheckResult.Warning, + string.Format(_localizationService.GetLocalizedString("NotificationStatusSingleClientHealthCheckMessage"), string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))), + "#notifications-are-unavailable-due-to-failures"); + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatus.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatus.cs new file mode 100644 index 000000000..cfc3e1f63 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatus.cs @@ -0,0 +1,27 @@ +using Dapper; +using NzbDrone.Core.Datastore; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class CleanupOrphanedNotificationStatus : IHousekeepingTask + { + private readonly IMainDatabase _database; + + public CleanupOrphanedNotificationStatus(IMainDatabase database) + { + _database = database; + } + + public void Clean() + { + using var mapper = _database.OpenConnection(); + + mapper.Execute(@"DELETE FROM ""NotificationStatus"" + WHERE ""Id"" IN ( + SELECT ""NotificationStatus"".""Id"" FROM ""NotificationStatus"" + LEFT OUTER JOIN ""Notifications"" + ON ""NotificationStatus"".""ProviderId"" = ""Notifications"".""Id"" + WHERE ""Notifications"".""Id"" IS NULL)"); + } + } +} diff --git a/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureNotificationStatusTimes.cs b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureNotificationStatusTimes.cs new file mode 100644 index 000000000..10af6ab42 --- /dev/null +++ b/src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureNotificationStatusTimes.cs @@ -0,0 +1,12 @@ +using NzbDrone.Core.Notifications; + +namespace NzbDrone.Core.Housekeeping.Housekeepers +{ + public class FixFutureNotificationStatusTimes : FixFutureProviderStatusTimes, IHousekeepingTask + { + public FixFutureNotificationStatusTimes(INotificationStatusRepository notificationStatusRepository) + : base(notificationStatusRepository) + { + } + } +} diff --git a/src/NzbDrone.Core/Notifications/NotificationFactory.cs b/src/NzbDrone.Core/Notifications/NotificationFactory.cs index 920033f52..db6b15320 100644 --- a/src/NzbDrone.Core/Notifications/NotificationFactory.cs +++ b/src/NzbDrone.Core/Notifications/NotificationFactory.cs @@ -9,87 +9,168 @@ namespace NzbDrone.Core.Notifications { public interface INotificationFactory : IProviderFactory { - List OnGrabEnabled(); - List OnReleaseImportEnabled(); - List OnUpgradeEnabled(); - List OnRenameEnabled(); - List OnAlbumDeleteEnabled(); - List OnArtistDeleteEnabled(); - List OnHealthIssueEnabled(); - List OnHealthRestoredEnabled(); - List OnDownloadFailureEnabled(); - List OnImportFailureEnabled(); - List OnTrackRetagEnabled(); - List OnApplicationUpdateEnabled(); + List OnGrabEnabled(bool filterBlockedNotifications = true); + List OnReleaseImportEnabled(bool filterBlockedNotifications = true); + List OnUpgradeEnabled(bool filterBlockedNotifications = true); + List OnRenameEnabled(bool filterBlockedNotifications = true); + List OnAlbumDeleteEnabled(bool filterBlockedNotifications = true); + List OnArtistDeleteEnabled(bool filterBlockedNotifications = true); + List OnHealthIssueEnabled(bool filterBlockedNotifications = true); + List OnHealthRestoredEnabled(bool filterBlockedNotifications = true); + List OnDownloadFailureEnabled(bool filterBlockedNotifications = true); + List OnImportFailureEnabled(bool filterBlockedNotifications = true); + List OnTrackRetagEnabled(bool filterBlockedNotifications = true); + List OnApplicationUpdateEnabled(bool filterBlockedNotifications = true); } public class NotificationFactory : ProviderFactory, INotificationFactory { - public NotificationFactory(INotificationRepository providerRepository, IEnumerable providers, IServiceProvider container, IEventAggregator eventAggregator, Logger logger) + private readonly INotificationStatusService _notificationStatusService; + private readonly Logger _logger; + + public NotificationFactory(INotificationStatusService notificationStatusService, INotificationRepository providerRepository, IEnumerable providers, IServiceProvider container, IEventAggregator eventAggregator, Logger logger) : base(providerRepository, providers, container, eventAggregator, logger) { + _notificationStatusService = notificationStatusService; + _logger = logger; } - public List OnGrabEnabled() + public List OnGrabEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnGrab)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnGrab).ToList(); } - public List OnReleaseImportEnabled() + public List OnReleaseImportEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnReleaseImport)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnReleaseImport).ToList(); } - public List OnUpgradeEnabled() + public List OnUpgradeEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnUpgrade)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnUpgrade).ToList(); } - public List OnRenameEnabled() + public List OnRenameEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnRename)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnRename).ToList(); } - public List OnAlbumDeleteEnabled() + public List OnAlbumDeleteEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnAlbumDelete)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnAlbumDelete).ToList(); } - public List OnArtistDeleteEnabled() + public List OnArtistDeleteEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnArtistDelete)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnArtistDelete).ToList(); } - public List OnHealthIssueEnabled() + public List OnHealthIssueEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthIssue)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthIssue).ToList(); } - public List OnHealthRestoredEnabled() + public List OnHealthRestoredEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthRestored)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthRestored).ToList(); } - public List OnDownloadFailureEnabled() + public List OnDownloadFailureEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnDownloadFailure)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnDownloadFailure).ToList(); } - public List OnImportFailureEnabled() + public List OnImportFailureEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnImportFailure)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnImportFailure).ToList(); } - public List OnTrackRetagEnabled() + public List OnTrackRetagEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnTrackRetag)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnTrackRetag).ToList(); } - public List OnApplicationUpdateEnabled() + public List OnApplicationUpdateEnabled(bool filterBlockedNotifications = true) { + if (filterBlockedNotifications) + { + return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnApplicationUpdate)).ToList(); + } + return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnApplicationUpdate).ToList(); } + private IEnumerable FilterBlockedNotifications(IEnumerable notifications) + { + var blockedNotifications = _notificationStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v); + + foreach (var notification in notifications) + { + if (blockedNotifications.TryGetValue(notification.Definition.Id, out var notificationStatus)) + { + _logger.Debug("Temporarily ignoring notification {0} till {1} due to recent failures.", notification.Definition.Name, notificationStatus.DisabledTill.Value.ToLocalTime()); + continue; + } + + yield return notification; + } + } + public override void SetProviderCharacteristics(INotification provider, NotificationDefinition definition) { base.SetProviderCharacteristics(provider, definition); diff --git a/src/NzbDrone.Core/Notifications/NotificationService.cs b/src/NzbDrone.Core/Notifications/NotificationService.cs index 2a7faba89..1a5408aee 100644 --- a/src/NzbDrone.Core/Notifications/NotificationService.cs +++ b/src/NzbDrone.Core/Notifications/NotificationService.cs @@ -33,11 +33,13 @@ namespace NzbDrone.Core.Notifications IHandleAsync { private readonly INotificationFactory _notificationFactory; + private readonly INotificationStatusService _notificationStatusService; private readonly Logger _logger; - public NotificationService(INotificationFactory notificationFactory, Logger logger) + public NotificationService(INotificationFactory notificationFactory, INotificationStatusService notificationStatusService, Logger logger) { _notificationFactory = notificationFactory; + _notificationStatusService = notificationStatusService; _logger = logger; } @@ -142,9 +144,11 @@ namespace NzbDrone.Core.Notifications } notification.OnGrab(grabMessage); + _notificationStatusService.RecordSuccess(notification.Definition.Id); } catch (Exception ex) { + _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Error(ex, "Unable to send OnGrab notification to {0}", notification.Definition.Name); } } @@ -178,11 +182,13 @@ namespace NzbDrone.Core.Notifications if (downloadMessage.OldFiles.Empty() || ((NotificationDefinition)notification.Definition).OnUpgrade) { notification.OnReleaseImport(downloadMessage); + _notificationStatusService.RecordSuccess(notification.Definition.Id); } } } catch (Exception ex) { + _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Warn(ex, "Unable to send OnReleaseImport notification to: " + notification.Definition.Name); } } @@ -197,10 +203,12 @@ namespace NzbDrone.Core.Notifications if (ShouldHandleArtist(notification.Definition, message.Artist)) { notification.OnRename(message.Artist, message.RenamedFiles); + _notificationStatusService.RecordSuccess(notification.Definition.Id); } } catch (Exception ex) { + _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Warn(ex, "Unable to send OnRename notification to: " + notification.Definition.Name); } } @@ -217,10 +225,12 @@ namespace NzbDrone.Core.Notifications if (ShouldHandleArtist(notification.Definition, deleteMessage.Album.Artist)) { notification.OnAlbumDelete(deleteMessage); + _notificationStatusService.RecordSuccess(notification.Definition.Id); } } catch (Exception ex) { + _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Warn(ex, "Unable to send OnAlbumDelete notification to: " + notification.Definition.Name); } } @@ -239,10 +249,12 @@ namespace NzbDrone.Core.Notifications if (ShouldHandleArtist(notification.Definition, deleteMessage.Artist)) { notification.OnArtistDelete(deleteMessage); + _notificationStatusService.RecordSuccess(notification.Definition.Id); } } catch (Exception ex) { + _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Warn(ex, "Unable to send OnArtistDelete notification to: " + notification.Definition.Name); } } @@ -265,10 +277,12 @@ namespace NzbDrone.Core.Notifications if (ShouldHandleHealthFailure(message.HealthCheck, ((NotificationDefinition)notification.Definition).IncludeHealthWarnings)) { notification.OnHealthIssue(message.HealthCheck); + _notificationStatusService.RecordSuccess(notification.Definition.Id); } } catch (Exception ex) { + _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Warn(ex, "Unable to send OnHealthIssue notification to: " + notification.Definition.Name); } } @@ -288,10 +302,12 @@ namespace NzbDrone.Core.Notifications if (ShouldHandleHealthFailure(message.PreviousCheck, ((NotificationDefinition)notification.Definition).IncludeHealthWarnings)) { notification.OnHealthRestored(message.PreviousCheck); + _notificationStatusService.RecordSuccess(notification.Definition.Id); } } catch (Exception ex) { + _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Warn(ex, "Unable to send OnHealthRestored notification to: " + notification.Definition.Name); } } @@ -310,9 +326,18 @@ namespace NzbDrone.Core.Notifications foreach (var notification in _notificationFactory.OnDownloadFailureEnabled()) { - if (ShouldHandleArtist(notification.Definition, message.TrackedDownload.RemoteAlbum.Artist)) + try + { + if (ShouldHandleArtist(notification.Definition, message.TrackedDownload.RemoteAlbum.Artist)) + { + notification.OnDownloadFailure(downloadFailedMessage); + _notificationStatusService.RecordSuccess(notification.Definition.Id); + } + } + catch (Exception ex) { - notification.OnDownloadFailure(downloadFailedMessage); + _notificationStatusService.RecordFailure(notification.Definition.Id); + _logger.Warn(ex, "Unable to send OnDownloadFailure notification to: " + notification.Definition.Name); } } } @@ -327,9 +352,18 @@ namespace NzbDrone.Core.Notifications foreach (var notification in _notificationFactory.OnImportFailureEnabled()) { - if (ShouldHandleArtist(notification.Definition, message.TrackedDownload.RemoteAlbum.Artist)) + try { - notification.OnImportFailure(downloadMessage); + if (ShouldHandleArtist(notification.Definition, message.TrackedDownload.RemoteAlbum.Artist)) + { + notification.OnImportFailure(downloadMessage); + _notificationStatusService.RecordSuccess(notification.Definition.Id); + } + } + catch (Exception ex) + { + _notificationStatusService.RecordFailure(notification.Definition.Id); + _logger.Warn(ex, "Unable to send OnImportFailure notification to: " + notification.Definition.Name); } } } @@ -349,9 +383,18 @@ namespace NzbDrone.Core.Notifications foreach (var notification in _notificationFactory.OnTrackRetagEnabled()) { - if (ShouldHandleArtist(notification.Definition, message.Artist)) + try + { + if (ShouldHandleArtist(notification.Definition, message.Artist)) + { + notification.OnTrackRetag(retagMessage); + _notificationStatusService.RecordSuccess(notification.Definition.Id); + } + } + catch (Exception ex) { - notification.OnTrackRetag(retagMessage); + _notificationStatusService.RecordFailure(notification.Definition.Id); + _logger.Warn(ex, "Unable to send OnTrackRetag notification to: " + notification.Definition.Name); } } } @@ -368,9 +411,11 @@ namespace NzbDrone.Core.Notifications try { notification.OnApplicationUpdate(updateMessage); + _notificationStatusService.RecordSuccess(notification.Definition.Id); } catch (Exception ex) { + _notificationStatusService.RecordFailure(notification.Definition.Id); _logger.Warn(ex, "Unable to send OnApplicationUpdate notification to: " + notification.Definition.Name); } } diff --git a/src/NzbDrone.Core/Notifications/NotificationStatus.cs b/src/NzbDrone.Core/Notifications/NotificationStatus.cs new file mode 100644 index 000000000..1cb6f4a2e --- /dev/null +++ b/src/NzbDrone.Core/Notifications/NotificationStatus.cs @@ -0,0 +1,8 @@ +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Notifications +{ + public class NotificationStatus : ProviderStatusBase + { + } +} diff --git a/src/NzbDrone.Core/Notifications/NotificationStatusRepository.cs b/src/NzbDrone.Core/Notifications/NotificationStatusRepository.cs new file mode 100644 index 000000000..c5e61647f --- /dev/null +++ b/src/NzbDrone.Core/Notifications/NotificationStatusRepository.cs @@ -0,0 +1,18 @@ +using NzbDrone.Core.Datastore; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Notifications +{ + public interface INotificationStatusRepository : IProviderStatusRepository + { + } + + public class NotificationStatusRepository : ProviderStatusRepository, INotificationStatusRepository + { + public NotificationStatusRepository(IMainDatabase database, IEventAggregator eventAggregator) + : base(database, eventAggregator) + { + } + } +} diff --git a/src/NzbDrone.Core/Notifications/NotificationStatusService.cs b/src/NzbDrone.Core/Notifications/NotificationStatusService.cs new file mode 100644 index 000000000..218b21ba8 --- /dev/null +++ b/src/NzbDrone.Core/Notifications/NotificationStatusService.cs @@ -0,0 +1,22 @@ +using System; +using NLog; +using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider.Status; + +namespace NzbDrone.Core.Notifications +{ + public interface INotificationStatusService : IProviderStatusServiceBase + { + } + + public class NotificationStatusService : ProviderStatusServiceBase, INotificationStatusService + { + public NotificationStatusService(INotificationStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger) + : base(providerStatusRepository, eventAggregator, runtimeInfo, logger) + { + MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5); + MaximumEscalationLevel = 5; + } + } +}