New: Notifications (Connect) Status

(cherry picked from commit e3545801721e00d4e5cac3fa534e66dcbe9d2d05)

frontend/src/System/Status/Health/Health.js

@@ -33,6 +33,14 @@ function getInternalLink(source) {
           to="/settings/downloadclients"
         />
       );
+    case 'NotificationStatusCheck':
+      return (
+        <IconButton
+          name={icons.SETTINGS}
+          title={translate('Settings')}
+          to="/settings/connect"
+        />
+      );
     case 'IndexerRssCheck':
     case 'IndexerSearchCheck':
     case 'IndexerStatusCheck':

src/NzbDrone.Core.Test/HealthCheck/Checks/NotificationStatusCheckFixture.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Collections.Generic;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.HealthCheck.Checks;
+using NzbDrone.Core.Localization;
+using NzbDrone.Core.Notifications;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.HealthCheck.Checks
+{
+    [TestFixture]
+    public class NotificationStatusCheckFixture : CoreTest<NotificationStatusCheck>
+    {
+        private List<INotification> _notifications = new List<INotification>();
+        private List<NotificationStatus> _blockedNotifications = new List<NotificationStatus>();
+
+        [SetUp]
+        public void SetUp()
+        {
+            Mocker.GetMock<INotificationFactory>()
+                  .Setup(v => v.GetAvailableProviders())
+                  .Returns(_notifications);
+
+            Mocker.GetMock<INotificationStatusService>()
+                   .Setup(v => v.GetBlockedProviders())
+                   .Returns(_blockedNotifications);
+
+            Mocker.GetMock<ILocalizationService>()
+                .Setup(s => s.GetLocalizedString(It.IsAny<string>()))
+                .Returns("Some Warning Message");
+        }
+
+        private Mock<INotification> GivenNotification(int id, double backoffHours, double failureHours)
+        {
+            var mockNotification = new Mock<INotification>();
+            mockNotification.SetupGet(s => s.Definition).Returns(new NotificationDefinition { Id = id });
+
+            _notifications.Add(mockNotification.Object);
+
+            if (backoffHours != 0.0)
+            {
+                _blockedNotifications.Add(new NotificationStatus
+                {
+                    ProviderId = id,
+                    InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
+                    MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
+                    EscalationLevel = 5,
+                    DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
+                });
+            }
+
+            return mockNotification;
+        }
+
+        [Test]
+        public void should_not_return_error_when_no_notifications()
+        {
+            Subject.Check().ShouldBeOk();
+        }
+
+        [Test]
+        public void should_return_warning_if_notification_unavailable()
+        {
+            GivenNotification(1, 10.0, 24.0);
+            GivenNotification(2, 0.0, 0.0);
+
+            Subject.Check().ShouldBeWarning();
+        }
+
+        [Test]
+        public void should_return_error_if_all_notifications_unavailable()
+        {
+            GivenNotification(1, 10.0, 24.0);
+
+            Subject.Check().ShouldBeError();
+        }
+
+        [Test]
+        public void should_return_warning_if_few_notifications_unavailable()
+        {
+            GivenNotification(1, 10.0, 24.0);
+            GivenNotification(2, 10.0, 24.0);
+            GivenNotification(3, 0.0, 0.0);
+
+            Subject.Check().ShouldBeWarning();
+        }
+    }
+}

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatusFixture.cs

@@ -0,0 +1,56 @@
+using FizzWare.NBuilder;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.Housekeeping.Housekeepers;
+using NzbDrone.Core.Notifications;
+using NzbDrone.Core.Notifications.Join;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
+{
+    [TestFixture]
+    public class CleanupOrphanedNotificationStatusFixture : DbTest<CleanupOrphanedNotificationStatus, NotificationStatus>
+    {
+        private NotificationDefinition _notification;
+
+        [SetUp]
+        public void Setup()
+        {
+            _notification = Builder<NotificationDefinition>.CreateNew()
+                                                           .With(s => s.Settings = new JoinSettings { })
+                                                           .BuildNew();
+        }
+
+        private void GivenNotification()
+        {
+            Db.Insert(_notification);
+        }
+
+        [Test]
+        public void should_delete_orphaned_notificationstatus()
+        {
+            var status = Builder<NotificationStatus>.CreateNew()
+                                                    .With(h => h.ProviderId = _notification.Id)
+                                                    .BuildNew();
+            Db.Insert(status);
+
+            Subject.Clean();
+            AllStoredModels.Should().BeEmpty();
+        }
+
+        [Test]
+        public void should_not_delete_unorphaned_notificationstatus()
+        {
+            GivenNotification();
+
+            var status = Builder<NotificationStatus>.CreateNew()
+                                                    .With(h => h.ProviderId = _notification.Id)
+                                                    .BuildNew();
+            Db.Insert(status);
+
+            Subject.Clean();
+            AllStoredModels.Should().HaveCount(1);
+            AllStoredModels.Should().Contain(h => h.ProviderId == _notification.Id);
+        }
+    }
+}

src/NzbDrone.Core.Test/NotificationTests/NotificationStatusServiceFixture.cs

@@ -0,0 +1,161 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Core.Notifications;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.NotificationTests
+{
+    public class NotificationStatusServiceFixture : CoreTest<NotificationStatusService>
+    {
+        private DateTime _epoch;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
+        }
+
+        private NotificationStatus WithStatus(NotificationStatus status)
+        {
+            Mocker.GetMock<INotificationStatusRepository>()
+                .Setup(v => v.FindByProviderId(1))
+                .Returns(status);
+
+            Mocker.GetMock<INotificationStatusRepository>()
+                .Setup(v => v.All())
+                .Returns(new[] { status });
+
+            return status;
+        }
+
+        private void VerifyUpdate()
+        {
+            Mocker.GetMock<INotificationStatusRepository>()
+                .Verify(v => v.Upsert(It.IsAny<NotificationStatus>()), Times.Once());
+        }
+
+        private void VerifyNoUpdate()
+        {
+            Mocker.GetMock<INotificationStatusRepository>()
+                .Verify(v => v.Upsert(It.IsAny<NotificationStatus>()), Times.Never());
+        }
+
+        [Test]
+        public void should_not_consider_blocked_within_5_minutes_since_initial_failure()
+        {
+            WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(4),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+
+            VerifyUpdate();
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().BeNull();
+        }
+
+        [Test]
+        public void should_consider_blocked_after_5_minutes_since_initial_failure()
+        {
+            WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+
+            VerifyUpdate();
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+        }
+
+        [Test]
+        public void should_not_escalate_further_till_after_5_minutes_since_initial_failure()
+        {
+            var origStatus = WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(4),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().BeNull();
+
+            origStatus.EscalationLevel.Should().Be(3);
+        }
+
+        [Test]
+        public void should_escalate_further_after_5_minutes_since_initial_failure()
+        {
+            WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+
+            status.EscalationLevel.Should().BeGreaterThan(3);
+        }
+
+        [Test]
+        public void should_not_escalate_beyond_3_hours()
+        {
+            WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+            status.DisabledTill.Should().HaveValue();
+            status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1));
+        }
+    }
+}

src/NzbDrone.Core/Datastore/Migration/037_add_notification_status.cs

@@ -0,0 +1,19 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(037)]
+    public class add_notification_status : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Create.TableForModel("NotificationStatus")
+                  .WithColumn("ProviderId").AsInt32().NotNullable().Unique()
+                  .WithColumn("InitialFailure").AsDateTimeOffset().Nullable()
+                  .WithColumn("MostRecentFailure").AsDateTimeOffset().Nullable()
+                  .WithColumn("EscalationLevel").AsInt32().NotNullable()
+                  .WithColumn("DisabledTill").AsDateTimeOffset().Nullable();
+        }
+    }
+}

src/NzbDrone.Core/Datastore/TableMapping.cs

@@ -91,10 +91,9 @@ namespace NzbDrone.Core.Datastore
                   .Ignore(c => c.Message);
 
             Mapper.Entity<IndexerStatus>("IndexerStatus").RegisterModel();
-
             Mapper.Entity<DownloadClientStatus>("DownloadClientStatus").RegisterModel();
-
             Mapper.Entity<ApplicationStatus>("ApplicationStatus").RegisterModel();
+            Mapper.Entity<NotificationStatus>("NotificationStatus").RegisterModel();
 
             Mapper.Entity<CustomFilter>("CustomFilters").RegisterModel();
             Mapper.Entity<UpdateHistory>("UpdateHistory").RegisterModel();

src/NzbDrone.Core/HealthCheck/Checks/NotificationStatusCheck.cs

@@ -0,0 +1,52 @@
+using System.Linq;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Localization;
+using NzbDrone.Core.Notifications;
+using NzbDrone.Core.ThingiProvider.Events;
+
+namespace NzbDrone.Core.HealthCheck.Checks
+{
+    [CheckOn(typeof(ProviderUpdatedEvent<INotification>))]
+    [CheckOn(typeof(ProviderDeletedEvent<INotification>))]
+    [CheckOn(typeof(ProviderStatusChangedEvent<INotification>))]
+    public class NotificationStatusCheck : HealthCheckBase
+    {
+        private readonly INotificationFactory _providerFactory;
+        private readonly INotificationStatusService _providerStatusService;
+
+        public NotificationStatusCheck(INotificationFactory providerFactory, INotificationStatusService providerStatusService, ILocalizationService localizationService)
+            : base(localizationService)
+        {
+            _providerFactory = providerFactory;
+            _providerStatusService = providerStatusService;
+        }
+
+        public override HealthCheck Check()
+        {
+            var enabledProviders = _providerFactory.GetAvailableProviders();
+            var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
+                    i => i.Definition.Id,
+                    s => s.ProviderId,
+                    (i, s) => new { Provider = i, Status = s })
+                .ToList();
+
+            if (backOffProviders.Empty())
+            {
+                return new HealthCheck(GetType());
+            }
+
+            if (backOffProviders.Count == enabledProviders.Count)
+            {
+                return new HealthCheck(GetType(),
+                    HealthCheckResult.Error,
+                    _localizationService.GetLocalizedString("NotificationStatusAllClientHealthCheckMessage"),
+                    "#notifications-are-unavailable-due-to-failures");
+            }
+
+            return new HealthCheck(GetType(),
+                HealthCheckResult.Warning,
+                string.Format(_localizationService.GetLocalizedString("NotificationStatusSingleClientHealthCheckMessage"), string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))),
+                "#notifications-are-unavailable-due-to-failures");
+        }
+    }
+}

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatus.cs

@@ -0,0 +1,27 @@
+using Dapper;
+using NzbDrone.Core.Datastore;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class CleanupOrphanedNotificationStatus : IHousekeepingTask
+    {
+        private readonly IMainDatabase _database;
+
+        public CleanupOrphanedNotificationStatus(IMainDatabase database)
+        {
+            _database = database;
+        }
+
+        public void Clean()
+        {
+            using var mapper = _database.OpenConnection();
+
+            mapper.Execute(@"DELETE FROM ""NotificationStatus""
+                                     WHERE ""Id"" IN (
+                                     SELECT ""NotificationStatus"".""Id"" FROM ""NotificationStatus""
+                                     LEFT OUTER JOIN ""Notifications""
+                                     ON ""NotificationStatus"".""ProviderId"" = ""Notifications"".""Id""
+                                     WHERE ""Notifications"".""Id"" IS NULL)");
+        }
+    }
+}

src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureNotificationStatusTimes.cs

@@ -0,0 +1,12 @@
+using NzbDrone.Core.Notifications;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class FixFutureNotificationStatusTimes : FixFutureProviderStatusTimes<NotificationStatus>, IHousekeepingTask
+    {
+        public FixFutureNotificationStatusTimes(INotificationStatusRepository notificationStatusRepository)
+            : base(notificationStatusRepository)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Localization/Core/en.json

@@ -333,6 +333,8 @@
   "NoUpdatesAreAvailable": "No updates are available",
   "NotSupported": "Not Supported",
   "Notification": "Notification",
+  "NotificationStatusAllClientHealthCheckMessage": "All notifications are unavailable due to failures",
+  "NotificationStatusSingleClientHealthCheckMessage": "Notifications unavailable due to failures: {0}",
   "NotificationTriggers": "Notification Triggers",
   "NotificationTriggersHelpText": "Select which events should trigger this notification",
   "Notifications": "Notifications",

src/NzbDrone.Core/Notifications/NotificationFactory.cs

@@ -9,39 +9,80 @@ namespace NzbDrone.Core.Notifications
 {
     public interface INotificationFactory : IProviderFactory<INotification, NotificationDefinition>
     {
-        List<INotification> OnGrabEnabled();
+        List<INotification> OnGrabEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnHealthIssueEnabled();
+        List<INotification> OnHealthIssueEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnHealthRestoredEnabled();
+        List<INotification> OnHealthRestoredEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnApplicationUpdateEnabled();
+        List<INotification> OnApplicationUpdateEnabled(bool filterBlockedNotifications = true);
     }
 
     public class NotificationFactory : ProviderFactory<INotification, NotificationDefinition>, INotificationFactory
     {
-        public NotificationFactory(INotificationRepository providerRepository, IEnumerable<INotification> providers, IServiceProvider container, IEventAggregator eventAggregator, Logger logger)
+        private readonly INotificationStatusService _notificationStatusService;
+        private readonly Logger _logger;
+
+        public NotificationFactory(INotificationStatusService notificationStatusService, INotificationRepository providerRepository, IEnumerable<INotification> providers, IServiceProvider container, IEventAggregator eventAggregator, Logger logger)
             : base(providerRepository, providers, container, eventAggregator, logger)
         {
+            _notificationStatusService = notificationStatusService;
+            _logger = logger;
         }
 
-        public List<INotification> OnGrabEnabled()
+        public List<INotification> OnGrabEnabled(bool filterBlockedNotifications = true)
         {
+            if (filterBlockedNotifications)
+            {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnGrab)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnGrab).ToList();
         }
 
-        public List<INotification> OnHealthIssueEnabled()
+        public List<INotification> OnHealthIssueEnabled(bool filterBlockedNotifications = true)
         {
+            if (filterBlockedNotifications)
+            {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthIssue)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthIssue).ToList();
         }
 
-        public List<INotification> OnHealthRestoredEnabled()
+        public List<INotification> OnHealthRestoredEnabled(bool filterBlockedNotifications = true)
         {
+            if (filterBlockedNotifications)
+            {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthRestored)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthRestored).ToList();
         }
 
-        public List<INotification> OnApplicationUpdateEnabled()
+        public List<INotification> OnApplicationUpdateEnabled(bool filterBlockedNotifications = true)
         {
+            if (filterBlockedNotifications)
+            {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnApplicationUpdate)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnApplicationUpdate).ToList();
         }
 
+        private IEnumerable<INotification> FilterBlockedNotifications(IEnumerable<INotification> notifications)
+        {
+            var blockedNotifications = _notificationStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v);
+
+            foreach (var notification in notifications)
+            {
+                if (blockedNotifications.TryGetValue(notification.Definition.Id, out var notificationStatus))
+                {
+                    _logger.Debug("Temporarily ignoring notification {0} till {1} due to recent failures.", notification.Definition.Name, notificationStatus.DisabledTill.Value.ToLocalTime());
+                    continue;
+                }
+
+                yield return notification;
+            }
+        }
+
         public override void SetProviderCharacteristics(INotification provider, NotificationDefinition definition)
         {
             base.SetProviderCharacteristics(provider, definition);

src/NzbDrone.Core/Notifications/NotificationService.cs

@@ -20,42 +20,34 @@ namespace NzbDrone.Core.Notifications
           IHandle<IndexerDownloadEvent>
     {
         private readonly INotificationFactory _notificationFactory;
+        private readonly INotificationStatusService _notificationStatusService;
         private readonly Logger _logger;
 
-        public NotificationService(INotificationFactory notificationFactory, Logger logger)
+        public NotificationService(INotificationFactory notificationFactory, INotificationStatusService notificationStatusService, Logger logger)
         {
             _notificationFactory = notificationFactory;
+            _notificationStatusService = notificationStatusService;
             _logger = logger;
         }
 
         private bool ShouldHandleHealthFailure(HealthCheck.HealthCheck healthCheck, bool includeWarnings)
         {
-            if (healthCheck.Type == HealthCheckResult.Error)
+            return healthCheck.Type switch
             {
-                return true;
+                HealthCheckResult.Error => true,
-            }
+                HealthCheckResult.Warning when includeWarnings => true,
-
+                _ => false
-            if (healthCheck.Type == HealthCheckResult.Warning && includeWarnings)
+            };
-            {
-                return true;
-            }
-
-            return false;
         }
 
         private bool ShouldHandleOnGrab(GrabMessage message, bool includeManual)
         {
-            if (message.GrabTrigger == GrabTrigger.Api)
+            return message.GrabTrigger switch
             {
-                return true;
+                GrabTrigger.Api => true,
-            }
+                GrabTrigger.Manual when includeManual => true,
-
+                _ => false
-            if (message.GrabTrigger == GrabTrigger.Manual && includeManual)
+            };
-            {
-                return true;
-            }
-
-            return false;
         }
 
         private string GetMessage(ReleaseInfo release, GrabTrigger grabTrigger, string source, string downloadClient)
@@ -96,10 +88,12 @@ namespace NzbDrone.Core.Notifications
                     if (ShouldHandleHealthFailure(message.HealthCheck, ((NotificationDefinition)notification.Definition).IncludeHealthWarnings))
                     {
                         notification.OnHealthIssue(message.HealthCheck);
+                        _notificationStatusService.RecordSuccess(notification.Definition.Id);
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnHealthIssue notification to: " + notification.Definition.Name);
                 }
             }
@@ -119,10 +113,12 @@ namespace NzbDrone.Core.Notifications
                     if (ShouldHandleHealthFailure(message.PreviousCheck, ((NotificationDefinition)notification.Definition).IncludeHealthWarnings))
                     {
                         notification.OnHealthRestored(message.PreviousCheck);
+                        _notificationStatusService.RecordSuccess(notification.Definition.Id);
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnHealthRestored notification to: " + notification.Definition.Name);
                 }
             }
@@ -145,9 +141,11 @@ namespace NzbDrone.Core.Notifications
                 try
                 {
                     notification.OnApplicationUpdate(updateMessage);
+                    _notificationStatusService.RecordSuccess(notification.Definition.Id);
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnApplicationUpdate notification to: " + notification.Definition.Name);
                 }
             }
@@ -192,10 +190,12 @@ namespace NzbDrone.Core.Notifications
                         ShouldHandleOnGrab(grabMessage, ((NotificationDefinition)notification.Definition).IncludeManualGrabs))
                     {
                         notification.OnGrab(grabMessage);
+                        _notificationStatusService.RecordSuccess(notification.Definition.Id);
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Error(ex, "Unable to send OnGrab notification to {0}", notification.Definition.Name);
                 }
             }

src/NzbDrone.Core/Notifications/NotificationStatus.cs

@@ -0,0 +1,8 @@
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Notifications
+{
+    public class NotificationStatus : ProviderStatusBase
+    {
+    }
+}

src/NzbDrone.Core/Notifications/NotificationStatusRepository.cs

@@ -0,0 +1,18 @@
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Notifications
+{
+    public interface INotificationStatusRepository : IProviderStatusRepository<NotificationStatus>
+    {
+    }
+
+    public class NotificationStatusRepository : ProviderStatusRepository<NotificationStatus>, INotificationStatusRepository
+    {
+        public NotificationStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
+            : base(database, eventAggregator)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Notifications/NotificationStatusService.cs

@@ -0,0 +1,22 @@
+using System;
+using NLog;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Notifications
+{
+    public interface INotificationStatusService : IProviderStatusServiceBase<NotificationStatus>
+    {
+    }
+
+    public class NotificationStatusService : ProviderStatusServiceBase<INotification, NotificationStatus>, INotificationStatusService
+    {
+        public NotificationStatusService(INotificationStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
+        {
+            MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5);
+            MaximumEscalationLevel = 5;
+        }
+    }
+}