New: Notifications (Connect) Status

(cherry picked from commit e3545801721e00d4e5cac3fa534e66dcbe9d2d05)
(cherry picked from commit cb27b05a6c046ca0a6e4998f7e7ecd6b45add1a2)

frontend/src/System/Status/Health/Health.js

@@ -38,6 +38,14 @@ function getInternalLink(source) {
           to="/settings/downloadclients"
         />
       );
+    case 'NotificationStatusCheck':
+      return (
+        <IconButton
+          name={icons.SETTINGS}
+          title={translate('Settings')}
+          to="/settings/connect"
+        />
+      );
     case 'RootFolderCheck':
       return (
         <IconButton

src/NzbDrone.Core.Test/HealthCheck/Checks/NotificationStatusCheckFixture.cs

@@ -0,0 +1,89 @@
+using System;
+using System.Collections.Generic;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Core.HealthCheck.Checks;
+using NzbDrone.Core.Localization;
+using NzbDrone.Core.Notifications;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.HealthCheck.Checks
+{
+    [TestFixture]
+    public class NotificationStatusCheckFixture : CoreTest<NotificationStatusCheck>
+    {
+        private List<INotification> _notifications = new List<INotification>();
+        private List<NotificationStatus> _blockedNotifications = new List<NotificationStatus>();
+
+        [SetUp]
+        public void SetUp()
+        {
+            Mocker.GetMock<INotificationFactory>()
+                  .Setup(v => v.GetAvailableProviders())
+                  .Returns(_notifications);
+
+            Mocker.GetMock<INotificationStatusService>()
+                   .Setup(v => v.GetBlockedProviders())
+                   .Returns(_blockedNotifications);
+
+            Mocker.GetMock<ILocalizationService>()
+                .Setup(s => s.GetLocalizedString(It.IsAny<string>()))
+                .Returns("Some Warning Message");
+        }
+
+        private Mock<INotification> GivenNotification(int id, double backoffHours, double failureHours)
+        {
+            var mockNotification = new Mock<INotification>();
+            mockNotification.SetupGet(s => s.Definition).Returns(new NotificationDefinition { Id = id });
+
+            _notifications.Add(mockNotification.Object);
+
+            if (backoffHours != 0.0)
+            {
+                _blockedNotifications.Add(new NotificationStatus
+                {
+                    ProviderId = id,
+                    InitialFailure = DateTime.UtcNow.AddHours(-failureHours),
+                    MostRecentFailure = DateTime.UtcNow.AddHours(-0.1),
+                    EscalationLevel = 5,
+                    DisabledTill = DateTime.UtcNow.AddHours(backoffHours)
+                });
+            }
+
+            return mockNotification;
+        }
+
+        [Test]
+        public void should_not_return_error_when_no_notifications()
+        {
+            Subject.Check().ShouldBeOk();
+        }
+
+        [Test]
+        public void should_return_warning_if_notification_unavailable()
+        {
+            GivenNotification(1, 10.0, 24.0);
+            GivenNotification(2, 0.0, 0.0);
+
+            Subject.Check().ShouldBeWarning();
+        }
+
+        [Test]
+        public void should_return_error_if_all_notifications_unavailable()
+        {
+            GivenNotification(1, 10.0, 24.0);
+
+            Subject.Check().ShouldBeError();
+        }
+
+        [Test]
+        public void should_return_warning_if_few_notifications_unavailable()
+        {
+            GivenNotification(1, 10.0, 24.0);
+            GivenNotification(2, 10.0, 24.0);
+            GivenNotification(3, 0.0, 0.0);
+
+            Subject.Check().ShouldBeWarning();
+        }
+    }
+}

src/NzbDrone.Core.Test/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatusFixture.cs

@@ -0,0 +1,56 @@
+using FizzWare.NBuilder;
+using FluentAssertions;
+using NUnit.Framework;
+using NzbDrone.Core.Housekeeping.Housekeepers;
+using NzbDrone.Core.Notifications;
+using NzbDrone.Core.Notifications.Join;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.Housekeeping.Housekeepers
+{
+    [TestFixture]
+    public class CleanupOrphanedNotificationStatusFixture : DbTest<CleanupOrphanedNotificationStatus, NotificationStatus>
+    {
+        private NotificationDefinition _notification;
+
+        [SetUp]
+        public void Setup()
+        {
+            _notification = Builder<NotificationDefinition>.CreateNew()
+                                                           .With(s => s.Settings = new JoinSettings { })
+                                                           .BuildNew();
+        }
+
+        private void GivenNotification()
+        {
+            Db.Insert(_notification);
+        }
+
+        [Test]
+        public void should_delete_orphaned_notificationstatus()
+        {
+            var status = Builder<NotificationStatus>.CreateNew()
+                                                    .With(h => h.ProviderId = _notification.Id)
+                                                    .BuildNew();
+            Db.Insert(status);
+
+            Subject.Clean();
+            AllStoredModels.Should().BeEmpty();
+        }
+
+        [Test]
+        public void should_not_delete_unorphaned_notificationstatus()
+        {
+            GivenNotification();
+
+            var status = Builder<NotificationStatus>.CreateNew()
+                                                    .With(h => h.ProviderId = _notification.Id)
+                                                    .BuildNew();
+            Db.Insert(status);
+
+            Subject.Clean();
+            AllStoredModels.Should().HaveCount(1);
+            AllStoredModels.Should().Contain(h => h.ProviderId == _notification.Id);
+        }
+    }
+}

src/NzbDrone.Core.Test/NotificationTests/NotificationStatusServiceFixture.cs

@@ -0,0 +1,161 @@
+using System;
+using System.Linq;
+using FluentAssertions;
+using Moq;
+using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Core.Notifications;
+using NzbDrone.Core.Test.Framework;
+
+namespace NzbDrone.Core.Test.NotificationTests
+{
+    public class NotificationStatusServiceFixture : CoreTest<NotificationStatusService>
+    {
+        private DateTime _epoch;
+
+        [SetUp]
+        public void SetUp()
+        {
+            _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
+        }
+
+        private NotificationStatus WithStatus(NotificationStatus status)
+        {
+            Mocker.GetMock<INotificationStatusRepository>()
+                .Setup(v => v.FindByProviderId(1))
+                .Returns(status);
+
+            Mocker.GetMock<INotificationStatusRepository>()
+                .Setup(v => v.All())
+                .Returns(new[] { status });
+
+            return status;
+        }
+
+        private void VerifyUpdate()
+        {
+            Mocker.GetMock<INotificationStatusRepository>()
+                .Verify(v => v.Upsert(It.IsAny<NotificationStatus>()), Times.Once());
+        }
+
+        private void VerifyNoUpdate()
+        {
+            Mocker.GetMock<INotificationStatusRepository>()
+                .Verify(v => v.Upsert(It.IsAny<NotificationStatus>()), Times.Never());
+        }
+
+        [Test]
+        public void should_not_consider_blocked_within_5_minutes_since_initial_failure()
+        {
+            WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(4),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+
+            VerifyUpdate();
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().BeNull();
+        }
+
+        [Test]
+        public void should_consider_blocked_after_5_minutes_since_initial_failure()
+        {
+            WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+
+            VerifyUpdate();
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+        }
+
+        [Test]
+        public void should_not_escalate_further_till_after_5_minutes_since_initial_failure()
+        {
+            var origStatus = WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(4),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(4),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().BeNull();
+
+            origStatus.EscalationLevel.Should().Be(3);
+        }
+
+        [Test]
+        public void should_escalate_further_after_5_minutes_since_initial_failure()
+        {
+            WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+
+            status.EscalationLevel.Should().BeGreaterThan(3);
+        }
+
+        [Test]
+        public void should_not_escalate_beyond_3_hours()
+        {
+            WithStatus(new NotificationStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+            status.DisabledTill.Should().HaveValue();
+            status.DisabledTill.Should().NotBeAfter(_epoch + TimeSpan.FromHours(3.1));
+        }
+    }
+}

src/NzbDrone.Core/Datastore/Migration/232_add_notification_status.cs

@@ -0,0 +1,19 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(232)]
+    public class add_notification_status : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Create.TableForModel("NotificationStatus")
+                  .WithColumn("ProviderId").AsInt32().NotNullable().Unique()
+                  .WithColumn("InitialFailure").AsDateTimeOffset().Nullable()
+                  .WithColumn("MostRecentFailure").AsDateTimeOffset().Nullable()
+                  .WithColumn("EscalationLevel").AsInt32().NotNullable()
+                  .WithColumn("DisabledTill").AsDateTimeOffset().Nullable();
+        }
+    }
+}

src/NzbDrone.Core/Datastore/TableMapping.cs

@@ -165,6 +165,7 @@ namespace NzbDrone.Core.Datastore
             Mapper.Entity<IndexerStatus>("IndexerStatus").RegisterModel();
             Mapper.Entity<DownloadClientStatus>("DownloadClientStatus").RegisterModel();
             Mapper.Entity<ImportListStatus>("ImportListStatus").RegisterModel();
+            Mapper.Entity<NotificationStatus>("NotificationStatus").RegisterModel();
 
             Mapper.Entity<CustomFilter>("CustomFilters").RegisterModel();
 

src/NzbDrone.Core/HealthCheck/Checks/NotificationStatusCheck.cs

@@ -0,0 +1,52 @@
+using System.Linq;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Localization;
+using NzbDrone.Core.Notifications;
+using NzbDrone.Core.ThingiProvider.Events;
+
+namespace NzbDrone.Core.HealthCheck.Checks
+{
+    [CheckOn(typeof(ProviderUpdatedEvent<INotification>))]
+    [CheckOn(typeof(ProviderDeletedEvent<INotification>))]
+    [CheckOn(typeof(ProviderStatusChangedEvent<INotification>))]
+    public class NotificationStatusCheck : HealthCheckBase
+    {
+        private readonly INotificationFactory _providerFactory;
+        private readonly INotificationStatusService _providerStatusService;
+
+        public NotificationStatusCheck(INotificationFactory providerFactory, INotificationStatusService providerStatusService, ILocalizationService localizationService)
+            : base(localizationService)
+        {
+            _providerFactory = providerFactory;
+            _providerStatusService = providerStatusService;
+        }
+
+        public override HealthCheck Check()
+        {
+            var enabledProviders = _providerFactory.GetAvailableProviders();
+            var backOffProviders = enabledProviders.Join(_providerStatusService.GetBlockedProviders(),
+                    i => i.Definition.Id,
+                    s => s.ProviderId,
+                    (i, s) => new { Provider = i, Status = s })
+                .ToList();
+
+            if (backOffProviders.Empty())
+            {
+                return new HealthCheck(GetType());
+            }
+
+            if (backOffProviders.Count == enabledProviders.Count)
+            {
+                return new HealthCheck(GetType(),
+                    HealthCheckResult.Error,
+                    _localizationService.GetLocalizedString("NotificationStatusAllClientHealthCheckMessage"),
+                    "#notifications-are-unavailable-due-to-failures");
+            }
+
+            return new HealthCheck(GetType(),
+                HealthCheckResult.Warning,
+                string.Format(_localizationService.GetLocalizedString("NotificationStatusSingleClientHealthCheckMessage"), string.Join(", ", backOffProviders.Select(v => v.Provider.Definition.Name))),
+                "#notifications-are-unavailable-due-to-failures");
+        }
+    }
+}

src/NzbDrone.Core/Housekeeping/Housekeepers/CleanupOrphanedNotificationStatus.cs

@@ -0,0 +1,27 @@
+using Dapper;
+using NzbDrone.Core.Datastore;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class CleanupOrphanedNotificationStatus : IHousekeepingTask
+    {
+        private readonly IMainDatabase _database;
+
+        public CleanupOrphanedNotificationStatus(IMainDatabase database)
+        {
+            _database = database;
+        }
+
+        public void Clean()
+        {
+            using var mapper = _database.OpenConnection();
+
+            mapper.Execute(@"DELETE FROM ""NotificationStatus""
+                                     WHERE ""Id"" IN (
+                                     SELECT ""NotificationStatus"".""Id"" FROM ""NotificationStatus""
+                                     LEFT OUTER JOIN ""Notifications""
+                                     ON ""NotificationStatus"".""ProviderId"" = ""Notifications"".""Id""
+                                     WHERE ""Notifications"".""Id"" IS NULL)");
+        }
+    }
+}

src/NzbDrone.Core/Housekeeping/Housekeepers/FixFutureNotificationStatusTimes.cs

@@ -0,0 +1,12 @@
+using NzbDrone.Core.Notifications;
+
+namespace NzbDrone.Core.Housekeeping.Housekeepers
+{
+    public class FixFutureNotificationStatusTimes : FixFutureProviderStatusTimes<NotificationStatus>, IHousekeepingTask
+    {
+        public FixFutureNotificationStatusTimes(INotificationStatusRepository notificationStatusRepository)
+            : base(notificationStatusRepository)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Localization/Core/en.json

@@ -705,6 +705,8 @@
   "None": "None",
   "NotAvailable": "Not Available",
   "NotMonitored": "Not Monitored",
+  "NotificationStatusAllClientHealthCheckMessage": "All notifications are unavailable due to failures",
+  "NotificationStatusSingleClientHealthCheckMessage": "Notifications unavailable due to failures: {0}",
   "NotificationTriggers": "Notification Triggers",
   "NotificationTriggersHelpText": "Select which events should trigger this notification",
   "OAuthPopupMessage": "Pop-ups are being blocked by your browser",

src/NzbDrone.Core/Notifications/NotificationFactory.cs

@@ -9,86 +9,167 @@ namespace NzbDrone.Core.Notifications
 {
     public interface INotificationFactory : IProviderFactory<INotification, NotificationDefinition>
     {
-        List<INotification> OnGrabEnabled();
+        List<INotification> OnGrabEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnDownloadEnabled();
+        List<INotification> OnDownloadEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnUpgradeEnabled();
+        List<INotification> OnUpgradeEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnRenameEnabled();
+        List<INotification> OnRenameEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnMovieAddedEnabled();
+        List<INotification> OnMovieAddedEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnMovieDeleteEnabled();
+        List<INotification> OnMovieDeleteEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnMovieFileDeleteEnabled();
+        List<INotification> OnMovieFileDeleteEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnHealthIssueEnabled();
+        List<INotification> OnHealthIssueEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnHealthRestoredEnabled();
+        List<INotification> OnHealthRestoredEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnApplicationUpdateEnabled();
+        List<INotification> OnApplicationUpdateEnabled(bool filterBlockedNotifications = true);
-        List<INotification> OnManualInteractionEnabled();
+        List<INotification> OnManualInteractionEnabled(bool filterBlockedNotifications = true);
     }
 
     public class NotificationFactory : ProviderFactory<INotification, NotificationDefinition>, INotificationFactory
     {
-        public NotificationFactory(INotificationRepository providerRepository, IEnumerable<INotification> providers, IServiceProvider container, IEventAggregator eventAggregator, Logger logger)
+        private readonly INotificationStatusService _notificationStatusService;
+        private readonly Logger _logger;
+
+        public NotificationFactory(INotificationStatusService notificationStatusService, INotificationRepository providerRepository, IEnumerable<INotification> providers, IServiceProvider container, IEventAggregator eventAggregator, Logger logger)
             : base(providerRepository, providers, container, eventAggregator, logger)
         {
+            _notificationStatusService = notificationStatusService;
+            _logger = logger;
         }
 
-        public List<INotification> OnGrabEnabled()
+        public List<INotification> OnGrabEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnGrab)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnGrab).ToList();
         }
 
-        public List<INotification> OnDownloadEnabled()
+        public List<INotification> OnDownloadEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnDownload)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnDownload).ToList();
         }
 
-        public List<INotification> OnUpgradeEnabled()
+        public List<INotification> OnUpgradeEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnUpgrade)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnUpgrade).ToList();
         }
 
-        public List<INotification> OnRenameEnabled()
+        public List<INotification> OnRenameEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnRename)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnRename).ToList();
         }
 
-        public List<INotification> OnMovieAddedEnabled()
+        public List<INotification> OnMovieAddedEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnMovieAdded)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnMovieAdded).ToList();
         }
 
-        public List<INotification> OnMovieDeleteEnabled()
+        public List<INotification> OnMovieDeleteEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnMovieDelete)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnMovieDelete).ToList();
         }
 
-        public List<INotification> OnMovieFileDeleteEnabled()
+        public List<INotification> OnMovieFileDeleteEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnMovieFileDelete)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnMovieFileDelete).ToList();
         }
 
-        public List<INotification> OnMovieFileDeleteForUpgradeEnabled()
+        public List<INotification> OnMovieFileDeleteForUpgradeEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnMovieFileDeleteForUpgrade)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnMovieFileDeleteForUpgrade).ToList();
         }
 
-        public List<INotification> OnHealthIssueEnabled()
+        public List<INotification> OnHealthIssueEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthIssue)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthIssue).ToList();
         }
 
-        public List<INotification> OnHealthRestoredEnabled()
+        public List<INotification> OnHealthRestoredEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthRestored)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnHealthRestored).ToList();
         }
 
-        public List<INotification> OnApplicationUpdateEnabled()
+        public List<INotification> OnApplicationUpdateEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnApplicationUpdate)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnApplicationUpdate).ToList();
         }
 
-        public List<INotification> OnManualInteractionEnabled()
+        public List<INotification> OnManualInteractionEnabled(bool filterBlockedNotifications = true)
+        {
+            if (filterBlockedNotifications)
             {
+                return FilterBlockedNotifications(GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnManualInteractionRequired)).ToList();
+            }
+
             return GetAvailableProviders().Where(n => ((NotificationDefinition)n.Definition).OnManualInteractionRequired).ToList();
         }
 
+        private IEnumerable<INotification> FilterBlockedNotifications(IEnumerable<INotification> notifications)
+        {
+            var blockedNotifications = _notificationStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v);
+
+            foreach (var notification in notifications)
+            {
+                if (blockedNotifications.TryGetValue(notification.Definition.Id, out var notificationStatus))
+                {
+                    _logger.Debug("Temporarily ignoring notification {0} till {1} due to recent failures.", notification.Definition.Name, notificationStatus.DisabledTill.Value.ToLocalTime());
+                    continue;
+                }
+
+                yield return notification;
+            }
+        }
+
         public override void SetProviderCharacteristics(INotification provider, NotificationDefinition definition)
         {
             base.SetProviderCharacteristics(provider, definition);

src/NzbDrone.Core/Notifications/NotificationService.cs

@@ -32,11 +32,13 @@ namespace NzbDrone.Core.Notifications
           IHandleAsync<HealthCheckCompleteEvent>
     {
         private readonly INotificationFactory _notificationFactory;
+        private readonly INotificationStatusService _notificationStatusService;
         private readonly Logger _logger;
 
-        public NotificationService(INotificationFactory notificationFactory, Logger logger)
+        public NotificationService(INotificationFactory notificationFactory, INotificationStatusService notificationStatusService, Logger logger)
         {
             _notificationFactory = notificationFactory;
+            _notificationStatusService = notificationStatusService;
             _logger = logger;
         }
 
@@ -114,9 +116,11 @@ namespace NzbDrone.Core.Notifications
                     }
 
                     notification.OnGrab(grabMessage);
+                    _notificationStatusService.RecordSuccess(notification.Definition.Id);
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Error(ex, "Unable to send OnGrab notification to {0}", notification.Definition.Name);
                 }
             }
@@ -151,11 +155,13 @@ namespace NzbDrone.Core.Notifications
                         if (downloadMessage.OldMovieFiles.Empty() || ((NotificationDefinition)notification.Definition).OnUpgrade)
                         {
                             notification.OnDownload(downloadMessage);
+                            _notificationStatusService.RecordSuccess(notification.Definition.Id);
                         }
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnDownload notification to: " + notification.Definition.Name);
                 }
             }
@@ -170,10 +176,12 @@ namespace NzbDrone.Core.Notifications
                     if (ShouldHandleMovie(notification.Definition, message.Movie))
                     {
                         notification.OnMovieAdded(message.Movie);
+                        _notificationStatusService.RecordSuccess(notification.Definition.Id);
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnMovieAdded notification to: " + notification.Definition.Name);
                 }
             }
@@ -190,11 +198,13 @@ namespace NzbDrone.Core.Notifications
                         if (ShouldHandleMovie(notification.Definition, movie))
                         {
                             notification.OnMovieAdded(movie);
+                            _notificationStatusService.RecordSuccess(notification.Definition.Id);
                         }
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnMovieAdded notification to: " + notification.Definition.Name);
                 }
             }
@@ -209,10 +219,12 @@ namespace NzbDrone.Core.Notifications
                     if (ShouldHandleMovie(notification.Definition, message.Movie))
                     {
                         notification.OnMovieRename(message.Movie, message.RenamedFiles);
+                        _notificationStatusService.RecordSuccess(notification.Definition.Id);
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnRename notification to: " + notification.Definition.Name);
                 }
             }
@@ -230,9 +242,11 @@ namespace NzbDrone.Core.Notifications
                 try
                 {
                     notification.OnApplicationUpdate(updateMessage);
+                    _notificationStatusService.RecordSuccess(notification.Definition.Id);
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnApplicationUpdate notification to: " + notification.Definition.Name);
                 }
             }
@@ -263,9 +277,11 @@ namespace NzbDrone.Core.Notifications
                     }
 
                     notification.OnManualInteractionRequired(manualInteractionMessage);
+                    _notificationStatusService.RecordSuccess(notification.Definition.Id);
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Error(ex, "Unable to send OnManualInteractionRequired notification to {0}", notification.Definition.Name);
                 }
             }
@@ -288,11 +304,13 @@ namespace NzbDrone.Core.Notifications
                         if (ShouldHandleMovie(notification.Definition, message.MovieFile.Movie))
                         {
                             notification.OnMovieFileDelete(deleteMessage);
+                            _notificationStatusService.RecordSuccess(notification.Definition.Id);
                         }
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnMovieFileDelete notification to: " + notification.Definition.Name);
                 }
             }
@@ -311,10 +329,12 @@ namespace NzbDrone.Core.Notifications
                         if (ShouldHandleMovie(notification.Definition, deleteMessage.Movie))
                         {
                             notification.OnMovieDelete(deleteMessage);
+                            _notificationStatusService.RecordSuccess(notification.Definition.Id);
                         }
                     }
                     catch (Exception ex)
                     {
+                        _notificationStatusService.RecordFailure(notification.Definition.Id);
                         _logger.Warn(ex, "Unable to send OnMovieDelete notification to: " + notification.Definition.Name);
                     }
                 }
@@ -337,10 +357,12 @@ namespace NzbDrone.Core.Notifications
                     if (ShouldHandleHealthFailure(message.HealthCheck, ((NotificationDefinition)notification.Definition).IncludeHealthWarnings))
                     {
                         notification.OnHealthIssue(message.HealthCheck);
+                        _notificationStatusService.RecordSuccess(notification.Definition.Id);
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnHealthIssue notification to: " + notification.Definition.Name);
                 }
             }
@@ -360,10 +382,12 @@ namespace NzbDrone.Core.Notifications
                     if (ShouldHandleHealthFailure(message.PreviousCheck, ((NotificationDefinition)notification.Definition).IncludeHealthWarnings))
                     {
                         notification.OnHealthRestored(message.PreviousCheck);
+                        _notificationStatusService.RecordSuccess(notification.Definition.Id);
                     }
                 }
                 catch (Exception ex)
                 {
+                    _notificationStatusService.RecordFailure(notification.Definition.Id);
                     _logger.Warn(ex, "Unable to send OnHealthRestored notification to: " + notification.Definition.Name);
                 }
             }

src/NzbDrone.Core/Notifications/NotificationStatus.cs

@@ -0,0 +1,8 @@
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Notifications
+{
+    public class NotificationStatus : ProviderStatusBase
+    {
+    }
+}

src/NzbDrone.Core/Notifications/NotificationStatusRepository.cs

@@ -0,0 +1,18 @@
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Notifications
+{
+    public interface INotificationStatusRepository : IProviderStatusRepository<NotificationStatus>
+    {
+    }
+
+    public class NotificationStatusRepository : ProviderStatusRepository<NotificationStatus>, INotificationStatusRepository
+    {
+        public NotificationStatusRepository(IMainDatabase database, IEventAggregator eventAggregator)
+            : base(database, eventAggregator)
+        {
+        }
+    }
+}

src/NzbDrone.Core/Notifications/NotificationStatusService.cs

@@ -0,0 +1,22 @@
+using System;
+using NLog;
+using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.ThingiProvider.Status;
+
+namespace NzbDrone.Core.Notifications
+{
+    public interface INotificationStatusService : IProviderStatusServiceBase<NotificationStatus>
+    {
+    }
+
+    public class NotificationStatusService : ProviderStatusServiceBase<INotification, NotificationStatus>, INotificationStatusService
+    {
+        public NotificationStatusService(INotificationStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
+        {
+            MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5);
+            MaximumEscalationLevel = 5;
+        }
+    }
+}