New: Limit indexer/download client backoff to 5 min during the first 15 min of application start.

closes #2366

src/NzbDrone.Common/EnvironmentInfo/IRuntimeInfo.cs

@@ -1,7 +1,10 @@
+using System;
+
 namespace NzbDrone.Common.EnvironmentInfo
 {
     public interface IRuntimeInfo
     {
+        DateTime StartTime { get; }
         bool IsUserInteractive { get; }
         bool IsAdmin { get; }
         bool IsWindowsService { get; }

src/NzbDrone.Common/EnvironmentInfo/RuntimeInfo.cs

@@ -12,6 +12,7 @@ namespace NzbDrone.Common.EnvironmentInfo
     public class RuntimeInfo : IRuntimeInfo
     {
         private readonly Logger _logger;
+        private readonly DateTime _startTime = DateTime.UtcNow;
 
         public RuntimeInfo(IServiceProvider serviceProvider, Logger logger)
         {
@@ -37,6 +38,14 @@ namespace NzbDrone.Common.EnvironmentInfo
             IsProduction = InternalIsProduction();
         }
 
+        public DateTime StartTime
+        {
+            get
+            {
+                return _startTime;
+            }
+        }
+
         public static bool IsUserInteractive => Environment.UserInteractive;
 
         bool IRuntimeInfo.IsUserInteractive => IsUserInteractive;

src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs

@@ -1,8 +1,9 @@
-using System;
+using System;
 using System.Linq;
 using FluentAssertions;
 using Moq;
 using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Download;
 using NzbDrone.Core.Test.Framework;
 
@@ -16,6 +17,10 @@ namespace NzbDrone.Core.Test.Download
         public void SetUp()
         {
             _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
         }
 
         private DownloadClientStatus WithStatus(DownloadClientStatus status)

src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs

@@ -1,8 +1,9 @@
-using System;
+using System;
 using System.Linq;
 using FluentAssertions;
 using Moq;
 using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Test.Framework;
 
@@ -16,6 +17,10 @@ namespace NzbDrone.Core.Test.IndexerTests
         public void SetUp()
         {
             _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
         }
 
         private void WithStatus(IndexerStatus status)

src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs

@@ -1,9 +1,10 @@
-using System;
+using System;
 using System.Linq;
 using FluentAssertions;
 using Moq;
 using NLog;
 using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Test.Framework;
 using NzbDrone.Core.ThingiProvider;
@@ -25,8 +26,8 @@ namespace NzbDrone.Core.Test.ThingiProviderTests
 
     public class MockProviderStatusService : ProviderStatusServiceBase<IMockProvider, MockProviderStatus>
     {
-        public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
-            : base(providerStatusRepository, eventAggregator, logger)
+        public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
         {
 
         }
@@ -40,9 +41,20 @@ namespace NzbDrone.Core.Test.ThingiProviderTests
         public void SetUp()
         {
             _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
+        }
+
+        private void GivenRecentStartup()
+        {
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromMinutes(12));
         }
 
-        private void WithStatus(MockProviderStatus status)
+        private MockProviderStatus WithStatus(MockProviderStatus status)
         {
             Mocker.GetMock<IMockProviderStatusRepository>()
                 .Setup(v => v.FindByProviderId(1))
@@ -51,6 +63,8 @@ namespace NzbDrone.Core.Test.ThingiProviderTests
             Mocker.GetMock<IMockProviderStatusRepository>()
                 .Setup(v => v.All())
                 .Returns(new[] { status });
+
+            return status;
         }
 
         private void VerifyUpdate()
@@ -122,5 +136,32 @@ namespace NzbDrone.Core.Test.ThingiProviderTests
             status.DisabledTill.Should().HaveValue();
             status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500);
         }
+
+        [Test]
+        public void should_not_escalate_further_till_after_5_minutes_since_startup()
+        {
+            GivenRecentStartup();
+
+            var origStatus = WithStatus(new MockProviderStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+
+            origStatus.EscalationLevel.Should().Be(3);
+            status.DisabledTill.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500);
+        }
     }
 }

src/NzbDrone.Core/Download/DownloadClientStatusService.cs

@@ -1,5 +1,6 @@
-using System;
+using System;
 using NLog;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider.Status;
 
@@ -12,8 +13,8 @@ namespace NzbDrone.Core.Download
 
     public class DownloadClientStatusService : ProviderStatusServiceBase<IDownloadClient, DownloadClientStatus>, IDownloadClientStatusService
     {
-        public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
-            : base(providerStatusRepository, eventAggregator, logger)
+        public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
         {
             MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5);
             MaximumEscalationLevel = 5;

src/NzbDrone.Core/Indexers/IndexerStatusService.cs

@@ -1,4 +1,5 @@
-using NLog;
+using NLog;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider.Status;
@@ -14,8 +15,8 @@ namespace NzbDrone.Core.Indexers
 
     public class IndexerStatusService : ProviderStatusServiceBase<IIndexer, IndexerStatus>, IIndexerStatusService
     {
-        public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
-            : base(providerStatusRepository, eventAggregator, logger)
+        public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
         {
 
         }

src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using NLog;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider.Events;
 
@@ -24,15 +25,18 @@ namespace NzbDrone.Core.ThingiProvider.Status
 
         protected readonly IProviderStatusRepository<TModel> _providerStatusRepository;
         protected readonly IEventAggregator _eventAggregator;
+        protected readonly IRuntimeInfo _runtimeInfo;
         protected readonly Logger _logger;
 
         protected int MaximumEscalationLevel { get; set; } = EscalationBackOff.Periods.Length - 1;
         protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero;
+        protected TimeSpan MinimumTimeSinceStartup { get; set; } = TimeSpan.FromMinutes(15);
 
-        public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
+        public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
         {
             _providerStatusRepository = providerStatusRepository;
             _eventAggregator = eventAggregator;
+            _runtimeInfo = runtimeInfo;
             _logger = logger;
         }
 
@@ -89,9 +93,10 @@ namespace NzbDrone.Core.ThingiProvider.Status
                     escalate = false;
                 }
 
+                var inStartupGracePeriod = (_runtimeInfo.StartTime + MinimumTimeSinceStartup) > now;
                 var inGracePeriod = (status.InitialFailure.Value + MinimumTimeSinceInitialFailure) > now;
 
-                if (escalate && !inGracePeriod)
+                if (escalate && !inGracePeriod && !inStartupGracePeriod)
                 {
                     status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1);
                 }
@@ -109,6 +114,15 @@ namespace NzbDrone.Core.ThingiProvider.Status
                     status.DisabledTill = now + CalculateBackOffPeriod(status);
                 }
 
+                if (inStartupGracePeriod && minimumBackOff == TimeSpan.Zero && status.DisabledTill.HasValue)
+                {
+                    var maximumDisabledTill = now + TimeSpan.FromSeconds(EscalationBackOff.Periods[1]);
+                    if (maximumDisabledTill < status.DisabledTill)
+                    {
+                        status.DisabledTill = maximumDisabledTill;
+                    }
+                }
+
                 _providerStatusRepository.Upsert(status);
 
                 _eventAggregator.PublishEvent(new ProviderStatusChangedEvent<TProvider>(providerId, status));