New: Limit provider backoff during the first 15 min of startup

Co-Authored-By: taloth <taloth@users.noreply.github.com>

src/NzbDrone.Core.Test/Download/DownloadClientStatusServiceFixture.cs

@@ -3,6 +3,7 @@ using System.Linq;
 using FluentAssertions;
 using Moq;
 using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Download;
 using NzbDrone.Core.Test.Framework;
 
@@ -16,6 +17,10 @@ namespace NzbDrone.Core.Test.Download
         public void SetUp()
         {
             _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
         }
 
         private DownloadClientStatus WithStatus(DownloadClientStatus status)

src/NzbDrone.Core.Test/ImportListTests/ImportListStatusServiceFixture.cs

@@ -3,6 +3,7 @@ using System.Linq;
 using FluentAssertions;
 using Moq;
 using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.ImportLists;
 using NzbDrone.Core.Test.Framework;
 
@@ -16,6 +17,10 @@ namespace NzbDrone.Core.Test.ImportListTests
         public void SetUp()
         {
             _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
         }
 
         private void WithStatus(ImportListStatus status)

src/NzbDrone.Core.Test/IndexerTests/IndexerStatusServiceFixture.cs

@@ -3,6 +3,7 @@ using System.Linq;
 using FluentAssertions;
 using Moq;
 using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.Test.Framework;
 
@@ -16,6 +17,10 @@ namespace NzbDrone.Core.Test.IndexerTests
         public void SetUp()
         {
             _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
         }
 
         private void WithStatus(IndexerStatus status)

src/NzbDrone.Core.Test/ThingiProviderTests/ProviderStatusServiceFixture.cs

@@ -4,6 +4,7 @@ using FluentAssertions;
 using Moq;
 using NLog;
 using NUnit.Framework;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Test.Framework;
 using NzbDrone.Core.ThingiProvider;
@@ -25,8 +26,8 @@ namespace NzbDrone.Core.Test.ThingiProviderTests
 
     public class MockProviderStatusService : ProviderStatusServiceBase<IMockProvider, MockProviderStatus>
     {
-        public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
-            : base(providerStatusRepository, eventAggregator, logger)
+        public MockProviderStatusService(IMockProviderStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
         {
 
         }
@@ -40,9 +41,20 @@ namespace NzbDrone.Core.Test.ThingiProviderTests
         public void SetUp()
         {
             _epoch = DateTime.UtcNow;
+
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromHours(1));
+        }
+
+        private void GivenRecentStartup()
+        {
+            Mocker.GetMock<IRuntimeInfo>()
+                .SetupGet(v => v.StartTime)
+                .Returns(_epoch - TimeSpan.FromMinutes(12));
         }
 
-        private void WithStatus(MockProviderStatus status)
+        private MockProviderStatus WithStatus(MockProviderStatus status)
         {
             Mocker.GetMock<IMockProviderStatusRepository>()
                 .Setup(v => v.FindByProviderId(1))
@@ -51,6 +63,8 @@ namespace NzbDrone.Core.Test.ThingiProviderTests
             Mocker.GetMock<IMockProviderStatusRepository>()
                 .Setup(v => v.All())
                 .Returns(new[] { status });
+
+            return status;
         }
 
         private void VerifyUpdate()
@@ -122,5 +136,32 @@ namespace NzbDrone.Core.Test.ThingiProviderTests
             status.DisabledTill.Should().HaveValue();
             status.DisabledTill.Value.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(15), 500);
         }
+
+        [Test]
+        public void should_not_escalate_further_than_5_minutes_for_15_min_after_startup()
+        {
+            GivenRecentStartup();
+
+            var origStatus = WithStatus(new MockProviderStatus
+            {
+                InitialFailure = _epoch - TimeSpan.FromMinutes(6),
+                MostRecentFailure = _epoch - TimeSpan.FromSeconds(120),
+                EscalationLevel = 3
+            });
+
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+            Subject.RecordFailure(1);
+
+            var status = Subject.GetBlockedProviders().FirstOrDefault();
+            status.Should().NotBeNull();
+
+            origStatus.EscalationLevel.Should().Be(3);
+            status.DisabledTill.Should().BeCloseTo(_epoch + TimeSpan.FromMinutes(5), 500);
+        }
     }
 }

src/NzbDrone.Core.Test/packages.config

@@ -12,7 +12,6 @@
   <package id="Newtonsoft.Json" version="11.0.2" targetFramework="net461" />
   <package id="NLog" version="4.5.4" targetFramework="net461" />
   <package id="NUnit" version="3.11.0" targetFramework="net461" />
-  <package id="NUnit3TestAdapter" version="3.11.0" targetFramework="net461" />
   <package id="Prowlin" version="0.9.4456.26422" targetFramework="net461" />
   <package id="Unity" version="2.1.505.2" targetFramework="net461" />
 </packages>

src/NzbDrone.Core/Download/DownloadClientStatusService.cs

@@ -1,5 +1,6 @@
 using System;
 using NLog;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider.Status;
 
@@ -12,8 +13,8 @@ namespace NzbDrone.Core.Download
 
     public class DownloadClientStatusService : ProviderStatusServiceBase<IDownloadClient, DownloadClientStatus>, IDownloadClientStatusService
     {
-        public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
-            : base(providerStatusRepository, eventAggregator, logger)
+        public DownloadClientStatusService(IDownloadClientStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
         {
             MinimumTimeSinceInitialFailure = TimeSpan.FromMinutes(5);
             MaximumEscalationLevel = 5;

src/NzbDrone.Core/ImportLists/ImportListStatusService.cs

@@ -1,4 +1,5 @@
 using NLog;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider.Status;
@@ -14,8 +15,8 @@ namespace NzbDrone.Core.ImportLists
 
     public class ImportListStatusService : ProviderStatusServiceBase<IImportList, ImportListStatus>, IImportListStatusService
     {
-        public ImportListStatusService(IImportListStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
-            : base(providerStatusRepository, eventAggregator, logger)
+        public ImportListStatusService(IImportListStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
         {
         }
 

src/NzbDrone.Core/Indexers/IndexerStatusService.cs

@@ -1,4 +1,5 @@
 using NLog;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider.Status;
@@ -14,8 +15,8 @@ namespace NzbDrone.Core.Indexers
 
     public class IndexerStatusService : ProviderStatusServiceBase<IIndexer, IndexerStatus>, IIndexerStatusService
     {
-        public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
-            : base(providerStatusRepository, eventAggregator, logger)
+        public IndexerStatusService(IIndexerStatusRepository providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
+            : base(providerStatusRepository, eventAggregator, runtimeInfo, logger)
         {
         }
 

src/NzbDrone.Core/ThingiProvider/Status/ProviderStatusServiceBase.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using NLog;
+using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider.Events;
 
@@ -25,15 +26,18 @@ namespace NzbDrone.Core.ThingiProvider.Status
 
         protected readonly IProviderStatusRepository<TModel> _providerStatusRepository;
         protected readonly IEventAggregator _eventAggregator;
+        protected readonly IRuntimeInfo _runtimeInfo;
         protected readonly Logger _logger;
 
         protected int MaximumEscalationLevel { get; set; } = EscalationBackOff.Periods.Length - 1;
         protected TimeSpan MinimumTimeSinceInitialFailure { get; set; } = TimeSpan.Zero;
+        protected TimeSpan MinimumTimeSinceStartup { get; set; } = TimeSpan.FromMinutes(15);
 
-        public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, IEventAggregator eventAggregator, Logger logger)
+        public ProviderStatusServiceBase(IProviderStatusRepository<TModel> providerStatusRepository, IEventAggregator eventAggregator, IRuntimeInfo runtimeInfo, Logger logger)
         {
             _providerStatusRepository = providerStatusRepository;
             _eventAggregator = eventAggregator;
+            _runtimeInfo = runtimeInfo;
             _logger = logger;
         }
 
@@ -90,9 +94,10 @@ namespace NzbDrone.Core.ThingiProvider.Status
                     escalate = false;
                 }
 
+                var inStartupGracePeriod = (_runtimeInfo.StartTime + MinimumTimeSinceStartup) > now;
                 var inGracePeriod = (status.InitialFailure.Value + MinimumTimeSinceInitialFailure) > now;
 
-                if (escalate && !inGracePeriod)
+                if (escalate && !inGracePeriod && !inStartupGracePeriod)
                 {
                     status.EscalationLevel = Math.Min(MaximumEscalationLevel, status.EscalationLevel + 1);
                 }
@@ -110,6 +115,15 @@ namespace NzbDrone.Core.ThingiProvider.Status
                     status.DisabledTill = now + CalculateBackOffPeriod(status);
                 }
 
+                if (inStartupGracePeriod && minimumBackOff == TimeSpan.Zero && status.DisabledTill.HasValue)
+                {
+                    var maximumDisabledTill = now + TimeSpan.FromSeconds(EscalationBackOff.Periods[1]);
+                    if (maximumDisabledTill < status.DisabledTill)
+                    {
+                        status.DisabledTill = maximumDisabledTill;
+                    }
+                }
+
                 _providerStatusRepository.Upsert(status);
 
                 _eventAggregator.PublishEvent(new ProviderStatusChangedEvent<TProvider>(providerId, status));