Generalized RateLimit logic to all indexers based on indexer id

src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs

@@ -1,30 +0,0 @@
-using FluentAssertions;
-using NUnit.Framework;
-using NzbDrone.Common.Http;
-
-namespace NzbDrone.Common.Test.Http
-{
-    [TestFixture]
-    public class HttpRateLimitKeyFactoryFixture
-    {
-        [TestCase("http://127.0.0.2:9117/jackett/api/v2.0/indexers/viva/results/torznab/api?t=search&cat=5000,5070,100030,100041", "127.0.0.2:9117/jackett/api/v2.0/indexers/viva")]
-        public void should_detect_jackett(string url, string expectedKey)
-        {
-            var request = new HttpRequest(url);
-
-            var key = HttpRateLimitKeyFactory.GetRateLimitKey(request);
-
-            key.Should().Be(expectedKey);
-        }
-
-        [TestCase("http://127.0.0.2:9117/jackett", "127.0.0.2")]
-        public void should_default_to_host(string url, string expectedKey)
-        {
-            var request = new HttpRequest(url);
-
-            var key = HttpRateLimitKeyFactory.GetRateLimitKey(request);
-
-            key.Should().Be(expectedKey);
-        }
-    }
-}

src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs

@@ -89,5 +89,38 @@ namespace NzbDrone.Common.Test.TPLTests
 
             (GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100));
         }
+
+        [Test]
+        public void should_extend_subkey_delay()
+        {
+            GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
+            GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(300));
+
+            Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
+
+            (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(400));
+        }
+
+        [Test]
+        public void should_honor_basekey_delay()
+        {
+            GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
+            GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(0));
+
+            Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
+
+            (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(200));
+        }
+
+        [Test]
+        public void should_not_extend_basekey_delay()
+        {
+            GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
+            GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(100));
+
+            Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
+
+            (GetRateLimitStore()["me"] - _epoch).Should().BeCloseTo(TimeSpan.FromMilliseconds(200));
+        }
     }
 }

src/NzbDrone.Common/Http/HttpClient.cs

@@ -107,7 +107,7 @@ namespace NzbDrone.Common.Http
 
             if (request.RateLimit != TimeSpan.Zero)
             {
-                _rateLimitService.WaitAndPulse(HttpRateLimitKeyFactory.GetRateLimitKey(request), request.RateLimit);
+                _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimitKey, request.RateLimit);
             }
 
             _logger.Trace(request);

src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs

@@ -1,28 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
-
-namespace NzbDrone.Common.Http
-{
-    public static class HttpRateLimitKeyFactory
-    {
-        // Use a different key for jackett instances to prevent hitting the ratelimit for multiple separate indexers.
-        private static readonly Regex _regex = new Regex(@"^https?://(.+/jackett/api/v2.0/indexers/\w+)/", RegexOptions.Compiled);
-
-        public static string GetRateLimitKey(HttpRequest request)
-        {
-            var match = _regex.Match(request.Url.ToString());
-
-            if (match.Success)
-            {
-                return match.Groups[1].Value;
-            }
-
-            return request.Url.Host;
-        }
-            
-    }
-}

src/NzbDrone.Common/Http/HttpRequest.cs

@@ -45,6 +45,7 @@ namespace NzbDrone.Common.Http
         public bool StoreResponseCookie { get; set; }
         public TimeSpan RequestTimeout { get; set; }
         public TimeSpan RateLimit { get; set; }
+        public string RateLimitKey { get; set; }
         public Stream ResponseStream { get; set; }
 
         public override string ToString()

src/NzbDrone.Common/TPL/RateLimitService.cs

@@ -1,13 +1,15 @@
-using System;
+using System;
 using System.Collections.Concurrent;
 using NLog;
 using NzbDrone.Common.Cache;
+using NzbDrone.Common.Extensions;
 
 namespace NzbDrone.Common.TPL
 {
     public interface IRateLimitService
     {
         void WaitAndPulse(string key, TimeSpan interval);
+        void WaitAndPulse(string key, string subKey, TimeSpan interval);
     }
 
     public class RateLimitService : IRateLimitService
@@ -23,9 +25,37 @@ namespace NzbDrone.Common.TPL
 
         public void WaitAndPulse(string key, TimeSpan interval)
         {
-            var waitUntil = _rateLimitStore.AddOrUpdate(key,
-                (s) => DateTime.UtcNow + interval,
-                (s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
+            WaitAndPulse(key, null, interval);
+        }
+
+        public void WaitAndPulse(string key, string subKey, TimeSpan interval)
+        {
+            var waitUntil = DateTime.UtcNow.Add(interval);
+
+            if (subKey.IsNotNullOrWhiteSpace())
+            {
+                // Expand the base key timer, but don't extend it beyond now+interval.
+                var baseUntil = _rateLimitStore.AddOrUpdate(key,
+                    (s) => waitUntil,
+                    (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Ticks), DateTimeKind.Utc));
+
+                if (baseUntil > waitUntil)
+                {
+                    waitUntil = baseUntil;
+                }
+
+                // Wait for the full key
+                var combinedKey = key + "-" + subKey;
+                waitUntil = _rateLimitStore.AddOrUpdate(combinedKey,
+                    (s) => waitUntil,
+                    (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc));
+            }
+            else
+            {
+                waitUntil = _rateLimitStore.AddOrUpdate(key,
+                    (s) => waitUntil,
+                    (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc));
+            }
 
             waitUntil -= interval;
 

src/NzbDrone.Core/Download/TorrentClientBase.cs

@@ -128,6 +128,7 @@ namespace NzbDrone.Core.Download
             try
             {
                 var request = new HttpRequest(torrentUrl);
+                request.RateLimitKey = remoteBook?.Release?.IndexerId.ToString();
                 request.Headers.Accept = "application/x-bittorrent";
                 request.AllowAutoRedirect = false;
 

src/NzbDrone.Core/Download/UsenetClientBase.cs

@@ -44,15 +44,16 @@ namespace NzbDrone.Core.Download
 
             try
             {
-                var nzbDataRequest = new HttpRequest(url);
+                var request = new HttpRequest(url);
+                request.RateLimitKey = remoteBook?.Release?.IndexerId.ToString();
 
                 // TODO: Look into moving download request handling to indexer
                 if (remoteBook.Release.BasicAuthString.IsNotNullOrWhiteSpace())
                 {
-                    nzbDataRequest.Headers.Set("Authorization", "Basic " + remoteBook.Release.BasicAuthString);
+                    request.Headers.Set("Authorization", "Basic " + remoteBook.Release.BasicAuthString);
                 }
 
-                nzbData = _httpClient.Get(nzbDataRequest).ResponseData;
+                nzbData = _httpClient.Get(request).ResponseData;
 
                 _logger.Debug("Downloaded nzb for release '{0}' finished ({1} bytes from {2})", remoteBook.Release.Title, nzbData.Length, url);
             }

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -280,6 +280,8 @@ namespace NzbDrone.Core.Indexers
                 request.HttpRequest.RateLimit = RateLimit;
             }
 
+            request.HttpRequest.RateLimitKey = Definition.Id.ToString();
+
             return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));
         }