diff --git a/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs b/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs deleted file mode 100644 index df08a7ef4..000000000 --- a/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs +++ /dev/null @@ -1,30 +0,0 @@ -using FluentAssertions; -using NUnit.Framework; -using NzbDrone.Common.Http; - -namespace NzbDrone.Common.Test.Http -{ - [TestFixture] - public class HttpRateLimitKeyFactoryFixture - { - [TestCase("http://127.0.0.2:9117/jackett/api/v2.0/indexers/viva/results/torznab/api?t=search&cat=5000,5070,100030,100041", "127.0.0.2:9117/jackett/api/v2.0/indexers/viva")] - public void should_detect_jackett(string url, string expectedKey) - { - var request = new HttpRequest(url); - - var key = HttpRateLimitKeyFactory.GetRateLimitKey(request); - - key.Should().Be(expectedKey); - } - - [TestCase("http://127.0.0.2:9117/jackett", "127.0.0.2")] - public void should_default_to_host(string url, string expectedKey) - { - var request = new HttpRequest(url); - - var key = HttpRateLimitKeyFactory.GetRateLimitKey(request); - - key.Should().Be(expectedKey); - } - } -} \ No newline at end of file diff --git a/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs b/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs index 919868458..a92eed1f1 100644 --- a/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs +++ b/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs @@ -89,5 +89,38 @@ namespace NzbDrone.Common.Test.TPLTests (GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100)); } + + [Test] + public void should_extend_subkey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(300)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(400)); + } + + [Test] + public void should_honor_basekey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(0)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(200)); + } + + [Test] + public void should_not_extend_basekey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(100)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me"] - _epoch).Should().BeCloseTo(TimeSpan.FromMilliseconds(200)); + } } } diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index 96ced471a..3cf6b1d0d 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -107,7 +107,7 @@ namespace NzbDrone.Common.Http if (request.RateLimit != TimeSpan.Zero) { - _rateLimitService.WaitAndPulse(HttpRateLimitKeyFactory.GetRateLimitKey(request), request.RateLimit); + _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimitKey, request.RateLimit); } _logger.Trace(request); diff --git a/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs b/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs deleted file mode 100644 index e4a627e75..000000000 --- a/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs +++ /dev/null @@ -1,28 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; -using System.Threading.Tasks; - -namespace NzbDrone.Common.Http -{ - public static class HttpRateLimitKeyFactory - { - // Use a different key for jackett instances to prevent hitting the ratelimit for multiple separate indexers. - private static readonly Regex _regex = new Regex(@"^https?://(.+/jackett/api/v2.0/indexers/\w+)/", RegexOptions.Compiled); - - public static string GetRateLimitKey(HttpRequest request) - { - var match = _regex.Match(request.Url.ToString()); - - if (match.Success) - { - return match.Groups[1].Value; - } - - return request.Url.Host; - } - - } -} diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs index ecb5784af..0f4f8cd3d 100644 --- a/src/NzbDrone.Common/Http/HttpRequest.cs +++ b/src/NzbDrone.Common/Http/HttpRequest.cs @@ -45,6 +45,7 @@ namespace NzbDrone.Common.Http public bool StoreResponseCookie { get; set; } public TimeSpan RequestTimeout { get; set; } public TimeSpan RateLimit { get; set; } + public string RateLimitKey { get; set; } public Stream ResponseStream { get; set; } public override string ToString() diff --git a/src/NzbDrone.Common/TPL/RateLimitService.cs b/src/NzbDrone.Common/TPL/RateLimitService.cs index f0d30b4ff..f006c378d 100644 --- a/src/NzbDrone.Common/TPL/RateLimitService.cs +++ b/src/NzbDrone.Common/TPL/RateLimitService.cs @@ -1,13 +1,15 @@ -using System; +using System; using System.Collections.Concurrent; using NLog; using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; namespace NzbDrone.Common.TPL { public interface IRateLimitService { void WaitAndPulse(string key, TimeSpan interval); + void WaitAndPulse(string key, string subKey, TimeSpan interval); } public class RateLimitService : IRateLimitService @@ -23,9 +25,37 @@ namespace NzbDrone.Common.TPL public void WaitAndPulse(string key, TimeSpan interval) { - var waitUntil = _rateLimitStore.AddOrUpdate(key, - (s) => DateTime.UtcNow + interval, - (s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval); + WaitAndPulse(key, null, interval); + } + + public void WaitAndPulse(string key, string subKey, TimeSpan interval) + { + var waitUntil = DateTime.UtcNow.Add(interval); + + if (subKey.IsNotNullOrWhiteSpace()) + { + // Expand the base key timer, but don't extend it beyond now+interval. + var baseUntil = _rateLimitStore.AddOrUpdate(key, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Ticks), DateTimeKind.Utc)); + + if (baseUntil > waitUntil) + { + waitUntil = baseUntil; + } + + // Wait for the full key + var combinedKey = key + "-" + subKey; + waitUntil = _rateLimitStore.AddOrUpdate(combinedKey, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc)); + } + else + { + waitUntil = _rateLimitStore.AddOrUpdate(key, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc)); + } waitUntil -= interval; diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs index 000961047..531bac025 100644 --- a/src/NzbDrone.Core/Download/TorrentClientBase.cs +++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs @@ -128,6 +128,7 @@ namespace NzbDrone.Core.Download try { var request = new HttpRequest(torrentUrl); + request.RateLimitKey = remoteBook?.Release?.IndexerId.ToString(); request.Headers.Accept = "application/x-bittorrent"; request.AllowAutoRedirect = false; diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs index f9e8ecaa9..a4a62744b 100644 --- a/src/NzbDrone.Core/Download/UsenetClientBase.cs +++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs @@ -44,15 +44,16 @@ namespace NzbDrone.Core.Download try { - var nzbDataRequest = new HttpRequest(url); + var request = new HttpRequest(url); + request.RateLimitKey = remoteBook?.Release?.IndexerId.ToString(); // TODO: Look into moving download request handling to indexer if (remoteBook.Release.BasicAuthString.IsNotNullOrWhiteSpace()) { - nzbDataRequest.Headers.Set("Authorization", "Basic " + remoteBook.Release.BasicAuthString); + request.Headers.Set("Authorization", "Basic " + remoteBook.Release.BasicAuthString); } - nzbData = _httpClient.Get(nzbDataRequest).ResponseData; + nzbData = _httpClient.Get(request).ResponseData; _logger.Debug("Downloaded nzb for release '{0}' finished ({1} bytes from {2})", remoteBook.Release.Title, nzbData.Length, url); } diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 6043f535b..b06ca74fe 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -280,6 +280,8 @@ namespace NzbDrone.Core.Indexers request.HttpRequest.RateLimit = RateLimit; } + request.HttpRequest.RateLimitKey = Definition.Id.ToString(); + return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest)); }