diff --git a/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs b/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs index 919868458..a92eed1f1 100644 --- a/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs +++ b/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs @@ -89,5 +89,38 @@ namespace NzbDrone.Common.Test.TPLTests (GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100)); } + + [Test] + public void should_extend_subkey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(300)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(400)); + } + + [Test] + public void should_honor_basekey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(0)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(200)); + } + + [Test] + public void should_not_extend_basekey_delay() + { + GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200)); + GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(100)); + + Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100)); + + (GetRateLimitStore()["me"] - _epoch).Should().BeCloseTo(TimeSpan.FromMilliseconds(200)); + } } } diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index f89307d5e..a7abca8df 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -109,7 +109,7 @@ namespace NzbDrone.Common.Http if (request.RateLimit != TimeSpan.Zero) { - _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimit); + _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimitKey, request.RateLimit); } _logger.Trace(request); diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs index 3674d61d5..f623c3fe5 100644 --- a/src/NzbDrone.Common/Http/HttpRequest.cs +++ b/src/NzbDrone.Common/Http/HttpRequest.cs @@ -44,6 +44,7 @@ namespace NzbDrone.Common.Http public bool StoreResponseCookie { get; set; } public TimeSpan RequestTimeout { get; set; } public TimeSpan RateLimit { get; set; } + public string RateLimitKey { get; set; } public override string ToString() { diff --git a/src/NzbDrone.Common/TPL/RateLimitService.cs b/src/NzbDrone.Common/TPL/RateLimitService.cs index f0d30b4ff..87b0ff22f 100644 --- a/src/NzbDrone.Common/TPL/RateLimitService.cs +++ b/src/NzbDrone.Common/TPL/RateLimitService.cs @@ -2,12 +2,14 @@ using System.Collections.Concurrent; using NLog; using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; namespace NzbDrone.Common.TPL { public interface IRateLimitService { void WaitAndPulse(string key, TimeSpan interval); + void WaitAndPulse(string key, string subKey, TimeSpan interval); } public class RateLimitService : IRateLimitService @@ -23,9 +25,37 @@ namespace NzbDrone.Common.TPL public void WaitAndPulse(string key, TimeSpan interval) { - var waitUntil = _rateLimitStore.AddOrUpdate(key, - (s) => DateTime.UtcNow + interval, - (s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval); + WaitAndPulse(key, null, interval); + } + + public void WaitAndPulse(string key, string subKey, TimeSpan interval) + { + var waitUntil = DateTime.UtcNow.Add(interval); + + if (subKey.IsNotNullOrWhiteSpace()) + { + // Expand the base key timer, but don't extend it beyond now+interval. + var baseUntil = _rateLimitStore.AddOrUpdate(key, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Ticks), DateTimeKind.Utc)); + + if (baseUntil > waitUntil) + { + waitUntil = baseUntil; + } + + // Wait for the full key + var combinedKey = key + "-" + subKey; + waitUntil = _rateLimitStore.AddOrUpdate(combinedKey, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc)); + } + else + { + waitUntil = _rateLimitStore.AddOrUpdate(key, + (s) => waitUntil, + (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc)); + } waitUntil -= interval; diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs index 44dd120cd..c043f6f6e 100644 --- a/src/NzbDrone.Core/Download/TorrentClientBase.cs +++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs @@ -129,6 +129,7 @@ namespace NzbDrone.Core.Download try { var request = new HttpRequest(torrentUrl); + request.RateLimitKey = remoteMovie?.Release?.IndexerId.ToString(); request.Headers.Accept = "application/x-bittorrent"; request.AllowAutoRedirect = false; diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs index 27afbd704..307ba75f3 100644 --- a/src/NzbDrone.Core/Download/UsenetClientBase.cs +++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs @@ -44,7 +44,9 @@ namespace NzbDrone.Core.Download try { - nzbData = _httpClient.Get(new HttpRequest(url)).ResponseData; + var request = new HttpRequest(url); + request.RateLimitKey = remoteMovie?.Release?.IndexerId.ToString(); + nzbData = _httpClient.Get(request).ResponseData; _logger.Debug("Downloaded nzb for movie '{0}' finished ({1} bytes from {2})", remoteMovie.Release.Title, nzbData.Length, url); } diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 260c92155..a0739b75a 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -301,6 +301,8 @@ namespace NzbDrone.Core.Indexers request.HttpRequest.RateLimit = RateLimit; } + request.HttpRequest.RateLimitKey = Definition.Id.ToString(); + request.HttpRequest.AllowAutoRedirect = true; return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));