diff --git a/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs b/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs
deleted file mode 100644
index df08a7ef4..000000000
--- a/src/NzbDrone.Common.Test/Http/HttpRateLimitKeyFactoryFixture.cs
+++ /dev/null
@@ -1,30 +0,0 @@
-using FluentAssertions;
-using NUnit.Framework;
-using NzbDrone.Common.Http;
-
-namespace NzbDrone.Common.Test.Http
-{
- [TestFixture]
- public class HttpRateLimitKeyFactoryFixture
- {
- [TestCase("http://127.0.0.2:9117/jackett/api/v2.0/indexers/viva/results/torznab/api?t=search&cat=5000,5070,100030,100041", "127.0.0.2:9117/jackett/api/v2.0/indexers/viva")]
- public void should_detect_jackett(string url, string expectedKey)
- {
- var request = new HttpRequest(url);
-
- var key = HttpRateLimitKeyFactory.GetRateLimitKey(request);
-
- key.Should().Be(expectedKey);
- }
-
- [TestCase("http://127.0.0.2:9117/jackett", "127.0.0.2")]
- public void should_default_to_host(string url, string expectedKey)
- {
- var request = new HttpRequest(url);
-
- var key = HttpRateLimitKeyFactory.GetRateLimitKey(request);
-
- key.Should().Be(expectedKey);
- }
- }
-}
\ No newline at end of file
diff --git a/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs b/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs
index e5ccc8244..83f3e95be 100644
--- a/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs
+++ b/src/NzbDrone.Common.Test/TPLTests/RateLimitServiceFixture.cs
@@ -88,5 +88,38 @@ namespace NzbDrone.Common.Test.TPLTests
(GetRateLimitStore()["me"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(100));
}
+
+ [Test]
+ public void should_extend_subkey_delay()
+ {
+ GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
+ GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(300));
+
+ Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
+
+ (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(400));
+ }
+
+ [Test]
+ public void should_honor_basekey_delay()
+ {
+ GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
+ GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(0));
+
+ Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
+
+ (GetRateLimitStore()["me-sub"] - _epoch).Should().BeGreaterOrEqualTo(TimeSpan.FromMilliseconds(200));
+ }
+
+ [Test]
+ public void should_not_extend_basekey_delay()
+ {
+ GivenExisting("me", _epoch + TimeSpan.FromMilliseconds(200));
+ GivenExisting("me-sub", _epoch + TimeSpan.FromMilliseconds(100));
+
+ Subject.WaitAndPulse("me", "sub", TimeSpan.FromMilliseconds(100));
+
+ (GetRateLimitStore()["me"] - _epoch).Should().BeCloseTo(TimeSpan.FromMilliseconds(200));
+ }
}
}
diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs
index 10e483076..7e7f72b12 100644
--- a/src/NzbDrone.Common/Http/HttpClient.cs
+++ b/src/NzbDrone.Common/Http/HttpClient.cs
@@ -111,7 +111,7 @@ namespace NzbDrone.Common.Http
if (request.RateLimit != TimeSpan.Zero)
{
- _rateLimitService.WaitAndPulse(HttpRateLimitKeyFactory.GetRateLimitKey(request), request.RateLimit);
+ _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimitKey, request.RateLimit);
}
_logger.Trace(request);
diff --git a/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs b/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs
deleted file mode 100644
index e4a627e75..000000000
--- a/src/NzbDrone.Common/Http/HttpRateLimitKeyFactory.cs
+++ /dev/null
@@ -1,28 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Text.RegularExpressions;
-using System.Threading.Tasks;
-
-namespace NzbDrone.Common.Http
-{
- public static class HttpRateLimitKeyFactory
- {
- // Use a different key for jackett instances to prevent hitting the ratelimit for multiple separate indexers.
- private static readonly Regex _regex = new Regex(@"^https?://(.+/jackett/api/v2.0/indexers/\w+)/", RegexOptions.Compiled);
-
- public static string GetRateLimitKey(HttpRequest request)
- {
- var match = _regex.Match(request.Url.ToString());
-
- if (match.Success)
- {
- return match.Groups[1].Value;
- }
-
- return request.Url.Host;
- }
-
- }
-}
diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs
index 2519531ca..355aa6dd6 100644
--- a/src/NzbDrone.Common/Http/HttpRequest.cs
+++ b/src/NzbDrone.Common/Http/HttpRequest.cs
@@ -44,6 +44,7 @@ namespace NzbDrone.Common.Http
public bool StoreResponseCookie { get; set; }
public TimeSpan RequestTimeout { get; set; }
public TimeSpan RateLimit { get; set; }
+ public string RateLimitKey { get; set; }
public Stream ResponseStream { get; set; }
public override string ToString()
diff --git a/src/NzbDrone.Common/TPL/RateLimitService.cs b/src/NzbDrone.Common/TPL/RateLimitService.cs
index 6d1b9ee17..5030ead23 100644
--- a/src/NzbDrone.Common/TPL/RateLimitService.cs
+++ b/src/NzbDrone.Common/TPL/RateLimitService.cs
@@ -2,12 +2,14 @@
using System.Collections.Concurrent;
using NLog;
using NzbDrone.Common.Cache;
+using NzbDrone.Common.Extensions;
namespace NzbDrone.Common.TPL
{
public interface IRateLimitService
{
void WaitAndPulse(string key, TimeSpan interval);
+ void WaitAndPulse(string key, string subKey, TimeSpan interval);
}
public class RateLimitService : IRateLimitService
@@ -23,9 +25,37 @@ namespace NzbDrone.Common.TPL
public void WaitAndPulse(string key, TimeSpan interval)
{
- var waitUntil = _rateLimitStore.AddOrUpdate(key,
- (s) => DateTime.UtcNow + interval,
- (s,i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
+ WaitAndPulse(key, null, interval);
+ }
+
+ public void WaitAndPulse(string key, string subKey, TimeSpan interval)
+ {
+ var waitUntil = DateTime.UtcNow.Add(interval);
+
+ if (subKey.IsNotNullOrWhiteSpace())
+ {
+ // Expand the base key timer, but don't extend it beyond now+interval.
+ var baseUntil = _rateLimitStore.AddOrUpdate(key,
+ (s) => waitUntil,
+ (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Ticks), DateTimeKind.Utc));
+
+ if (baseUntil > waitUntil)
+ {
+ waitUntil = baseUntil;
+ }
+
+ // Wait for the full key
+ var combinedKey = key + "-" + subKey;
+ waitUntil = _rateLimitStore.AddOrUpdate(combinedKey,
+ (s) => waitUntil,
+ (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc));
+ }
+ else
+ {
+ waitUntil = _rateLimitStore.AddOrUpdate(key,
+ (s) => waitUntil,
+ (s, i) => new DateTime(Math.Max(waitUntil.Ticks, i.Add(interval).Ticks), DateTimeKind.Utc));
+ }
waitUntil -= interval;
diff --git a/src/NzbDrone.Core/Download/TorrentClientBase.cs b/src/NzbDrone.Core/Download/TorrentClientBase.cs
index 07f0bd42c..72b47a2db 100644
--- a/src/NzbDrone.Core/Download/TorrentClientBase.cs
+++ b/src/NzbDrone.Core/Download/TorrentClientBase.cs
@@ -128,6 +128,7 @@ namespace NzbDrone.Core.Download
try
{
var request = new HttpRequest(torrentUrl);
+ request.RateLimitKey = remoteEpisode?.Release?.IndexerId.ToString();
request.Headers.Accept = "application/x-bittorrent";
request.AllowAutoRedirect = false;
diff --git a/src/NzbDrone.Core/Download/UsenetClientBase.cs b/src/NzbDrone.Core/Download/UsenetClientBase.cs
index 68135bd22..9dc35ce2f 100644
--- a/src/NzbDrone.Core/Download/UsenetClientBase.cs
+++ b/src/NzbDrone.Core/Download/UsenetClientBase.cs
@@ -43,7 +43,9 @@ namespace NzbDrone.Core.Download
try
{
- nzbData = _httpClient.Get(new HttpRequest(url)).ResponseData;
+ var request = new HttpRequest(url);
+ request.RateLimitKey = remoteEpisode?.Release?.IndexerId.ToString();
+ nzbData = _httpClient.Get(request).ResponseData;
_logger.Debug("Downloaded nzb for episode '{0}' finished ({1} bytes from {2})", remoteEpisode.Release.Title, nzbData.Length, url);
}
diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
index 75310a5e4..d55b3a4a7 100644
--- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
+++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
@@ -317,6 +317,7 @@ namespace NzbDrone.Core.Indexers
{
request.HttpRequest.RateLimit = RateLimit;
}
+ request.HttpRequest.RateLimitKey = Definition.Id.ToString();
return new IndexerResponse(request, _httpClient.Execute(request.HttpRequest));
}