New: Updated Rarbg request limits

(cherry picked from commit 47cf8e6430b7f7704ce2f1524fa9e3c8e6f9b47a)

Close #3526

src/NzbDrone.Common/Http/HttpRequestBuilder.cs

@@ -20,6 +20,7 @@ namespace NzbDrone.Common.Http
         public Dictionary<string, string> Segments { get; private set; }
         public HttpHeader Headers { get; private set; }
         public bool SuppressHttpError { get; set; }
+        public IEnumerable<HttpStatusCode> SuppressHttpErrorStatusCodes { get; set; }
         public bool LogHttpError { get; set; }
         public bool UseSimplifiedUserAgent { get; set; }
         public bool AllowAutoRedirect { get; set; }
@@ -102,6 +103,7 @@ namespace NzbDrone.Common.Http
         {
             request.Method = Method;
             request.SuppressHttpError = SuppressHttpError;
+            request.SuppressHttpErrorStatusCodes = SuppressHttpErrorStatusCodes;
             request.LogHttpError = LogHttpError;
             request.UseSimplifiedUserAgent = UseSimplifiedUserAgent;
             request.AllowAutoRedirect = AllowAutoRedirect;

src/NzbDrone.Common/Http/TooManyRequestsException.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 
 namespace NzbDrone.Common.Http
 {

src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Linq;
+using System.Net;
 using System.Net.Http;
 using FluentAssertions;
 using Moq;
@@ -84,5 +85,57 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests
 
             ExceptionVerification.ExpectedWarns(1);
         }
+
+        [Test]
+        public void should_warn_and_record_failure_on_429_response()
+        {
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", HttpStatusCode.TooManyRequests));
+
+            var releases = Subject.FetchRecent();
+
+            releases.Should().HaveCount(0);
+
+            ExceptionVerification.ExpectedWarns(1);
+
+            Mocker.GetMock<IIndexerStatusService>()
+                  .Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(2))));
+        }
+
+        [Test]
+        public void should_warn_and_record_failure_on_520_response()
+        {
+            Mocker.GetMock<IHttpClient>()
+                .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
+                .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "", (HttpStatusCode)520));
+
+            var releases = Subject.FetchRecent();
+
+            releases.Should().HaveCount(0);
+
+            ExceptionVerification.ExpectedWarns(1);
+
+            Mocker.GetMock<IIndexerStatusService>()
+                  .Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(3))));
+        }
+
+        // Uncomment when RarbgParser is updated
+        // [Test]
+        // public void should_warn_and_record_failure_on_200_response_with_rate_limit()
+        // {
+        //     Mocker.GetMock<IHttpClient>()
+        //         .Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
+        //         .Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), "{ rate_limit: 1 }"));
+        //
+        //     var releases = Subject.FetchRecent();
+        //
+        //     releases.Should().HaveCount(0);
+        //
+        //     ExceptionVerification.ExpectedWarns(1);
+        //
+        //     Mocker.GetMock<IIndexerStatusService>()
+        //         .Verify(v => v.RecordFailure(It.IsAny<int>(), It.Is<TimeSpan>(t => t == TimeSpan.FromMinutes(5))));
+        // }
     }
 }

src/NzbDrone.Core/Indexers/Exceptions/RequestLimitReachedException.cs

@@ -1,9 +1,12 @@
-using NzbDrone.Common.Exceptions;
+using System;
+using NzbDrone.Common.Exceptions;
 
 namespace NzbDrone.Core.Indexers.Exceptions
 {
     public class RequestLimitReachedException : NzbDroneException
     {
+        public TimeSpan RetryAfter { get; private set; }
+
         public RequestLimitReachedException(string message, params object[] args)
             : base(message, args)
         {
@@ -13,5 +16,11 @@ namespace NzbDrone.Core.Indexers.Exceptions
             : base(message)
         {
         }
+
+        public RequestLimitReachedException(string message, TimeSpan retryAfter)
+            : base(message)
+        {
+            RetryAfter = retryAfter;
+        }
     }
 }

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -207,9 +207,17 @@ namespace NzbDrone.Core.Indexers
                 _indexerStatusService.RecordFailure(Definition.Id);
                 _logger.Warn("{0} {1}", this, ex.Message);
             }
-            catch (RequestLimitReachedException)
+            catch (RequestLimitReachedException ex)
             {
-                _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
+                if (ex.RetryAfter != TimeSpan.Zero)
+                {
+                    _indexerStatusService.RecordFailure(Definition.Id, ex.RetryAfter);
+                }
+                else
+                {
+                    _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
+                }
+
                 _logger.Warn("API Request Limit reached for {0}", this);
             }
             catch (ApiKeyException)

src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs

@@ -18,8 +18,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
         public override string Name => "Rarbg";
 
         public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
-
-        public override TimeSpan RateLimit => TimeSpan.FromSeconds(2);
+        public override TimeSpan RateLimit => TimeSpan.FromSeconds(4);
 
         public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IIndexerStatusService indexerStatusService, IConfigService configService, IParsingService parsingService, Logger logger)
             : base(httpClient, indexerStatusService, configService, parsingService, logger)

src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Net;
 using System.Text.RegularExpressions;
 using NzbDrone.Common.Http;
@@ -17,10 +18,14 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
             switch (indexerResponse.HttpResponse.StatusCode)
             {
+                case HttpStatusCode.TooManyRequests:
+                    throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(2));
+                case (HttpStatusCode)520:
+                    throw new RequestLimitReachedException("Indexer API error. Likely rate limited by origin server", TimeSpan.FromMinutes(3));
                 default:
                     if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
                     {
-                        throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected StatusCode [{0}]", indexerResponse.HttpResponse.StatusCode);
+                        throw new IndexerException(indexerResponse, "Indexer API call returned an unexpected status code [{0}]", indexerResponse.HttpResponse.StatusCode);
                     }
 
                     break;
@@ -44,6 +49,12 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
             if (jsonResponse.Resource.torrent_results == null)
             {
+                // Despite this being the requested behaviour it appears to be problematic, commenting it out for now
+                // if (jsonResponse.Resource.rate_limit == 1)
+                // {
+                //     throw new RequestLimitReachedException("Indexer API limit reached", TimeSpan.FromMinutes(5));
+                // }
+
                 return results;
             }
 

src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs

@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Net;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
@@ -50,6 +51,8 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 .Resource("/pubapi_v2.php")
                 .Accept(HttpAccept.Json);
 
+            requestBuilder.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.TooManyRequests, (HttpStatusCode)520 };
+
             if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
             {
                 requestBuilder.UseSimplifiedUserAgent = true;

src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 
 namespace NzbDrone.Core.Indexers.Rarbg
@@ -7,6 +7,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
     {
         public string error { get; set; }
         public int? error_code { get; set; }
+        public int? rate_limit { get; set; }
         public List<RarbgTorrent> torrent_results { get; set; }
     }
 

src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using Newtonsoft.Json.Linq;
 using NLog;
 using NzbDrone.Common.Cache;