Fixed: (Rarbg) More reliable token handling and retry

Fixes #1148

src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs

@@ -1,6 +1,11 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
+using System.Net.Http.Json;
+using System.Threading.Tasks;
+using System.Web;
+using Newtonsoft.Json;
 using NLog;
 using NzbDrone.Common.EnvironmentInfo;
 using NzbDrone.Common.Extensions;
@@ -8,7 +13,9 @@ using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Http.CloudFlare;
+using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.Parser;
 using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.Indexers.Rarbg
@@ -95,6 +102,57 @@ namespace NzbDrone.Core.Indexers.Rarbg
             return caps;
         }
 
+        protected override async Task<IndexerQueryResult> FetchPage(IndexerRequest request, IParseIndexerResponse parser)
+        {
+            var response = await FetchIndexerResponse(request);
+
+            // try and recover from token or rate limit errors
+            var jsonResponse = new HttpResponse<RarbgResponse>(response.HttpResponse);
+
+            if (jsonResponse.Resource.error_code.HasValue)
+            {
+                if (jsonResponse.Resource.error_code == 4 || jsonResponse.Resource.error_code == 2)
+                {
+                    _logger.Debug("Invalid or expired token, refreshing token from Rarbg");
+                    _tokenProvider.ExpireToken(Settings);
+                    var newToken = _tokenProvider.GetToken(Settings);
+
+                    var qs = HttpUtility.ParseQueryString(request.HttpRequest.Url.Query);
+                    qs.Set("token", newToken);
+
+                    request.HttpRequest.Url = request.Url.SetQuery(qs.GetQueryString());
+                    response = await FetchIndexerResponse(request);
+                }
+                else if (jsonResponse.Resource.error_code == 5 || jsonResponse.Resource.rate_limit.HasValue)
+                {
+                    _logger.Debug("Rarbg rate limit hit, retying request");
+                    response = await FetchIndexerResponse(request);
+                }
+            }
+
+            try
+            {
+                var releases = parser.ParseResponse(response).ToList();
+
+                if (releases.Count == 0)
+                {
+                    _logger.Trace(response.Content);
+                }
+
+                return new IndexerQueryResult
+                {
+                    Releases = releases,
+                    Response = response.HttpResponse
+                };
+            }
+            catch (Exception ex)
+            {
+                ex.WithData(response.HttpResponse, 128 * 1024);
+                _logger.Trace("Unexpected Response content ({0} bytes): {1}", response.HttpResponse.ResponseData.Length, response.HttpResponse.Content);
+                throw;
+            }
+        }
+
         public override object RequestAction(string action, IDictionary<string, string> query)
         {
             if (action == "checkCaptcha")

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs

@@ -41,9 +41,10 @@ namespace NzbDrone.Core.Indexers.Rarbg
             {
                 if (jsonResponse.Resource.error_code == 20 || jsonResponse.Resource.error_code == 8
                     || jsonResponse.Resource.error_code == 9 || jsonResponse.Resource.error_code == 10
-                    || jsonResponse.Resource.error_code == 5)
+                    || jsonResponse.Resource.error_code == 5 || jsonResponse.Resource.error_code == 13
+                    || jsonResponse.Resource.error_code == 14)
                 {
-                    // No results, rate limit, or imdbid not found
+                    // No results, rate limit, or imdbid/tvdb not found
                     return results;
                 }
 

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs

@@ -69,42 +69,36 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
         public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
         {
-            var request = GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories, searchCriteria.FullImdbId, searchCriteria.TmdbId);
-            return GetRequestChain(request, 2);
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories, searchCriteria.FullImdbId, searchCriteria.TmdbId));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
         {
-            var request = GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories);
-            return GetRequestChain(request, 2);
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
         {
-            var request = GetRequest(searchCriteria.SanitizedTvSearchString, searchCriteria.Categories, searchCriteria.FullImdbId, tvdbId: searchCriteria.TvdbId);
-            return GetRequestChain(request, 2);
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SanitizedTvSearchString, searchCriteria.Categories, searchCriteria.FullImdbId, tvdbId: searchCriteria.TvdbId));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
         {
-            var request = GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories);
-            return GetRequestChain(request, 2);
+            var pageableRequests = new IndexerPageableRequestChain();
+            pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
+            return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
-        {
-            var request = GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories);
-            return GetRequestChain(request, 2);
-        }
-
-        private IndexerPageableRequestChain GetRequestChain(IEnumerable<IndexerRequest> requests, int retry)
         {
             var pageableRequests = new IndexerPageableRequestChain();
-
-            for (int i = 0; i < retry; i++)
-            {
-                pageableRequests.AddTier(requests);
-            }
+            pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories));
 
             return pageableRequests;
         }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgResponse.cs

@@ -7,6 +7,7 @@ namespace NzbDrone.Core.Indexers.Rarbg
     {
         public string error { get; set; }
         public int? error_code { get; set; }
+        public int? rate_limit { get; set; }
         public List<RarbgTorrent> torrent_results { get; set; }
     }
 

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs

@@ -3,14 +3,14 @@ using Newtonsoft.Json.Linq;
 using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.EnvironmentInfo;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 
 namespace NzbDrone.Core.Indexers.Rarbg
 {
     public interface IRarbgTokenProvider
     {
-        string GetToken(RarbgSettings settings, string baseUrl);
+        string GetToken(RarbgSettings settings);
+        void ExpireToken(RarbgSettings settings);
     }
 
     public class RarbgTokenProvider : IRarbgTokenProvider
@@ -26,12 +26,17 @@ namespace NzbDrone.Core.Indexers.Rarbg
             _logger = logger;
         }
 
-        public string GetToken(RarbgSettings settings, string baseUrl)
+        public void ExpireToken(RarbgSettings settings)
         {
-            return _tokenCache.Get(baseUrl,
+            _tokenCache.Remove(settings.BaseUrl);
+        }
+
+        public string GetToken(RarbgSettings settings)
+        {
+            return _tokenCache.Get(settings.BaseUrl,
                 () =>
                 {
-                    var requestBuilder = new HttpRequestBuilder(baseUrl.Trim('/'))
+                    var requestBuilder = new HttpRequestBuilder(settings.BaseUrl.Trim('/'))
                         .WithRateLimit(3.0)
                         .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}")
                         .Accept(HttpAccept.Json);