diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs index 4fc04fbf4..3f3befd76 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs @@ -1,6 +1,11 @@ using System; +using System.Collections; using System.Collections.Generic; using System.Linq; +using System.Net.Http.Json; +using System.Threading.Tasks; +using System.Web; +using Newtonsoft.Json; using NLog; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; @@ -8,7 +13,9 @@ using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Exceptions; using NzbDrone.Core.Http.CloudFlare; +using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.Parser; using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Rarbg @@ -95,6 +102,57 @@ namespace NzbDrone.Core.Indexers.Rarbg return caps; } + protected override async Task FetchPage(IndexerRequest request, IParseIndexerResponse parser) + { + var response = await FetchIndexerResponse(request); + + // try and recover from token or rate limit errors + var jsonResponse = new HttpResponse(response.HttpResponse); + + if (jsonResponse.Resource.error_code.HasValue) + { + if (jsonResponse.Resource.error_code == 4 || jsonResponse.Resource.error_code == 2) + { + _logger.Debug("Invalid or expired token, refreshing token from Rarbg"); + _tokenProvider.ExpireToken(Settings); + var newToken = _tokenProvider.GetToken(Settings); + + var qs = HttpUtility.ParseQueryString(request.HttpRequest.Url.Query); + qs.Set("token", newToken); + + request.HttpRequest.Url = request.Url.SetQuery(qs.GetQueryString()); + response = await FetchIndexerResponse(request); + } + else if (jsonResponse.Resource.error_code == 5 || jsonResponse.Resource.rate_limit.HasValue) + { + _logger.Debug("Rarbg rate limit hit, retying request"); + response = await FetchIndexerResponse(request); + } + } + + try + { + var releases = parser.ParseResponse(response).ToList(); + + if (releases.Count == 0) + { + _logger.Trace(response.Content); + } + + return new IndexerQueryResult + { + Releases = releases, + Response = response.HttpResponse + }; + } + catch (Exception ex) + { + ex.WithData(response.HttpResponse, 128 * 1024); + _logger.Trace("Unexpected Response content ({0} bytes): {1}", response.HttpResponse.ResponseData.Length, response.HttpResponse.Content); + throw; + } + } + public override object RequestAction(string action, IDictionary query) { if (action == "checkCaptcha") diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs index 690c82d6d..e2fcf3a7c 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgParser.cs @@ -41,9 +41,10 @@ namespace NzbDrone.Core.Indexers.Rarbg { if (jsonResponse.Resource.error_code == 20 || jsonResponse.Resource.error_code == 8 || jsonResponse.Resource.error_code == 9 || jsonResponse.Resource.error_code == 10 - || jsonResponse.Resource.error_code == 5) + || jsonResponse.Resource.error_code == 5 || jsonResponse.Resource.error_code == 13 + || jsonResponse.Resource.error_code == 14) { - // No results, rate limit, or imdbid not found + // No results, rate limit, or imdbid/tvdb not found return results; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs index cef7b90f0..d4d9548c6 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs @@ -69,42 +69,36 @@ namespace NzbDrone.Core.Indexers.Rarbg public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) { - var request = GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories, searchCriteria.FullImdbId, searchCriteria.TmdbId); - return GetRequestChain(request, 2); + var pageableRequests = new IndexerPageableRequestChain(); + pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories, searchCriteria.FullImdbId, searchCriteria.TmdbId)); + return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria) { - var request = GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories); - return GetRequestChain(request, 2); + var pageableRequests = new IndexerPageableRequestChain(); + pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories)); + return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria) { - var request = GetRequest(searchCriteria.SanitizedTvSearchString, searchCriteria.Categories, searchCriteria.FullImdbId, tvdbId: searchCriteria.TvdbId); - return GetRequestChain(request, 2); + var pageableRequests = new IndexerPageableRequestChain(); + pageableRequests.Add(GetRequest(searchCriteria.SanitizedTvSearchString, searchCriteria.Categories, searchCriteria.FullImdbId, tvdbId: searchCriteria.TvdbId)); + return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria) { - var request = GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories); - return GetRequestChain(request, 2); + var pageableRequests = new IndexerPageableRequestChain(); + pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories)); + return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria) - { - var request = GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories); - return GetRequestChain(request, 2); - } - - private IndexerPageableRequestChain GetRequestChain(IEnumerable requests, int retry) { var pageableRequests = new IndexerPageableRequestChain(); - - for (int i = 0; i < retry; i++) - { - pageableRequests.AddTier(requests); - } + pageableRequests.Add(GetRequest(searchCriteria.SanitizedSearchTerm, searchCriteria.Categories)); return pageableRequests; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgResponse.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgResponse.cs index 2ba32d32b..c93535400 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgResponse.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgResponse.cs @@ -7,6 +7,7 @@ namespace NzbDrone.Core.Indexers.Rarbg { public string error { get; set; } public int? error_code { get; set; } + public int? rate_limit { get; set; } public List torrent_results { get; set; } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs index 60069d089..5e2c37d74 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs @@ -3,14 +3,14 @@ using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.EnvironmentInfo; -using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; namespace NzbDrone.Core.Indexers.Rarbg { public interface IRarbgTokenProvider { - string GetToken(RarbgSettings settings, string baseUrl); + string GetToken(RarbgSettings settings); + void ExpireToken(RarbgSettings settings); } public class RarbgTokenProvider : IRarbgTokenProvider @@ -26,12 +26,17 @@ namespace NzbDrone.Core.Indexers.Rarbg _logger = logger; } - public string GetToken(RarbgSettings settings, string baseUrl) + public void ExpireToken(RarbgSettings settings) { - return _tokenCache.Get(baseUrl, + _tokenCache.Remove(settings.BaseUrl); + } + + public string GetToken(RarbgSettings settings) + { + return _tokenCache.Get(settings.BaseUrl, () => { - var requestBuilder = new HttpRequestBuilder(baseUrl.Trim('/')) + var requestBuilder = new HttpRequestBuilder(settings.BaseUrl.Trim('/')) .WithRateLimit(3.0) .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}") .Accept(HttpAccept.Json);