open-source-mirrors
/
Prowlarr
mirror of https://github.com/Prowlarr/Prowlarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed: Better Indexer Response Handling

Browse Source
pull/825/head
bakerboy448 3 years ago
parent a0f7d5e309
commit 646709560d
8 changed files with 120 additions and 47 deletions
Show Stats Download Patch File Download Diff File

19
src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs
Unescape View File

@ -1,4 +1,6 @@
using System.Net;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text.RegularExpressions;
using NLog;
using NzbDrone.Common.Http;
@ -10,6 +12,7 @@ namespace NzbDrone.Core.Http.CloudFlare
private const string _cloudFlareChallengeScript = "cdn-cgi/scripts/cf.challenge.js";
private readonly Logger _logger;
private static readonly Regex _cloudFlareRegex = new Regex(@"data-ray=""(?<Ray>[\w-_]+)"".*?data-sitekey=""(?<SiteKey>[\w-_]+)"".*?data-stoken=""(?<SecretToken>[\w-_]+)""", RegexOptions.Compiled);
private static readonly HashSet<string> CloudflareServerNames = new HashSet<string> { "cloudflare", "cloudflare-nginx" };
public CloudFlareHttpInterceptor(Logger logger)
{
@ -23,12 +26,19 @@ namespace NzbDrone.Core.Http.CloudFlare
public HttpResponse PostResponse(HttpResponse response)
{
//ToDo: Determine if the ChallengeScript is still valid and update if needed
if (response.StatusCode == HttpStatusCode.Forbidden && response.Content.Contains(_cloudFlareChallengeScript))
{
_logger.Debug("CloudFlare CAPTCHA block on {0}", response.Request.Url);
throw new CloudFlareCaptchaException(response, CreateCaptchaRequest(response));
}
if (response.StatusCode == HttpStatusCode.ServiceUnavailable && IsCloudflareProtected(response))
{
_logger.Warn("Unable to connect. CloudFlare detected FlareSolver may be needed", response.Request.Url);
throw new CloudFlareProtectedException(response);
}
return response;
}
@ -50,5 +60,12 @@ namespace NzbDrone.Core.Http.CloudFlare
ResponseUrl = response.Request.Url + new HttpUri("/cdn-cgi/l/chk_captcha")
};
}
private bool IsCloudflareProtected(HttpResponse response)
{
// check response headers for cloudflare
return response.Headers.Any(i =>
i.Key != null && i.Key.ToLower() == "server" && CloudflareServerNames.Contains(i.Value.ToLower()));
}
}
}

15
src/NzbDrone.Core/Http/CloudFlare/CloudFlareProtectedException.cs
Unescape View File

@ -0,0 +1,15 @@
using NzbDrone.Common.Exceptions;
using NzbDrone.Common.Http;
namespace NzbDrone.Core.Http.CloudFlare
{
public class CloudFlareProtectedException : NzbDroneException
{
public HttpResponse Response { get; set; }
public CloudFlareProtectedException(HttpResponse response)
: base("Cloudflare Detected. Flaresolverr may be required. {0} has been blocked by CloudFlare", response.Request.Url.Host)
{
Response = response;
}
}
}

12
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
Unescape View File

@ -188,19 +188,21 @@ namespace NzbDrone.Core.Indexers.Cardigann
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
var response = ex.Response;
var responseStatus = response.StatusCode;
if (responseStatus == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if ((int)ex.Response.StatusCode == 429)
if (responseStatus == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);

19
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs
Unescape View File

@ -9,6 +9,7 @@ using AngleSharp.Xml.Parser;
using Newtonsoft.Json.Linq;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Indexers.Definitions.Cardigann.Exceptions;
using NzbDrone.Core.Indexers.Exceptions;
@ -37,8 +38,9 @@ namespace NzbDrone.Core.Indexers.Cardigann
_logger.Debug("Parsing");
var indexerLogging = _configService.LogIndexerResponse;
var indexerResponseStatus = indexerResponse.HttpResponse.StatusCode;
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
if (indexerResponseStatus != HttpStatusCode.OK)
{
// Remove cookie cache
if (indexerResponse.HttpResponse.HasHttpRedirect && indexerResponse.HttpResponse.RedirectUrl
@ -48,7 +50,20 @@ namespace NzbDrone.Core.Indexers.Cardigann
throw new IndexerException(indexerResponse, "We are being redirected to the login page. Most likely your session expired or was killed. Try testing the indexer in the settings.");
}
throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
// Catch common http exceptions before parsing
switch (indexerResponseStatus)
{
case HttpStatusCode.BadRequest:
case HttpStatusCode.Forbidden:
case HttpStatusCode.BadGateway:
case HttpStatusCode.ServiceUnavailable:
case HttpStatusCode.GatewayTimeout:
throw new HttpException(indexerResponse.HttpResponse);
case HttpStatusCode.Unauthorized:
throw new IndexerAuthException(indexerResponse.HttpResponse.Content.ToString());
default:
throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
}
}
var results = indexerResponse.Content;

1
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs
Unescape View File

@ -4,7 +4,6 @@ using System.Collections.Specialized;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Text;
using System.Threading.Tasks;
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;

6
src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs
Unescape View File

@ -2,6 +2,7 @@ using System;
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Net;
using AngleSharp.Html.Parser;
using FluentValidation;
using NLog;
@ -264,6 +265,11 @@ namespace NzbDrone.Core.Indexers.Definitions
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
{
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
{
throw new HttpException(indexerResponse.HttpResponse);
}
var torrentInfos = new List<TorrentInfo>();
var parser = new HtmlParser();

85
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
Unescape View File

@ -197,28 +197,6 @@ namespace NzbDrone.Core.Indexers
_indexerStatusService.RecordSuccess(Definition.Id);
}
catch (WebException webException)
{
if (webException.Status == WebExceptionStatus.NameResolutionFailure ||
webException.Status == WebExceptionStatus.ConnectFailure)
{
_indexerStatusService.RecordConnectionFailure(Definition.Id);
}
else
{
_indexerStatusService.RecordFailure(Definition.Id);
}
if (webException.Message.Contains("502") || webException.Message.Contains("503") ||
webException.Message.Contains("timed out"))
{
_logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message);
}
else
{
_logger.Warn("{0} {1} {2}", this, url, webException.Message);
}
}
catch (TooManyRequestsException ex)
{
result.Queries.Add(new IndexerQueryResult { Response = ex.Response });
@ -234,11 +212,34 @@ namespace NzbDrone.Core.Indexers
_logger.Warn("API Request Limit reached for {0}", this);
}
catch (WebException ex)
{
var indexerResponseStatus = ex.Status;
if (indexerResponseStatus == WebExceptionStatus.NameResolutionFailure ||
indexerResponseStatus == WebExceptionStatus.ConnectFailure)
{
_indexerStatusService.RecordConnectionFailure(Definition.Id);
_logger.Warn("Error Resolving (DNS Name Resolution Failure) or Connecting to {0}", ex);
}
}
catch (HttpException ex)
{
result.Queries.Add(new IndexerQueryResult { Response = ex.Response });
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn("{0} {1}", this, ex.Message);
var indexerResponse = ex.Response;
var indexerResponseStatus = indexerResponse.StatusCode;
if (indexerResponseStatus == HttpStatusCode.BadGateway || indexerResponseStatus == HttpStatusCode.ServiceUnavailable)
{
_indexerStatusService.RecordConnectionFailure(Definition.Id);
_logger.Warn("{0} server is currently unavailable. {1} {2}", this, ex.Request.Url, ex.Message);
}
if (indexerResponseStatus == HttpStatusCode.BadRequest && indexerResponse.Content.Contains("not support the requested query"))
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Warn(ex, "Indexer does not support the current query. Check the log for more details.");
}
_indexerStatusService.RecordConnectionFailure(Definition.Id);
_logger.Warn(ex, "Unknown HTTP Exception - Unable to connect to indexer");
}
catch (RequestLimitReachedException ex)
{
@ -399,7 +400,7 @@ namespace NzbDrone.Core.Indexers
{
_logger.Warn("HTTP Error - {0}", response);
if ((int)response.StatusCode == 429)
if (response.StatusCode == HttpStatusCode.TooManyRequests)
{
throw new TooManyRequestsException(request.HttpRequest, response);
}
@ -482,6 +483,10 @@ namespace NzbDrone.Core.Indexers
return new ValidationFailure("CaptchaToken", "Site protected by CloudFlare CAPTCHA. Valid CAPTCHA token required.");
}
}
catch (CloudFlareProtectedException ex)
{
return new ValidationFailure(string.Empty, "CloudFlare detected - Unable to connect to indexer. FlareSolver may be needed or your Cookie has expired [" + ex.Response.Request.Url.Host.ToString() + "]");
}
catch (UnsupportedFeedException ex)
{
_logger.Warn(ex, "Indexer feed is not supported");
@ -494,20 +499,32 @@ namespace NzbDrone.Core.Indexers
return new ValidationFailure(string.Empty, "Unable to connect to indexer. " + ex.Message);
}
catch (HttpException ex)
catch (WebException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.BadRequest &&
ex.Response.Content.Contains("not support the requested query"))
if (ex.Status == WebExceptionStatus.NameResolutionFailure || ex.Status == WebExceptionStatus.ConnectFailure)
{
_logger.Warn(ex, "Indexer does not support the query");
return new ValidationFailure(string.Empty, "Indexer does not support the current query. Check if the categories and or searching for movies are supported. Check the log for more details.");
_logger.Warn("{0} server could not be reached. {1}", this, ex.Message);
return new ValidationFailure(string.Empty, string.Format("{0} server could not be reached. {1}", this, ex.Message));
}
else
}
catch (HttpException ex)
{
var indexerResponse = ex.Response;
var indexerResponseStatus = indexerResponse.StatusCode;
if (indexerResponseStatus == HttpStatusCode.BadGateway || indexerResponseStatus == HttpStatusCode.ServiceUnavailable)
{
_logger.Warn(ex, "Unable to connect to indexer");
_logger.Warn("{0} server is currently unavailable. {1} {2}", this, ex.Request.Url, ex.Message);
return new ValidationFailure(string.Empty, string.Format("{0} server is currently unavailable. {1} {2}", this, ex.Request.Url, ex.Message));
}
return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details");
if (indexerResponseStatus == HttpStatusCode.BadRequest && indexerResponse.Content.Contains("not support the requested query"))
{
_logger.Warn(ex, "Indexer does not support the query");
return new ValidationFailure(string.Empty, "Indexer does not support the current query. Check the log for more details.");
}
_logger.Warn(ex, "Unknown HTTP Exception - Unable to connect to indexer");
return new ValidationFailure(string.Empty, "Unknown HTTP Exception - Unable to connect to indexer");
}
catch (Exception ex)
{

10
src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs
Unescape View File

@ -48,22 +48,24 @@ namespace NzbDrone.Core.Indexers
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
var indexerResponse = ex.Response;
var indexerResponseStatus = indexerResponse.StatusCode;
if (indexerResponseStatus == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if ((int)ex.Response.StatusCode == 429)
if (indexerResponseStatus == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading torrent failed - Request Limit Reached", ex);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{

Write Preview
Loading…
Cancel
Save