diff --git a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs index dca1d12c0..ec593996c 100644 --- a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs +++ b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs @@ -1,4 +1,6 @@ -using System.Net; +using System.Collections.Generic; +using System.Linq; +using System.Net; using System.Text.RegularExpressions; using NLog; using NzbDrone.Common.Http; @@ -10,6 +12,7 @@ namespace NzbDrone.Core.Http.CloudFlare private const string _cloudFlareChallengeScript = "cdn-cgi/scripts/cf.challenge.js"; private readonly Logger _logger; private static readonly Regex _cloudFlareRegex = new Regex(@"data-ray=""(?[\w-_]+)"".*?data-sitekey=""(?[\w-_]+)"".*?data-stoken=""(?[\w-_]+)""", RegexOptions.Compiled); + private static readonly HashSet CloudflareServerNames = new HashSet { "cloudflare", "cloudflare-nginx" }; public CloudFlareHttpInterceptor(Logger logger) { @@ -23,12 +26,19 @@ namespace NzbDrone.Core.Http.CloudFlare public HttpResponse PostResponse(HttpResponse response) { + //ToDo: Determine if the ChallengeScript is still valid and update if needed if (response.StatusCode == HttpStatusCode.Forbidden && response.Content.Contains(_cloudFlareChallengeScript)) { _logger.Debug("CloudFlare CAPTCHA block on {0}", response.Request.Url); throw new CloudFlareCaptchaException(response, CreateCaptchaRequest(response)); } + if (response.StatusCode == HttpStatusCode.ServiceUnavailable && IsCloudflareProtected(response)) + { + _logger.Warn("Unable to connect. CloudFlare detected FlareSolver may be needed", response.Request.Url); + throw new CloudFlareProtectedException(response); + } + return response; } @@ -50,5 +60,12 @@ namespace NzbDrone.Core.Http.CloudFlare ResponseUrl = response.Request.Url + new HttpUri("/cdn-cgi/l/chk_captcha") }; } + + private bool IsCloudflareProtected(HttpResponse response) + { + // check response headers for cloudflare + return response.Headers.Any(i => + i.Key != null && i.Key.ToLower() == "server" && CloudflareServerNames.Contains(i.Value.ToLower())); + } } } diff --git a/src/NzbDrone.Core/Http/CloudFlare/CloudFlareProtectedException.cs b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareProtectedException.cs new file mode 100644 index 000000000..ba858b24b --- /dev/null +++ b/src/NzbDrone.Core/Http/CloudFlare/CloudFlareProtectedException.cs @@ -0,0 +1,15 @@ +using NzbDrone.Common.Exceptions; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.Http.CloudFlare +{ + public class CloudFlareProtectedException : NzbDroneException + { + public HttpResponse Response { get; set; } + public CloudFlareProtectedException(HttpResponse response) + : base("Cloudflare Detected. Flaresolverr may be required. {0} has been blocked by CloudFlare", response.Request.Url.Host) + { + Response = response; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index 23285b6c3..a2db87984 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -188,19 +188,21 @@ namespace NzbDrone.Core.Indexers.Cardigann } catch (HttpException ex) { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) + var response = ex.Response; + var responseStatus = response.StatusCode; + if (responseStatus == HttpStatusCode.NotFound) { - _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri); + _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri); throw new ReleaseUnavailableException("Downloading torrent failed", ex); } - if ((int)ex.Response.StatusCode == 429) + if (responseStatus == HttpStatusCode.TooManyRequests) { - _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri); + _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); } else { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri); + _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); } throw new ReleaseDownloadException("Downloading torrent failed", ex); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs index 1916a01ed..1c5deb77c 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs @@ -9,6 +9,7 @@ using AngleSharp.Xml.Parser; using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers.Definitions.Cardigann.Exceptions; using NzbDrone.Core.Indexers.Exceptions; @@ -37,8 +38,9 @@ namespace NzbDrone.Core.Indexers.Cardigann _logger.Debug("Parsing"); var indexerLogging = _configService.LogIndexerResponse; + var indexerResponseStatus = indexerResponse.HttpResponse.StatusCode; - if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + if (indexerResponseStatus != HttpStatusCode.OK) { // Remove cookie cache if (indexerResponse.HttpResponse.HasHttpRedirect && indexerResponse.HttpResponse.RedirectUrl @@ -48,7 +50,20 @@ namespace NzbDrone.Core.Indexers.Cardigann throw new IndexerException(indexerResponse, "We are being redirected to the login page. Most likely your session expired or was killed. Try testing the indexer in the settings."); } - throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); + // Catch common http exceptions before parsing + switch (indexerResponseStatus) + { + case HttpStatusCode.BadRequest: + case HttpStatusCode.Forbidden: + case HttpStatusCode.BadGateway: + case HttpStatusCode.ServiceUnavailable: + case HttpStatusCode.GatewayTimeout: + throw new HttpException(indexerResponse.HttpResponse); + case HttpStatusCode.Unauthorized: + throw new IndexerAuthException(indexerResponse.HttpResponse.Content.ToString()); + default: + throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); + } } var results = indexerResponse.Content; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs index 30ccb3ad0..475e76bf6 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs @@ -4,7 +4,6 @@ using System.Collections.Specialized; using System.Linq; using System.Net; using System.Net.Http; -using System.Text; using System.Threading.Tasks; using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; diff --git a/src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs b/src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs index 445a0980a..a313cdd72 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/IPTorrents.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Collections.Specialized; using System.Linq; +using System.Net; using AngleSharp.Html.Parser; using FluentValidation; using NLog; @@ -264,6 +265,11 @@ namespace NzbDrone.Core.Indexers.Definitions public IList ParseResponse(IndexerResponse indexerResponse) { + if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new HttpException(indexerResponse.HttpResponse); + } + var torrentInfos = new List(); var parser = new HtmlParser(); diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 0e0d435fb..1c8474f19 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -197,28 +197,6 @@ namespace NzbDrone.Core.Indexers _indexerStatusService.RecordSuccess(Definition.Id); } - catch (WebException webException) - { - if (webException.Status == WebExceptionStatus.NameResolutionFailure || - webException.Status == WebExceptionStatus.ConnectFailure) - { - _indexerStatusService.RecordConnectionFailure(Definition.Id); - } - else - { - _indexerStatusService.RecordFailure(Definition.Id); - } - - if (webException.Message.Contains("502") || webException.Message.Contains("503") || - webException.Message.Contains("timed out")) - { - _logger.Warn("{0} server is currently unavailable. {1} {2}", this, url, webException.Message); - } - else - { - _logger.Warn("{0} {1} {2}", this, url, webException.Message); - } - } catch (TooManyRequestsException ex) { result.Queries.Add(new IndexerQueryResult { Response = ex.Response }); @@ -234,11 +212,34 @@ namespace NzbDrone.Core.Indexers _logger.Warn("API Request Limit reached for {0}", this); } + catch (WebException ex) + { + var indexerResponseStatus = ex.Status; + if (indexerResponseStatus == WebExceptionStatus.NameResolutionFailure || + indexerResponseStatus == WebExceptionStatus.ConnectFailure) + { + _indexerStatusService.RecordConnectionFailure(Definition.Id); + _logger.Warn("Error Resolving (DNS Name Resolution Failure) or Connecting to {0}", ex); + } + } catch (HttpException ex) { - result.Queries.Add(new IndexerQueryResult { Response = ex.Response }); - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Warn("{0} {1}", this, ex.Message); + var indexerResponse = ex.Response; + var indexerResponseStatus = indexerResponse.StatusCode; + if (indexerResponseStatus == HttpStatusCode.BadGateway || indexerResponseStatus == HttpStatusCode.ServiceUnavailable) + { + _indexerStatusService.RecordConnectionFailure(Definition.Id); + _logger.Warn("{0} server is currently unavailable. {1} {2}", this, ex.Request.Url, ex.Message); + } + + if (indexerResponseStatus == HttpStatusCode.BadRequest && indexerResponse.Content.Contains("not support the requested query")) + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Warn(ex, "Indexer does not support the current query. Check the log for more details."); + } + + _indexerStatusService.RecordConnectionFailure(Definition.Id); + _logger.Warn(ex, "Unknown HTTP Exception - Unable to connect to indexer"); } catch (RequestLimitReachedException ex) { @@ -399,7 +400,7 @@ namespace NzbDrone.Core.Indexers { _logger.Warn("HTTP Error - {0}", response); - if ((int)response.StatusCode == 429) + if (response.StatusCode == HttpStatusCode.TooManyRequests) { throw new TooManyRequestsException(request.HttpRequest, response); } @@ -482,6 +483,10 @@ namespace NzbDrone.Core.Indexers return new ValidationFailure("CaptchaToken", "Site protected by CloudFlare CAPTCHA. Valid CAPTCHA token required."); } } + catch (CloudFlareProtectedException ex) + { + return new ValidationFailure(string.Empty, "CloudFlare detected - Unable to connect to indexer. FlareSolver may be needed or your Cookie has expired [" + ex.Response.Request.Url.Host.ToString() + "]"); + } catch (UnsupportedFeedException ex) { _logger.Warn(ex, "Indexer feed is not supported"); @@ -494,20 +499,32 @@ namespace NzbDrone.Core.Indexers return new ValidationFailure(string.Empty, "Unable to connect to indexer. " + ex.Message); } - catch (HttpException ex) + catch (WebException ex) { - if (ex.Response.StatusCode == HttpStatusCode.BadRequest && - ex.Response.Content.Contains("not support the requested query")) + if (ex.Status == WebExceptionStatus.NameResolutionFailure || ex.Status == WebExceptionStatus.ConnectFailure) { - _logger.Warn(ex, "Indexer does not support the query"); - return new ValidationFailure(string.Empty, "Indexer does not support the current query. Check if the categories and or searching for movies are supported. Check the log for more details."); + _logger.Warn("{0} server could not be reached. {1}", this, ex.Message); + return new ValidationFailure(string.Empty, string.Format("{0} server could not be reached. {1}", this, ex.Message)); } - else + } + catch (HttpException ex) + { + var indexerResponse = ex.Response; + var indexerResponseStatus = indexerResponse.StatusCode; + if (indexerResponseStatus == HttpStatusCode.BadGateway || indexerResponseStatus == HttpStatusCode.ServiceUnavailable) { - _logger.Warn(ex, "Unable to connect to indexer"); + _logger.Warn("{0} server is currently unavailable. {1} {2}", this, ex.Request.Url, ex.Message); + return new ValidationFailure(string.Empty, string.Format("{0} server is currently unavailable. {1} {2}", this, ex.Request.Url, ex.Message)); + } - return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details"); + if (indexerResponseStatus == HttpStatusCode.BadRequest && indexerResponse.Content.Contains("not support the requested query")) + { + _logger.Warn(ex, "Indexer does not support the query"); + return new ValidationFailure(string.Empty, "Indexer does not support the current query. Check the log for more details."); } + + _logger.Warn(ex, "Unknown HTTP Exception - Unable to connect to indexer"); + return new ValidationFailure(string.Empty, "Unknown HTTP Exception - Unable to connect to indexer"); } catch (Exception ex) { diff --git a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs index 451d001a7..f3a49824d 100644 --- a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs @@ -48,22 +48,24 @@ namespace NzbDrone.Core.Indexers } catch (HttpException ex) { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) + var indexerResponse = ex.Response; + var indexerResponseStatus = indexerResponse.StatusCode; + if (indexerResponseStatus == HttpStatusCode.NotFound) { _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri); throw new ReleaseUnavailableException("Downloading torrent failed", ex); } - if ((int)ex.Response.StatusCode == 429) + if (indexerResponseStatus == HttpStatusCode.TooManyRequests) { _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); + throw new ReleaseDownloadException("Downloading torrent failed - Request Limit Reached", ex); } else { _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); + throw new ReleaseDownloadException("Downloading torrent failed", ex); } - - throw new ReleaseDownloadException("Downloading torrent failed", ex); } catch (WebException ex) {