Rework Cloudflare Protection Detection

src/NzbDrone.Common.Test/Http/HttpClientFixture.cs

@@ -67,7 +67,7 @@ namespace NzbDrone.Common.Test.Http
 
                 res = _httpClient.GetAsync($"https://{site}/status/429").GetAwaiter().GetResult();
 
-                if (res == null || res.StatusCode != (HttpStatusCode)429)
+                if (res == null || res.StatusCode != HttpStatusCode.TooManyRequests)
                 {
                     return false;
                 }

src/NzbDrone.Common/Http/HttpClient.cs

@@ -118,7 +118,7 @@ namespace NzbDrone.Common.Http
                     _logger.Warn("HTTP Error - {0}", response);
                 }
 
-                if ((int)response.StatusCode == 429)
+                if (response.StatusCode == HttpStatusCode.TooManyRequests)
                 {
                     throw new TooManyRequestsException(request, response);
                 }

src/NzbDrone.Core/Http/CloudFlare/CloudFlareCaptchaException.cs

@@ -1,21 +0,0 @@
-using NzbDrone.Common.Exceptions;
-using NzbDrone.Common.Http;
-
-namespace NzbDrone.Core.Http.CloudFlare
-{
-    public class CloudFlareCaptchaException : NzbDroneException
-    {
-        public HttpResponse Response { get; set; }
-
-        public CloudFlareCaptchaRequest CaptchaRequest { get; set; }
-
-        public CloudFlareCaptchaException(HttpResponse response, CloudFlareCaptchaRequest captchaRequest)
-            : base("Unable to access {0}, blocked by CloudFlare CAPTCHA. Likely due to shared-IP VPN.", response.Request.Url.Host)
-        {
-            Response = response;
-            CaptchaRequest = captchaRequest;
-        }
-
-        public bool IsExpired => Response.Request.Cookies.ContainsKey("cf_clearance");
-    }
-}

src/NzbDrone.Core/Http/CloudFlare/CloudFlareCaptchaRequest.cs

@@ -1,15 +0,0 @@
-using NzbDrone.Common.Http;
-
-namespace NzbDrone.Core.Http.CloudFlare
-{
-    public class CloudFlareCaptchaRequest
-    {
-        public string Host { get; set; }
-        public string SiteKey { get; set; }
-
-        public string Ray { get; set; }
-        public string SecretToken { get; set; }
-
-        public HttpUri ResponseUrl { get; set; }
-    }
-}

src/NzbDrone.Core/Http/CloudFlare/CloudFlareDetectionService.cs

@@ -0,0 +1,45 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Text.RegularExpressions;
+using NLog;
+using NzbDrone.Common.Http;
+
+namespace NzbDrone.Core.Http.CloudFlare
+{
+    public class CloudFlareDetectionService
+    {
+        private static readonly HashSet<string> CloudflareServerNames = new HashSet<string> { "cloudflare", "cloudflare-nginx", "ddos-guard" };
+        private readonly Logger _logger;
+
+        public CloudFlareDetectionService(Logger logger)
+        {
+            _logger = logger;
+        }
+
+        public static bool IsCloudflareProtected(HttpResponse response)
+        {
+            if (!response.Headers.Any(i => i.Key != null && i.Key.ToLower() == "server" && CloudflareServerNames.Contains(i.Value.ToLower())))
+            {
+                return false;
+            }
+
+            // detect CloudFlare and DDoS-GUARD
+            if (response.StatusCode.Equals(HttpStatusCode.ServiceUnavailable) ||
+                response.StatusCode.Equals(HttpStatusCode.Forbidden))
+            {
+                return true; // Defected CloudFlare and DDoS-GUARD
+            }
+
+            // detect Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
+            if (response.Headers.Vary.ToString() == "Accept-Encoding,User-Agent" &&
+                response.Headers.ContentEncoding.ToString() == "" &&
+                response.Content.ToLower().Contains("ddos"))
+            {
+                return true;
+            }
+
+            return false;
+        }
+    }
+}

src/NzbDrone.Core/Http/CloudFlare/CloudFlareHttpInterceptor.cs

@@ -1,54 +0,0 @@
-using System.Net;
-using System.Text.RegularExpressions;
-using NLog;
-using NzbDrone.Common.Http;
-
-namespace NzbDrone.Core.Http.CloudFlare
-{
-    public class CloudFlareHttpInterceptor : IHttpRequestInterceptor
-    {
-        private const string _cloudFlareChallengeScript = "cdn-cgi/scripts/cf.challenge.js";
-        private readonly Logger _logger;
-        private static readonly Regex _cloudFlareRegex = new Regex(@"data-ray=""(?<Ray>[\w-_]+)"".*?data-sitekey=""(?<SiteKey>[\w-_]+)"".*?data-stoken=""(?<SecretToken>[\w-_]+)""", RegexOptions.Compiled);
-
-        public CloudFlareHttpInterceptor(Logger logger)
-        {
-            _logger = logger;
-        }
-
-        public HttpRequest PreRequest(HttpRequest request)
-        {
-            return request;
-        }
-
-        public HttpResponse PostResponse(HttpResponse response)
-        {
-            if (response.StatusCode == HttpStatusCode.Forbidden && response.Content.Contains(_cloudFlareChallengeScript))
-            {
-                _logger.Debug("CloudFlare CAPTCHA block on {0}", response.Request.Url);
-                throw new CloudFlareCaptchaException(response, CreateCaptchaRequest(response));
-            }
-
-            return response;
-        }
-
-        private CloudFlareCaptchaRequest CreateCaptchaRequest(HttpResponse response)
-        {
-            var match = _cloudFlareRegex.Match(response.Content);
-
-            if (!match.Success)
-            {
-                return null;
-            }
-
-            return new CloudFlareCaptchaRequest
-            {
-                Host = response.Request.Url.Host,
-                SiteKey = match.Groups["SiteKey"].Value,
-                Ray = match.Groups["Ray"].Value,
-                SecretToken = match.Groups["SecretToken"].Value,
-                ResponseUrl = response.Request.Url + new HttpUri("/cdn-cgi/l/chk_captcha")
-            };
-        }
-    }
-}

src/NzbDrone.Core/Http/CloudFlare/CloudFlareProtectionException.cs

@@ -0,0 +1,16 @@
+using NzbDrone.Common.Exceptions;
+using NzbDrone.Common.Http;
+
+namespace NzbDrone.Core.Http.CloudFlare
+{
+    public class CloudFlareProtectionException : NzbDroneException
+    {
+        public HttpResponse Response { get; set; }
+
+        public CloudFlareProtectionException(HttpResponse response)
+            : base("Unable to access {0}, blocked by CloudFlare Protection.", response.Request.Url.Host)
+        {
+            Response = response;
+        }
+    }
+}

src/NzbDrone.Core/IndexerProxies/FlareSolverr/FlareSolverr.cs

@@ -11,6 +11,7 @@ using NzbDrone.Common.Cloud;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Common.Serializer;
+using NzbDrone.Core.Http.CloudFlare;
 using NzbDrone.Core.Localization;
 using NzbDrone.Core.Validation;
 
@@ -45,7 +46,7 @@ namespace NzbDrone.Core.IndexerProxies.FlareSolverr
 
         public override HttpResponse PostResponse(HttpResponse response)
         {
-            if (!IsCloudflareProtected(response))
+            if (!CloudFlareDetectionService.IsCloudflareProtected(response))
             {
                 _logger.Debug("CF Protection not detected, returning original response");
                 return response;
@@ -53,14 +54,12 @@ namespace NzbDrone.Core.IndexerProxies.FlareSolverr
 
             var flaresolverrResponse = _httpClient.Execute(GenerateFlareSolverrRequest(response.Request));
 
-            FlareSolverrResponse result = null;
-
             if (flaresolverrResponse.StatusCode != HttpStatusCode.OK && flaresolverrResponse.StatusCode != HttpStatusCode.InternalServerError)
             {
                 throw new FlareSolverrException("HTTP StatusCode not 200 or 500. Status is :" + response.StatusCode);
             }
 
-            result = JsonConvert.DeserializeObject<FlareSolverrResponse>(flaresolverrResponse.Content);
+            var result = JsonConvert.DeserializeObject<FlareSolverrResponse>(flaresolverrResponse.Content);
 
             var newRequest = response.Request;
 
@@ -76,31 +75,6 @@ namespace NzbDrone.Core.IndexerProxies.FlareSolverr
             return finalResponse;
         }
 
-        private static bool IsCloudflareProtected(HttpResponse response)
-        {
-            if (!response.Headers.Any(i => i.Key != null && i.Key.ToLower() == "server" && CloudflareServerNames.Contains(i.Value.ToLower())))
-            {
-                return false;
-            }
-
-            // detect CloudFlare and DDoS-GUARD
-            if (response.StatusCode.Equals(HttpStatusCode.ServiceUnavailable) ||
-                response.StatusCode.Equals(HttpStatusCode.Forbidden))
-            {
-                return true; // Defected CloudFlare and DDoS-GUARD
-            }
-
-            // detect Custom CloudFlare for EbookParadijs, Film-Paleis, MuziekFabriek and Puur-Hollands
-            if (response.Headers.Vary.ToString() == "Accept-Encoding,User-Agent" &&
-                response.Headers.ContentEncoding.ToString() == "" &&
-                response.Content.ToLower().Contains("ddos"))
-            {
-                return true;
-            }
-
-            return false;
-        }
-
         private void InjectCookies(HttpRequest request, FlareSolverrResponse flareSolverrResponse)
         {
             var rCookies = flareSolverrResponse.Solution.Cookies;

src/NzbDrone.Core/Indexers/Definitions/Anidub.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Collections.Specialized;
 using System.Globalization;
 using System.Linq;
+using System.Net;
 using System.Net.Http;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -493,7 +494,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 // Throw common http errors here before we try to parse
                 if (releaseResponse.HttpResponse.HasHttpError)
                 {
-                    if ((int)releaseResponse.HttpResponse.StatusCode == 429)
+                    if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests)
                     {
                         throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse);
                     }

src/NzbDrone.Core/Indexers/Definitions/Animedia.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Collections.Specialized;
+using System.Net;
 using System.Text;
 using System.Text.RegularExpressions;
 using AngleSharp.Html.Parser;
@@ -307,7 +308,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 // Throw common http errors here before we try to parse
                 if (releaseResponse.HttpResponse.HasHttpError)
                 {
-                    if ((int)releaseResponse.HttpResponse.StatusCode == 429)
+                    if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests)
                     {
                         throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse);
                     }

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -194,7 +194,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     throw new ReleaseUnavailableException("Downloading torrent failed", ex);
                 }
 
-                if ((int)ex.Response.StatusCode == 429)
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
                 {
                     _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
                 }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/Rarbg.cs

@@ -101,29 +101,12 @@ namespace NzbDrone.Core.Indexers.Rarbg
             {
                 Settings.Validate().Filter("BaseUrl").ThrowOnError();
 
-                try
-                {
-                    var request = new HttpRequestBuilder(Settings.BaseUrl.Trim('/'))
+                var request = new HttpRequestBuilder(Settings.BaseUrl.Trim('/'))
                            .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}")
                            .Accept(HttpAccept.Json)
                            .Build();
 
-                    _httpClient.Get(request);
-                }
-                catch (CloudFlareCaptchaException ex)
-                {
-                    return new
-                    {
-                        captchaRequest = new
-                        {
-                            host = ex.CaptchaRequest.Host,
-                            ray = ex.CaptchaRequest.Ray,
-                            siteKey = ex.CaptchaRequest.SiteKey,
-                            secretToken = ex.CaptchaRequest.SecretToken,
-                            responseUrl = ex.CaptchaRequest.ResponseUrl.FullUri,
-                        }
-                    };
-                }
+                _httpClient.Get(request);
 
                 return new
                 {

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgRequestGenerator.cs

@@ -26,12 +26,6 @@ namespace NzbDrone.Core.Indexers.Rarbg
                 .Resource("/pubapi_v2.php")
                 .Accept(HttpAccept.Json);
 
-            if (Settings.CaptchaToken.IsNotNullOrWhiteSpace())
-            {
-                requestBuilder.UseSimplifiedUserAgent = true;
-                requestBuilder.SetCookie("cf_clearance", Settings.CaptchaToken);
-            }
-
             requestBuilder.AddQueryParam("mode", "search");
 
             if (imdbId.IsNotNullOrWhiteSpace())

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgSettings.cs

@@ -14,8 +14,5 @@ namespace NzbDrone.Core.Indexers.Rarbg
 
         [FieldDefinition(2, Type = FieldType.Checkbox, Label = "Ranked Only", HelpText = "Only include ranked results.")]
         public bool RankedOnly { get; set; }
-
-        [FieldDefinition(3, Type = FieldType.Captcha, Label = "CAPTCHA Token", HelpText = "CAPTCHA Clearance token used to handle CloudFlare Anti-DDOS measures on shared-ip VPNs.")]
-        public string CaptchaToken { get; set; }
     }
 }

src/NzbDrone.Core/Indexers/Definitions/Rarbg/RarbgTokenProvider.cs

@@ -36,12 +36,6 @@ namespace NzbDrone.Core.Indexers.Rarbg
                         .Resource($"/pubapi_v2.php?get_token=get_token&app_id={BuildInfo.AppName}")
                         .Accept(HttpAccept.Json);
 
-                    if (settings.CaptchaToken.IsNotNullOrWhiteSpace())
-                    {
-                        requestBuilder.UseSimplifiedUserAgent = true;
-                        requestBuilder.SetCookie("cf_clearance", settings.CaptchaToken);
-                    }
-
                     var response = _httpClient.Get<JObject>(requestBuilder.Build());
 
                     return response.Resource["token"].ToString();

src/NzbDrone.Core/Indexers/Definitions/SpeedApp.cs

@@ -154,7 +154,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                     throw new ReleaseUnavailableException("Downloading torrent failed", ex);
                 }
 
-                if ((int)ex.Response.StatusCode == 429)
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
                 {
                     _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
                 }

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -232,7 +232,7 @@ namespace NzbDrone.Core.Indexers
                     _indexerStatusService.RecordFailure(Definition.Id, TimeSpan.FromHours(1));
                 }
 
-                _logger.Warn("API Request Limit reached for {0}", this);
+                _logger.Warn("Request Limit reached for {0}", this);
             }
             catch (HttpException ex)
             {
@@ -251,19 +251,12 @@ namespace NzbDrone.Core.Indexers
                 _indexerStatusService.RecordFailure(Definition.Id);
                 _logger.Warn("Invalid Credentials for {0} {1}", this, url);
             }
-            catch (CloudFlareCaptchaException ex)
+            catch (CloudFlareProtectionException ex)
             {
                 result.Queries.Add(new IndexerQueryResult { Response = ex.Response });
                 _indexerStatusService.RecordFailure(Definition.Id);
                 ex.WithData("FeedUrl", url);
-                if (ex.IsExpired)
-                {
-                    _logger.Error(ex, "Expired CAPTCHA token for {0}, please refresh in indexer settings.", this);
-                }
-                else
-                {
-                    _logger.Error(ex, "CAPTCHA token required for {0}, check indexer settings.", this);
-                }
+                _logger.Error(ex, "Cloudflare protection detected for {0}, Flaresolverr may be required.", this);
             }
             catch (IndexerException ex)
             {
@@ -399,12 +392,17 @@ namespace NzbDrone.Core.Indexers
             {
                 _logger.Warn("HTTP Error - {0}", response);
 
-                if ((int)response.StatusCode == 429)
+                if (response.StatusCode == HttpStatusCode.TooManyRequests)
                 {
                     throw new TooManyRequestsException(request.HttpRequest, response);
                 }
             }
 
+            if (CloudFlareDetectionService.IsCloudflareProtected(response))
+            {
+                throw new CloudFlareProtectionException(response);
+            }
+
             UpdateCookies(Cookies, DateTime.Now + TimeSpan.FromDays(30));
 
             return new IndexerResponse(request, response);
@@ -471,16 +469,9 @@ namespace NzbDrone.Core.Indexers
             {
                 _logger.Warn("Request limit reached: " + ex.Message);
             }
-            catch (CloudFlareCaptchaException ex)
+            catch (CloudFlareProtectionException ex)
             {
-                if (ex.IsExpired)
-                {
-                    return new ValidationFailure("CaptchaToken", "CloudFlare CAPTCHA token expired, please Refresh.");
-                }
-                else
-                {
-                    return new ValidationFailure("CaptchaToken", "Site protected by CloudFlare CAPTCHA. Valid CAPTCHA token required.");
-                }
+                return new ValidationFailure(string.Empty, ex.Message);
             }
             catch (UnsupportedFeedException ex)
             {

src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs

@@ -54,7 +54,7 @@ namespace NzbDrone.Core.Indexers
                     throw new ReleaseUnavailableException("Downloading torrent failed", ex);
                 }
 
-                if ((int)ex.Response.StatusCode == 429)
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
                 {
                     _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
                 }

src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs

@@ -52,7 +52,7 @@ namespace NzbDrone.Core.Indexers
                     throw new ReleaseUnavailableException("Downloading nzb failed", ex);
                 }
 
-                if ((int)ex.Response.StatusCode == 429)
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
                 {
                     _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
                 }