Fixed: (Indexers) Rate limit for download and auth

src/NzbDrone.Core/Download/DownloadService.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.EnsureThat;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Common.Instrumentation.Extensions;
@@ -61,14 +60,6 @@ namespace NzbDrone.Core.Download
 
             // Get the seed configuration for this release.
             // remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie);
-
-            // Limit grabs to 2 per second.
-            if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:"))
-            {
-                var url = new HttpUri(release.DownloadUrl);
-                _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
-            }
-
             var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId));
 
             string downloadClientId;

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -1,15 +1,12 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Net;
-using System.Text;
 using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider;
@@ -53,7 +50,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
             var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
                 new CardigannRequestGenerator(_configService,
                     _definitionService.GetCachedDefinition(Settings.DefinitionFile),
-                    _logger)
+                    _logger,
+                    RateLimit)
                 {
                     HttpClient = _httpClient,
                     Definition = Definition,
@@ -180,63 +178,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
             await generator.DoLogin();
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override async Task<HttpRequest> GetDownloadRequest(Uri link)
         {
             var generator = (CardigannRequestGenerator)GetRequestGenerator();
 
             var request = await generator.DownloadRequest(link);
 
-            if (request.Url.Scheme == "magnet")
-            {
-                ValidateMagnet(request.Url.FullUri);
-
-                return Encoding.UTF8.GetBytes(request.Url.FullUri);
-            }
-
             request.AllowAutoRedirect = true;
 
-            var downloadBytes = Array.Empty<byte>();
+            return request;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            ValidateTorrent(downloadBytes);
-
-            return downloadBytes;
         }
 
         protected override async Task Test(List<ValidationFailure> failures)

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs

@@ -4,7 +4,6 @@ using System.Collections.Specialized;
 using System.Linq;
 using System.Net;
 using System.Net.Http;
-using System.Text;
 using System.Threading.Tasks;
 using AngleSharp.Html.Dom;
 using AngleSharp.Html.Parser;
@@ -29,11 +28,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
         protected IHtmlDocument landingResultDocument;
         protected override string SiteLink => ResolveSiteLink();
 
+        private readonly TimeSpan _rateLimit;
+
         public CardigannRequestGenerator(IConfigService configService,
                                          CardigannDefinition definition,
-                                         Logger logger)
+                                         Logger logger,
+                                         TimeSpan rateLimit)
         : base(configService, definition, logger)
         {
+            _rateLimit = rateLimit;
         }
 
         public Func<IDictionary<string, string>> GetCookies { get; set; }
@@ -218,9 +221,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     requestBuilder.AddFormParameter(pair.Key, pair.Value);
                 }
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                var request = requestBuilder
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -356,11 +362,13 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.SetCookies(Cookies);
+                    var request = requestBuilder
-
+                        .SetCookies(Cookies)
-                    requestBuilder.Headers.Add("Referer", loginUrl);
+                        .SetHeader("Referer", loginUrl)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
 
-                    var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                    var simpleCaptchaResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                     var simpleCaptchaJSON = JObject.Parse(simpleCaptchaResult.Content);
                     var captchaSelection = simpleCaptchaJSON["images"][0]["hash"].ToString();
@@ -424,10 +432,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.Headers.Add("Referer", SiteLink);
-
-                    requestBuilder.SetCookies(Cookies);
-
                     foreach (var pair in pairs)
                     {
                         requestBuilder.AddFormParameter(pair.Key, pair.Value);
@@ -438,7 +442,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         requestBuilder.SetHeader(header.Key, header.Value);
                     }
 
-                    var request = requestBuilder.Build();
+                    var request = requestBuilder
+                        .SetCookies(Cookies)
+                        .SetHeader("Referer", SiteLink)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
+
                     request.SetContent(body);
 
                     loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -454,15 +463,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         Encoding = _encoding
                     };
 
-                    requestBuilder.SetCookies(Cookies);
-                    requestBuilder.Headers.Add("Referer", loginUrl);
-
                     foreach (var pair in pairs)
                     {
                         requestBuilder.AddFormParameter(pair.Key, pair.Value);
                     }
 
-                    loginResult = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                    var request = requestBuilder
+                        .SetCookies(Cookies)
+                        .SetHeader("Referer", loginUrl)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
+                        .Build();
+
+                    loginResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
                 }
 
                 Cookies = loginResult.GetCookies();
@@ -496,9 +508,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     Encoding = _encoding
                 };
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                var request = requestBuilder
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -521,9 +536,12 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     Encoding = _encoding
                 };
 
-                requestBuilder.Headers.Add("Referer", SiteLink);
+                var request = requestBuilder
+                    .SetHeader("Referer", SiteLink)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                var response = await HttpClient.ExecuteProxiedAsync(requestBuilder.Build(), Definition);
+                var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
                 Cookies = response.GetCookies();
 
@@ -594,14 +612,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 Encoding = _encoding
             };
 
-            requestBuilder.Headers.Add("Referer", SiteLink);
-
             if (Cookies != null)
             {
                 requestBuilder.SetCookies(Cookies);
             }
 
-            var request = requestBuilder.Build();
+            var request = requestBuilder
+                .SetHeader("Referer", SiteLink)
+                .WithRateLimit(_rateLimit.TotalSeconds)
+                .Build();
 
             landingResult = await HttpClient.ExecuteProxiedAsync(request, Definition);
 
@@ -646,6 +665,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         .SetCookies(landingResult.GetCookies())
                         .SetHeader("Referer", loginUrl.AbsoluteUri)
                         .SetEncoding(_encoding)
+                        .WithRateLimit(_rateLimit.TotalSeconds)
                         .Build();
 
                     var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
@@ -656,10 +676,8 @@ namespace NzbDrone.Core.Indexers.Cardigann
                         ImageData = response.ResponseData
                     };
                 }
-                else
+
-                {
+                _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
-                    _logger.Debug("CardigannIndexer ({0}): No captcha image found", _definition.Id);
-                }
             }
             else
             {
@@ -724,23 +742,28 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 requestLinkStr += queryCollection.GetQueryString(_encoding, separator: request.Queryseparator);
             }
 
-            var httpRequest = new HttpRequestBuilder(requestLinkStr)
+            var httpRequestBuilder = new HttpRequestBuilder(requestLinkStr)
-                .SetCookies(Cookies ?? new Dictionary<string, string>())
+            {
-                .SetEncoding(_encoding)
+                Method = method,
-                .SetHeader("Referer", referer);
+                Encoding = _encoding
-
+            };
-            httpRequest.Method = method;
 
             // Add form data for POST requests
             if (method == HttpMethod.Post)
             {
                 foreach (var param in pairs)
                 {
-                    httpRequest.AddFormParameter(param.Key, param.Value);
+                    httpRequestBuilder.AddFormParameter(param.Key, param.Value);
                 }
             }
 
-            var response = await HttpClient.ExecuteProxiedAsync(httpRequest.Build(), Definition);
+            var httpRequest = httpRequestBuilder
+                .SetCookies(Cookies ?? new Dictionary<string, string>())
+                .SetHeader("Referer", referer)
+                .WithRateLimit(_rateLimit.TotalSeconds)
+                .Build();
+
+            var response = await HttpClient.ExecuteProxiedAsync(httpRequest, Definition);
 
             _logger.Debug("CardigannIndexer ({0}): handleRequest() remote server returned {1}", _definition.Id, response.StatusCode);
             return response;
@@ -766,6 +789,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                     .SetCookies(Cookies ?? new Dictionary<string, string>())
                     .SetHeaders(headers ?? new Dictionary<string, string>())
                     .SetEncoding(_encoding)
+                    .WithRateLimit(_rateLimit.TotalSeconds)
                     .Build();
 
                 request.AllowAutoRedirect = true;
@@ -856,6 +880,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                                     .SetCookies(Cookies ?? new Dictionary<string, string>())
                                     .SetHeaders(headers ?? new Dictionary<string, string>())
                                     .SetEncoding(_encoding)
+                                    .WithRateLimit(_rateLimit.TotalSeconds)
                                     .Build();
 
                                 response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition);
@@ -875,6 +900,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                                 .SetCookies(Cookies ?? new Dictionary<string, string>())
                                 .SetHeaders(headers ?? new Dictionary<string, string>())
                                 .SetEncoding(_encoding)
+                                .WithRateLimit(_rateLimit.TotalSeconds)
                                 .Build();
 
                             selectorDownloadRequest.Method = method;
@@ -895,6 +921,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 .SetCookies(Cookies ?? new Dictionary<string, string>())
                 .SetHeaders(headers ?? new Dictionary<string, string>())
                 .SetEncoding(_encoding)
+                .WithRateLimit(_rateLimit.TotalSeconds)
                 .Build();
 
             downloadRequest.Method = method;
@@ -1096,16 +1123,18 @@ namespace NzbDrone.Core.Indexers.Cardigann
 
                 _logger.Info($"Adding request: {searchUrl}");
 
-                var requestbuilder = new HttpRequestBuilder(searchUrl);
+                var requestBuilder = new HttpRequestBuilder(searchUrl)
-
+                {
-                requestbuilder.Method = method;
+                    Method = method,
+                    Encoding = _encoding
+                };
 
                 // Add FormData for searchs that POST
                 if (method == HttpMethod.Post)
                 {
                     foreach (var param in queryCollection)
                     {
-                        requestbuilder.AddFormParameter(param.Key, param.Value);
+                        requestBuilder.AddFormParameter(param.Key, param.Value);
                     }
                 }
 
@@ -1113,14 +1142,22 @@ namespace NzbDrone.Core.Indexers.Cardigann
                 if (search.Headers != null)
                 {
                     var headers = ParseCustomHeaders(search.Headers, variables);
-                    requestbuilder.SetHeaders(headers ?? new Dictionary<string, string>());
+                    requestBuilder.SetHeaders(headers ?? new Dictionary<string, string>());
                 }
 
-                var request = new CardigannRequest(requestbuilder.SetEncoding(_encoding).Build(), variables, searchPath);
+                var request = requestBuilder
+                    .WithRateLimit(_rateLimit.TotalSeconds)
+                    .Build();
 
-                request.HttpRequest.AllowAutoRedirect = searchPath.Followredirect;
+                var cardigannRequest = new CardigannRequest(request, variables, searchPath)
+                    {
+                        HttpRequest =
+                        {
+                            AllowAutoRedirect = searchPath.Followredirect
+                        }
+                    };
 
-                yield return request;
+                yield return cardigannRequest;
             }
         }
     }

src/NzbDrone.Core/Indexers/Definitions/Nebulance.cs

@@ -46,12 +46,17 @@ namespace NzbDrone.Core.Indexers.Definitions
             return new NebulanceParser(Settings);
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            // Invalidate cookies before downloading to prevent redirect to login page.
+            // Avoid using cookies to prevent redirects to login page
-            UpdateCookies(null, null);
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
+
+            var request = requestBuilder.Build();
 
-            return await base.Download(link);
+            return Task.FromResult(request);
         }
 
         private IndexerCapabilities SetCapabilities()

src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs

@@ -112,7 +112,7 @@ namespace NzbDrone.Core.Indexers.Definitions
                 _logger.Error("Download failed");
             }
 
-            ValidateTorrent(downloadBytes);
+            ValidateDownloadData(downloadBytes);
 
             return downloadBytes;
         }

src/NzbDrone.Core/Indexers/Definitions/Redacted.cs

@@ -50,6 +50,20 @@ namespace NzbDrone.Core.Indexers.Definitions
             return new RedactedParser(Settings, Capabilities.Categories);
         }
 
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
+
+            var request = requestBuilder
+                .SetHeader("Authorization", Settings.Apikey)
+                .Build();
+
+            return Task.FromResult(request);
+        }
+
         private IndexerCapabilities SetCapabilities()
         {
             var caps = new IndexerCapabilities
@@ -74,30 +88,6 @@ namespace NzbDrone.Core.Indexers.Definitions
 
             return caps;
         }
-
-        public override async Task<byte[]> Download(Uri link)
-        {
-            var request = new HttpRequestBuilder(link.AbsoluteUri)
-                .SetHeader("Authorization", Settings.Apikey)
-                .Build();
-
-            var downloadBytes = Array.Empty<byte>();
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                downloadBytes = response.ResponseData;
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Download failed");
-            }
-
-            ValidateTorrent(downloadBytes);
-
-            return downloadBytes;
-        }
     }
 
     public class RedactedRequestGenerator : IIndexerRequestGenerator
@@ -188,18 +178,12 @@ namespace NzbDrone.Core.Indexers.Definitions
                 queryCats.ForEach(cat => parameters.Set($"filter_cat[{cat}]", "1"));
             }
 
-            var request = RequestBuilder()
+            var searchUrl = _settings.BaseUrl.TrimEnd('/') + $"/ajax.php?{parameters.GetQueryString()}";
-                .Resource($"/ajax.php?{parameters.GetQueryString()}")
-                .Build();
 
-            yield return new IndexerRequest(request);
+            var request = new IndexerRequest(searchUrl, HttpAccept.Json);
-        }
+            request.HttpRequest.Headers.Set("Authorization", _settings.Apikey);
 
-        private HttpRequestBuilder RequestBuilder()
+            yield return request;
-        {
-            return new HttpRequestBuilder($"{_settings.BaseUrl.TrimEnd('/')}")
-                .Accept(HttpAccept.Json)
-                .SetHeader("Authorization", _settings.Apikey);
         }
     }
 

src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs

@@ -15,7 +15,6 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Annotations;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.Indexers.Settings;
 using NzbDrone.Core.IndexerSearch.Definitions;
@@ -104,70 +103,18 @@ namespace NzbDrone.Core.Indexers.Definitions
             request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override Task<HttpRequest> GetDownloadRequest(Uri link)
         {
-            Cookies = GetCookies();
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
-
-            if (link.Scheme == "magnet")
-            {
-                ValidateMagnet(link.OriginalString);
-
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-            request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
-
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
             {
-                _indexerStatusService.RecordFailure(Definition.Id);
+                AllowAutoRedirect = FollowRedirect
-                _logger.Error("Downloading torrent failed");
+            };
-                throw;
-            }
 
-            ValidateTorrent(torrentData);
+            var request = requestBuilder
+                .SetHeader("Authorization", $"Bearer {Settings.ApiKey}")
+                .Build();
 
-            return torrentData;
+            return Task.FromResult(request);
         }
 
         protected virtual IndexerCapabilities SetCapabilities()
@@ -276,7 +223,6 @@ namespace NzbDrone.Core.Indexers.Definitions
             var searchUrl = _settings.BaseUrl + "api/torrent?" + parameters.GetQueryString(duplicateKeysIfMulti: true);
 
             var request = new IndexerRequest(searchUrl, HttpAccept.Json);
-
             request.HttpRequest.Headers.Set("Authorization", $"Bearer {_settings.ApiKey}");
 
             yield return request;

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -6,10 +6,12 @@ using System.Net.Http;
 using System.Text;
 using System.Threading.Tasks;
 using FluentValidation.Results;
+using MonoTorrent;
 using NLog;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Http.CloudFlare;
 using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Indexers.Exceptions;
@@ -101,6 +103,95 @@ namespace NzbDrone.Core.Indexers
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
         }
 
+        public override async Task<byte[]> Download(Uri link)
+        {
+            Cookies = GetCookies();
+
+            var request = await GetDownloadRequest(link);
+
+            if (request.Url.Scheme == "magnet")
+            {
+                ValidateMagnet(request.Url.FullUri);
+                return Encoding.UTF8.GetBytes(request.Url.FullUri);
+            }
+
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
+            byte[] fileData;
+
+            try
+            {
+                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
+                fileData = response.ResponseData;
+
+                _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri);
+            }
+            catch (HttpException ex)
+            {
+                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
+                {
+                    _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri);
+                    throw new ReleaseUnavailableException("Download failed", ex);
+                }
+
+                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
+                {
+                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
+                }
+                else
+                {
+                    _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+                }
+
+                throw new ReleaseDownloadException("Download failed", ex);
+            }
+            catch (WebException ex)
+            {
+                _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
+
+                throw new ReleaseDownloadException("Download failed", ex);
+            }
+            catch (Exception)
+            {
+                _indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Error("Download failed");
+                throw;
+            }
+
+            ValidateDownloadData(fileData);
+
+            return fileData;
+        }
+
+        protected virtual Task<HttpRequest> GetDownloadRequest(Uri link)
+        {
+            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri)
+            {
+                AllowAutoRedirect = FollowRedirect
+            };
+
+            if (Cookies != null)
+            {
+                requestBuilder.SetCookies(Cookies);
+            }
+
+            var request = requestBuilder.Build();
+
+            return Task.FromResult(request);
+        }
+
+        protected virtual void ValidateDownloadData(byte[] fileData)
+        {
+        }
+
+        protected void ValidateMagnet(string link)
+        {
+            MagnetLink.Parse(link);
+        }
+
         protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
         {
             //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page.
@@ -420,6 +511,11 @@ namespace NzbDrone.Core.Indexers
                 request.RequestTimeout = TimeSpan.FromSeconds(15);
             }
 
+            if (request.RateLimit < RateLimit)
+            {
+                request.RateLimit = RateLimit;
+            }
+
             var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
 
             _eventAggregator.PublishEvent(new IndexerAuthEvent(Definition.Id, !response.HasHttpError, response.ElapsedTime));

src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs

@@ -1,12 +1,7 @@
-using System;
-using System.Net;
 using System.Text;
-using System.Threading.Tasks;
 using MonoTorrent;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -19,85 +14,15 @@ namespace NzbDrone.Core.Indexers
         {
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override void ValidateDownloadData(byte[] fileData)
-        {
-            Cookies = GetCookies();
-
-            if (link.Scheme == "magnet")
-            {
-                ValidateMagnet(link.OriginalString);
-
-                return Encoding.UTF8.GetBytes(link.OriginalString);
-            }
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] torrentData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                torrentData = response.ResponseData;
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading torrent failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading torrent failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading torrent failed");
-                throw;
-            }
-
-            ValidateTorrent(torrentData);
-
-            return torrentData;
-        }
-
-        protected void ValidateMagnet(string link)
-        {
-            MagnetLink.Parse(link);
-        }
-
-        protected void ValidateTorrent(byte[] torrentData)
         {
             try
             {
-                Torrent.Load(torrentData);
+                Torrent.Load(fileData);
             }
             catch
             {
-                _logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(torrentData));
+                _logger.Trace("Invalid torrent file contents: {0}", Encoding.ASCII.GetString(fileData));
                 throw;
             }
         }

src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs

@@ -1,11 +1,6 @@
-using System;
-using System.Net;
-using System.Threading.Tasks;
 using NLog;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Download;
-using NzbDrone.Core.Exceptions;
 using NzbDrone.Core.Messaging.Events;
 
 namespace NzbDrone.Core.Indexers
@@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers
             _nzbValidationService = nzbValidationService;
         }
 
-        public override async Task<byte[]> Download(Uri link)
+        protected override void ValidateDownloadData(byte[] fileData)
         {
-            Cookies = GetCookies();
+            _nzbValidationService.Validate(fileData);
-
-            var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
-
-            if (Cookies != null)
-            {
-                requestBuilder.SetCookies(Cookies);
-            }
-
-            var request = requestBuilder.Build();
-            request.AllowAutoRedirect = FollowRedirect;
-
-            byte[] nzbData;
-
-            try
-            {
-                var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
-                nzbData = response.ResponseData;
-
-                _logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri);
-            }
-            catch (HttpException ex)
-            {
-                if (ex.Response.StatusCode == HttpStatusCode.NotFound)
-                {
-                    _logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri);
-                    throw new ReleaseUnavailableException("Downloading nzb failed", ex);
-                }
-
-                if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
-                {
-                    _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
-                }
-                else
-                {
-                    _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-                }
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (WebException ex)
-            {
-                _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
-
-                throw new ReleaseDownloadException("Downloading nzb failed", ex);
-            }
-            catch (Exception)
-            {
-                _indexerStatusService.RecordFailure(Definition.Id);
-                _logger.Error("Downloading nzb failed");
-                throw;
-            }
-
-            _nzbValidationService.Validate(nzbData);
-
-            return nzbData;
         }
     }
 }