diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index 6c6f67f0a..df6fa6cb5 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -61,14 +61,6 @@ namespace NzbDrone.Core.Download // Get the seed configuration for this release. // remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie); - - // Limit grabs to 2 per second. - if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:")) - { - var url = new HttpUri(release.DownloadUrl); - _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2)); - } - var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId)); string downloadClientId; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index f86724a03..d5dba03b3 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -180,60 +180,15 @@ namespace NzbDrone.Core.Indexers.Cardigann await generator.DoLogin(); } - public override async Task Download(Uri link) + protected override async Task GetDownloadRequest(Uri link) { var generator = (CardigannRequestGenerator)GetRequestGenerator(); var request = await generator.DownloadRequest(link); - if (request.Url.Scheme == "magnet") - { - ValidateMagnet(request.Url.FullUri); - return Encoding.UTF8.GetBytes(request.Url.FullUri); - } - request.AllowAutoRedirect = true; - var downloadBytes = Array.Empty(); - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - downloadBytes = response.ResponseData; - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri); - throw new ReleaseUnavailableException("Downloading torrent failed", ex); - } - - if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) - { - _logger.Error("API Grab Limit reached for {0}", request.Url.FullUri); - } - else - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri); - } - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri); - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Downloading torrent failed"); - throw; - } - - return downloadBytes; + return request; } protected override async Task Test(List failures) diff --git a/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs b/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs index 6fe62739e..bfbaec6bb 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs @@ -80,26 +80,18 @@ namespace NzbDrone.Core.Indexers.Definitions return caps; } - public override async Task Download(Uri link) + protected override Task GetDownloadRequest(Uri link) { - var request = new HttpRequestBuilder(link.AbsoluteUri) - .SetHeader("Authorization", Settings.Apikey) - .Build(); - - var downloadBytes = Array.Empty(); + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); - try + if (Cookies != null) { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - downloadBytes = response.ResponseData; - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Download failed"); + requestBuilder.SetCookies(Cookies); } - return downloadBytes; + var request = requestBuilder.SetHeader("Authorization", Settings.Apikey).Build(); + + return Task.FromResult(request); } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs b/src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs index e0569bc09..5869f60df 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs @@ -103,16 +103,8 @@ namespace NzbDrone.Core.Indexers.Definitions request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}"); } - public override async Task Download(Uri link) + protected override Task GetDownloadRequest(Uri link) { - Cookies = GetCookies(); - - if (link.Scheme == "magnet") - { - ValidateMagnet(link.OriginalString); - return Encoding.UTF8.GetBytes(link.OriginalString); - } - var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); if (Cookies != null) @@ -124,46 +116,7 @@ namespace NzbDrone.Core.Indexers.Definitions request.AllowAutoRedirect = FollowRedirect; request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}"); - byte[] torrentData; - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - torrentData = response.ResponseData; - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri); - throw new ReleaseUnavailableException("Downloading torrent failed", ex); - } - - if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) - { - _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); - } - else - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); - } - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Downloading torrent failed"); - throw; - } - - return torrentData; + return Task.FromResult(request); } protected virtual IndexerCapabilities SetCapabilities() diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 24044a858..e966ec3ba 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -5,10 +5,12 @@ using System.Net; using System.Text; using System.Threading.Tasks; using FluentValidation.Results; +using MonoTorrent; using NLog; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.Http.CloudFlare; using NzbDrone.Core.Indexers.Events; using NzbDrone.Core.Indexers.Exceptions; @@ -100,6 +102,93 @@ namespace NzbDrone.Core.Indexers return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } + public override async Task Download(Uri link) + { + Cookies = GetCookies(); + + var request = await GetDownloadRequest(link); + + if (request.Url.Scheme == "magnet") + { + ValidateMagnet(request.Url.FullUri); + return Encoding.UTF8.GetBytes(request.Url.FullUri); + } + + if (request.RateLimit < RateLimit) + { + request.RateLimit = RateLimit; + } + + byte[] fileData; + + try + { + var response = await _httpClient.ExecuteProxiedAsync(request, Definition); + fileData = response.ResponseData; + + _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri); + } + catch (HttpException ex) + { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri); + throw new ReleaseUnavailableException("Downloading nzb failed", ex); + } + + if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) + { + _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); + } + else + { + _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri); + } + + throw new ReleaseDownloadException("Downloading failed", ex); + } + catch (WebException ex) + { + _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri); + + throw new ReleaseDownloadException("Downloading failed", ex); + } + catch (Exception) + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Error("Downloading failed"); + throw; + } + + ValidateDownloadData(fileData); + + return fileData; + } + + protected virtual Task GetDownloadRequest(Uri link) + { + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); + + if (Cookies != null) + { + requestBuilder.SetCookies(Cookies); + } + + var request = requestBuilder.Build(); + request.AllowAutoRedirect = FollowRedirect; + + return Task.FromResult(request); + } + + protected virtual void ValidateDownloadData(byte[] fileData) + { + } + + protected void ValidateMagnet(string link) + { + MagnetLink.Parse(link); + } + protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator) { //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page. diff --git a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs index b804c2903..35395f5b5 100644 --- a/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs @@ -1,12 +1,5 @@ -using System; -using System.Net; -using System.Text; -using System.Threading.Tasks; -using MonoTorrent; using NLog; -using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; -using NzbDrone.Core.Exceptions; using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers @@ -18,72 +11,5 @@ namespace NzbDrone.Core.Indexers : base(httpClient, eventAggregator, indexerStatusService, configService, logger) { } - - public override async Task Download(Uri link) - { - Cookies = GetCookies(); - - if (link.Scheme == "magnet") - { - ValidateMagnet(link.OriginalString); - return Encoding.UTF8.GetBytes(link.OriginalString); - } - - var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); - - if (Cookies != null) - { - requestBuilder.SetCookies(Cookies); - } - - var request = requestBuilder.Build(); - request.AllowAutoRedirect = FollowRedirect; - - byte[] torrentData; - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - torrentData = response.ResponseData; - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri); - throw new ReleaseUnavailableException("Downloading torrent failed", ex); - } - - if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) - { - _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); - } - else - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); - } - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri); - - throw new ReleaseDownloadException("Downloading torrent failed", ex); - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Downloading torrent failed"); - throw; - } - - return torrentData; - } - - protected void ValidateMagnet(string link) - { - MagnetLink.Parse(link); - } } } diff --git a/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs b/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs index f8104a844..b40d8fba8 100644 --- a/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs @@ -1,11 +1,6 @@ -using System; -using System.Net; -using System.Threading.Tasks; using NLog; -using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; -using NzbDrone.Core.Exceptions; using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Indexers @@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers _nzbValidationService = nzbValidationService; } - public override async Task Download(Uri link) + protected override void ValidateDownloadData(byte[] fileData) { - Cookies = GetCookies(); - - var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); - - if (Cookies != null) - { - requestBuilder.SetCookies(Cookies); - } - - var request = requestBuilder.Build(); - request.AllowAutoRedirect = FollowRedirect; - - byte[] nzbData; - - try - { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - nzbData = response.ResponseData; - - _logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri); - } - catch (HttpException ex) - { - if (ex.Response.StatusCode == HttpStatusCode.NotFound) - { - _logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri); - throw new ReleaseUnavailableException("Downloading nzb failed", ex); - } - - if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) - { - _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); - } - else - { - _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri); - } - - throw new ReleaseDownloadException("Downloading nzb failed", ex); - } - catch (WebException ex) - { - _logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri); - - throw new ReleaseDownloadException("Downloading nzb failed", ex); - } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Downloading nzb failed"); - throw; - } - - _nzbValidationService.Validate(nzbData); - - return nzbData; + _nzbValidationService.Validate(fileData); } } }