open-source-mirrors
/
Prowlarr
mirror of https://github.com/Prowlarr/Prowlarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed: (Indexers) Rate limit Download requests by Indexer

Browse Source
pull/1300/head
Qstick 2 years ago
parent c28f9b6bcd
commit e3480d1143
7 changed files with 102 additions and 255 deletions
Show Stats Download Patch File Download Diff File

8
src/NzbDrone.Core/Download/DownloadService.cs
Unescape View File

@ -61,14 +61,6 @@ namespace NzbDrone.Core.Download
// Get the seed configuration for this release.
// remoteMovie.SeedConfiguration = _seedConfigProvider.GetSeedConfiguration(remoteMovie);
// Limit grabs to 2 per second.
if (release.DownloadUrl.IsNotNullOrWhiteSpace() && !release.DownloadUrl.StartsWith("magnet:"))
{
var url = new HttpUri(release.DownloadUrl);
_rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
}
var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(release.IndexerId));
string downloadClientId;

49
src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs
Unescape View File

@ -180,60 +180,15 @@ namespace NzbDrone.Core.Indexers.Cardigann
await generator.DoLogin();
}
public override async Task<byte[]> Download(Uri link)
protected override async Task<HttpRequest> GetDownloadRequest(Uri link)
{
var generator = (CardigannRequestGenerator)GetRequestGenerator();
var request = await generator.DownloadRequest(link);
if (request.Url.Scheme == "magnet")
{
ValidateMagnet(request.Url.FullUri);
return Encoding.UTF8.GetBytes(request.Url.FullUri);
}
request.AllowAutoRedirect = true;
var downloadBytes = Array.Empty<byte>();
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
downloadBytes = response.ResponseData;
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", request.Url.FullUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", request.Url.FullUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", request.Url.FullUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading torrent failed");
throw;
}
return downloadBytes;
return request;
}
protected override async Task Test(List<ValidationFailure> failures)

22
src/NzbDrone.Core/Indexers/Definitions/Redacted.cs
Unescape View File

@ -80,26 +80,18 @@ namespace NzbDrone.Core.Indexers.Definitions
return caps;
}
public override async Task<byte[]> Download(Uri link)
protected override Task<HttpRequest> GetDownloadRequest(Uri link)
{
var request = new HttpRequestBuilder(link.AbsoluteUri)
.SetHeader("Authorization", Settings.Apikey)
.Build();
var downloadBytes = Array.Empty<byte>();
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
try
if (Cookies != null)
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
downloadBytes = response.ResponseData;
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Download failed");
requestBuilder.SetCookies(Cookies);
}
return downloadBytes;
var request = requestBuilder.SetHeader("Authorization", Settings.Apikey).Build();
return Task.FromResult(request);
}
}

51
src/NzbDrone.Core/Indexers/Definitions/SpeedApp/SpeedAppBase.cs
Unescape View File

@ -103,16 +103,8 @@ namespace NzbDrone.Core.Indexers.Definitions
request.HttpRequest.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
}
public override async Task<byte[]> Download(Uri link)
protected override Task<HttpRequest> GetDownloadRequest(Uri link)
{
Cookies = GetCookies();
if (link.Scheme == "magnet")
{
ValidateMagnet(link.OriginalString);
return Encoding.UTF8.GetBytes(link.OriginalString);
}
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
@ -124,46 +116,7 @@ namespace NzbDrone.Core.Indexers.Definitions
request.AllowAutoRedirect = FollowRedirect;
request.Headers.Set("Authorization", $"Bearer {Settings.ApiKey}");
byte[] torrentData;
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
torrentData = response.ResponseData;
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading torrent failed");
throw;
}
return torrentData;
return Task.FromResult(request);
}
protected virtual IndexerCapabilities SetCapabilities()

89
src/NzbDrone.Core/Indexers/HttpIndexerBase.cs
Unescape View File

@ -5,10 +5,12 @@ using System.Net;
using System.Text;
using System.Threading.Tasks;
using FluentValidation.Results;
using MonoTorrent;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Http.CloudFlare;
using NzbDrone.Core.Indexers.Events;
using NzbDrone.Core.Indexers.Exceptions;
@ -100,6 +102,93 @@ namespace NzbDrone.Core.Indexers
return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
}
public override async Task<byte[]> Download(Uri link)
{
Cookies = GetCookies();
var request = await GetDownloadRequest(link);
if (request.Url.Scheme == "magnet")
{
ValidateMagnet(request.Url.FullUri);
return Encoding.UTF8.GetBytes(request.Url.FullUri);
}
if (request.RateLimit < RateLimit)
{
request.RateLimit = RateLimit;
}
byte[] fileData;
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
fileData = response.ResponseData;
_logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri);
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading nzb failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading failed");
throw;
}
ValidateDownloadData(fileData);
return fileData;
}
protected virtual Task<HttpRequest> GetDownloadRequest(Uri link)
{
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
return Task.FromResult(request);
}
protected virtual void ValidateDownloadData(byte[] fileData)
{
}
protected void ValidateMagnet(string link)
{
MagnetLink.Parse(link);
}
protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
{
//A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page.

74
src/NzbDrone.Core/Indexers/TorrentIndexerBase.cs
Unescape View File

@ -1,12 +1,5 @@
using System;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using MonoTorrent;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers
@ -18,72 +11,5 @@ namespace NzbDrone.Core.Indexers
: base(httpClient, eventAggregator, indexerStatusService, configService, logger)
{
}
public override async Task<byte[]> Download(Uri link)
{
Cookies = GetCookies();
if (link.Scheme == "magnet")
{
ValidateMagnet(link.OriginalString);
return Encoding.UTF8.GetBytes(link.OriginalString);
}
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
byte[] torrentData;
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
torrentData = response.ResponseData;
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading torrent file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading torrent failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading torrent file for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading torrent failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading torrent failed");
throw;
}
return torrentData;
}
protected void ValidateMagnet(string link)
{
MagnetLink.Parse(link);
}
}
}

64
src/NzbDrone.Core/Indexers/UsenetIndexerBase.cs
Unescape View File

@ -1,11 +1,6 @@
using System;
using System.Net;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Download;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Indexers
@ -21,64 +16,9 @@ namespace NzbDrone.Core.Indexers
_nzbValidationService = nzbValidationService;
}
public override async Task<byte[]> Download(Uri link)
protected override void ValidateDownloadData(byte[] fileData)
{
Cookies = GetCookies();
var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri);
if (Cookies != null)
{
requestBuilder.SetCookies(Cookies);
}
var request = requestBuilder.Build();
request.AllowAutoRedirect = FollowRedirect;
byte[] nzbData;
try
{
var response = await _httpClient.ExecuteProxiedAsync(request, Definition);
nzbData = response.ResponseData;
_logger.Debug("Downloaded nzb for release finished ({0} bytes from {1})", nzbData.Length, link.AbsoluteUri);
}
catch (HttpException ex)
{
if (ex.Response.StatusCode == HttpStatusCode.NotFound)
{
_logger.Error(ex, "Downloading nzb file for release failed since it no longer exists ({0})", link.AbsoluteUri);
throw new ReleaseUnavailableException("Downloading nzb failed", ex);
}
if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests)
{
_logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri);
}
else
{
_logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
}
throw new ReleaseDownloadException("Downloading nzb failed", ex);
}
catch (WebException ex)
{
_logger.Error(ex, "Downloading nzb for release failed ({0})", link.AbsoluteUri);
throw new ReleaseDownloadException("Downloading nzb failed", ex);
}
catch (Exception)
{
_indexerStatusService.RecordFailure(Definition.Id);
_logger.Error("Downloading nzb failed");
throw;
}
_nzbValidationService.Validate(nzbData);
return nzbData;
_nzbValidationService.Validate(fileData);
}
}
}

Write Preview
Loading…
Cancel
Save