open-source-mirrors
/
Prowlarr
mirror of https://github.com/Prowlarr/Prowlarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed: (Newznab API) Response with StatusCode 429 when limits are reached

Browse Source
pull/1444/head
Bogdan 2 years ago
parent 773e8ff1f4
commit caa8bb05a7
6 changed files with 188 additions and 39 deletions
Show Stats Download Patch File Download Diff File

8
src/NzbDrone.Common/Http/TooManyRequestsException.cs
Unescape View File

@ -11,15 +11,13 @@ namespace NzbDrone.Common.Http
{
if (response.Headers.ContainsKey("Retry-After"))
{
var retryAfter = response.Headers["Retry-After"].ToString();
int seconds;
DateTime date;
var retryAfter = response.Headers["Retry-After"];
if (int.TryParse(retryAfter, out seconds))
if (int.TryParse(retryAfter, out var seconds))
{
RetryAfter = TimeSpan.FromSeconds(seconds);
}
else if (DateTime.TryParse(retryAfter, out date))
else if (DateTime.TryParse(retryAfter, out var date))
{
RetryAfter = date.ToUniversalTime() - DateTime.UtcNow;
}

6
src/NzbDrone.Core/Download/DownloadService.cs
Unescape View File

@ -92,8 +92,7 @@ namespace NzbDrone.Core.Download
}
catch (ReleaseDownloadException ex)
{
var http429 = ex.InnerException as TooManyRequestsException;
if (http429 != null)
if (ex.InnerException is TooManyRequestsException http429)
{
_indexerStatusService.RecordFailure(release.IndexerId, http429.RetryAfter);
}
@ -141,8 +140,7 @@ namespace NzbDrone.Core.Download
}
catch (ReleaseDownloadException ex)
{
var http429 = ex.InnerException as TooManyRequestsException;
if (http429 != null)
if (ex.InnerException is TooManyRequestsException http429)
{
_indexerStatusService.RecordFailure(indexerId, http429.RetryAfter);
}

20
src/NzbDrone.Core/History/HistoryRepository.cs
Unescape View File

@ -19,6 +19,7 @@ namespace NzbDrone.Core.History
List<History> Since(DateTime date, HistoryEventType? eventType);
void Cleanup(int days);
int CountSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes);
History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit);
}
public class HistoryRepository : BasicRepository<History>, IHistoryRepository
@ -115,5 +116,24 @@ namespace NzbDrone.Core.History
return conn.ExecuteScalar<int>(sql.RawSql, sql.Parameters);
}
}
public History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit)
{
var intEvents = eventTypes.Select(t => (int)t).ToList();
var builder = Builder()
.Where<History>(x => x.IndexerId == indexerId)
.Where<History>(x => x.Date >= date)
.Where<History>(x => intEvents.Contains((int)x.EventType));
var query = Query(builder);
if (limit > 0)
{
query = query.OrderByDescending(h => h.Date).Take(limit).ToList();
}
return query.MinBy(h => h.Date);
}
}
}

6
src/NzbDrone.Core/History/HistoryService.cs
Unescape View File

@ -27,6 +27,7 @@ namespace NzbDrone.Core.History
List<History> Between(DateTime start, DateTime end);
List<History> Since(DateTime date, HistoryEventType? eventType);
int CountSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes);
History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit);
}
public class HistoryService : IHistoryService,
@ -232,5 +233,10 @@ namespace NzbDrone.Core.History
{
return _historyRepository.CountSince(indexerId, date, eventTypes);
}
public History FindFirstForIndexerSince(int indexerId, DateTime date, List<HistoryEventType> eventTypes, int limit)
{
return _historyRepository.FindFirstForIndexerSince(indexerId, date, eventTypes, limit);
}
}
}

45
src/NzbDrone.Core/Indexers/IndexerLimitService.cs
Unescape View File

@ -1,7 +1,6 @@
using System;
using System.Collections.Generic;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.History;
namespace NzbDrone.Core.Indexers
@ -10,6 +9,8 @@ namespace NzbDrone.Core.Indexers
{
bool AtDownloadLimit(IndexerDefinition indexer);
bool AtQueryLimit(IndexerDefinition indexer);
int CalculateRetryAfterDownloadLimit(IndexerDefinition indexer);
int CalculateRetryAfterQueryLimit(IndexerDefinition indexer);
}
public class IndexerLimitService : IIndexerLimitService
@ -31,9 +32,9 @@ namespace NzbDrone.Core.Indexers
var grabCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.ReleaseGrabbed });
var grabLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit;
if (grabCount > grabLimit)
if (grabCount >= grabLimit)
{
_logger.Info("Indexer {0} has exceeded maximum grab limit for last 24 hours", indexer.Name);
_logger.Info("Indexer {0} has performed {1} of possible {2} grabs in last 24 hours, exceeding the maximum grab limit", indexer.Name, grabCount, grabLimit);
return true;
}
@ -51,9 +52,9 @@ namespace NzbDrone.Core.Indexers
var queryCount = _historyService.CountSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss });
var queryLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit;
if (queryCount > queryLimit)
if (queryCount >= queryLimit)
{
_logger.Info("Indexer {0} has exceeded maximum query limit for last 24 hours", indexer.Name);
_logger.Info("Indexer {0} has performed {1} of possible {2} queries in last 24 hours, exceeding the maximum query limit", indexer.Name, queryCount, queryLimit);
return true;
}
@ -63,5 +64,39 @@ namespace NzbDrone.Core.Indexers
return false;
}
public int CalculateRetryAfterDownloadLimit(IndexerDefinition indexer)
{
if (indexer.Id > 0 && ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.HasValue)
{
var grabLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.GrabLimit.GetValueOrDefault();
var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.ReleaseGrabbed }, grabLimit);
if (firstHistorySince != null)
{
return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddHours(24).Subtract(DateTime.Now).TotalSeconds);
}
}
return 0;
}
public int CalculateRetryAfterQueryLimit(IndexerDefinition indexer)
{
if (indexer.Id > 0 && ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.HasValue)
{
var queryLimit = ((IIndexerSettings)indexer.Settings).BaseSettings.QueryLimit.GetValueOrDefault();
var firstHistorySince = _historyService.FindFirstForIndexerSince(indexer.Id, DateTime.Now.AddHours(-24), new List<HistoryEventType> { HistoryEventType.IndexerQuery, HistoryEventType.IndexerRss }, queryLimit);
if (firstHistorySince != null)
{
return Convert.ToInt32(firstHistorySince.Date.ToLocalTime().AddHours(24).Subtract(DateTime.Now).TotalSeconds);
}
}
return 0;
}
}
}

132
src/Prowlarr.Api.V1/Indexers/NewznabController.cs
Unescape View File

@ -1,21 +1,25 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using System.Threading.Tasks;
using System.Xml.Linq;
using Microsoft.AspNetCore.Cors;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Download;
using NzbDrone.Core.History;
using NzbDrone.Core.Exceptions;
using NzbDrone.Core.Indexers;
using NzbDrone.Core.IndexerSearch;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.ThingiProvider.Status;
using Prowlarr.Http.Extensions;
using Prowlarr.Http.REST;
using BadRequestException = NzbDrone.Core.Exceptions.BadRequestException;
namespace NzbDrone.Api.V1.Indexers
{
@ -27,18 +31,21 @@ namespace NzbDrone.Api.V1.Indexers
private IIndexerFactory _indexerFactory { get; set; }
private ISearchForNzb _nzbSearchService { get; set; }
private IIndexerLimitService _indexerLimitService { get; set; }
private IIndexerStatusService _indexerStatusService;
private IDownloadMappingService _downloadMappingService { get; set; }
private IDownloadService _downloadService { get; set; }
public NewznabController(IndexerFactory indexerFactory,
ISearchForNzb nzbSearchService,
IIndexerLimitService indexerLimitService,
IIndexerStatusService indexerStatusService,
IDownloadMappingService downloadMappingService,
IDownloadService downloadService)
{
_indexerFactory = indexerFactory;
_nzbSearchService = nzbSearchService;
_indexerLimitService = indexerLimitService;
_indexerStatusService = indexerStatusService;
_downloadMappingService = downloadMappingService;
_downloadService = downloadService;
}
@ -54,7 +61,7 @@ namespace NzbDrone.Api.V1.Indexers
if (requestType.IsNullOrWhiteSpace())
{
return Content(CreateErrorXML(200, "Missing parameter (t)"), "application/rss+xml");
return CreateResponse(CreateErrorXML(200, "Missing parameter (t)"), statusCode: StatusCodes.Status400BadRequest);
}
request.imdbid = request.imdbid?.TrimStart('t') ?? null;
@ -63,7 +70,7 @@ namespace NzbDrone.Api.V1.Indexers
{
if (!int.TryParse(request.imdbid, out var imdb) || imdb == 0)
{
return Content(CreateErrorXML(201, "Incorrect parameter (imdbid)"), "application/rss+xml");
return CreateResponse(CreateErrorXML(201, "Incorrect parameter (imdbid)"), statusCode: StatusCodes.Status400BadRequest);
}
}
@ -97,25 +104,27 @@ namespace NzbDrone.Api.V1.Indexers
caps.Categories.AddCategoryMapping(1, cat);
}
return Content(caps.ToXml(), "application/rss+xml");
return CreateResponse(caps.ToXml());
case "search":
case "tvsearch":
case "music":
case "book":
case "movie":
var results = new NewznabResults();
results.Releases = new List<ReleaseInfo>
var results = new NewznabResults
{
new ReleaseInfo
Releases = new List<ReleaseInfo>
{
new ()
{
Title = "Test Release",
Guid = "https://prowlarr.com",
DownloadUrl = "https://prowlarr.com",
PublishDate = DateTime.Now
}
}
};
return Content(results.ToXml(DownloadProtocol.Usenet), "application/rss+xml");
return CreateResponse(results.ToXml(DownloadProtocol.Usenet));
}
}
@ -126,19 +135,37 @@ namespace NzbDrone.Api.V1.Indexers
throw new NotFoundException("Indexer Not Found");
}
if (!indexerDef.Enable)
{
return CreateResponse(CreateErrorXML(410, "Indexer is disabled"), statusCode: StatusCodes.Status410Gone);
}
var indexer = _indexerFactory.GetInstance(indexerDef);
var blockedIndexerStatus = GetBlockedIndexerStatus(indexer);
if (blockedIndexerStatus?.DisabledTill != null)
{
var retryAfterDisabledTill = Convert.ToInt32(blockedIndexerStatus.DisabledTill.Value.ToLocalTime().Subtract(DateTime.Now).TotalSeconds);
AddRetryAfterHeader(retryAfterDisabledTill);
return CreateResponse(CreateErrorXML(429, $"Indexer is disabled till {blockedIndexerStatus.DisabledTill.Value.ToLocalTime()} due to recent failures."), statusCode: StatusCodes.Status429TooManyRequests);
}
//TODO Optimize this so it's not called here and in NzbSearchService (for manual search)
if (_indexerLimitService.AtQueryLimit(indexerDef))
{
return Content(CreateErrorXML(429, $"Request limit reached ({((IIndexerSettings)indexer.Definition.Settings).BaseSettings.QueryLimit})"), "application/rss+xml");
var retryAfterQueryLimit = _indexerLimitService.CalculateRetryAfterQueryLimit(indexerDef);
AddRetryAfterHeader(retryAfterQueryLimit);
return CreateResponse(CreateErrorXML(429, $"User configurable Indexer Query Limit of {((IIndexerSettings)indexer.Definition.Settings).BaseSettings.QueryLimit} reached."), statusCode: StatusCodes.Status429TooManyRequests);
}
switch (requestType)
{
case "caps":
var caps = indexer.GetCapabilities();
return Content(caps.ToXml(), "application/rss+xml");
return CreateResponse(caps.ToXml());
case "search":
case "tvsearch":
case "music":
@ -156,9 +183,9 @@ namespace NzbDrone.Api.V1.Indexers
}
}
return Content(results.ToXml(indexer.Protocol), "application/rss+xml");
return CreateResponse(results.ToXml(indexer.Protocol));
default:
return Content(CreateErrorXML(202, $"No such function ({requestType})"), "application/rss+xml");
return CreateResponse(CreateErrorXML(202, $"No such function ({requestType})"), statusCode: StatusCodes.Status400BadRequest);
}
}
@ -167,11 +194,35 @@ namespace NzbDrone.Api.V1.Indexers
public async Task<object> GetDownload(int id, string link, string file)
{
var indexerDef = _indexerFactory.Get(id);
if (indexerDef == null)
{
throw new NotFoundException("Indexer Not Found");
}
if (!indexerDef.Enable)
{
return CreateResponse(CreateErrorXML(410, "Indexer is disabled"), statusCode: StatusCodes.Status410Gone);
}
var indexer = _indexerFactory.GetInstance(indexerDef);
var blockedIndexerStatus = GetBlockedIndexerStatus(indexer);
if (blockedIndexerStatus?.DisabledTill != null)
{
var retryAfterDisabledTill = Convert.ToInt32(blockedIndexerStatus.DisabledTill.Value.ToLocalTime().Subtract(DateTime.Now).TotalSeconds);
AddRetryAfterHeader(retryAfterDisabledTill);
return CreateResponse(CreateErrorXML(429, $"Indexer is disabled till {blockedIndexerStatus.DisabledTill.Value.ToLocalTime()} due to recent failures."), statusCode: StatusCodes.Status429TooManyRequests);
}
if (_indexerLimitService.AtDownloadLimit(indexerDef))
{
return Content(CreateErrorXML(429, $"Grab limit reached ({((IIndexerSettings)indexer.Definition.Settings).BaseSettings.GrabLimit})"), "application/rss+xml");
var retryAfterDownloadLimit = _indexerLimitService.CalculateRetryAfterDownloadLimit(indexerDef);
AddRetryAfterHeader(retryAfterDownloadLimit);
return CreateResponse(CreateErrorXML(429, $"User configurable Indexer Grab Limit of {((IIndexerSettings)indexer.Definition.Settings).BaseSettings.GrabLimit} reached."), statusCode: StatusCodes.Status429TooManyRequests);
}
if (link.IsNullOrWhiteSpace() || file.IsNullOrWhiteSpace())
@ -181,11 +232,6 @@ namespace NzbDrone.Api.V1.Indexers
file = WebUtility.UrlDecode(file);
if (indexerDef == null)
{
throw new NotFoundException("Indexer Not Found");
}
var source = UserAgentParser.ParseSource(Request.Headers["User-Agent"]);
var host = Request.GetHostName();
@ -198,8 +244,27 @@ namespace NzbDrone.Api.V1.Indexers
return RedirectPermanent(unprotectedlLink);
}
var downloadBytes = Array.Empty<byte>();
byte[] downloadBytes;
try
{
downloadBytes = await _downloadService.DownloadReport(unprotectedlLink, id, source, host, file);
}
catch (ReleaseUnavailableException ex)
{
return CreateResponse(CreateErrorXML(410, ex.Message), statusCode: StatusCodes.Status410Gone);
}
catch (ReleaseDownloadException ex) when (ex.InnerException is TooManyRequestsException http429)
{
var http429RetryAfter = Convert.ToInt32(http429.RetryAfter.TotalSeconds);
AddRetryAfterHeader(http429RetryAfter);
return CreateResponse(CreateErrorXML(429, ex.Message), statusCode: StatusCodes.Status429TooManyRequests);
}
catch (Exception ex)
{
return CreateResponse(CreateErrorXML(500, ex.Message), statusCode: StatusCodes.Status500InternalServerError);
}
// handle magnet URLs
if (downloadBytes.Length >= 7
@ -232,5 +297,32 @@ namespace NzbDrone.Api.V1.Indexers
return xdoc.Declaration + Environment.NewLine + xdoc;
}
private ContentResult CreateResponse(string content, string contentType = "application/rss+xml", int statusCode = StatusCodes.Status200OK)
{
var mediaTypeHeaderValue = MediaTypeHeaderValue.Parse(contentType);
return new ContentResult
{
StatusCode = statusCode,
Content = content,
ContentType = mediaTypeHeaderValue.ToString()
};
}
private ProviderStatusBase GetBlockedIndexerStatus(IIndexer indexer)
{
var blockedIndexers = _indexerStatusService.GetBlockedProviders().ToDictionary(v => v.ProviderId, v => v);
return blockedIndexers.TryGetValue(indexer.Definition.Id, out var blockedIndexerStatus) ? blockedIndexerStatus : null;
}
private void AddRetryAfterHeader(int retryAfterSeconds)
{
if (!HttpContext.Response.Headers.ContainsKey("Retry-After") && retryAfterSeconds > 0)
{
HttpContext.Response.Headers.Add("Retry-After", $"{retryAfterSeconds}");
}
}
}
}

Write Preview
Loading…
Cancel
Save