open-source-mirrors
/
Prowlarr
mirror of https://github.com/Prowlarr/Prowlarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed: (Anidub/Animedia) Use rate limit in sub-requests

Browse Source
pull/1405/head
Bogdan 1 year ago
parent f8082047a5
commit 94c45541ae
2 changed files with 136 additions and 128 deletions
Show Stats Download Patch File Download Diff File

132
src/NzbDrone.Core/Indexers/Definitions/Anidub.cs
Unescape View File

@ -39,12 +39,12 @@ namespace NzbDrone.Core.Indexers.Definitions
public override IIndexerRequestGenerator GetRequestGenerator() public override IIndexerRequestGenerator GetRequestGenerator()
{ {
return new AnidubRequestGenerator { Settings = Settings, Capabilities = Capabilities }; return new AnidubRequestGenerator(Settings);
} }
public override IParseIndexerResponse GetParser() public override IParseIndexerResponse GetParser()
{ {
return new AnidubParser(Settings, Capabilities.Categories) { HttpClient = _httpClient, Logger = _logger }; return new AnidubParser(Settings, Capabilities.Categories, RateLimit, _httpClient, _logger);
} }
protected override async Task DoLogin() protected override async Task DoLogin()
@ -135,21 +135,25 @@ namespace NzbDrone.Core.Indexers.Definitions
public class AnidubRequestGenerator : IIndexerRequestGenerator public class AnidubRequestGenerator : IIndexerRequestGenerator
{ {
public UserPassTorrentBaseSettings Settings { get; set; } private readonly UserPassTorrentBaseSettings _settings;
public IndexerCapabilities Capabilities { get; set; }
public AnidubRequestGenerator(UserPassTorrentBaseSettings settings)
{
_settings = settings;
}
private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories) private IEnumerable<IndexerRequest> GetPagedRequests(string term)
{ {
var requestUrl = string.Empty; string requestUrl;
var isSearch = !string.IsNullOrWhiteSpace(term); var isSearch = !string.IsNullOrWhiteSpace(term);
if (isSearch) if (isSearch)
{ {
requestUrl = string.Format("{0}/index.php?do=search", Settings.BaseUrl.TrimEnd('/')); requestUrl = $"{_settings.BaseUrl.TrimEnd('/')}/index.php?do=search";
} }
else else
{ {
requestUrl = Settings.BaseUrl; requestUrl = _settings.BaseUrl;
} }
var request = new IndexerRequest(requestUrl, HttpAccept.Html); var request = new IndexerRequest(requestUrl, HttpAccept.Html);
@ -194,7 +198,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
var pageableRequests = new IndexerPageableRequestChain(); var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests; return pageableRequests;
} }
@ -203,7 +207,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
var pageableRequests = new IndexerPageableRequestChain(); var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories)); pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}"));
return pageableRequests; return pageableRequests;
} }
@ -212,7 +216,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
var pageableRequests = new IndexerPageableRequestChain(); var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests; return pageableRequests;
} }
@ -221,7 +225,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
var pageableRequests = new IndexerPageableRequestChain(); var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests; return pageableRequests;
} }
@ -230,7 +234,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
var pageableRequests = new IndexerPageableRequestChain(); var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests; return pageableRequests;
} }
@ -243,33 +247,37 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
private readonly UserPassTorrentBaseSettings _settings; private readonly UserPassTorrentBaseSettings _settings;
private readonly IndexerCapabilitiesCategories _categories; private readonly IndexerCapabilitiesCategories _categories;
public IIndexerHttpClient HttpClient { get; set; } private readonly TimeSpan _rateLimit;
public Logger Logger { get; set; } private readonly IIndexerHttpClient _httpClient;
private readonly Logger _logger;
private static Dictionary<string, string> CategoriesMap => new Dictionary<string, string> private static Dictionary<string, string> CategoriesMap => new ()
{ {
{ "/anime_tv/full", "14" }, { "/anime_tv/full", "14" },
{ "/anime_tv/anime_ongoing", "10" }, { "/anime_tv/anime_ongoing", "10" },
{ "/anime_tv/shonen", "11" }, { "/anime_tv/shonen", "11" },
{ "/anime_tv", "2" }, { "/anime_tv", "2" },
{ "/xxx", "13" }, { "/xxx", "13" },
{ "/manga", "15" }, { "/manga", "15" },
{ "/ost", "16" }, { "/ost", "16" },
{ "/podcast", "17" }, { "/podcast", "17" },
{ "/anime_movie", "3" }, { "/anime_movie", "3" },
{ "/anime_ova/anime_ona", "5" }, { "/anime_ova/anime_ona", "5" },
{ "/anime_ova", "4" }, { "/anime_ova", "4" },
{ "/dorama/japan_dorama", "6" }, { "/dorama/japan_dorama", "6" },
{ "/dorama/korea_dorama", "7" }, { "/dorama/korea_dorama", "7" },
{ "/dorama/china_dorama", "8" }, { "/dorama/china_dorama", "8" },
{ "/dorama", "9" }, { "/dorama", "9" },
{ "/anons_ongoing", "12" } { "/anons_ongoing", "12" }
}; };
public AnidubParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories) public AnidubParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories, TimeSpan rateLimit, IIndexerHttpClient httpClient, Logger logger)
{ {
_settings = settings; _settings = settings;
_categories = categories; _categories = categories;
_rateLimit = rateLimit;
_httpClient = httpClient;
_logger = logger;
} }
private static string GetTitle(AngleSharp.Html.Dom.IHtmlDocument content, AngleSharp.Dom.IElement tabNode) private static string GetTitle(AngleSharp.Html.Dom.IHtmlDocument content, AngleSharp.Dom.IElement tabNode)
@ -314,9 +322,9 @@ namespace NzbDrone.Core.Indexers.Definitions
private static int GetReleaseLeechers(AngleSharp.Dom.IElement tabNode) private static int GetReleaseLeechers(AngleSharp.Dom.IElement tabNode)
{ {
const string LeechersSelector = ".list.down > .li_swing_m"; const string leechersSelector = ".list.down > .li_swing_m";
var leechersStr = tabNode.QuerySelector(LeechersSelector).TextContent; var leechersStr = tabNode.QuerySelector(leechersSelector).TextContent;
int.TryParse(leechersStr, out var leechers); int.TryParse(leechersStr, out var leechers);
return leechers; return leechers;
} }
@ -332,18 +340,18 @@ namespace NzbDrone.Core.Indexers.Definitions
private static int GetReleaseGrabs(AngleSharp.Dom.IElement tabNode) private static int GetReleaseGrabs(AngleSharp.Dom.IElement tabNode)
{ {
const string GrabsSelector = ".list.down > .li_download_m"; const string grabsSelector = ".list.down > .li_download_m";
var grabsStr = tabNode.QuerySelector(GrabsSelector).TextContent; var grabsStr = tabNode.QuerySelector(grabsSelector).TextContent;
int.TryParse(grabsStr, out var grabs); int.TryParse(grabsStr, out var grabs);
return grabs; return grabs;
} }
private static string GetDateFromDocument(AngleSharp.Html.Dom.IHtmlDocument content) private static string GetDateFromDocument(AngleSharp.Html.Dom.IHtmlDocument content)
{ {
const string DateSelector = ".story_inf > li:nth-child(2)"; const string dateSelector = ".story_inf > li:nth-child(2)";
var domDate = content.QuerySelector(DateSelector).LastChild; var domDate = content.QuerySelector(dateSelector).LastChild;
if (domDate?.NodeName != "#text") if (domDate?.NodeName != "#text")
{ {
@ -384,16 +392,16 @@ namespace NzbDrone.Core.Indexers.Definitions
return utcDate.AddHours(-russianStandardTimeDiff); return utcDate.AddHours(-russianStandardTimeDiff);
} }
Logger.Warn($"[AniDub] Date time couldn't be parsed on. Date text: {dateText}"); _logger.Warn($"[AniDub] Date time couldn't be parsed on. Date text: {dateText}");
return DateTime.UtcNow; return DateTime.UtcNow;
} }
private static long GetReleaseSize(AngleSharp.Dom.IElement tabNode) private static long GetReleaseSize(AngleSharp.Dom.IElement tabNode)
{ {
const string SizeSelector = ".list.down > .red"; const string sizeSelector = ".list.down > .red";
var sizeStr = tabNode.QuerySelector(SizeSelector).TextContent; var sizeStr = tabNode.QuerySelector(sizeSelector).TextContent;
return ParseUtil.GetBytes(sizeStr); return ParseUtil.GetBytes(sizeStr);
} }
@ -433,11 +441,11 @@ namespace NzbDrone.Core.Indexers.Definitions
var release = new TorrentInfo var release = new TorrentInfo
{ {
Title = GetTitle(dom, t), Title = GetTitle(dom, t),
InfoUrl = indexerResponse.Request.Url.ToString(), InfoUrl = indexerResponse.Request.Url.FullUri,
DownloadVolumeFactor = 0, DownloadVolumeFactor = 0,
UploadVolumeFactor = 1, UploadVolumeFactor = 1,
Guid = indexerResponse.Request.Url.ToString() + t.Id, Guid = indexerResponse.Request.Url.FullUri + t.Id,
Seeders = GetReleaseSeeders(t), Seeders = GetReleaseSeeders(t),
Peers = GetReleaseSeeders(t) + GetReleaseLeechers(t), Peers = GetReleaseSeeders(t) + GetReleaseLeechers(t),
Grabs = GetReleaseGrabs(t), Grabs = GetReleaseGrabs(t),
@ -459,36 +467,30 @@ namespace NzbDrone.Core.Indexers.Definitions
var parser = new HtmlParser(); var parser = new HtmlParser();
var dom = parser.ParseDocument(indexerResponse.Content); var dom = parser.ParseDocument(indexerResponse.Content);
var domQuery = string.Empty;
if (indexerResponse.Request.Url.Query.Contains("do=search")) var links = dom.QuerySelectorAll(".searchitem > h3 > a[href], #dle-content > .story > .story_h > .lcol > h2 > a[href]");
{
domQuery = ".searchitem > h3 > a";
}
else
{
domQuery = "#dle-content > .story > .story_h > .lcol > h2 > a";
}
var links = dom.QuerySelectorAll(domQuery);
foreach (var link in links) foreach (var link in links)
{ {
var url = link.GetAttribute("href"); var url = link.GetAttribute("href");
var releaseRequest = new IndexerRequest(url, HttpAccept.Html); var releaseRequest = new HttpRequestBuilder(url)
var releaseResponse = new IndexerResponse(releaseRequest, HttpClient.Execute(releaseRequest.HttpRequest)); .WithRateLimit(_rateLimit.TotalSeconds)
.SetHeader("Referer", _settings.BaseUrl)
.Accept(HttpAccept.Html)
.Build();
var releaseIndexerRequest = new IndexerRequest(releaseRequest);
var releaseResponse = new IndexerResponse(releaseIndexerRequest, _httpClient.Execute(releaseIndexerRequest.HttpRequest));
// Throw common http errors here before we try to parse // Throw common http errors here before we try to parse
if (releaseResponse.HttpResponse.HasHttpError) if (releaseResponse.HttpResponse.HasHttpError)
{ {
if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests) if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests)
{ {
throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse); throw new TooManyRequestsException(releaseResponse.HttpRequest, releaseResponse.HttpResponse);
}
else
{
throw new IndexerException(releaseResponse, "Http error code: " + releaseResponse.HttpResponse.StatusCode);
} }
throw new IndexerException(releaseResponse, $"HTTP Error - {releaseResponse.HttpResponse.StatusCode}. {url}");
} }
torrentInfos.AddRange(ParseRelease(releaseResponse)); torrentInfos.AddRange(ParseRelease(releaseResponse));

132
src/NzbDrone.Core/Indexers/Definitions/Animedia.cs
Unescape View File

@ -5,10 +5,8 @@ using System.Net;
using System.Text; using System.Text;
using System.Text.RegularExpressions; using System.Text.RegularExpressions;
using AngleSharp.Html.Parser; using AngleSharp.Html.Parser;
using FluentValidation;
using NLog; using NLog;
using NzbDrone.Common.Http; using NzbDrone.Common.Http;
using NzbDrone.Core.Annotations;
using NzbDrone.Core.Configuration; using NzbDrone.Core.Configuration;
using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.Indexers.Exceptions;
using NzbDrone.Core.Indexers.Settings; using NzbDrone.Core.Indexers.Settings;
@ -16,14 +14,13 @@ using NzbDrone.Core.IndexerSearch.Definitions;
using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.Parser; using NzbDrone.Core.Parser;
using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Parser.Model;
using NzbDrone.Core.Validation;
namespace NzbDrone.Core.Indexers.Definitions namespace NzbDrone.Core.Indexers.Definitions
{ {
public class Animedia : TorrentIndexerBase<NoAuthTorrentBaseSettings> public class Animedia : TorrentIndexerBase<NoAuthTorrentBaseSettings>
{ {
public override string Name => "Animedia"; public override string Name => "Animedia";
public override string[] IndexerUrls => new string[] { "https://tt.animedia.tv/" }; public override string[] IndexerUrls => new[] { "https://tt.animedia.tv/" };
public override string Description => "Animedia is russian anime voiceover group and eponymous anime tracker."; public override string Description => "Animedia is russian anime voiceover group and eponymous anime tracker.";
public override string Language => "ru-RU"; public override string Language => "ru-RU";
public override Encoding Encoding => Encoding.UTF8; public override Encoding Encoding => Encoding.UTF8;
@ -38,12 +35,12 @@ namespace NzbDrone.Core.Indexers.Definitions
public override IIndexerRequestGenerator GetRequestGenerator() public override IIndexerRequestGenerator GetRequestGenerator()
{ {
return new AnimediaRequestGenerator() { Settings = Settings, Capabilities = Capabilities }; return new AnimediaRequestGenerator(Settings);
} }
public override IParseIndexerResponse GetParser() public override IParseIndexerResponse GetParser()
{ {
return new AnimediaParser(Settings, Capabilities.Categories) { HttpClient = _httpClient, Logger = _logger }; return new AnimediaParser(Settings, Capabilities.Categories, RateLimit, _httpClient);
} }
private IndexerCapabilities SetCapabilities() private IndexerCapabilities SetCapabilities()
@ -51,38 +48,40 @@ namespace NzbDrone.Core.Indexers.Definitions
var caps = new IndexerCapabilities var caps = new IndexerCapabilities
{ {
TvSearchParams = new List<TvSearchParam> TvSearchParams = new List<TvSearchParam>
{ {
TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep
}, },
MovieSearchParams = new List<MovieSearchParam> MovieSearchParams = new List<MovieSearchParam>
{ {
MovieSearchParam.Q MovieSearchParam.Q
} }
}; };
caps.Categories.AddCategoryMapping(1, NewznabStandardCategory.TVAnime, "TV Anime"); caps.Categories.AddCategoryMapping(1, NewznabStandardCategory.TVAnime, "TV Anime");
caps.Categories.AddCategoryMapping(2, NewznabStandardCategory.TVAnime, "OVA/ONA/Special"); caps.Categories.AddCategoryMapping(2, NewznabStandardCategory.TVAnime, "OVA/ONA/Special");
caps.Categories.AddCategoryMapping(3, NewznabStandardCategory.TV, "Dorama"); caps.Categories.AddCategoryMapping(3, NewznabStandardCategory.TV, "Dorama");
caps.Categories.AddCategoryMapping(4, NewznabStandardCategory.Movies, "Movies"); caps.Categories.AddCategoryMapping(4, NewznabStandardCategory.Movies, "Movies");
return caps; return caps;
} }
} }
public class AnimediaRequestGenerator : IIndexerRequestGenerator public class AnimediaRequestGenerator : IIndexerRequestGenerator
{ {
public NoAuthTorrentBaseSettings Settings { get; set; } private readonly NoAuthTorrentBaseSettings _settings;
public IndexerCapabilities Capabilities { get; set; }
public AnimediaRequestGenerator() public AnimediaRequestGenerator(NoAuthTorrentBaseSettings settings)
{ {
_settings = settings;
} }
private IEnumerable<IndexerRequest> GetPagedRequests(string term, int[] categories) private IEnumerable<IndexerRequest> GetPagedRequests(string term)
{ {
var requestUrl = string.Empty; string requestUrl;
if (string.IsNullOrWhiteSpace(term)) if (string.IsNullOrWhiteSpace(term))
{ {
requestUrl = Settings.BaseUrl; requestUrl = _settings.BaseUrl;
} }
else else
{ {
@ -94,18 +93,17 @@ namespace NzbDrone.Core.Indexers.Definitions
{ "orderby_sort", "entry_date|desc" } { "orderby_sort", "entry_date|desc" }
}; };
requestUrl = string.Format("{0}/ajax/search_result/P0?{1}", Settings.BaseUrl.TrimEnd('/'), queryCollection.GetQueryString()); requestUrl = $"{_settings.BaseUrl.TrimEnd('/')}/ajax/search_result/P0?{queryCollection.GetQueryString()}";
} }
var request = new IndexerRequest(requestUrl, HttpAccept.Html); yield return new IndexerRequest(requestUrl, HttpAccept.Html);
yield return request;
} }
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
{ {
var pageableRequests = new IndexerPageableRequestChain(); var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests; return pageableRequests;
} }
@ -114,7 +112,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
var pageableRequests = new IndexerPageableRequestChain(); var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories)); pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}"));
return pageableRequests; return pageableRequests;
} }
@ -123,7 +121,7 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
var pageableRequests = new IndexerPageableRequestChain(); var pageableRequests = new IndexerPageableRequestChain();
pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}"));
return pageableRequests; return pageableRequests;
} }
@ -148,6 +146,9 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
private readonly NoAuthTorrentBaseSettings _settings; private readonly NoAuthTorrentBaseSettings _settings;
private readonly IndexerCapabilitiesCategories _categories; private readonly IndexerCapabilitiesCategories _categories;
private readonly TimeSpan _rateLimit;
private readonly IIndexerHttpClient _httpClient;
private static readonly Regex EpisodesInfoQueryRegex = new Regex(@"сери[ия] (\d+)(?:-(\d+))? из.*", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex EpisodesInfoQueryRegex = new Regex(@"сери[ия] (\d+)(?:-(\d+))? из.*", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex ResolutionInfoQueryRegex = new Regex(@"качество (\d+)", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex ResolutionInfoQueryRegex = new Regex(@"качество (\d+)", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex SizeInfoQueryRegex = new Regex(@"размер:(.*)\n", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex SizeInfoQueryRegex = new Regex(@"размер:(.*)\n", RegexOptions.Compiled | RegexOptions.IgnoreCase);
@ -155,25 +156,25 @@ namespace NzbDrone.Core.Indexers.Definitions
private static readonly Regex CategorieMovieRegex = new Regex(@"Фильм", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex CategorieMovieRegex = new Regex(@"Фильм", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex CategorieOVARegex = new Regex(@"ОВА|OVA|ОНА|ONA|Special", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex CategorieOVARegex = new Regex(@"ОВА|OVA|ОНА|ONA|Special", RegexOptions.Compiled | RegexOptions.IgnoreCase);
private static readonly Regex CategorieDoramaRegex = new Regex(@"Дорама", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex CategorieDoramaRegex = new Regex(@"Дорама", RegexOptions.Compiled | RegexOptions.IgnoreCase);
public IIndexerHttpClient HttpClient { get; set; }
public Logger Logger { get; set; }
public AnimediaParser(NoAuthTorrentBaseSettings settings, IndexerCapabilitiesCategories categories) public AnimediaParser(NoAuthTorrentBaseSettings settings, IndexerCapabilitiesCategories categories, TimeSpan rateLimit, IIndexerHttpClient httpClient)
{ {
_settings = settings; _settings = settings;
_categories = categories; _categories = categories;
_rateLimit = rateLimit;
_httpClient = httpClient;
} }
private string composeTitle(AngleSharp.Html.Dom.IHtmlDocument dom, AngleSharp.Dom.IElement t, AngleSharp.Dom.IElement tr) private string ComposeTitle(AngleSharp.Html.Dom.IHtmlDocument dom, AngleSharp.Dom.IElement t, AngleSharp.Dom.IElement tr)
{ {
var name_ru = dom.QuerySelector("div.media__post__header > h1").TextContent.Trim(); var nameRu = dom.QuerySelector("div.media__post__header > h1")?.TextContent.Trim() ?? string.Empty;
var name_en = dom.QuerySelector("div.media__panel > div:nth-of-type(1) > div.col-l:nth-of-type(1) > div > span").TextContent.Trim(); var nameEn = dom.QuerySelector("div.media__panel > div:nth-of-type(1) > div.col-l:nth-of-type(1) > div > span")?.TextContent.Trim() ?? string.Empty;
var name_orig = dom.QuerySelector("div.media__panel > div:nth-of-type(1) > div.col-l:nth-of-type(2) > div > span").TextContent.Trim(); var nameOrig = dom.QuerySelector("div.media__panel > div:nth-of-type(1) > div.col-l:nth-of-type(2) > div > span")?.TextContent.Trim() ?? string.Empty;
var title = name_ru + " / " + name_en; var title = nameRu + " / " + nameEn;
if (name_en != name_orig) if (nameEn != nameOrig)
{ {
title += " / " + name_orig; title += " / " + nameOrig;
} }
var tabName = t.TextContent; var tabName = t.TextContent;
@ -183,7 +184,7 @@ namespace NzbDrone.Core.Indexers.Definitions
tabName = ""; tabName = "";
} }
var heading = tr.QuerySelector("h3.tracker_info_bold").TextContent; var heading = tr.QuerySelector("h3.tracker_info_bold")?.TextContent.Trim() ?? string.Empty;
// Parse episodes info from heading if episods info present // Parse episodes info from heading if episods info present
var match = EpisodesInfoQueryRegex.Match(heading); var match = EpisodesInfoQueryRegex.Match(heading);
@ -192,40 +193,40 @@ namespace NzbDrone.Core.Indexers.Definitions
{ {
if (string.IsNullOrEmpty(match.Groups[2].Value)) if (string.IsNullOrEmpty(match.Groups[2].Value))
{ {
heading += " E" + match.Groups[1].Value; heading += $" E{match.Groups[1].Value}";
} }
else else
{ {
heading += string.Format(" E{0}-{1}", match.Groups[1].Value, match.Groups[2].Value); heading += $" E{match.Groups[1].Value}-{match.Groups[2].Value}";
} }
} }
return title + " - " + heading + " [" + getResolution(tr) + "p]"; return title + " - " + heading + " [" + GetResolution(tr) + "p]";
} }
private string getResolution(AngleSharp.Dom.IElement tr) private string GetResolution(AngleSharp.Dom.IElement tr)
{ {
var resolution = tr.QuerySelector("div.tracker_info_left").TextContent; var resolution = tr.QuerySelector("div.tracker_info_left")?.TextContent.Trim() ?? string.Empty;
return ResolutionInfoQueryRegex.Match(resolution).Groups[1].Value; return ResolutionInfoQueryRegex.Match(resolution).Groups[1].Value;
} }
private long getReleaseSize(AngleSharp.Dom.IElement tr) private long GetReleaseSize(AngleSharp.Dom.IElement tr)
{ {
var sizeStr = tr.QuerySelector("div.tracker_info_left").TextContent; var sizeStr = tr.QuerySelector("div.tracker_info_left")?.TextContent.Trim() ?? string.Empty;
return ParseUtil.GetBytes(SizeInfoQueryRegex.Match(sizeStr).Groups[1].Value.Trim()); return ParseUtil.GetBytes(SizeInfoQueryRegex.Match(sizeStr).Groups[1].Value.Trim());
} }
private DateTime getReleaseDate(AngleSharp.Dom.IElement tr) private DateTime GetReleaseDate(AngleSharp.Dom.IElement tr)
{ {
var sizeStr = tr.QuerySelector("div.tracker_info_left").TextContent; var sizeStr = tr.QuerySelector("div.tracker_info_left")?.TextContent.Trim() ?? string.Empty;
return DateTime.Parse(ReleaseDateInfoQueryRegex.Match(sizeStr).Groups[1].Value.Trim()); return DateTime.Parse(ReleaseDateInfoQueryRegex.Match(sizeStr).Groups[1].Value.Trim());
} }
private ICollection<IndexerCategory> MapCategories(AngleSharp.Html.Dom.IHtmlDocument dom, AngleSharp.Dom.IElement t, AngleSharp.Dom.IElement tr) private ICollection<IndexerCategory> MapCategories(AngleSharp.Html.Dom.IHtmlDocument dom, AngleSharp.Dom.IElement t, AngleSharp.Dom.IElement tr)
{ {
var rName = t.TextContent; var rName = t.TextContent;
var rDesc = tr.QuerySelector("h3.tracker_info_bold").TextContent; var rDesc = tr.QuerySelector("h3.tracker_info_bold")?.TextContent.Trim() ?? string.Empty;
var type = dom.QuerySelector("div.releases-date:contains('Тип:')").TextContent; var type = dom.QuerySelector("div.releases-date:contains('Тип:')")?.TextContent.Trim() ?? string.Empty;
// Check OVA first cause OVA looks like anime with OVA in release name or description // Check OVA first cause OVA looks like anime with OVA in release name or description
if (CategorieOVARegex.IsMatch(rName) || CategorieOVARegex.IsMatch(rDesc)) if (CategorieOVARegex.IsMatch(rName) || CategorieOVARegex.IsMatch(rDesc))
@ -256,28 +257,28 @@ namespace NzbDrone.Core.Indexers.Definitions
foreach (var t in dom.QuerySelectorAll("ul.media__tabs__nav > li > a")) foreach (var t in dom.QuerySelectorAll("ul.media__tabs__nav > li > a"))
{ {
var tr_id = t.Attributes["href"].Value; var trId = t.GetAttribute("href");
var tr = dom.QuerySelector("div" + tr_id); var tr = dom.QuerySelector("div" + trId);
var seeders = int.Parse(tr.QuerySelector("div.circle_green_text_top").TextContent); var seeders = int.Parse(tr.QuerySelector("div.circle_green_text_top").TextContent);
var url = indexerResponse.HttpRequest.Url.ToString(); var url = indexerResponse.HttpRequest.Url.FullUri;
var release = new TorrentInfo var release = new TorrentInfo
{ {
Title = composeTitle(dom, t, tr), Title = ComposeTitle(dom, t, tr),
InfoUrl = url, InfoUrl = url,
DownloadVolumeFactor = 0, DownloadVolumeFactor = 0,
UploadVolumeFactor = 1, UploadVolumeFactor = 1,
Guid = url + tr_id, Guid = url + trId,
Seeders = seeders, Seeders = seeders,
Peers = seeders + int.Parse(tr.QuerySelector("div.circle_red_text_top").TextContent), Peers = seeders + int.Parse(tr.QuerySelector("div.circle_red_text_top").TextContent),
Grabs = int.Parse(tr.QuerySelector("div.circle_grey_text_top").TextContent), Grabs = int.Parse(tr.QuerySelector("div.circle_grey_text_top").TextContent),
Categories = MapCategories(dom, t, tr), Categories = MapCategories(dom, t, tr),
PublishDate = getReleaseDate(tr), PublishDate = GetReleaseDate(tr),
DownloadUrl = tr.QuerySelector("div.download_tracker > a.btn__green").Attributes["href"].Value, DownloadUrl = tr.QuerySelector("div.download_tracker > a.btn__green").GetAttribute("href"),
MagnetUrl = tr.QuerySelector("div.download_tracker > a.btn__d-gray").Attributes["href"].Value, MagnetUrl = tr.QuerySelector("div.download_tracker > a.btn__d-gray").GetAttribute("href"),
Size = getReleaseSize(tr), Size = GetReleaseSize(tr),
Resolution = getResolution(tr) Resolution = GetResolution(tr)
}; };
torrentInfos.Add(release); torrentInfos.Add(release);
} }
@ -291,6 +292,7 @@ namespace NzbDrone.Core.Indexers.Definitions
var parser = new HtmlParser(); var parser = new HtmlParser();
var dom = parser.ParseDocument(indexerResponse.Content); var dom = parser.ParseDocument(indexerResponse.Content);
var links = dom.QuerySelectorAll("a.ads-list__item__title"); var links = dom.QuerySelectorAll("a.ads-list__item__title");
foreach (var link in links) foreach (var link in links)
{ {
@ -302,20 +304,24 @@ namespace NzbDrone.Core.Indexers.Definitions
url = "https:" + url; url = "https:" + url;
} }
var releaseRequest = new IndexerRequest(url, HttpAccept.Html); var releaseRequest = new HttpRequestBuilder(url)
var releaseResponse = new IndexerResponse(releaseRequest, HttpClient.Execute(releaseRequest.HttpRequest)); .WithRateLimit(_rateLimit.TotalSeconds)
.SetHeader("Referer", _settings.BaseUrl)
.Accept(HttpAccept.Html)
.Build();
var releaseIndexerRequest = new IndexerRequest(releaseRequest);
var releaseResponse = new IndexerResponse(releaseIndexerRequest, _httpClient.Execute(releaseIndexerRequest.HttpRequest));
// Throw common http errors here before we try to parse // Throw common http errors here before we try to parse
if (releaseResponse.HttpResponse.HasHttpError) if (releaseResponse.HttpResponse.HasHttpError)
{ {
if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests) if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests)
{ {
throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse); throw new TooManyRequestsException(releaseResponse.HttpRequest, releaseResponse.HttpResponse);
}
else
{
throw new IndexerException(releaseResponse, "Http error code: " + releaseResponse.HttpResponse.StatusCode);
} }
throw new IndexerException(releaseResponse, $"HTTP Error - {releaseResponse.HttpResponse.StatusCode}. {url}");
} }
torrentInfos.AddRange(ParseRelease(releaseResponse)); torrentInfos.AddRange(ParseRelease(releaseResponse));

Write Preview
Loading…
Cancel
Save