diff --git a/src/NzbDrone.Core/Indexers/Definitions/Anidub.cs b/src/NzbDrone.Core/Indexers/Definitions/Anidub.cs index 1e7c820d8..5ff70ac1a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Anidub.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Anidub.cs @@ -39,12 +39,12 @@ namespace NzbDrone.Core.Indexers.Definitions public override IIndexerRequestGenerator GetRequestGenerator() { - return new AnidubRequestGenerator { Settings = Settings, Capabilities = Capabilities }; + return new AnidubRequestGenerator(Settings); } public override IParseIndexerResponse GetParser() { - return new AnidubParser(Settings, Capabilities.Categories) { HttpClient = _httpClient, Logger = _logger }; + return new AnidubParser(Settings, Capabilities.Categories, RateLimit, _httpClient, _logger); } protected override async Task DoLogin() @@ -135,21 +135,25 @@ namespace NzbDrone.Core.Indexers.Definitions public class AnidubRequestGenerator : IIndexerRequestGenerator { - public UserPassTorrentBaseSettings Settings { get; set; } - public IndexerCapabilities Capabilities { get; set; } + private readonly UserPassTorrentBaseSettings _settings; + + public AnidubRequestGenerator(UserPassTorrentBaseSettings settings) + { + _settings = settings; + } - private IEnumerable GetPagedRequests(string term, int[] categories) + private IEnumerable GetPagedRequests(string term) { - var requestUrl = string.Empty; + string requestUrl; var isSearch = !string.IsNullOrWhiteSpace(term); if (isSearch) { - requestUrl = string.Format("{0}/index.php?do=search", Settings.BaseUrl.TrimEnd('/')); + requestUrl = $"{_settings.BaseUrl.TrimEnd('/')}/index.php?do=search"; } else { - requestUrl = Settings.BaseUrl; + requestUrl = _settings.BaseUrl; } var request = new IndexerRequest(requestUrl, HttpAccept.Html); @@ -194,7 +198,7 @@ namespace NzbDrone.Core.Indexers.Definitions { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); + pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}")); return pageableRequests; } @@ -203,7 +207,7 @@ namespace NzbDrone.Core.Indexers.Definitions { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories)); + pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}")); return pageableRequests; } @@ -212,7 +216,7 @@ namespace NzbDrone.Core.Indexers.Definitions { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); + pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}")); return pageableRequests; } @@ -221,7 +225,7 @@ namespace NzbDrone.Core.Indexers.Definitions { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); + pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}")); return pageableRequests; } @@ -230,7 +234,7 @@ namespace NzbDrone.Core.Indexers.Definitions { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); + pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}")); return pageableRequests; } @@ -243,33 +247,37 @@ namespace NzbDrone.Core.Indexers.Definitions { private readonly UserPassTorrentBaseSettings _settings; private readonly IndexerCapabilitiesCategories _categories; - public IIndexerHttpClient HttpClient { get; set; } - public Logger Logger { get; set; } + private readonly TimeSpan _rateLimit; + private readonly IIndexerHttpClient _httpClient; + private readonly Logger _logger; - private static Dictionary CategoriesMap => new Dictionary - { - { "/anime_tv/full", "14" }, - { "/anime_tv/anime_ongoing", "10" }, - { "/anime_tv/shonen", "11" }, - { "/anime_tv", "2" }, - { "/xxx", "13" }, - { "/manga", "15" }, - { "/ost", "16" }, - { "/podcast", "17" }, - { "/anime_movie", "3" }, - { "/anime_ova/anime_ona", "5" }, - { "/anime_ova", "4" }, - { "/dorama/japan_dorama", "6" }, - { "/dorama/korea_dorama", "7" }, - { "/dorama/china_dorama", "8" }, - { "/dorama", "9" }, - { "/anons_ongoing", "12" } - }; - - public AnidubParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories) + private static Dictionary CategoriesMap => new () + { + { "/anime_tv/full", "14" }, + { "/anime_tv/anime_ongoing", "10" }, + { "/anime_tv/shonen", "11" }, + { "/anime_tv", "2" }, + { "/xxx", "13" }, + { "/manga", "15" }, + { "/ost", "16" }, + { "/podcast", "17" }, + { "/anime_movie", "3" }, + { "/anime_ova/anime_ona", "5" }, + { "/anime_ova", "4" }, + { "/dorama/japan_dorama", "6" }, + { "/dorama/korea_dorama", "7" }, + { "/dorama/china_dorama", "8" }, + { "/dorama", "9" }, + { "/anons_ongoing", "12" } + }; + + public AnidubParser(UserPassTorrentBaseSettings settings, IndexerCapabilitiesCategories categories, TimeSpan rateLimit, IIndexerHttpClient httpClient, Logger logger) { _settings = settings; _categories = categories; + _rateLimit = rateLimit; + _httpClient = httpClient; + _logger = logger; } private static string GetTitle(AngleSharp.Html.Dom.IHtmlDocument content, AngleSharp.Dom.IElement tabNode) @@ -314,9 +322,9 @@ namespace NzbDrone.Core.Indexers.Definitions private static int GetReleaseLeechers(AngleSharp.Dom.IElement tabNode) { - const string LeechersSelector = ".list.down > .li_swing_m"; + const string leechersSelector = ".list.down > .li_swing_m"; - var leechersStr = tabNode.QuerySelector(LeechersSelector).TextContent; + var leechersStr = tabNode.QuerySelector(leechersSelector).TextContent; int.TryParse(leechersStr, out var leechers); return leechers; } @@ -332,18 +340,18 @@ namespace NzbDrone.Core.Indexers.Definitions private static int GetReleaseGrabs(AngleSharp.Dom.IElement tabNode) { - const string GrabsSelector = ".list.down > .li_download_m"; + const string grabsSelector = ".list.down > .li_download_m"; - var grabsStr = tabNode.QuerySelector(GrabsSelector).TextContent; + var grabsStr = tabNode.QuerySelector(grabsSelector).TextContent; int.TryParse(grabsStr, out var grabs); return grabs; } private static string GetDateFromDocument(AngleSharp.Html.Dom.IHtmlDocument content) { - const string DateSelector = ".story_inf > li:nth-child(2)"; + const string dateSelector = ".story_inf > li:nth-child(2)"; - var domDate = content.QuerySelector(DateSelector).LastChild; + var domDate = content.QuerySelector(dateSelector).LastChild; if (domDate?.NodeName != "#text") { @@ -384,16 +392,16 @@ namespace NzbDrone.Core.Indexers.Definitions return utcDate.AddHours(-russianStandardTimeDiff); } - Logger.Warn($"[AniDub] Date time couldn't be parsed on. Date text: {dateText}"); + _logger.Warn($"[AniDub] Date time couldn't be parsed on. Date text: {dateText}"); return DateTime.UtcNow; } private static long GetReleaseSize(AngleSharp.Dom.IElement tabNode) { - const string SizeSelector = ".list.down > .red"; + const string sizeSelector = ".list.down > .red"; - var sizeStr = tabNode.QuerySelector(SizeSelector).TextContent; + var sizeStr = tabNode.QuerySelector(sizeSelector).TextContent; return ParseUtil.GetBytes(sizeStr); } @@ -433,11 +441,11 @@ namespace NzbDrone.Core.Indexers.Definitions var release = new TorrentInfo { Title = GetTitle(dom, t), - InfoUrl = indexerResponse.Request.Url.ToString(), + InfoUrl = indexerResponse.Request.Url.FullUri, DownloadVolumeFactor = 0, UploadVolumeFactor = 1, - Guid = indexerResponse.Request.Url.ToString() + t.Id, + Guid = indexerResponse.Request.Url.FullUri + t.Id, Seeders = GetReleaseSeeders(t), Peers = GetReleaseSeeders(t) + GetReleaseLeechers(t), Grabs = GetReleaseGrabs(t), @@ -459,36 +467,30 @@ namespace NzbDrone.Core.Indexers.Definitions var parser = new HtmlParser(); var dom = parser.ParseDocument(indexerResponse.Content); - var domQuery = string.Empty; - if (indexerResponse.Request.Url.Query.Contains("do=search")) - { - domQuery = ".searchitem > h3 > a"; - } - else - { - domQuery = "#dle-content > .story > .story_h > .lcol > h2 > a"; - } - - var links = dom.QuerySelectorAll(domQuery); + var links = dom.QuerySelectorAll(".searchitem > h3 > a[href], #dle-content > .story > .story_h > .lcol > h2 > a[href]"); foreach (var link in links) { var url = link.GetAttribute("href"); - var releaseRequest = new IndexerRequest(url, HttpAccept.Html); - var releaseResponse = new IndexerResponse(releaseRequest, HttpClient.Execute(releaseRequest.HttpRequest)); + var releaseRequest = new HttpRequestBuilder(url) + .WithRateLimit(_rateLimit.TotalSeconds) + .SetHeader("Referer", _settings.BaseUrl) + .Accept(HttpAccept.Html) + .Build(); + + var releaseIndexerRequest = new IndexerRequest(releaseRequest); + var releaseResponse = new IndexerResponse(releaseIndexerRequest, _httpClient.Execute(releaseIndexerRequest.HttpRequest)); // Throw common http errors here before we try to parse if (releaseResponse.HttpResponse.HasHttpError) { if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests) { - throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse); - } - else - { - throw new IndexerException(releaseResponse, "Http error code: " + releaseResponse.HttpResponse.StatusCode); + throw new TooManyRequestsException(releaseResponse.HttpRequest, releaseResponse.HttpResponse); } + + throw new IndexerException(releaseResponse, $"HTTP Error - {releaseResponse.HttpResponse.StatusCode}. {url}"); } torrentInfos.AddRange(ParseRelease(releaseResponse)); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Animedia.cs b/src/NzbDrone.Core/Indexers/Definitions/Animedia.cs index 0710550ba..eabb933e7 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Animedia.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Animedia.cs @@ -5,10 +5,8 @@ using System.Net; using System.Text; using System.Text.RegularExpressions; using AngleSharp.Html.Parser; -using FluentValidation; using NLog; using NzbDrone.Common.Http; -using NzbDrone.Core.Annotations; using NzbDrone.Core.Configuration; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.Indexers.Settings; @@ -16,14 +14,13 @@ using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.Parser; using NzbDrone.Core.Parser.Model; -using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Definitions { public class Animedia : TorrentIndexerBase { public override string Name => "Animedia"; - public override string[] IndexerUrls => new string[] { "https://tt.animedia.tv/" }; + public override string[] IndexerUrls => new[] { "https://tt.animedia.tv/" }; public override string Description => "Animedia is russian anime voiceover group and eponymous anime tracker."; public override string Language => "ru-RU"; public override Encoding Encoding => Encoding.UTF8; @@ -38,12 +35,12 @@ namespace NzbDrone.Core.Indexers.Definitions public override IIndexerRequestGenerator GetRequestGenerator() { - return new AnimediaRequestGenerator() { Settings = Settings, Capabilities = Capabilities }; + return new AnimediaRequestGenerator(Settings); } public override IParseIndexerResponse GetParser() { - return new AnimediaParser(Settings, Capabilities.Categories) { HttpClient = _httpClient, Logger = _logger }; + return new AnimediaParser(Settings, Capabilities.Categories, RateLimit, _httpClient); } private IndexerCapabilities SetCapabilities() @@ -51,38 +48,40 @@ namespace NzbDrone.Core.Indexers.Definitions var caps = new IndexerCapabilities { TvSearchParams = new List - { - TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep - }, + { + TvSearchParam.Q, TvSearchParam.Season, TvSearchParam.Ep + }, MovieSearchParams = new List - { - MovieSearchParam.Q - } + { + MovieSearchParam.Q + } }; + caps.Categories.AddCategoryMapping(1, NewznabStandardCategory.TVAnime, "TV Anime"); caps.Categories.AddCategoryMapping(2, NewznabStandardCategory.TVAnime, "OVA/ONA/Special"); caps.Categories.AddCategoryMapping(3, NewznabStandardCategory.TV, "Dorama"); caps.Categories.AddCategoryMapping(4, NewznabStandardCategory.Movies, "Movies"); + return caps; } } public class AnimediaRequestGenerator : IIndexerRequestGenerator { - public NoAuthTorrentBaseSettings Settings { get; set; } - public IndexerCapabilities Capabilities { get; set; } + private readonly NoAuthTorrentBaseSettings _settings; - public AnimediaRequestGenerator() + public AnimediaRequestGenerator(NoAuthTorrentBaseSettings settings) { + _settings = settings; } - private IEnumerable GetPagedRequests(string term, int[] categories) + private IEnumerable GetPagedRequests(string term) { - var requestUrl = string.Empty; + string requestUrl; if (string.IsNullOrWhiteSpace(term)) { - requestUrl = Settings.BaseUrl; + requestUrl = _settings.BaseUrl; } else { @@ -94,18 +93,17 @@ namespace NzbDrone.Core.Indexers.Definitions { "orderby_sort", "entry_date|desc" } }; - requestUrl = string.Format("{0}/ajax/search_result/P0?{1}", Settings.BaseUrl.TrimEnd('/'), queryCollection.GetQueryString()); + requestUrl = $"{_settings.BaseUrl.TrimEnd('/')}/ajax/search_result/P0?{queryCollection.GetQueryString()}"; } - var request = new IndexerRequest(requestUrl, HttpAccept.Html); - yield return request; + yield return new IndexerRequest(requestUrl, HttpAccept.Html); } public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); + pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}")); return pageableRequests; } @@ -114,7 +112,7 @@ namespace NzbDrone.Core.Indexers.Definitions { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedTvSearchString), searchCriteria.Categories)); + pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedTvSearchString}")); return pageableRequests; } @@ -123,7 +121,7 @@ namespace NzbDrone.Core.Indexers.Definitions { var pageableRequests = new IndexerPageableRequestChain(); - pageableRequests.Add(GetPagedRequests(string.Format("{0}", searchCriteria.SanitizedSearchTerm), searchCriteria.Categories)); + pageableRequests.Add(GetPagedRequests($"{searchCriteria.SanitizedSearchTerm}")); return pageableRequests; } @@ -148,6 +146,9 @@ namespace NzbDrone.Core.Indexers.Definitions { private readonly NoAuthTorrentBaseSettings _settings; private readonly IndexerCapabilitiesCategories _categories; + private readonly TimeSpan _rateLimit; + private readonly IIndexerHttpClient _httpClient; + private static readonly Regex EpisodesInfoQueryRegex = new Regex(@"сери[ия] (\d+)(?:-(\d+))? из.*", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex ResolutionInfoQueryRegex = new Regex(@"качество (\d+)", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex SizeInfoQueryRegex = new Regex(@"размер:(.*)\n", RegexOptions.Compiled | RegexOptions.IgnoreCase); @@ -155,25 +156,25 @@ namespace NzbDrone.Core.Indexers.Definitions private static readonly Regex CategorieMovieRegex = new Regex(@"Фильм", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex CategorieOVARegex = new Regex(@"ОВА|OVA|ОНА|ONA|Special", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex CategorieDoramaRegex = new Regex(@"Дорама", RegexOptions.Compiled | RegexOptions.IgnoreCase); - public IIndexerHttpClient HttpClient { get; set; } - public Logger Logger { get; set; } - public AnimediaParser(NoAuthTorrentBaseSettings settings, IndexerCapabilitiesCategories categories) + public AnimediaParser(NoAuthTorrentBaseSettings settings, IndexerCapabilitiesCategories categories, TimeSpan rateLimit, IIndexerHttpClient httpClient) { _settings = settings; _categories = categories; + _rateLimit = rateLimit; + _httpClient = httpClient; } - private string composeTitle(AngleSharp.Html.Dom.IHtmlDocument dom, AngleSharp.Dom.IElement t, AngleSharp.Dom.IElement tr) + private string ComposeTitle(AngleSharp.Html.Dom.IHtmlDocument dom, AngleSharp.Dom.IElement t, AngleSharp.Dom.IElement tr) { - var name_ru = dom.QuerySelector("div.media__post__header > h1").TextContent.Trim(); - var name_en = dom.QuerySelector("div.media__panel > div:nth-of-type(1) > div.col-l:nth-of-type(1) > div > span").TextContent.Trim(); - var name_orig = dom.QuerySelector("div.media__panel > div:nth-of-type(1) > div.col-l:nth-of-type(2) > div > span").TextContent.Trim(); + var nameRu = dom.QuerySelector("div.media__post__header > h1")?.TextContent.Trim() ?? string.Empty; + var nameEn = dom.QuerySelector("div.media__panel > div:nth-of-type(1) > div.col-l:nth-of-type(1) > div > span")?.TextContent.Trim() ?? string.Empty; + var nameOrig = dom.QuerySelector("div.media__panel > div:nth-of-type(1) > div.col-l:nth-of-type(2) > div > span")?.TextContent.Trim() ?? string.Empty; - var title = name_ru + " / " + name_en; - if (name_en != name_orig) + var title = nameRu + " / " + nameEn; + if (nameEn != nameOrig) { - title += " / " + name_orig; + title += " / " + nameOrig; } var tabName = t.TextContent; @@ -183,7 +184,7 @@ namespace NzbDrone.Core.Indexers.Definitions tabName = ""; } - var heading = tr.QuerySelector("h3.tracker_info_bold").TextContent; + var heading = tr.QuerySelector("h3.tracker_info_bold")?.TextContent.Trim() ?? string.Empty; // Parse episodes info from heading if episods info present var match = EpisodesInfoQueryRegex.Match(heading); @@ -192,40 +193,40 @@ namespace NzbDrone.Core.Indexers.Definitions { if (string.IsNullOrEmpty(match.Groups[2].Value)) { - heading += " E" + match.Groups[1].Value; + heading += $" E{match.Groups[1].Value}"; } else { - heading += string.Format(" E{0}-{1}", match.Groups[1].Value, match.Groups[2].Value); + heading += $" E{match.Groups[1].Value}-{match.Groups[2].Value}"; } } - return title + " - " + heading + " [" + getResolution(tr) + "p]"; + return title + " - " + heading + " [" + GetResolution(tr) + "p]"; } - private string getResolution(AngleSharp.Dom.IElement tr) + private string GetResolution(AngleSharp.Dom.IElement tr) { - var resolution = tr.QuerySelector("div.tracker_info_left").TextContent; + var resolution = tr.QuerySelector("div.tracker_info_left")?.TextContent.Trim() ?? string.Empty; return ResolutionInfoQueryRegex.Match(resolution).Groups[1].Value; } - private long getReleaseSize(AngleSharp.Dom.IElement tr) + private long GetReleaseSize(AngleSharp.Dom.IElement tr) { - var sizeStr = tr.QuerySelector("div.tracker_info_left").TextContent; + var sizeStr = tr.QuerySelector("div.tracker_info_left")?.TextContent.Trim() ?? string.Empty; return ParseUtil.GetBytes(SizeInfoQueryRegex.Match(sizeStr).Groups[1].Value.Trim()); } - private DateTime getReleaseDate(AngleSharp.Dom.IElement tr) + private DateTime GetReleaseDate(AngleSharp.Dom.IElement tr) { - var sizeStr = tr.QuerySelector("div.tracker_info_left").TextContent; + var sizeStr = tr.QuerySelector("div.tracker_info_left")?.TextContent.Trim() ?? string.Empty; return DateTime.Parse(ReleaseDateInfoQueryRegex.Match(sizeStr).Groups[1].Value.Trim()); } private ICollection MapCategories(AngleSharp.Html.Dom.IHtmlDocument dom, AngleSharp.Dom.IElement t, AngleSharp.Dom.IElement tr) { var rName = t.TextContent; - var rDesc = tr.QuerySelector("h3.tracker_info_bold").TextContent; - var type = dom.QuerySelector("div.releases-date:contains('Тип:')").TextContent; + var rDesc = tr.QuerySelector("h3.tracker_info_bold")?.TextContent.Trim() ?? string.Empty; + var type = dom.QuerySelector("div.releases-date:contains('Тип:')")?.TextContent.Trim() ?? string.Empty; // Check OVA first cause OVA looks like anime with OVA in release name or description if (CategorieOVARegex.IsMatch(rName) || CategorieOVARegex.IsMatch(rDesc)) @@ -256,28 +257,28 @@ namespace NzbDrone.Core.Indexers.Definitions foreach (var t in dom.QuerySelectorAll("ul.media__tabs__nav > li > a")) { - var tr_id = t.Attributes["href"].Value; - var tr = dom.QuerySelector("div" + tr_id); + var trId = t.GetAttribute("href"); + var tr = dom.QuerySelector("div" + trId); var seeders = int.Parse(tr.QuerySelector("div.circle_green_text_top").TextContent); - var url = indexerResponse.HttpRequest.Url.ToString(); + var url = indexerResponse.HttpRequest.Url.FullUri; var release = new TorrentInfo { - Title = composeTitle(dom, t, tr), + Title = ComposeTitle(dom, t, tr), InfoUrl = url, DownloadVolumeFactor = 0, UploadVolumeFactor = 1, - Guid = url + tr_id, + Guid = url + trId, Seeders = seeders, Peers = seeders + int.Parse(tr.QuerySelector("div.circle_red_text_top").TextContent), Grabs = int.Parse(tr.QuerySelector("div.circle_grey_text_top").TextContent), Categories = MapCategories(dom, t, tr), - PublishDate = getReleaseDate(tr), - DownloadUrl = tr.QuerySelector("div.download_tracker > a.btn__green").Attributes["href"].Value, - MagnetUrl = tr.QuerySelector("div.download_tracker > a.btn__d-gray").Attributes["href"].Value, - Size = getReleaseSize(tr), - Resolution = getResolution(tr) + PublishDate = GetReleaseDate(tr), + DownloadUrl = tr.QuerySelector("div.download_tracker > a.btn__green").GetAttribute("href"), + MagnetUrl = tr.QuerySelector("div.download_tracker > a.btn__d-gray").GetAttribute("href"), + Size = GetReleaseSize(tr), + Resolution = GetResolution(tr) }; torrentInfos.Add(release); } @@ -291,6 +292,7 @@ namespace NzbDrone.Core.Indexers.Definitions var parser = new HtmlParser(); var dom = parser.ParseDocument(indexerResponse.Content); + var links = dom.QuerySelectorAll("a.ads-list__item__title"); foreach (var link in links) { @@ -302,20 +304,24 @@ namespace NzbDrone.Core.Indexers.Definitions url = "https:" + url; } - var releaseRequest = new IndexerRequest(url, HttpAccept.Html); - var releaseResponse = new IndexerResponse(releaseRequest, HttpClient.Execute(releaseRequest.HttpRequest)); + var releaseRequest = new HttpRequestBuilder(url) + .WithRateLimit(_rateLimit.TotalSeconds) + .SetHeader("Referer", _settings.BaseUrl) + .Accept(HttpAccept.Html) + .Build(); + + var releaseIndexerRequest = new IndexerRequest(releaseRequest); + var releaseResponse = new IndexerResponse(releaseIndexerRequest, _httpClient.Execute(releaseIndexerRequest.HttpRequest)); // Throw common http errors here before we try to parse if (releaseResponse.HttpResponse.HasHttpError) { if (releaseResponse.HttpResponse.StatusCode == HttpStatusCode.TooManyRequests) { - throw new TooManyRequestsException(releaseRequest.HttpRequest, releaseResponse.HttpResponse); - } - else - { - throw new IndexerException(releaseResponse, "Http error code: " + releaseResponse.HttpResponse.StatusCode); + throw new TooManyRequestsException(releaseResponse.HttpRequest, releaseResponse.HttpResponse); } + + throw new IndexerException(releaseResponse, $"HTTP Error - {releaseResponse.HttpResponse.StatusCode}. {url}"); } torrentInfos.AddRange(ParseRelease(releaseResponse));