diff --git a/src/NzbDrone.Common/Http/HttpRequestBuilder.cs b/src/NzbDrone.Common/Http/HttpRequestBuilder.cs index 2307658d4..249a403e9 100644 --- a/src/NzbDrone.Common/Http/HttpRequestBuilder.cs +++ b/src/NzbDrone.Common/Http/HttpRequestBuilder.cs @@ -291,6 +291,16 @@ namespace NzbDrone.Common.Http return this; } + public virtual HttpRequestBuilder SetHeaders(Dictionary headers) + { + foreach (var header in headers) + { + Headers.Set(header.Key, header.Value); + } + + return this; + } + public virtual HttpRequestBuilder AddPrefixQueryParam(string key, object value, bool replace = false) { if (replace) diff --git a/src/NzbDrone.Core/Download/DownloadService.cs b/src/NzbDrone.Core/Download/DownloadService.cs index ddb10cc3f..c25b5ab9e 100644 --- a/src/NzbDrone.Core/Download/DownloadService.cs +++ b/src/NzbDrone.Core/Download/DownloadService.cs @@ -112,7 +112,7 @@ namespace NzbDrone.Core.Download public async Task DownloadReport(string link, int indexerId, string source, string title) { - var url = new HttpUri(link); + var url = new Uri(link); // Limit grabs to 2 per second. if (link.IsNotNullOrWhiteSpace() && !link.StartsWith("magnet:")) diff --git a/src/NzbDrone.Core/History/HistoryService.cs b/src/NzbDrone.Core/History/HistoryService.cs index 9a65c79f6..0f9c56f88 100644 --- a/src/NzbDrone.Core/History/HistoryService.cs +++ b/src/NzbDrone.Core/History/HistoryService.cs @@ -142,7 +142,7 @@ namespace NzbDrone.Core.History if (message.Query is BookSearchCriteria) { history.Data.Add("Author", ((BookSearchCriteria)message.Query).Author ?? string.Empty); - history.Data.Add("Title", ((BookSearchCriteria)message.Query).Title ?? string.Empty); + history.Data.Add("BookTitle", ((BookSearchCriteria)message.Query).Title ?? string.Empty); } history.Data.Add("ElapsedTime", message.Time.ToString()); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index cae457831..d5418ebbf 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -37,6 +37,8 @@ namespace NzbDrone.Core.Indexers.Cardigann Settings = Settings }); + generator = (CardigannRequestGenerator)SetCookieFunctions(generator); + _generatorCache.ClearExpired(); return generator; @@ -134,6 +136,29 @@ namespace NzbDrone.Core.Indexers.Cardigann await generator.DoLogin(); } + public override async Task Download(Uri link) + { + var generator = (CardigannRequestGenerator)GetRequestGenerator(); + + var request = await generator.DownloadRequest(link); + request.AllowAutoRedirect = true; + + var downloadBytes = Array.Empty(); + + try + { + var response = await _httpClient.ExecuteAsync(request); + downloadBytes = response.ResponseData; + } + catch (Exception) + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Error("Download failed"); + } + + return downloadBytes; + } + protected override async Task Test(List failures) { await base.Test(failures); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs index e04a17aa3..a33faad5a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs @@ -6,6 +6,7 @@ using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; using AngleSharp.Dom; +using Microsoft.AspNetCore.WebUtilities; using Newtonsoft.Json.Linq; using NLog; using NzbDrone.Common.Extensions; @@ -706,6 +707,44 @@ namespace NzbDrone.Core.Indexers.Cardigann return data; } + protected Dictionary ParseCustomHeaders(Dictionary> customHeaders, + Dictionary variables) + { + if (customHeaders == null) + { + return null; + } + + // FIXME: fix jackett header handling (allow it to specifiy the same header multipe times) + var headers = new Dictionary(); + foreach (var header in customHeaders) + { + headers.Add(header.Key, ApplyGoTemplateText(header.Value[0], variables)); + } + + return headers; + } + + protected IDictionary AddTemplateVariablesFromUri(IDictionary variables, Uri uri, string prefix = "") + { + variables[prefix + ".AbsoluteUri"] = uri.AbsoluteUri; + variables[prefix + ".AbsolutePath"] = uri.AbsolutePath; + variables[prefix + ".Scheme"] = uri.Scheme; + variables[prefix + ".Host"] = uri.Host; + variables[prefix + ".Port"] = uri.Port.ToString(); + variables[prefix + ".PathAndQuery"] = uri.PathAndQuery; + variables[prefix + ".Query"] = uri.Query; + var queryString = QueryHelpers.ParseQuery(uri.Query); + + foreach (var key in queryString.Keys) + { + //If we have supplied the same query string multiple time, just take the first. + variables[prefix + ".Query." + key] = queryString[key].First(); + } + + return variables; + } + protected Uri ResolvePath(string path, Uri currentUrl = null) { return new Uri(currentUrl ?? new Uri(SiteLink), path); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs index 51bb3c9f1..17eadaf8b 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs @@ -607,7 +607,7 @@ namespace NzbDrone.Core.Indexers.Cardigann var request = new HttpRequestBuilder(captchaUrl.ToString()) .SetCookies(landingResult.GetCookies()) - .SetHeader("Referrer", loginUrl.AbsoluteUri) + .SetHeader("Referer", loginUrl.AbsoluteUri) .Build(); var response = await HttpClient.ExecuteAsync(request); @@ -644,6 +644,139 @@ namespace NzbDrone.Core.Indexers.Cardigann protected string GetRedirectDomainHint(HttpResponse result) => GetRedirectDomainHint(result.Request.Url.ToString(), result.Headers.GetSingleValue("Location")); + protected async Task HandleRequest(RequestBlock request, Dictionary variables = null, string referer = null) + { + var requestLinkStr = ResolvePath(ApplyGoTemplateText(request.Path, variables)).ToString(); + _logger.Debug($"CardigannIndexer ({_definition.Id}): handleRequest() requestLinkStr= {requestLinkStr}"); + + Dictionary pairs = null; + var queryCollection = new NameValueCollection(); + + var method = HttpMethod.GET; + if (string.Equals(request.Method, "post", StringComparison.OrdinalIgnoreCase)) + { + method = HttpMethod.POST; + pairs = new Dictionary(); + } + + foreach (var input in request.Inputs) + { + var value = ApplyGoTemplateText(input.Value, variables); + if (method == HttpMethod.GET) + { + queryCollection.Add(input.Key, value); + } + else if (method == HttpMethod.POST) + { + pairs.Add(input.Key, value); + } + } + + if (queryCollection.Count > 0) + { + if (!requestLinkStr.Contains("?")) + { + // TODO Need Encoding here if we add it back + requestLinkStr += "?" + queryCollection.GetQueryString(separator: request.Queryseparator).Substring(1); + } + else + { + requestLinkStr += queryCollection.GetQueryString(separator: request.Queryseparator); + } + } + + var httpRequest = new HttpRequestBuilder(requestLinkStr) + .SetCookies(Cookies ?? new Dictionary()) + .SetHeaders(pairs ?? new Dictionary()) + .SetHeader("Referer", referer) + .Build(); + + httpRequest.Method = method; + + var response = await HttpClient.ExecuteAsync(httpRequest); + + _logger.Debug($"CardigannIndexer ({_definition.Id}): handleRequest() remote server returned {response.StatusCode.ToString()}"); + return response; + } + + public async Task DownloadRequest(Uri link) + { + Cookies = GetCookies(); + var method = HttpMethod.GET; + + if (_definition.Download != null) + { + var download = _definition.Download; + var variables = GetBaseTemplateVariables(); + + AddTemplateVariablesFromUri(variables, link, ".DownloadUri"); + + if (download.Before != null) + { + await HandleRequest(download.Before, variables, link.ToString()); + } + + if (download.Method == "post") + { + method = HttpMethod.POST; + } + + if (download.Selector != null) + { + var selector = ApplyGoTemplateText(download.Selector, variables); + var headers = ParseCustomHeaders(_definition.Search?.Headers, variables); + + var request = new HttpRequestBuilder(link.ToString()) + .SetCookies(Cookies ?? new Dictionary()) + .SetHeaders(headers ?? new Dictionary()) + .Build(); + + request.AllowAutoRedirect = true; + + var response = await HttpClient.ExecuteAsync(request); + + var results = response.Content; + var searchResultParser = new HtmlParser(); + var searchResultDocument = searchResultParser.ParseDocument(results); + var downloadElement = searchResultDocument.QuerySelector(selector); + if (downloadElement != null) + { + _logger.Debug(string.Format("CardigannIndexer ({0}): Download selector {1} matched:{2}", _definition.Id, selector, downloadElement.ToHtmlPretty())); + + var href = ""; + if (download.Attribute != null) + { + href = downloadElement.GetAttribute(download.Attribute); + if (href == null) + { + throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", download.Attribute, downloadElement.ToHtmlPretty())); + } + } + else + { + href = downloadElement.TextContent; + } + + href = ApplyFilters(href, download.Filters, variables); + link = ResolvePath(href, link); + } + else + { + _logger.Error(string.Format("CardigannIndexer ({0}): Download selector {1} didn't match:\n{2}", _definition.Id, download.Selector, results)); + throw new Exception(string.Format("Download selector {0} didn't match", download.Selector)); + } + } + } + + var downloadRequest = new HttpRequestBuilder(link.AbsoluteUri) + .SetCookies(Cookies ?? new Dictionary()) + .Build(); + + downloadRequest.Method = method; + + return downloadRequest; + } + public bool CheckIfLoginIsNeeded(HttpResponse response) { if (response.HasHttpRedirect) diff --git a/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs b/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs index 9504007a7..84de801b2 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs @@ -50,9 +50,9 @@ namespace NzbDrone.Core.Indexers.Headphones } } - public override async Task Download(HttpUri link) + public override async Task Download(Uri link) { - var requestBuilder = new HttpRequestBuilder(link.FullUri); + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); var downloadBytes = Array.Empty(); diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 11541064b..fd6665637 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -97,11 +97,11 @@ namespace NzbDrone.Core.Indexers return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria)); } - public override async Task Download(HttpUri link) + public override async Task Download(Uri link) { Cookies = GetCookies(); - var requestBuilder = new HttpRequestBuilder(link.FullUri); + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri); if (Cookies != null) { diff --git a/src/NzbDrone.Core/Indexers/IIndexer.cs b/src/NzbDrone.Core/Indexers/IIndexer.cs index 65ab4f7d2..dfca11891 100644 --- a/src/NzbDrone.Core/Indexers/IIndexer.cs +++ b/src/NzbDrone.Core/Indexers/IIndexer.cs @@ -1,5 +1,5 @@ +using System; using System.Threading.Tasks; -using NzbDrone.Common.Http; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.ThingiProvider; @@ -22,7 +22,7 @@ namespace NzbDrone.Core.Indexers Task Fetch(BookSearchCriteria searchCriteria); Task Fetch(BasicSearchCriteria searchCriteria); - Task Download(HttpUri link); + Task Download(Uri link); IndexerCapabilities GetCapabilities(); } diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index b53fe8cd7..8a7623d4f 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -74,7 +74,7 @@ namespace NzbDrone.Core.Indexers public abstract Task Fetch(TvSearchCriteria searchCriteria); public abstract Task Fetch(BookSearchCriteria searchCriteria); public abstract Task Fetch(BasicSearchCriteria searchCriteria); - public abstract Task Download(HttpUri searchCriteria); + public abstract Task Download(Uri searchCriteria); public abstract IndexerCapabilities GetCapabilities();