diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs index 8f861e017..d12d9a042 100644 --- a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs +++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs @@ -24,7 +24,9 @@ namespace NzbDrone.Core.IndexerVersions public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, IExecute { - private const int DEFINITION_VERSION = 1; + /* Update Service will fall back if version # does not exist for an indexer per Ta */ + + private const int DEFINITION_VERSION = 2; private readonly List _defintionBlocklist = new List() { "aither", diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs index e930f1488..c666af501 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs @@ -39,6 +39,7 @@ namespace NzbDrone.Core.Indexers.Cardigann public List Links { get; set; } public List Legacylinks { get; set; } public bool Followredirect { get; set; } = false; + public bool TestLinkTorrent { get; set; } = true; public List Certificates { get; set; } public CapabilitiesBlock Caps { get; set; } public LoginBlock Login { get; set; } @@ -167,11 +168,30 @@ namespace NzbDrone.Core.Indexers.Cardigann } public class DownloadBlock + { + public List Selectors { get; set; } + public string Method { get; set; } + public BeforeBlock Before { get; set; } + public InfohashBlock Infohash { get; set; } + } + + public class InfohashBlock + { + public SelectorField Hash { get; set; } + public SelectorField Title { get; set; } + public bool UseBeforeResponse { get; set; } + } + + public class SelectorField { public string Selector { get; set; } public string Attribute { get; set; } + public bool UseBeforeResponse { get; set; } public List Filters { get; set; } - public string Method { get; set; } - public RequestBlock Before { get; set; } + } + + public class BeforeBlock : RequestBlock + { + public SelectorField Pathselector { get; set; } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs index 9fe82b64d..283481615 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs @@ -721,9 +721,26 @@ namespace NzbDrone.Core.Indexers.Cardigann AddTemplateVariablesFromUri(variables, link, ".DownloadUri"); - if (download.Before != null) + var headers = ParseCustomHeaders(_definition.Search?.Headers, variables); + HttpResponse response = null; + + var request = new HttpRequestBuilder(link.ToString()) + .SetCookies(Cookies ?? new Dictionary()) + .SetHeaders(headers ?? new Dictionary()) + .Build(); + + request.AllowAutoRedirect = true; + + var beforeBlock = download.Before; + if (beforeBlock != null) { - await HandleRequest(download.Before, variables, link.ToString()); + if (beforeBlock.Pathselector != null) + { + response = await HttpClient.ExecuteProxiedAsync(request, Definition); + beforeBlock.Path = MatchSelector(response, beforeBlock.Pathselector, variables); + } + + response = await HandleRequest(beforeBlock, variables, link.ToString()); } if (download.Method == "post") @@ -731,49 +748,105 @@ namespace NzbDrone.Core.Indexers.Cardigann method = HttpMethod.POST; } - if (download.Selector != null) + if (download.Infohash != null) { - var selector = ApplyGoTemplateText(download.Selector, variables); - var headers = ParseCustomHeaders(_definition.Search?.Headers, variables); + try + { + headers = ParseCustomHeaders(_definition.Search?.Headers, variables); - var request = new HttpRequestBuilder(link.ToString()) - .SetCookies(Cookies ?? new Dictionary()) - .SetHeaders(headers ?? new Dictionary()) - .Build(); + if (!download.Infohash.UseBeforeResponse || download.Before == null || response == null) + { + response = await HttpClient.ExecuteProxiedAsync(request, Definition); + } - request.AllowAutoRedirect = true; + var hash = MatchSelector(response, download.Infohash.Hash, variables); + if (hash == null) + { + throw new Exception($"InfoHash selectors didn't match"); + } - var response = await HttpClient.ExecuteProxiedAsync(request, Definition); + var title = MatchSelector(response, download.Infohash.Title, variables); + if (title == null) + { + throw new Exception($"InfoHash selectors didn't match"); + } + + var magnet = MagnetLinkBuilder.BuildPublicMagnetLink(hash, title); + var torrentLink = ResolvePath(magnet, link); + + var hashDownloadRequest = new HttpRequestBuilder(torrentLink.AbsoluteUri) + .SetCookies(Cookies ?? new Dictionary()) + .Build(); - var results = response.Content; - var searchResultParser = new HtmlParser(); - var searchResultDocument = searchResultParser.ParseDocument(results); - var downloadElement = searchResultDocument.QuerySelector(selector); - if (downloadElement != null) + hashDownloadRequest.Method = method; + + return hashDownloadRequest; + } + catch (Exception) + { + _logger.Error("CardigannIndexer ({0}): Exception with InfoHash block with hashSelector {1} and titleSelector {2}", + _definition.Id, + download.Infohash.Hash.Selector, + download.Infohash.Title.Selector); + } + } + else if (download.Selectors != null) + { + headers = ParseCustomHeaders(_definition.Search?.Headers, variables); + + foreach (var selector in download.Selectors) { - _logger.Debug(string.Format("CardigannIndexer ({0}): Download selector {1} matched:{2}", _definition.Id, selector, downloadElement.ToHtmlPretty())); + var queryselector = ApplyGoTemplateText(selector.Selector, variables); - var href = ""; - if (download.Attribute != null) + try { - href = downloadElement.GetAttribute(download.Attribute); + if (!selector.UseBeforeResponse || download.Before == null || response == null) + { + response = await HttpClient.ExecuteProxiedAsync(request, Definition); + } + + var href = MatchSelector(response, selector, variables, debugMatch: true); if (href == null) { - throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", download.Attribute, downloadElement.ToHtmlPretty())); + continue; } + + var torrentLink = ResolvePath(href, link); + if (torrentLink.Scheme != "magnet" && _definition.TestLinkTorrent) + { + // Test link + var testLinkRequest = new HttpRequestBuilder(torrentLink.ToString()) + .SetCookies(Cookies ?? new Dictionary()) + .SetHeaders(headers ?? new Dictionary()) + .Build(); + + response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition); + + var content = response.Content; + if (content.Length >= 1 && content[0] != 'd') + { + _logger.Debug("CardigannIndexer ({0}): Download selector {1}'s torrent file is invalid, retrying with next available selector", _definition.Id, queryselector); + + continue; + } + } + + link = torrentLink; + + var selectorDownloadRequest = new HttpRequestBuilder(link.AbsoluteUri) + .SetCookies(Cookies ?? new Dictionary()) + .Build(); + + selectorDownloadRequest.Method = method; + + return selectorDownloadRequest; } - else + catch (Exception e) { - href = downloadElement.TextContent; - } + _logger.Error("{0} CardigannIndexer ({1}): An exception occurred while trying selector {2}, retrying with next available selector", e, _definition.Id, queryselector); - href = ApplyFilters(href, download.Filters, variables); - link = ResolvePath(href, link); - } - else - { - _logger.Error(string.Format("CardigannIndexer ({0}): Download selector {1} didn't match:\n{2}", _definition.Id, download.Selector, results)); - throw new Exception(string.Format("Download selector {0} didn't match", download.Selector)); + throw new Exception(string.Format("An exception occurred while trying selector {0}", queryselector)); + } } } } @@ -787,6 +860,44 @@ namespace NzbDrone.Core.Indexers.Cardigann return downloadRequest; } + protected string MatchSelector(HttpResponse response, SelectorField selector, Dictionary variables, bool debugMatch = false) + { + var selectorText = ApplyGoTemplateText(selector.Selector, variables); + var parser = new HtmlParser(); + + var results = response.Content; + var resultDocument = parser.ParseDocument(results); + + var element = resultDocument.QuerySelector(selectorText); + if (element == null) + { + _logger.Debug($"CardigannIndexer ({_definition.Id}): Selector {selectorText} could not match any elements."); + return null; + } + + if (debugMatch) + { + _logger.Debug($"CardigannIndexer ({_definition.Id}): Download selector {selector} matched:{element.ToHtmlPretty()}"); + } + + string val; + if (selector.Attribute != null) + { + val = element.GetAttribute(selector.Attribute); + if (val == null) + { + throw new Exception($"Attribute \"{selector.Attribute}\" is not set for element {element.ToHtmlPretty()}"); + } + } + else + { + val = element.TextContent; + } + + val = ApplyFilters(val, selector.Filters, variables); + return val; + } + public bool CheckIfLoginIsNeeded(HttpResponse response) { if (response.HasHttpRedirect)