open-source-mirrors
/
Prowlarr
mirror of https://github.com/Prowlarr/Prowlarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

New: Support for Prowlarr Definitions v2

Browse Source 
New: Support for Updated yml Definitions
Fixes: #298
pull/543/head
bakerboy448 3 years ago committed by Qstick
parent 293b32ea0e
commit c5caf22375
3 changed files with 167 additions and 34 deletions
Show Stats Download Patch File Download Diff File

4
src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs
Unescape View File

@ -24,7 +24,9 @@ namespace NzbDrone.Core.IndexerVersions
public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, IExecute<IndexerDefinitionUpdateCommand>
{
private const int DEFINITION_VERSION = 1;
/* Update Service will fall back if version # does not exist for an indexer per Ta */
private const int DEFINITION_VERSION = 2;
private readonly List<string> _defintionBlocklist = new List<string>()
{
"aither",

24
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinition.cs
Unescape View File

@ -39,6 +39,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
public List<string> Links { get; set; }
public List<string> Legacylinks { get; set; }
public bool Followredirect { get; set; } = false;
public bool TestLinkTorrent { get; set; } = true;
public List<string> Certificates { get; set; }
public CapabilitiesBlock Caps { get; set; }
public LoginBlock Login { get; set; }
@ -167,11 +168,30 @@ namespace NzbDrone.Core.Indexers.Cardigann
}
public class DownloadBlock
{
public List<SelectorField> Selectors { get; set; }
public string Method { get; set; }
public BeforeBlock Before { get; set; }
public InfohashBlock Infohash { get; set; }
}
public class InfohashBlock
{
public SelectorField Hash { get; set; }
public SelectorField Title { get; set; }
public bool UseBeforeResponse { get; set; }
}
public class SelectorField
{
public string Selector { get; set; }
public string Attribute { get; set; }
public bool UseBeforeResponse { get; set; }
public List<FilterBlock> Filters { get; set; }
public string Method { get; set; }
public RequestBlock Before { get; set; }
}
public class BeforeBlock : RequestBlock
{
public SelectorField Pathselector { get; set; }
}
}

165
src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs
Unescape View File

@ -721,9 +721,26 @@ namespace NzbDrone.Core.Indexers.Cardigann
AddTemplateVariablesFromUri(variables, link, ".DownloadUri");
if (download.Before != null)
var headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
HttpResponse response = null;
var request = new HttpRequestBuilder(link.ToString())
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(headers ?? new Dictionary<string, string>())
.Build();
request.AllowAutoRedirect = true;
var beforeBlock = download.Before;
if (beforeBlock != null)
{
await HandleRequest(download.Before, variables, link.ToString());
if (beforeBlock.Pathselector != null)
{
response = await HttpClient.ExecuteProxiedAsync(request, Definition);
beforeBlock.Path = MatchSelector(response, beforeBlock.Pathselector, variables);
}
response = await HandleRequest(beforeBlock, variables, link.ToString());
}
if (download.Method == "post")
@ -731,49 +748,105 @@ namespace NzbDrone.Core.Indexers.Cardigann
method = HttpMethod.POST;
}
if (download.Selector != null)
if (download.Infohash != null)
{
var selector = ApplyGoTemplateText(download.Selector, variables);
var headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
try
{
headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
var request = new HttpRequestBuilder(link.ToString())
if (!download.Infohash.UseBeforeResponse || download.Before == null || response == null)
{
response = await HttpClient.ExecuteProxiedAsync(request, Definition);
}
var hash = MatchSelector(response, download.Infohash.Hash, variables);
if (hash == null)
{
throw new Exception($"InfoHash selectors didn't match");
}
var title = MatchSelector(response, download.Infohash.Title, variables);
if (title == null)
{
throw new Exception($"InfoHash selectors didn't match");
}
var magnet = MagnetLinkBuilder.BuildPublicMagnetLink(hash, title);
var torrentLink = ResolvePath(magnet, link);
var hashDownloadRequest = new HttpRequestBuilder(torrentLink.AbsoluteUri)
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(headers ?? new Dictionary<string, string>())
.Build();
request.AllowAutoRedirect = true;
hashDownloadRequest.Method = method;
var response = await HttpClient.ExecuteProxiedAsync(request, Definition);
return hashDownloadRequest;
}
catch (Exception)
{
_logger.Error("CardigannIndexer ({0}): Exception with InfoHash block with hashSelector {1} and titleSelector {2}",
_definition.Id,
download.Infohash.Hash.Selector,
download.Infohash.Title.Selector);
}
}
else if (download.Selectors != null)
{
headers = ParseCustomHeaders(_definition.Search?.Headers, variables);
var results = response.Content;
var searchResultParser = new HtmlParser();
var searchResultDocument = searchResultParser.ParseDocument(results);
var downloadElement = searchResultDocument.QuerySelector(selector);
if (downloadElement != null)
foreach (var selector in download.Selectors)
{
_logger.Debug(string.Format("CardigannIndexer ({0}): Download selector {1} matched:{2}", _definition.Id, selector, downloadElement.ToHtmlPretty()));
var queryselector = ApplyGoTemplateText(selector.Selector, variables);
var href = "";
if (download.Attribute != null)
try
{
href = downloadElement.GetAttribute(download.Attribute);
if (href == null)
if (!selector.UseBeforeResponse || download.Before == null || response == null)
{
throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", download.Attribute, downloadElement.ToHtmlPretty()));
response = await HttpClient.ExecuteProxiedAsync(request, Definition);
}
var href = MatchSelector(response, selector, variables, debugMatch: true);
if (href == null)
{
continue;
}
else
var torrentLink = ResolvePath(href, link);
if (torrentLink.Scheme != "magnet" && _definition.TestLinkTorrent)
{
href = downloadElement.TextContent;
// Test link
var testLinkRequest = new HttpRequestBuilder(torrentLink.ToString())
.SetCookies(Cookies ?? new Dictionary<string, string>())
.SetHeaders(headers ?? new Dictionary<string, string>())
.Build();
response = await HttpClient.ExecuteProxiedAsync(testLinkRequest, Definition);
var content = response.Content;
if (content.Length >= 1 && content[0] != 'd')
{
_logger.Debug("CardigannIndexer ({0}): Download selector {1}'s torrent file is invalid, retrying with next available selector", _definition.Id, queryselector);
continue;
}
}
href = ApplyFilters(href, download.Filters, variables);
link = ResolvePath(href, link);
link = torrentLink;
var selectorDownloadRequest = new HttpRequestBuilder(link.AbsoluteUri)
.SetCookies(Cookies ?? new Dictionary<string, string>())
.Build();
selectorDownloadRequest.Method = method;
return selectorDownloadRequest;
}
else
catch (Exception e)
{
_logger.Error(string.Format("CardigannIndexer ({0}): Download selector {1} didn't match:\n{2}", _definition.Id, download.Selector, results));
throw new Exception(string.Format("Download selector {0} didn't match", download.Selector));
_logger.Error("{0} CardigannIndexer ({1}): An exception occurred while trying selector {2}, retrying with next available selector", e, _definition.Id, queryselector);
throw new Exception(string.Format("An exception occurred while trying selector {0}", queryselector));
}
}
}
}
@ -787,6 +860,44 @@ namespace NzbDrone.Core.Indexers.Cardigann
return downloadRequest;
}
protected string MatchSelector(HttpResponse response, SelectorField selector, Dictionary<string, object> variables, bool debugMatch = false)
{
var selectorText = ApplyGoTemplateText(selector.Selector, variables);
var parser = new HtmlParser();
var results = response.Content;
var resultDocument = parser.ParseDocument(results);
var element = resultDocument.QuerySelector(selectorText);
if (element == null)
{
_logger.Debug($"CardigannIndexer ({_definition.Id}): Selector {selectorText} could not match any elements.");
return null;
}
if (debugMatch)
{
_logger.Debug($"CardigannIndexer ({_definition.Id}): Download selector {selector} matched:{element.ToHtmlPretty()}");
}
string val;
if (selector.Attribute != null)
{
val = element.GetAttribute(selector.Attribute);
if (val == null)
{
throw new Exception($"Attribute \"{selector.Attribute}\" is not set for element {element.ToHtmlPretty()}");
}
}
else
{
val = element.TextContent;
}
val = ApplyFilters(val, selector.Filters, variables);
return val;
}
public bool CheckIfLoginIsNeeded(HttpResponse response)
{
if (response.HasHttpRedirect)

Write Preview
Loading…
Cancel
Save