ta264
4 years ago
committed by
Qstick
parent
4b66d99029
commit
d041b152d6
@ -0,0 +1,105 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace NzbDrone.Core.Datastore.Converters
|
||||
{
|
||||
public class DictionaryStringObjectJsonConverter : JsonConverter<Dictionary<string, object>>
|
||||
{
|
||||
public override Dictionary<string, object> Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
|
||||
{
|
||||
if (reader.TokenType != JsonTokenType.StartObject)
|
||||
{
|
||||
throw new JsonException($"JsonTokenType was of type {reader.TokenType}, only objects are supported");
|
||||
}
|
||||
|
||||
var dictionary = new Dictionary<string, object>();
|
||||
while (reader.Read())
|
||||
{
|
||||
if (reader.TokenType == JsonTokenType.EndObject)
|
||||
{
|
||||
return dictionary;
|
||||
}
|
||||
|
||||
if (reader.TokenType != JsonTokenType.PropertyName)
|
||||
{
|
||||
throw new JsonException("JsonTokenType was not PropertyName");
|
||||
}
|
||||
|
||||
var propertyName = reader.GetString();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(propertyName))
|
||||
{
|
||||
throw new JsonException("Failed to get property name");
|
||||
}
|
||||
|
||||
reader.Read();
|
||||
|
||||
dictionary.Add(propertyName, ExtractValue(ref reader, options));
|
||||
}
|
||||
|
||||
return dictionary;
|
||||
}
|
||||
|
||||
public override void Write(Utf8JsonWriter writer, Dictionary<string, object> dictionary, JsonSerializerOptions options)
|
||||
{
|
||||
writer.WriteStartObject();
|
||||
|
||||
foreach (var kvp in dictionary)
|
||||
{
|
||||
writer.WritePropertyName(kvp.Key.ToString());
|
||||
|
||||
JsonSerializer.Serialize(writer, kvp.Value, options);
|
||||
}
|
||||
|
||||
writer.WriteEndObject();
|
||||
}
|
||||
|
||||
private object ExtractValue(ref Utf8JsonReader reader, JsonSerializerOptions options)
|
||||
{
|
||||
switch (reader.TokenType)
|
||||
{
|
||||
case JsonTokenType.String:
|
||||
if (reader.TryGetDateTime(out var date))
|
||||
{
|
||||
return date;
|
||||
}
|
||||
|
||||
return reader.GetString();
|
||||
|
||||
case JsonTokenType.False:
|
||||
return false;
|
||||
|
||||
case JsonTokenType.True:
|
||||
return true;
|
||||
|
||||
case JsonTokenType.Null:
|
||||
return null;
|
||||
|
||||
case JsonTokenType.Number:
|
||||
if (reader.TryGetInt64(out var result))
|
||||
{
|
||||
return result;
|
||||
}
|
||||
|
||||
return reader.GetDecimal();
|
||||
|
||||
case JsonTokenType.StartObject:
|
||||
return Read(ref reader, null, options);
|
||||
|
||||
case JsonTokenType.StartArray:
|
||||
var list = new List<object>();
|
||||
while (reader.Read() && reader.TokenType != JsonTokenType.EndArray)
|
||||
{
|
||||
list.Add(ExtractValue(ref reader, options));
|
||||
}
|
||||
|
||||
return list;
|
||||
|
||||
default:
|
||||
throw new JsonException($"'{reader.TokenType}' is not supported");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,108 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using FluentValidation.Results;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Http;
|
||||
using NzbDrone.Core.Configuration;
|
||||
using NzbDrone.Core.ThingiProvider;
|
||||
using NzbDrone.Core.Validation;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public class Cardigann : HttpIndexerBase<CardigannSettings>
|
||||
{
|
||||
private readonly ICardigannDefinitionService _definitionService;
|
||||
|
||||
public override string Name => "Cardigann";
|
||||
|
||||
public override DownloadProtocol Protocol => DownloadProtocol.Torrent;
|
||||
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
|
||||
public override int PageSize => 100;
|
||||
|
||||
public override IIndexerRequestGenerator GetRequestGenerator()
|
||||
{
|
||||
return new CardigannRequestGenerator(_definitionService.GetDefinition(Settings.DefinitionFile),
|
||||
Settings,
|
||||
_logger);
|
||||
}
|
||||
|
||||
public override IParseIndexerResponse GetParser()
|
||||
{
|
||||
return new CardigannParser(_definitionService.GetDefinition(Settings.DefinitionFile),
|
||||
Settings,
|
||||
_logger);
|
||||
}
|
||||
|
||||
public override IndexerCapabilities GetCapabilities()
|
||||
{
|
||||
// TODO: This uses indexer capabilities when called so we don't have to keep up with all of them
|
||||
// however, this is not pulled on a all pull from UI, doing so will kill the UI load if an indexer is down
|
||||
// should we just purge and manage
|
||||
return new IndexerCapabilities();
|
||||
}
|
||||
|
||||
public override IEnumerable<ProviderDefinition> DefaultDefinitions
|
||||
{
|
||||
get
|
||||
{
|
||||
foreach (var def in _definitionService.All())
|
||||
{
|
||||
yield return GetDefinition(def);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Cardigann(ICardigannDefinitionService definitionService,
|
||||
IHttpClient httpClient,
|
||||
IIndexerStatusService indexerStatusService,
|
||||
IConfigService configService,
|
||||
Logger logger)
|
||||
: base(httpClient, indexerStatusService, configService, logger)
|
||||
{
|
||||
_definitionService = definitionService;
|
||||
}
|
||||
|
||||
private IndexerDefinition GetDefinition(CardigannMetaDefinition definition)
|
||||
{
|
||||
return new IndexerDefinition
|
||||
{
|
||||
EnableRss = false,
|
||||
EnableAutomaticSearch = false,
|
||||
EnableInteractiveSearch = false,
|
||||
Name = definition.Name,
|
||||
Implementation = GetType().Name,
|
||||
Settings = new CardigannSettings { DefinitionFile = definition.File },
|
||||
Protocol = DownloadProtocol.Torrent,
|
||||
Privacy = definition.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public,
|
||||
SupportsRss = SupportsRss,
|
||||
SupportsSearch = SupportsSearch,
|
||||
Capabilities = Capabilities,
|
||||
ExtraFields = definition.Settings
|
||||
};
|
||||
}
|
||||
|
||||
protected override void Test(List<ValidationFailure> failures)
|
||||
{
|
||||
base.Test(failures);
|
||||
if (failures.HasErrors())
|
||||
{
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
protected static List<int> CategoryIds(List<IndexerCategory> categories)
|
||||
{
|
||||
var l = categories.Select(c => c.Id).ToList();
|
||||
|
||||
foreach (var category in categories)
|
||||
{
|
||||
if (category.SubCategories != null)
|
||||
{
|
||||
l.AddRange(CategoryIds(category.SubCategories));
|
||||
}
|
||||
}
|
||||
|
||||
return l;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,613 @@
|
||||
using System;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
using AngleSharp.Dom;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Common.Serializer;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public class CardigannBase
|
||||
{
|
||||
protected readonly CardigannDefinition _definition;
|
||||
protected readonly CardigannSettings _settings;
|
||||
protected readonly Logger _logger;
|
||||
protected readonly Encoding _encoding;
|
||||
|
||||
protected string SiteLink { get; private set; }
|
||||
|
||||
/* protected readonly List<CategoryMapping> categoryMapping = new List<CategoryMapping>(); */
|
||||
|
||||
protected readonly string[] OptionalFields = new string[] { "imdb", "rageid", "tvdbid", "banner" };
|
||||
|
||||
protected static readonly string[] _SupportedLogicFunctions =
|
||||
{
|
||||
"and",
|
||||
"or",
|
||||
"eq",
|
||||
"ne"
|
||||
};
|
||||
|
||||
protected static readonly string[] _LogicFunctionsUsingStringLiterals =
|
||||
{
|
||||
"eq",
|
||||
"ne"
|
||||
};
|
||||
|
||||
// Matches a logic function above and 2 or more of (.varname) or .varname or "string literal" in any combination
|
||||
protected static readonly Regex _LogicFunctionRegex = new Regex(
|
||||
$@"\b({string.Join("|", _SupportedLogicFunctions.Select(Regex.Escape))})(?:\s+(\(?\.[^\)\s]+\)?|""[^""]+"")){{2,}}");
|
||||
|
||||
public CardigannBase(CardigannDefinition definition,
|
||||
CardigannSettings settings,
|
||||
Logger logger)
|
||||
{
|
||||
_definition = definition;
|
||||
_settings = settings;
|
||||
_encoding = Encoding.GetEncoding(definition.Encoding);
|
||||
_logger = logger;
|
||||
|
||||
SiteLink = definition.Links.First();
|
||||
}
|
||||
|
||||
protected IElement QuerySelector(IElement element, string selector)
|
||||
{
|
||||
// AngleSharp doesn't support the :root pseudo selector, so we check for it manually
|
||||
if (selector.StartsWith(":root"))
|
||||
{
|
||||
selector = selector.Substring(5);
|
||||
while (element.ParentElement != null)
|
||||
{
|
||||
element = element.ParentElement;
|
||||
}
|
||||
}
|
||||
|
||||
return element.QuerySelector(selector);
|
||||
}
|
||||
|
||||
protected string HandleSelector(SelectorBlock selector, IElement dom, Dictionary<string, object> variables = null)
|
||||
{
|
||||
if (selector.Text != null)
|
||||
{
|
||||
return ApplyFilters(ApplyGoTemplateText(selector.Text, variables), selector.Filters, variables);
|
||||
}
|
||||
|
||||
var selection = dom;
|
||||
string value = null;
|
||||
|
||||
if (selector.Selector != null)
|
||||
{
|
||||
if (dom.Matches(selector.Selector))
|
||||
{
|
||||
selection = dom;
|
||||
}
|
||||
else
|
||||
{
|
||||
selection = QuerySelector(dom, selector.Selector);
|
||||
}
|
||||
|
||||
if (selection == null)
|
||||
{
|
||||
throw new Exception(string.Format("Selector \"{0}\" didn't match {1}", selector.Selector, dom.ToHtmlPretty()));
|
||||
}
|
||||
}
|
||||
|
||||
if (selector.Remove != null)
|
||||
{
|
||||
foreach (var i in selection.QuerySelectorAll(selector.Remove))
|
||||
{
|
||||
i.Remove();
|
||||
}
|
||||
}
|
||||
|
||||
if (selector.Case != null)
|
||||
{
|
||||
foreach (var @case in selector.Case)
|
||||
{
|
||||
if (selection.Matches(@case.Key) || QuerySelector(selection, @case.Key) != null)
|
||||
{
|
||||
value = @case.Value;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (value == null)
|
||||
{
|
||||
throw new Exception(string.Format("None of the case selectors \"{0}\" matched {1}", string.Join(",", selector.Case), selection.ToHtmlPretty()));
|
||||
}
|
||||
}
|
||||
else if (selector.Attribute != null)
|
||||
{
|
||||
value = selection.GetAttribute(selector.Attribute);
|
||||
if (value == null)
|
||||
{
|
||||
throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", selector.Attribute, selection.ToHtmlPretty()));
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
value = selection.TextContent;
|
||||
}
|
||||
|
||||
return ApplyFilters(ParseUtil.NormalizeSpace(value), selector.Filters, variables);
|
||||
}
|
||||
|
||||
protected Dictionary<string, object> GetBaseTemplateVariables()
|
||||
{
|
||||
var variables = new Dictionary<string, object>
|
||||
{
|
||||
[".Config.sitelink"] = SiteLink,
|
||||
[".True"] = "True",
|
||||
[".False"] = null,
|
||||
[".Today.Year"] = DateTime.Today.Year.ToString()
|
||||
};
|
||||
|
||||
_logger.Debug("Populating config vars");
|
||||
|
||||
foreach (var setting in _definition.Settings)
|
||||
{
|
||||
var name = ".Config." + setting.Name;
|
||||
var value = _settings.ExtraFieldData.GetValueOrDefault(setting.Name, setting.Default);
|
||||
|
||||
_logger.Debug($"{name} got value {value.ToJson()}");
|
||||
|
||||
if (setting.Type == "text")
|
||||
{
|
||||
variables[name] = value;
|
||||
}
|
||||
else if (setting.Type == "checkbox")
|
||||
{
|
||||
variables[name] = ((bool)value) ? ".True" : ".False";
|
||||
}
|
||||
else if (setting.Type == "select")
|
||||
{
|
||||
_logger.Debug($"setting options: {setting.Options.ToJson()}");
|
||||
var sorted = setting.Options.OrderBy(x => x.Key).ToList();
|
||||
var selected = sorted[(int)(long)value];
|
||||
|
||||
_logger.Debug($"selected option: {selected.ToJson()}");
|
||||
|
||||
variables[name] = selected.Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new NotSupportedException();
|
||||
}
|
||||
|
||||
_logger.Debug($"Setting {setting.Name} to {variables[name]}");
|
||||
}
|
||||
|
||||
return variables;
|
||||
}
|
||||
|
||||
/*
|
||||
protected ICollection<int> MapTrackerCatToNewznab(string input)
|
||||
{
|
||||
if (input == null)
|
||||
return new List<int>();
|
||||
|
||||
var cats = categoryMapping.Where(m => m.TrackerCategory != null && m.TrackerCategory.ToLowerInvariant() == input.ToLowerInvariant()).Select(c => c.NewzNabCategory).ToList();
|
||||
|
||||
// 1:1 category mapping
|
||||
try
|
||||
{
|
||||
var trackerCategoryInt = int.Parse(input);
|
||||
cats.Add(trackerCategoryInt + 100000);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
// input is not an integer, continue
|
||||
}
|
||||
|
||||
return cats;
|
||||
}*/
|
||||
|
||||
protected delegate string TemplateTextModifier(string str);
|
||||
|
||||
protected string ApplyGoTemplateText(string template, Dictionary<string, object> variables = null, TemplateTextModifier modifier = null)
|
||||
{
|
||||
if (variables == null)
|
||||
{
|
||||
variables = GetBaseTemplateVariables();
|
||||
}
|
||||
|
||||
// handle re_replace expression
|
||||
// Example: {{ re_replace .Query.Keywords "[^a-zA-Z0-9]+" "%" }}
|
||||
var reReplaceRegex = new Regex(@"{{\s*re_replace\s+(\..+?)\s+""(.*?)""\s+""(.*?)""\s*}}");
|
||||
var reReplaceRegexMatches = reReplaceRegex.Match(template);
|
||||
|
||||
while (reReplaceRegexMatches.Success)
|
||||
{
|
||||
var all = reReplaceRegexMatches.Groups[0].Value;
|
||||
var variable = reReplaceRegexMatches.Groups[1].Value;
|
||||
var regexp = reReplaceRegexMatches.Groups[2].Value;
|
||||
var newvalue = reReplaceRegexMatches.Groups[3].Value;
|
||||
|
||||
var replaceRegex = new Regex(regexp);
|
||||
var input = (string)variables[variable];
|
||||
var expanded = replaceRegex.Replace(input, newvalue);
|
||||
|
||||
if (modifier != null)
|
||||
{
|
||||
expanded = modifier(expanded);
|
||||
}
|
||||
|
||||
template = template.Replace(all, expanded);
|
||||
reReplaceRegexMatches = reReplaceRegexMatches.NextMatch();
|
||||
}
|
||||
|
||||
// handle join expression
|
||||
// Example: {{ join .Categories "," }}
|
||||
var joinRegex = new Regex(@"{{\s*join\s+(\..+?)\s+""(.*?)""\s*}}");
|
||||
var joinMatches = joinRegex.Match(template);
|
||||
|
||||
while (joinMatches.Success)
|
||||
{
|
||||
var all = joinMatches.Groups[0].Value;
|
||||
var variable = joinMatches.Groups[1].Value;
|
||||
var delimiter = joinMatches.Groups[2].Value;
|
||||
|
||||
var input = (ICollection<string>)variables[variable];
|
||||
var expanded = string.Join(delimiter, input);
|
||||
|
||||
if (modifier != null)
|
||||
{
|
||||
expanded = modifier(expanded);
|
||||
}
|
||||
|
||||
template = template.Replace(all, expanded);
|
||||
joinMatches = joinMatches.NextMatch();
|
||||
}
|
||||
|
||||
var logicMatch = _LogicFunctionRegex.Match(template);
|
||||
|
||||
while (logicMatch.Success)
|
||||
{
|
||||
var functionStartIndex = logicMatch.Groups[0].Index;
|
||||
var functionLength = logicMatch.Groups[0].Length;
|
||||
var functionName = logicMatch.Groups[1].Value;
|
||||
|
||||
// Use Group.Captures to get each matching string in a repeating Match.Group
|
||||
// Strip () around variable names here, as they are optional. Use quotes to differentiate variables and literals
|
||||
var parameters = logicMatch.Groups[2].Captures.Cast<Capture>().Select(c => c.Value.Trim('(', ')')).ToList();
|
||||
var functionResult = "";
|
||||
|
||||
// If the function can't use string literals, fail silently by removing the literals.
|
||||
if (!_LogicFunctionsUsingStringLiterals.Contains(functionName))
|
||||
{
|
||||
parameters.RemoveAll(param => param.StartsWith("\""));
|
||||
}
|
||||
|
||||
switch (functionName)
|
||||
{
|
||||
case "and": // returns first null or empty, else last variable
|
||||
case "or": // returns first not null or empty, else last variable
|
||||
var isAnd = functionName == "and";
|
||||
foreach (var parameter in parameters)
|
||||
{
|
||||
functionResult = parameter;
|
||||
|
||||
// (null as string) == null
|
||||
// (if null or empty) break if and, continue if or
|
||||
// (if neither null nor empty) continue if and, break if or
|
||||
if (string.IsNullOrWhiteSpace(variables[parameter] as string) == isAnd)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
break;
|
||||
case "eq": // Returns .True if equal
|
||||
case "ne": // Returns .False if equal
|
||||
{
|
||||
var wantEqual = functionName == "eq";
|
||||
|
||||
// eq/ne take exactly 2 params. Update the length to match
|
||||
// This removes the whitespace between params 2 and 3.
|
||||
// It shouldn't matter because the match starts at a word boundary
|
||||
if (parameters.Count > 2)
|
||||
{
|
||||
functionLength = logicMatch.Groups[2].Captures[2].Index - functionStartIndex;
|
||||
}
|
||||
|
||||
// Take first two parameters, convert vars to values and strip quotes on string literals
|
||||
// Counting distinct gives us 1 if equal and 2 if not.
|
||||
var isEqual =
|
||||
parameters.Take(2).Select(param => param.StartsWith("\"") ? param.Trim('"') : variables[param] as string)
|
||||
.Distinct().Count() == 1;
|
||||
|
||||
functionResult = isEqual == wantEqual ? ".True" : ".False";
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
template = template.Remove(functionStartIndex, functionLength)
|
||||
.Insert(functionStartIndex, functionResult);
|
||||
|
||||
// Rerunning match instead of using nextMatch allows us to support nested functions
|
||||
// like {{if and eq (.Var1) "string1" eq (.Var2) "string2"}}
|
||||
// No performance is lost because Match/NextMatch are lazy evaluated and pause execution after first match
|
||||
logicMatch = _LogicFunctionRegex.Match(template);
|
||||
}
|
||||
|
||||
// handle if ... else ... expression
|
||||
var ifElseRegex = new Regex(@"{{\s*if\s*(.+?)\s*}}(.*?){{\s*else\s*}}(.*?){{\s*end\s*}}");
|
||||
var ifElseRegexMatches = ifElseRegex.Match(template);
|
||||
|
||||
while (ifElseRegexMatches.Success)
|
||||
{
|
||||
string conditionResult = null;
|
||||
|
||||
var all = ifElseRegexMatches.Groups[0].Value;
|
||||
var condition = ifElseRegexMatches.Groups[1].Value;
|
||||
var onTrue = ifElseRegexMatches.Groups[2].Value;
|
||||
var onFalse = ifElseRegexMatches.Groups[3].Value;
|
||||
|
||||
if (condition.StartsWith("."))
|
||||
{
|
||||
var conditionResultState = false;
|
||||
var value = variables[condition];
|
||||
|
||||
if (value == null)
|
||||
{
|
||||
conditionResultState = false;
|
||||
}
|
||||
else if (value is string)
|
||||
{
|
||||
conditionResultState = !string.IsNullOrWhiteSpace((string)value);
|
||||
}
|
||||
else if (value is ICollection)
|
||||
{
|
||||
conditionResultState = ((ICollection)value).Count > 0;
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception(string.Format("Unexpceted type for variable {0}: {1}", condition, value.GetType()));
|
||||
}
|
||||
|
||||
if (conditionResultState)
|
||||
{
|
||||
conditionResult = onTrue;
|
||||
}
|
||||
else
|
||||
{
|
||||
conditionResult = onFalse;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new NotImplementedException("CardigannIndexer: Condition operation '" + condition + "' not implemented");
|
||||
}
|
||||
|
||||
template = template.Replace(all, conditionResult);
|
||||
ifElseRegexMatches = ifElseRegexMatches.NextMatch();
|
||||
}
|
||||
|
||||
// handle range expression
|
||||
var rangeRegex = new Regex(@"{{\s*range\s*(.+?)\s*}}(.*?){{\.}}(.*?){{end}}");
|
||||
var rangeRegexMatches = rangeRegex.Match(template);
|
||||
|
||||
while (rangeRegexMatches.Success)
|
||||
{
|
||||
var expanded = string.Empty;
|
||||
|
||||
var all = rangeRegexMatches.Groups[0].Value;
|
||||
var variable = rangeRegexMatches.Groups[1].Value;
|
||||
var prefix = rangeRegexMatches.Groups[2].Value;
|
||||
var postfix = rangeRegexMatches.Groups[3].Value;
|
||||
|
||||
foreach (var value in (ICollection<string>)variables[variable])
|
||||
{
|
||||
var newvalue = value;
|
||||
if (modifier != null)
|
||||
{
|
||||
newvalue = modifier(newvalue);
|
||||
}
|
||||
|
||||
expanded += prefix + newvalue + postfix;
|
||||
}
|
||||
|
||||
template = template.Replace(all, expanded);
|
||||
rangeRegexMatches = rangeRegexMatches.NextMatch();
|
||||
}
|
||||
|
||||
// handle simple variables
|
||||
var variablesRegEx = new Regex(@"{{\s*(\..+?)\s*}}");
|
||||
var variablesRegExMatches = variablesRegEx.Match(template);
|
||||
|
||||
while (variablesRegExMatches.Success)
|
||||
{
|
||||
var expanded = string.Empty;
|
||||
|
||||
var all = variablesRegExMatches.Groups[0].Value;
|
||||
var variable = variablesRegExMatches.Groups[1].Value;
|
||||
|
||||
var value = (string)variables[variable];
|
||||
if (modifier != null)
|
||||
{
|
||||
value = modifier(value);
|
||||
}
|
||||
|
||||
template = template.Replace(all, value);
|
||||
variablesRegExMatches = variablesRegExMatches.NextMatch();
|
||||
}
|
||||
|
||||
return template;
|
||||
}
|
||||
|
||||
protected string ApplyFilters(string data, List<FilterBlock> filters, Dictionary<string, object> variables = null)
|
||||
{
|
||||
if (filters == null)
|
||||
{
|
||||
return data;
|
||||
}
|
||||
|
||||
foreach (var filter in filters)
|
||||
{
|
||||
switch (filter.Name)
|
||||
{
|
||||
case "querystring":
|
||||
var param = (string)filter.Args;
|
||||
|
||||
// data = ParseUtil.GetArgumentFromQueryString(data, param);
|
||||
break;
|
||||
case "timeparse":
|
||||
case "dateparse":
|
||||
var layout = (string)filter.Args;
|
||||
try
|
||||
{
|
||||
var date = DateTimeUtil.ParseDateTimeGoLang(data, layout);
|
||||
data = date.ToString(DateTimeUtil.Rfc1123ZPattern);
|
||||
}
|
||||
catch (FormatException ex)
|
||||
{
|
||||
_logger.Debug(ex.Message);
|
||||
}
|
||||
|
||||
break;
|
||||
case "regexp":
|
||||
var pattern = (string)filter.Args;
|
||||
var regexp = new Regex(pattern);
|
||||
var match = regexp.Match(data);
|
||||
data = match.Groups[1].Value;
|
||||
break;
|
||||
case "re_replace":
|
||||
var regexpreplace_pattern = (string)filter.Args[0];
|
||||
var regexpreplace_replacement = (string)filter.Args[1];
|
||||
regexpreplace_replacement = ApplyGoTemplateText(regexpreplace_replacement, variables);
|
||||
var regexpreplace_regex = new Regex(regexpreplace_pattern);
|
||||
data = regexpreplace_regex.Replace(data, regexpreplace_replacement);
|
||||
break;
|
||||
case "split":
|
||||
var sep = (string)filter.Args[0];
|
||||
var pos = (string)filter.Args[1];
|
||||
var posInt = int.Parse(pos);
|
||||
var strParts = data.Split(sep[0]);
|
||||
if (posInt < 0)
|
||||
{
|
||||
posInt += strParts.Length;
|
||||
}
|
||||
|
||||
data = strParts[posInt];
|
||||
break;
|
||||
case "replace":
|
||||
var from = (string)filter.Args[0];
|
||||
var to = (string)filter.Args[1];
|
||||
to = ApplyGoTemplateText(to, variables);
|
||||
data = data.Replace(from, to);
|
||||
break;
|
||||
case "trim":
|
||||
var cutset = (string)filter.Args;
|
||||
if (cutset != null)
|
||||
{
|
||||
data = data.Trim(cutset[0]);
|
||||
}
|
||||
else
|
||||
{
|
||||
data = data.Trim();
|
||||
}
|
||||
|
||||
break;
|
||||
case "prepend":
|
||||
var prependstr = (string)filter.Args;
|
||||
data = ApplyGoTemplateText(prependstr, variables) + data;
|
||||
break;
|
||||
case "append":
|
||||
var str = (string)filter.Args;
|
||||
data += ApplyGoTemplateText(str, variables);
|
||||
break;
|
||||
case "tolower":
|
||||
data = data.ToLower();
|
||||
break;
|
||||
case "toupper":
|
||||
data = data.ToUpper();
|
||||
break;
|
||||
case "urldecode":
|
||||
data = WebUtilityHelpers.UrlDecode(data, _encoding);
|
||||
break;
|
||||
case "urlencode":
|
||||
data = WebUtilityHelpers.UrlEncode(data, _encoding);
|
||||
break;
|
||||
case "timeago":
|
||||
case "reltime":
|
||||
data = DateTimeUtil.FromTimeAgo(data).ToString(DateTimeUtil.Rfc1123ZPattern);
|
||||
break;
|
||||
case "fuzzytime":
|
||||
data = DateTimeUtil.FromUnknown(data).ToString(DateTimeUtil.Rfc1123ZPattern);
|
||||
break;
|
||||
case "validfilename":
|
||||
data = StringUtil.MakeValidFileName(data, '_', false);
|
||||
break;
|
||||
case "diacritics":
|
||||
var diacriticsOp = (string)filter.Args;
|
||||
if (diacriticsOp == "replace")
|
||||
{
|
||||
// Should replace diacritics charcaters with their base character
|
||||
// It's not perfect, e.g. "ŠĐĆŽ - šđčćž" becomes "SĐCZ-sđccz"
|
||||
var stFormD = data.Normalize(NormalizationForm.FormD);
|
||||
var len = stFormD.Length;
|
||||
var sb = new StringBuilder();
|
||||
for (var i = 0; i < len; i++)
|
||||
{
|
||||
var uc = System.Globalization.CharUnicodeInfo.GetUnicodeCategory(stFormD[i]);
|
||||
if (uc != System.Globalization.UnicodeCategory.NonSpacingMark)
|
||||
{
|
||||
sb.Append(stFormD[i]);
|
||||
}
|
||||
}
|
||||
|
||||
data = sb.ToString().Normalize(NormalizationForm.FormC);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception("unsupported diacritics filter argument");
|
||||
}
|
||||
|
||||
break;
|
||||
case "jsonjoinarray":
|
||||
var jsonjoinarrayJSONPath = (string)filter.Args[0];
|
||||
var jsonjoinarraySeparator = (string)filter.Args[1];
|
||||
var jsonjoinarrayO = JObject.Parse(data);
|
||||
var jsonjoinarrayOResult = jsonjoinarrayO.SelectToken(jsonjoinarrayJSONPath);
|
||||
var jsonjoinarrayOResultStrings = jsonjoinarrayOResult.Select(j => j.ToString());
|
||||
data = string.Join(jsonjoinarraySeparator, jsonjoinarrayOResultStrings);
|
||||
break;
|
||||
case "hexdump":
|
||||
// this is mainly for debugging invisible special char related issues
|
||||
var hexData = string.Join("", data.Select(c => c + "(" + ((int)c).ToString("X2") + ")"));
|
||||
_logger.Debug(string.Format("CardigannIndexer ({0}): strdump: {1}", _definition.Id, hexData));
|
||||
break;
|
||||
case "strdump":
|
||||
// for debugging
|
||||
var debugData = data.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\xA0", "\\xA0");
|
||||
var strTag = (string)filter.Args;
|
||||
if (strTag != null)
|
||||
{
|
||||
strTag = string.Format("({0}):", strTag);
|
||||
}
|
||||
else
|
||||
{
|
||||
strTag = ":";
|
||||
}
|
||||
|
||||
_logger.Debug(string.Format("CardigannIndexer ({0}): strdump{1} {2}", _definition.Id, strTag, debugData));
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return data;
|
||||
}
|
||||
|
||||
protected Uri ResolvePath(string path, Uri currentUrl = null)
|
||||
{
|
||||
return new Uri(currentUrl ?? new Uri(SiteLink), path);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,44 @@
|
||||
using System.Collections.Generic;
|
||||
using NzbDrone.Common.Http;
|
||||
using YamlDotNet.Serialization;
|
||||
using YamlDotNet.Serialization.NamingConventions;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public interface ICardigannDefinitionService
|
||||
{
|
||||
List<CardigannMetaDefinition> All();
|
||||
CardigannDefinition GetDefinition(string id);
|
||||
}
|
||||
|
||||
public class CardigannDefinitionService : ICardigannDefinitionService
|
||||
{
|
||||
private const int DEFINITION_VERSION = 1;
|
||||
|
||||
private readonly IHttpClient _httpClient;
|
||||
|
||||
private readonly IDeserializer _deserializer = new DeserializerBuilder()
|
||||
.IgnoreUnmatchedProperties()
|
||||
.WithNamingConvention(CamelCaseNamingConvention.Instance)
|
||||
.Build();
|
||||
|
||||
public CardigannDefinitionService(IHttpClient httpClient)
|
||||
{
|
||||
_httpClient = httpClient;
|
||||
}
|
||||
|
||||
public List<CardigannMetaDefinition> All()
|
||||
{
|
||||
var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}");
|
||||
var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
|
||||
return response.Resource;
|
||||
}
|
||||
|
||||
public CardigannDefinition GetDefinition(string id)
|
||||
{
|
||||
var req = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{id}");
|
||||
var response = _httpClient.Get(req);
|
||||
return _deserializer.Deserialize<CardigannDefinition>(response.Content);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,18 @@
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public class CardigannMetaDefinition
|
||||
{
|
||||
public string Id { get; set; }
|
||||
public string File { get; set; }
|
||||
public string Name { get; set; }
|
||||
public string Description { get; set; }
|
||||
public string Type { get; set; }
|
||||
public string Language { get; set; }
|
||||
public string Encoding { get; set; }
|
||||
public List<string> Links { get; set; }
|
||||
public List<string> Legacylinks { get; set; }
|
||||
public List<SettingsField> Settings { get; set; }
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,493 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using System.Text.RegularExpressions;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Core.Indexers.Exceptions;
|
||||
using NzbDrone.Core.Parser.Model;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public class CardigannParser : CardigannBase, IParseIndexerResponse
|
||||
{
|
||||
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
|
||||
|
||||
public CardigannParser(CardigannDefinition definition,
|
||||
CardigannSettings settings,
|
||||
Logger logger)
|
||||
: base(definition, settings, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public IList<ReleaseInfo> ParseResponse(IndexerResponse indexerResponse)
|
||||
{
|
||||
var releases = new List<CardigannReleaseInfo>();
|
||||
|
||||
_logger.Debug("Parsing");
|
||||
|
||||
if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK)
|
||||
{
|
||||
// Remove cookie cache
|
||||
if (indexerResponse.HttpResponse.HasHttpRedirect && indexerResponse.HttpResponse.Headers["Location"]
|
||||
.ContainsIgnoreCase("login.php"))
|
||||
{
|
||||
CookiesUpdater(null, null);
|
||||
throw new IndexerException(indexerResponse, "We are being redirected to the PTP login page. Most likely your session expired or was killed. Try testing the indexer in the settings.");
|
||||
}
|
||||
|
||||
throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request");
|
||||
}
|
||||
|
||||
var results = indexerResponse.Content;
|
||||
var request = indexerResponse.Request as CardigannRequest;
|
||||
var variables = request.Variables;
|
||||
var search = _definition.Search;
|
||||
|
||||
var searchUrlUri = new Uri(request.Url.FullUri);
|
||||
|
||||
try
|
||||
{
|
||||
var searchResultParser = new HtmlParser();
|
||||
var searchResultDocument = searchResultParser.ParseDocument(results);
|
||||
|
||||
/* checkForError(response, Definition.Search.Error); */
|
||||
|
||||
if (search.Preprocessingfilters != null)
|
||||
{
|
||||
results = ApplyFilters(results, search.Preprocessingfilters, variables);
|
||||
searchResultDocument = searchResultParser.ParseDocument(results);
|
||||
_logger.Debug(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results));
|
||||
}
|
||||
|
||||
var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables);
|
||||
var rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector);
|
||||
var rows = new List<IElement>();
|
||||
foreach (var rowDom in rowsDom)
|
||||
{
|
||||
rows.Add(rowDom);
|
||||
}
|
||||
|
||||
// merge following rows for After selector
|
||||
var after = search.Rows.After;
|
||||
if (after > 0)
|
||||
{
|
||||
for (var i = 0; i < rows.Count; i += 1)
|
||||
{
|
||||
var currentRow = rows[i];
|
||||
for (var j = 0; j < after; j += 1)
|
||||
{
|
||||
var mergeRowIndex = i + j + 1;
|
||||
var mergeRow = rows[mergeRowIndex];
|
||||
var mergeNodes = new List<INode>();
|
||||
foreach (var node in mergeRow.ChildNodes)
|
||||
{
|
||||
mergeNodes.Add(node);
|
||||
}
|
||||
|
||||
currentRow.Append(mergeNodes.ToArray());
|
||||
}
|
||||
|
||||
rows.RemoveRange(i + 1, after);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var row in rows)
|
||||
{
|
||||
try
|
||||
{
|
||||
var release = new CardigannReleaseInfo();
|
||||
|
||||
// Parse fields
|
||||
foreach (var field in search.Fields)
|
||||
{
|
||||
var fieldParts = field.Key.Split('|');
|
||||
var fieldName = fieldParts[0];
|
||||
var fieldModifiers = new List<string>();
|
||||
for (var i = 1; i < fieldParts.Length; i++)
|
||||
{
|
||||
fieldModifiers.Add(fieldParts[i]);
|
||||
}
|
||||
|
||||
string value = null;
|
||||
var variablesKey = ".Result." + fieldName;
|
||||
try
|
||||
{
|
||||
value = HandleSelector(field.Value, row, variables);
|
||||
switch (fieldName)
|
||||
{
|
||||
case "download":
|
||||
if (string.IsNullOrEmpty(value))
|
||||
{
|
||||
value = null;
|
||||
release.Link = null;
|
||||
break;
|
||||
}
|
||||
|
||||
if (value.StartsWith("magnet:"))
|
||||
{
|
||||
release.MagnetUri = new Uri(value);
|
||||
value = release.MagnetUri.ToString();
|
||||
}
|
||||
else
|
||||
{
|
||||
release.Link = ResolvePath(value, searchUrlUri);
|
||||
value = release.Link.ToString();
|
||||
}
|
||||
|
||||
break;
|
||||
case "magnet":
|
||||
var magnetUri = new Uri(value);
|
||||
release.MagnetUri = magnetUri;
|
||||
value = magnetUri.ToString();
|
||||
if (release.Guid == null)
|
||||
{
|
||||
release.Guid = magnetUri;
|
||||
}
|
||||
|
||||
break;
|
||||
case "details":
|
||||
var url = ResolvePath(value, searchUrlUri);
|
||||
release.Guid = url;
|
||||
release.Comments = url;
|
||||
if (release.Guid == null)
|
||||
{
|
||||
release.Guid = url;
|
||||
}
|
||||
|
||||
value = url.ToString();
|
||||
break;
|
||||
case "comments":
|
||||
var commentsUrl = ResolvePath(value, searchUrlUri);
|
||||
if (release.Comments == null)
|
||||
{
|
||||
release.Comments = commentsUrl;
|
||||
}
|
||||
|
||||
if (release.Guid == null)
|
||||
{
|
||||
release.Guid = commentsUrl;
|
||||
}
|
||||
|
||||
value = commentsUrl.ToString();
|
||||
break;
|
||||
case "title":
|
||||
if (fieldModifiers.Contains("append"))
|
||||
{
|
||||
release.Title += value;
|
||||
}
|
||||
else
|
||||
{
|
||||
release.Title = value;
|
||||
}
|
||||
|
||||
value = release.Title;
|
||||
break;
|
||||
case "description":
|
||||
if (fieldModifiers.Contains("append"))
|
||||
{
|
||||
release.Description += value;
|
||||
}
|
||||
else
|
||||
{
|
||||
release.Description = value;
|
||||
}
|
||||
|
||||
value = release.Description;
|
||||
break;
|
||||
case "category":
|
||||
// var cats = MapTrackerCatToNewznab(value);
|
||||
var cats = new List<int> { 2000 };
|
||||
if (cats.Any())
|
||||
{
|
||||
if (release.Category == null || fieldModifiers.Contains("noappend"))
|
||||
{
|
||||
release.Category = cats;
|
||||
}
|
||||
else
|
||||
{
|
||||
release.Category = release.Category.Union(cats).ToList();
|
||||
}
|
||||
}
|
||||
|
||||
value = release.Category.ToString();
|
||||
break;
|
||||
case "size":
|
||||
release.Size = CardigannReleaseInfo.GetBytes(value);
|
||||
value = release.Size.ToString();
|
||||
break;
|
||||
case "leechers":
|
||||
var leechers = ParseUtil.CoerceLong(value);
|
||||
leechers = leechers < 5000000L ? leechers : 0; // to fix #6558
|
||||
if (release.Peers == null)
|
||||
{
|
||||
release.Peers = leechers;
|
||||
}
|
||||
else
|
||||
{
|
||||
release.Peers += leechers;
|
||||
}
|
||||
|
||||
value = leechers.ToString();
|
||||
break;
|
||||
case "seeders":
|
||||
release.Seeders = ParseUtil.CoerceLong(value);
|
||||
release.Seeders = release.Seeders < 5000000L ? release.Seeders : 0; // to fix #6558
|
||||
if (release.Peers == null)
|
||||
{
|
||||
release.Peers = release.Seeders;
|
||||
}
|
||||
else
|
||||
{
|
||||
release.Peers += release.Seeders;
|
||||
}
|
||||
|
||||
value = release.Seeders.ToString();
|
||||
break;
|
||||
case "date":
|
||||
release.PublishDate = DateTimeUtil.FromUnknown(value);
|
||||
value = release.PublishDate.ToString(DateTimeUtil.Rfc1123ZPattern);
|
||||
break;
|
||||
case "files":
|
||||
release.Files = ParseUtil.CoerceLong(value);
|
||||
value = release.Files.ToString();
|
||||
break;
|
||||
case "grabs":
|
||||
release.Grabs = ParseUtil.CoerceLong(value);
|
||||
value = release.Grabs.ToString();
|
||||
break;
|
||||
case "downloadvolumefactor":
|
||||
release.DownloadVolumeFactor = ParseUtil.CoerceDouble(value);
|
||||
value = release.DownloadVolumeFactor.ToString();
|
||||
break;
|
||||
case "uploadvolumefactor":
|
||||
release.UploadVolumeFactor = ParseUtil.CoerceDouble(value);
|
||||
value = release.UploadVolumeFactor.ToString();
|
||||
break;
|
||||
case "minimumratio":
|
||||
release.MinimumRatio = ParseUtil.CoerceDouble(value);
|
||||
value = release.MinimumRatio.ToString();
|
||||
break;
|
||||
case "minimumseedtime":
|
||||
release.MinimumSeedTime = ParseUtil.CoerceLong(value);
|
||||
value = release.MinimumSeedTime.ToString();
|
||||
break;
|
||||
case "imdb":
|
||||
release.Imdb = ParseUtil.GetLongFromString(value);
|
||||
value = release.Imdb.ToString();
|
||||
break;
|
||||
case "tmdbid":
|
||||
var tmdbIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
|
||||
var tmdbIDMatch = tmdbIDRegEx.Match(value);
|
||||
var tmdbID = tmdbIDMatch.Groups[1].Value;
|
||||
release.TMDb = ParseUtil.CoerceLong(tmdbID);
|
||||
value = release.TMDb.ToString();
|
||||
break;
|
||||
case "rageid":
|
||||
var rageIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
|
||||
var rageIDMatch = rageIDRegEx.Match(value);
|
||||
var rageID = rageIDMatch.Groups[1].Value;
|
||||
release.RageID = ParseUtil.CoerceLong(rageID);
|
||||
value = release.RageID.ToString();
|
||||
break;
|
||||
case "tvdbid":
|
||||
var tvdbIdRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
|
||||
var tvdbIdMatch = tvdbIdRegEx.Match(value);
|
||||
var tvdbId = tvdbIdMatch.Groups[1].Value;
|
||||
release.TVDBId = ParseUtil.CoerceLong(tvdbId);
|
||||
value = release.TVDBId.ToString();
|
||||
break;
|
||||
case "author":
|
||||
release.Author = value;
|
||||
break;
|
||||
case "booktitle":
|
||||
release.BookTitle = value;
|
||||
break;
|
||||
case "banner":
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
var bannerurl = ResolvePath(value, searchUrlUri);
|
||||
release.BannerUrl = bannerurl;
|
||||
}
|
||||
|
||||
value = release.BannerUrl.ToString();
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
variables[variablesKey] = value;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
if (!variables.ContainsKey(variablesKey))
|
||||
{
|
||||
variables[variablesKey] = null;
|
||||
}
|
||||
|
||||
if (OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional)
|
||||
{
|
||||
variables[variablesKey] = null;
|
||||
continue;
|
||||
}
|
||||
|
||||
_logger.Error("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value == null ? "<null>" : value, ex.Message);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
var filters = search.Rows.Filters;
|
||||
var skipRelease = false;
|
||||
if (filters != null)
|
||||
{
|
||||
foreach (var filter in filters)
|
||||
{
|
||||
switch (filter.Name)
|
||||
{
|
||||
case "andmatch":
|
||||
var characterLimit = -1;
|
||||
if (filter.Args != null)
|
||||
{
|
||||
characterLimit = int.Parse(filter.Args);
|
||||
}
|
||||
|
||||
/*
|
||||
if (query.ImdbID != null && TorznabCaps.SupportsImdbMovieSearch)
|
||||
{
|
||||
break; // skip andmatch filter for imdb searches
|
||||
}
|
||||
|
||||
if (query.TmdbID != null && TorznabCaps.SupportsTmdbMovieSearch)
|
||||
{
|
||||
break; // skip andmatch filter for tmdb searches
|
||||
}
|
||||
|
||||
if (query.TvdbID != null && TorznabCaps.SupportsTvdbSearch)
|
||||
{
|
||||
break; // skip andmatch filter for tvdb searches
|
||||
}
|
||||
|
||||
var queryKeywords = variables[".Keywords"] as string;
|
||||
|
||||
if (!query.MatchQueryStringAND(release.Title, characterLimit, queryKeywords))
|
||||
{
|
||||
_logger.Debug(string.Format("CardigannIndexer ({0}): skipping {1} (andmatch filter)", _definition.Id, release.Title));
|
||||
skipRelease = true;
|
||||
}
|
||||
*/
|
||||
|
||||
break;
|
||||
case "strdump":
|
||||
// for debugging
|
||||
_logger.Debug(string.Format("CardigannIndexer ({0}): row strdump: {1}", _definition.Id, row.ToHtmlPretty()));
|
||||
break;
|
||||
default:
|
||||
_logger.Error(string.Format("CardigannIndexer ({0}): Unsupported rows filter: {1}", _definition.Id, filter.Name));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (skipRelease)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// if DateHeaders is set go through the previous rows and look for the header selector
|
||||
var dateHeaders = _definition.Search.Rows.Dateheaders;
|
||||
if (release.PublishDate == DateTime.MinValue && dateHeaders != null)
|
||||
{
|
||||
var prevRow = row.PreviousElementSibling;
|
||||
string value = null;
|
||||
if (prevRow == null)
|
||||
{
|
||||
// continue with parent
|
||||
var parent = row.ParentElement;
|
||||
if (parent != null)
|
||||
{
|
||||
prevRow = parent.PreviousElementSibling;
|
||||
}
|
||||
}
|
||||
|
||||
while (prevRow != null)
|
||||
{
|
||||
var curRow = prevRow;
|
||||
_logger.Debug(prevRow.OuterHtml);
|
||||
try
|
||||
{
|
||||
value = HandleSelector(dateHeaders, curRow);
|
||||
break;
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// do nothing
|
||||
}
|
||||
|
||||
prevRow = curRow.PreviousElementSibling;
|
||||
if (prevRow == null)
|
||||
{
|
||||
// continue with parent
|
||||
var parent = curRow.ParentElement;
|
||||
if (parent != null)
|
||||
{
|
||||
prevRow = parent.PreviousElementSibling;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (value == null && dateHeaders.Optional == false)
|
||||
{
|
||||
throw new Exception(string.Format("No date header row found for {0}", release.ToString()));
|
||||
}
|
||||
|
||||
if (value != null)
|
||||
{
|
||||
release.PublishDate = DateTimeUtil.FromUnknown(value);
|
||||
}
|
||||
}
|
||||
|
||||
releases.Add(release);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.Error(ex, "CardigannIndexer ({0}): Error while parsing row '{1}':\n\n{2}", _definition.Id, row.ToHtmlPretty());
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// OnParseError(results, ex);
|
||||
throw;
|
||||
}
|
||||
|
||||
/*
|
||||
if (query.Limit > 0)
|
||||
{
|
||||
releases = releases.Take(query.Limit).ToList();
|
||||
}*/
|
||||
|
||||
var result = new List<ReleaseInfo>();
|
||||
|
||||
result.AddRange(releases.Select(x => new TorrentInfo
|
||||
{
|
||||
Guid = x.Guid.ToString(),
|
||||
Title = x.Title,
|
||||
Size = x.Size.Value,
|
||||
DownloadUrl = x.Link?.ToString(),
|
||||
CommentUrl = x.Comments?.ToString(),
|
||||
InfoUrl = x.Link?.ToString(),
|
||||
MagnetUrl = x.MagnetUri?.ToString(),
|
||||
InfoHash = x.InfoHash,
|
||||
Seeders = (int?)x.Seeders,
|
||||
Peers = (int?)x.Peers
|
||||
}));
|
||||
|
||||
_logger.Debug($"Got {result.Count} releases");
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,77 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public class CardigannReleaseInfo
|
||||
{
|
||||
public string Title { get; set; }
|
||||
public Uri Guid { get; set; }
|
||||
public Uri Link { get; set; }
|
||||
public Uri Comments { get; set; }
|
||||
public DateTime PublishDate { get; set; }
|
||||
public ICollection<int> Category { get; set; }
|
||||
public long? Size { get; set; }
|
||||
public long? Files { get; set; }
|
||||
public long? Grabs { get; set; }
|
||||
public string Description { get; set; }
|
||||
public long? RageID { get; set; }
|
||||
public long? TVDBId { get; set; }
|
||||
public long? Imdb { get; set; }
|
||||
public long? TMDb { get; set; }
|
||||
public string Author { get; set; }
|
||||
public string BookTitle { get; set; }
|
||||
public long? Seeders { get; set; }
|
||||
public long? Peers { get; set; }
|
||||
public Uri BannerUrl { get; set; }
|
||||
public string InfoHash { get; set; }
|
||||
public Uri MagnetUri { get; set; }
|
||||
public double? MinimumRatio { get; set; }
|
||||
public long? MinimumSeedTime { get; set; }
|
||||
public double? DownloadVolumeFactor { get; set; }
|
||||
public double? UploadVolumeFactor { get; set; }
|
||||
|
||||
public static long GetBytes(string str)
|
||||
{
|
||||
var valStr = new string(str.Where(c => char.IsDigit(c) || c == '.').ToArray());
|
||||
var unit = new string(str.Where(char.IsLetter).ToArray());
|
||||
var val = ParseUtil.CoerceFloat(valStr);
|
||||
return GetBytes(unit, val);
|
||||
}
|
||||
|
||||
public static long GetBytes(string unit, float value)
|
||||
{
|
||||
unit = unit.Replace("i", "").ToLowerInvariant();
|
||||
if (unit.Contains("kb"))
|
||||
{
|
||||
return BytesFromKB(value);
|
||||
}
|
||||
|
||||
if (unit.Contains("mb"))
|
||||
{
|
||||
return BytesFromMB(value);
|
||||
}
|
||||
|
||||
if (unit.Contains("gb"))
|
||||
{
|
||||
return BytesFromGB(value);
|
||||
}
|
||||
|
||||
if (unit.Contains("tb"))
|
||||
{
|
||||
return BytesFromTB(value);
|
||||
}
|
||||
|
||||
return (long)value;
|
||||
}
|
||||
|
||||
public static long BytesFromTB(float tb) => BytesFromGB(tb * 1024f);
|
||||
|
||||
public static long BytesFromGB(float gb) => BytesFromMB(gb * 1024f);
|
||||
|
||||
public static long BytesFromMB(float mb) => BytesFromKB(mb * 1024f);
|
||||
|
||||
public static long BytesFromKB(float kb) => (long)(kb * 1024f);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
using System.Collections.Generic;
|
||||
using NzbDrone.Common.Http;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public class CardigannRequest : IndexerRequest
|
||||
{
|
||||
public Dictionary<string, object> Variables { get; private set; }
|
||||
|
||||
public CardigannRequest(string url, HttpAccept httpAccept, Dictionary<string, object> variables)
|
||||
: base(url, httpAccept)
|
||||
{
|
||||
Variables = variables;
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,216 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net;
|
||||
using NLog;
|
||||
using NzbDrone.Common.Http;
|
||||
using NzbDrone.Core.IndexerSearch.Definitions;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public class CardigannRequestGenerator : CardigannBase, IIndexerRequestGenerator
|
||||
{
|
||||
private List<string> _defaultCategories = new List<string>();
|
||||
|
||||
public CardigannRequestGenerator(CardigannDefinition definition,
|
||||
CardigannSettings settings,
|
||||
Logger logger)
|
||||
: base(definition, settings, logger)
|
||||
{
|
||||
}
|
||||
|
||||
public Func<IDictionary<string, string>> GetCookies { get; set; }
|
||||
public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
|
||||
|
||||
public virtual IndexerPageableRequestChain GetRecentRequests()
|
||||
{
|
||||
_logger.Trace("Getting recent");
|
||||
|
||||
var pageableRequests = new IndexerPageableRequestChain();
|
||||
|
||||
pageableRequests.Add(GetRequest(null));
|
||||
|
||||
return pageableRequests;
|
||||
}
|
||||
|
||||
public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
|
||||
{
|
||||
_logger.Trace("Getting search");
|
||||
|
||||
var pageableRequests = new IndexerPageableRequestChain();
|
||||
|
||||
foreach (var queryTitle in searchCriteria.QueryTitles)
|
||||
{
|
||||
pageableRequests.Add(GetRequest(string.Format("{0}", queryTitle)));
|
||||
}
|
||||
|
||||
return pageableRequests;
|
||||
}
|
||||
|
||||
private IEnumerable<IndexerRequest> GetRequest(string searchCriteria)
|
||||
{
|
||||
var search = _definition.Search;
|
||||
|
||||
// init template context
|
||||
var variables = GetBaseTemplateVariables();
|
||||
|
||||
variables[".Query.Type"] = null;
|
||||
variables[".Query.Q"] = searchCriteria;
|
||||
variables[".Query.Series"] = null;
|
||||
variables[".Query.Ep"] = null;
|
||||
variables[".Query.Season"] = null;
|
||||
variables[".Query.Movie"] = null;
|
||||
variables[".Query.Year"] = null;
|
||||
variables[".Query.Limit"] = null;
|
||||
variables[".Query.Offset"] = null;
|
||||
variables[".Query.Extended"] = null;
|
||||
variables[".Query.Categories"] = null;
|
||||
variables[".Query.APIKey"] = null;
|
||||
variables[".Query.TVDBID"] = null;
|
||||
variables[".Query.TVRageID"] = null;
|
||||
variables[".Query.IMDBID"] = null;
|
||||
variables[".Query.IMDBIDShort"] = null;
|
||||
variables[".Query.TMDBID"] = null;
|
||||
variables[".Query.TVMazeID"] = null;
|
||||
variables[".Query.TraktID"] = null;
|
||||
variables[".Query.Album"] = null;
|
||||
variables[".Query.Artist"] = null;
|
||||
variables[".Query.Label"] = null;
|
||||
variables[".Query.Track"] = null;
|
||||
variables[".Query.Episode"] = null;
|
||||
variables[".Query.Author"] = null;
|
||||
variables[".Query.Title"] = null;
|
||||
|
||||
/*
|
||||
var mappedCategories = MapTorznabCapsToTrackers(query);
|
||||
if (mappedCategories.Count == 0)
|
||||
{
|
||||
mappedCategories = _defaultCategories;
|
||||
}
|
||||
*/
|
||||
|
||||
var mappedCategories = _defaultCategories;
|
||||
|
||||
variables[".Categories"] = mappedCategories;
|
||||
|
||||
var keywordTokens = new List<string>();
|
||||
var keywordTokenKeys = new List<string> { "Q", "Series", "Movie", "Year" };
|
||||
foreach (var key in keywordTokenKeys)
|
||||
{
|
||||
var value = (string)variables[".Query." + key];
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
keywordTokens.Add(value);
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"]))
|
||||
{
|
||||
keywordTokens.Add((string)variables[".Query.Episode"]);
|
||||
}
|
||||
|
||||
variables[".Query.Keywords"] = string.Join(" ", keywordTokens);
|
||||
variables[".Keywords"] = ApplyFilters((string)variables[".Query.Keywords"], search.Keywordsfilters);
|
||||
|
||||
// TODO: prepare queries first and then send them parallel
|
||||
var searchPaths = search.Paths;
|
||||
foreach (var searchPath in searchPaths)
|
||||
{
|
||||
// skip path if categories don't match
|
||||
if (searchPath.Categories != null && mappedCategories.Count > 0)
|
||||
{
|
||||
var invertMatch = searchPath.Categories[0] == "!";
|
||||
var hasIntersect = mappedCategories.Intersect(searchPath.Categories).Any();
|
||||
if (invertMatch)
|
||||
{
|
||||
hasIntersect = !hasIntersect;
|
||||
}
|
||||
|
||||
if (!hasIntersect)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
// build search URL
|
||||
// HttpUtility.UrlPathEncode seems to only encode spaces, we use UrlEncode and replace + with %20 as a workaround
|
||||
var searchUrl = ResolvePath(ApplyGoTemplateText(searchPath.Path, variables, WebUtility.UrlEncode).Replace("+", "%20")).AbsoluteUri;
|
||||
var queryCollection = new List<KeyValuePair<string, string>>();
|
||||
var method = HttpMethod.GET;
|
||||
|
||||
if (string.Equals(searchPath.Method, "post", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
method = HttpMethod.POST;
|
||||
}
|
||||
|
||||
var inputsList = new List<Dictionary<string, string>>();
|
||||
if (searchPath.Inheritinputs)
|
||||
{
|
||||
inputsList.Add(search.Inputs);
|
||||
}
|
||||
|
||||
inputsList.Add(searchPath.Inputs);
|
||||
|
||||
foreach (var inputs in inputsList)
|
||||
{
|
||||
if (inputs != null)
|
||||
{
|
||||
foreach (var input in inputs)
|
||||
{
|
||||
if (input.Key == "$raw")
|
||||
{
|
||||
var rawStr = ApplyGoTemplateText(input.Value, variables, WebUtility.UrlEncode);
|
||||
foreach (var part in rawStr.Split('&'))
|
||||
{
|
||||
var parts = part.Split(new char[] { '=' }, 2);
|
||||
var key = parts[0];
|
||||
if (key.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var value = "";
|
||||
if (parts.Length == 2)
|
||||
{
|
||||
value = parts[1];
|
||||
}
|
||||
|
||||
queryCollection.Add(key, value);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
queryCollection.Add(input.Key, ApplyGoTemplateText(input.Value, variables));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (method == HttpMethod.GET)
|
||||
{
|
||||
if (queryCollection.Count > 0)
|
||||
{
|
||||
searchUrl += "?" + queryCollection.GetQueryString(_encoding);
|
||||
}
|
||||
}
|
||||
|
||||
_logger.Info($"Adding request: {searchUrl}");
|
||||
|
||||
var request = new CardigannRequest(searchUrl, HttpAccept.Html, variables);
|
||||
|
||||
// send HTTP request
|
||||
if (search.Headers != null)
|
||||
{
|
||||
foreach (var header in search.Headers)
|
||||
{
|
||||
request.HttpRequest.Headers.Add(header.Key, header.Value[0]);
|
||||
}
|
||||
}
|
||||
|
||||
request.HttpRequest.Method = method;
|
||||
|
||||
yield return request;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,57 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using FluentValidation;
|
||||
using NzbDrone.Common.Extensions;
|
||||
using NzbDrone.Core.Annotations;
|
||||
using NzbDrone.Core.Languages;
|
||||
using NzbDrone.Core.Validation;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public class CardigannSettingsValidator : AbstractValidator<CardigannSettings>
|
||||
{
|
||||
public CardigannSettingsValidator()
|
||||
{
|
||||
RuleFor(c => c).Custom((c, context) =>
|
||||
{
|
||||
if (c.Categories.Empty())
|
||||
{
|
||||
context.AddFailure("'Categories' must be provided");
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public class CardigannSettings : IIndexerSettings
|
||||
{
|
||||
private static readonly CardigannSettingsValidator Validator = new CardigannSettingsValidator();
|
||||
|
||||
public CardigannSettings()
|
||||
{
|
||||
Categories = new[] { 2000, 2010, 2020, 2030, 2035, 2040, 2045, 2050, 2060 };
|
||||
MultiLanguages = new List<int>();
|
||||
ExtraFieldData = new Dictionary<string, object>();
|
||||
}
|
||||
|
||||
[FieldDefinition(0, Hidden = HiddenType.Hidden)]
|
||||
public string DefinitionFile { get; set; }
|
||||
|
||||
public Dictionary<string, object> ExtraFieldData { get; set; }
|
||||
|
||||
public string BaseUrl { get; set; }
|
||||
|
||||
[FieldDefinition(1000, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)]
|
||||
public IEnumerable<int> MultiLanguages { get; set; }
|
||||
|
||||
[FieldDefinition(1001, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable all categories", Advanced = true)]
|
||||
public IEnumerable<int> Categories { get; set; }
|
||||
|
||||
// Field 8 is used by TorznabSettings MinimumSeeders
|
||||
// If you need to add another field here, update TorznabSettings as well and this comment
|
||||
public virtual NzbDroneValidationResult Validate()
|
||||
{
|
||||
return new NzbDroneValidationResult(Validator.Validate(this));
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,325 @@
|
||||
using System;
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public static class DateTimeUtil
|
||||
{
|
||||
public const string Rfc1123ZPattern = "ddd, dd MMM yyyy HH':'mm':'ss z";
|
||||
|
||||
private static readonly Regex _TimeAgoRegexp = new Regex(@"(?i)\bago", RegexOptions.Compiled);
|
||||
private static readonly Regex _TodayRegexp = new Regex(@"(?i)\btoday(?:[\s,]+(?:at){0,1}\s*|[\s,]*|$)", RegexOptions.Compiled);
|
||||
private static readonly Regex _TomorrowRegexp = new Regex(@"(?i)\btomorrow(?:[\s,]+(?:at){0,1}\s*|[\s,]*|$)", RegexOptions.Compiled);
|
||||
private static readonly Regex _YesterdayRegexp = new Regex(@"(?i)\byesterday(?:[\s,]+(?:at){0,1}\s*|[\s,]*|$)", RegexOptions.Compiled);
|
||||
private static readonly Regex _DaysOfWeekRegexp = new Regex(@"(?i)\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday)\s+at\s+", RegexOptions.Compiled);
|
||||
private static readonly Regex _MissingYearRegexp = new Regex(@"^(\d{1,2}-\d{1,2})(\s|$)", RegexOptions.Compiled);
|
||||
private static readonly Regex _MissingYearRegexp2 = new Regex(@"^(\d{1,2}\s+\w{3})\s+(\d{1,2}\:\d{1,2}.*)$", RegexOptions.Compiled); // 1 Jan 10:30
|
||||
|
||||
public static DateTime UnixTimestampToDateTime(long unixTime)
|
||||
{
|
||||
var dt = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
||||
dt = dt.AddSeconds(unixTime).ToLocalTime();
|
||||
return dt;
|
||||
}
|
||||
|
||||
public static DateTime UnixTimestampToDateTime(double unixTime)
|
||||
{
|
||||
var unixStart = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
|
||||
var unixTimeStampInTicks = (long)(unixTime * TimeSpan.TicksPerSecond);
|
||||
return new DateTime(unixStart.Ticks + unixTimeStampInTicks);
|
||||
}
|
||||
|
||||
public static double DateTimeToUnixTimestamp(DateTime dt)
|
||||
{
|
||||
var date = dt.ToUniversalTime();
|
||||
var ticks = date.Ticks - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc).Ticks;
|
||||
var ts = ticks / TimeSpan.TicksPerSecond;
|
||||
return ts;
|
||||
}
|
||||
|
||||
// ex: "2 hours 1 day"
|
||||
public static DateTime FromTimeAgo(string str)
|
||||
{
|
||||
str = str.ToLowerInvariant();
|
||||
if (str.Contains("now"))
|
||||
{
|
||||
return DateTime.SpecifyKind(DateTime.Now, DateTimeKind.Local);
|
||||
}
|
||||
|
||||
str = str.Replace(",", "");
|
||||
str = str.Replace("ago", "");
|
||||
str = str.Replace("and", "");
|
||||
|
||||
var timeAgo = TimeSpan.Zero;
|
||||
var timeagoRegex = new Regex(@"\s*?([\d\.]+)\s*?([^\d\s\.]+)\s*?");
|
||||
var timeagoMatches = timeagoRegex.Match(str);
|
||||
|
||||
while (timeagoMatches.Success)
|
||||
{
|
||||
var val = ParseUtil.CoerceFloat(timeagoMatches.Groups[1].Value);
|
||||
var unit = timeagoMatches.Groups[2].Value;
|
||||
timeagoMatches = timeagoMatches.NextMatch();
|
||||
|
||||
if (unit.Contains("sec") || unit == "s")
|
||||
{
|
||||
timeAgo += TimeSpan.FromSeconds(val);
|
||||
}
|
||||
else if (unit.Contains("min") || unit == "m")
|
||||
{
|
||||
timeAgo += TimeSpan.FromMinutes(val);
|
||||
}
|
||||
else if (unit.Contains("hour") || unit.Contains("hr") || unit == "h")
|
||||
{
|
||||
timeAgo += TimeSpan.FromHours(val);
|
||||
}
|
||||
else if (unit.Contains("day") || unit == "d")
|
||||
{
|
||||
timeAgo += TimeSpan.FromDays(val);
|
||||
}
|
||||
else if (unit.Contains("week") || unit.Contains("wk") || unit == "w")
|
||||
{
|
||||
timeAgo += TimeSpan.FromDays(val * 7);
|
||||
}
|
||||
else if (unit.Contains("month") || unit == "mo")
|
||||
{
|
||||
timeAgo += TimeSpan.FromDays(val * 30);
|
||||
}
|
||||
else if (unit.Contains("year") || unit == "y")
|
||||
{
|
||||
timeAgo += TimeSpan.FromDays(val * 365);
|
||||
}
|
||||
else
|
||||
{
|
||||
throw new Exception("TimeAgo parsing failed, unknown unit: " + unit);
|
||||
}
|
||||
}
|
||||
|
||||
return DateTime.SpecifyKind(DateTime.Now - timeAgo, DateTimeKind.Local);
|
||||
}
|
||||
|
||||
// Uses the DateTimeRoutines library to parse the date
|
||||
// http://www.codeproject.com/Articles/33298/C-Date-Time-Parser
|
||||
public static DateTime FromFuzzyTime(string str, string format = null)
|
||||
{
|
||||
/*var dtFormat = format == "UK" ?
|
||||
DateTimeRoutines.DateTimeRoutines.DateTimeFormat.UkDate :
|
||||
DateTimeRoutines.DateTimeRoutines.DateTimeFormat.UsaDate;
|
||||
|
||||
if (DateTimeRoutines.DateTimeRoutines.TryParseDateOrTime(
|
||||
str, dtFormat, out DateTimeRoutines.DateTimeRoutines.ParsedDateTime dt))
|
||||
return dt.DateTime;*/
|
||||
|
||||
throw new Exception("FromFuzzyTime parsing failed");
|
||||
}
|
||||
|
||||
public static DateTime FromUnknown(string str, string format = null)
|
||||
{
|
||||
try
|
||||
{
|
||||
str = ParseUtil.NormalizeSpace(str);
|
||||
if (str.ToLower().Contains("now"))
|
||||
{
|
||||
return DateTime.UtcNow;
|
||||
}
|
||||
|
||||
// ... ago
|
||||
var match = _TimeAgoRegexp.Match(str);
|
||||
if (match.Success)
|
||||
{
|
||||
var timeago = str;
|
||||
return FromTimeAgo(timeago);
|
||||
}
|
||||
|
||||
// Today ...
|
||||
match = _TodayRegexp.Match(str);
|
||||
if (match.Success)
|
||||
{
|
||||
var time = str.Replace(match.Groups[0].Value, "");
|
||||
var dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
|
||||
dt += ParseTimeSpan(time);
|
||||
return dt;
|
||||
}
|
||||
|
||||
// Yesterday ...
|
||||
match = _YesterdayRegexp.Match(str);
|
||||
if (match.Success)
|
||||
{
|
||||
var time = str.Replace(match.Groups[0].Value, "");
|
||||
var dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
|
||||
dt += ParseTimeSpan(time);
|
||||
dt -= TimeSpan.FromDays(1);
|
||||
return dt;
|
||||
}
|
||||
|
||||
// Tomorrow ...
|
||||
match = _TomorrowRegexp.Match(str);
|
||||
if (match.Success)
|
||||
{
|
||||
var time = str.Replace(match.Groups[0].Value, "");
|
||||
var dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
|
||||
dt += ParseTimeSpan(time);
|
||||
dt += TimeSpan.FromDays(1);
|
||||
return dt;
|
||||
}
|
||||
|
||||
// [day of the week] at ... (eg: Saturday at 14:22)
|
||||
match = _DaysOfWeekRegexp.Match(str);
|
||||
if (match.Success)
|
||||
{
|
||||
var time = str.Replace(match.Groups[0].Value, "");
|
||||
var dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified);
|
||||
dt += ParseTimeSpan(time);
|
||||
|
||||
DayOfWeek dow;
|
||||
var groupMatchLower = match.Groups[1].Value.ToLower();
|
||||
if (groupMatchLower.StartsWith("monday"))
|
||||
{
|
||||
dow = DayOfWeek.Monday;
|
||||
}
|
||||
else if (groupMatchLower.StartsWith("tuesday"))
|
||||
{
|
||||
dow = DayOfWeek.Tuesday;
|
||||
}
|
||||
else if (groupMatchLower.StartsWith("wednesday"))
|
||||
{
|
||||
dow = DayOfWeek.Wednesday;
|
||||
}
|
||||
else if (groupMatchLower.StartsWith("thursday"))
|
||||
{
|
||||
dow = DayOfWeek.Thursday;
|
||||
}
|
||||
else if (groupMatchLower.StartsWith("friday"))
|
||||
{
|
||||
dow = DayOfWeek.Friday;
|
||||
}
|
||||
else if (groupMatchLower.StartsWith("saturday"))
|
||||
{
|
||||
dow = DayOfWeek.Saturday;
|
||||
}
|
||||
else
|
||||
{
|
||||
dow = DayOfWeek.Sunday;
|
||||
}
|
||||
|
||||
while (dt.DayOfWeek != dow)
|
||||
{
|
||||
dt = dt.AddDays(-1);
|
||||
}
|
||||
|
||||
return dt;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
// try parsing the str as an unix timestamp
|
||||
var unixTimeStamp = long.Parse(str);
|
||||
return UnixTimestampToDateTime(unixTimeStamp);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
// it wasn't a timestamp, continue....
|
||||
}
|
||||
|
||||
// add missing year
|
||||
match = _MissingYearRegexp.Match(str);
|
||||
if (match.Success)
|
||||
{
|
||||
var date = match.Groups[1].Value;
|
||||
var newDate = DateTime.Now.Year + "-" + date;
|
||||
str = str.Replace(date, newDate);
|
||||
}
|
||||
|
||||
// add missing year 2
|
||||
match = _MissingYearRegexp2.Match(str);
|
||||
if (match.Success)
|
||||
{
|
||||
var date = match.Groups[1].Value;
|
||||
var time = match.Groups[2].Value;
|
||||
str = date + " " + DateTime.Now.Year + " " + time;
|
||||
}
|
||||
|
||||
return FromFuzzyTime(str, format);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
throw new Exception($"DateTime parsing failed for \"{str}\": {ex}");
|
||||
}
|
||||
}
|
||||
|
||||
// converts a date/time string to a DateTime object using a GoLang layout
|
||||
public static DateTime ParseDateTimeGoLang(string date, string layout)
|
||||
{
|
||||
date = ParseUtil.NormalizeSpace(date);
|
||||
var pattern = layout;
|
||||
|
||||
// year
|
||||
pattern = pattern.Replace("2006", "yyyy");
|
||||
pattern = pattern.Replace("06", "yy");
|
||||
|
||||
// month
|
||||
pattern = pattern.Replace("January", "MMMM");
|
||||
pattern = pattern.Replace("Jan", "MMM");
|
||||
pattern = pattern.Replace("01", "MM");
|
||||
|
||||
// day
|
||||
pattern = pattern.Replace("Monday", "dddd");
|
||||
pattern = pattern.Replace("Mon", "ddd");
|
||||
pattern = pattern.Replace("02", "dd");
|
||||
pattern = pattern.Replace("2", "d");
|
||||
|
||||
// hours/minutes/seconds
|
||||
pattern = pattern.Replace("05", "ss");
|
||||
|
||||
pattern = pattern.Replace("15", "HH");
|
||||
pattern = pattern.Replace("03", "hh");
|
||||
pattern = pattern.Replace("3", "h");
|
||||
|
||||
pattern = pattern.Replace("04", "mm");
|
||||
pattern = pattern.Replace("4", "m");
|
||||
|
||||
pattern = pattern.Replace("5", "s");
|
||||
|
||||
// month again
|
||||
pattern = pattern.Replace("1", "M");
|
||||
|
||||
// fractional seconds
|
||||
pattern = pattern.Replace(".0000", "ffff");
|
||||
pattern = pattern.Replace(".000", "fff");
|
||||
pattern = pattern.Replace(".00", "ff");
|
||||
pattern = pattern.Replace(".0", "f");
|
||||
|
||||
pattern = pattern.Replace(".9999", "FFFF");
|
||||
pattern = pattern.Replace(".999", "FFF");
|
||||
pattern = pattern.Replace(".99", "FF");
|
||||
pattern = pattern.Replace(".9", "F");
|
||||
|
||||
// AM/PM
|
||||
pattern = pattern.Replace("PM", "tt");
|
||||
pattern = pattern.Replace("pm", "tt"); // not sure if this works
|
||||
|
||||
// timezones
|
||||
// these might need further tuning
|
||||
pattern = pattern.Replace("Z07:00", "'Z'zzz");
|
||||
pattern = pattern.Replace("Z07", "'Z'zz");
|
||||
pattern = pattern.Replace("Z07:00", "'Z'zzz");
|
||||
pattern = pattern.Replace("Z07", "'Z'zz");
|
||||
pattern = pattern.Replace("-07:00", "zzz");
|
||||
pattern = pattern.Replace("-07", "zz");
|
||||
|
||||
try
|
||||
{
|
||||
return DateTime.ParseExact(date, pattern, CultureInfo.InvariantCulture);
|
||||
}
|
||||
catch (FormatException ex)
|
||||
{
|
||||
throw new FormatException($"Error while parsing DateTime \"{date}\", using layout \"{layout}\" ({pattern}): {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static TimeSpan ParseTimeSpan(string time) =>
|
||||
string.IsNullOrWhiteSpace(time)
|
||||
? TimeSpan.Zero
|
||||
: DateTime.Parse(time).TimeOfDay;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,176 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
// A Dictionary allowing the same key multiple times
|
||||
public class KeyValuePairList : List<KeyValuePair<string, SelectorBlock>>, IDictionary<string, SelectorBlock>
|
||||
{
|
||||
public SelectorBlock this[string key]
|
||||
{
|
||||
get => throw new NotImplementedException();
|
||||
|
||||
set => Add(new KeyValuePair<string, SelectorBlock>(key, value));
|
||||
}
|
||||
|
||||
public ICollection<string> Keys => throw new NotImplementedException();
|
||||
|
||||
public ICollection<SelectorBlock> Values => throw new NotImplementedException();
|
||||
|
||||
public void Add(string key, SelectorBlock value) => Add(new KeyValuePair<string, SelectorBlock>(key, value));
|
||||
|
||||
public bool ContainsKey(string key) => throw new NotImplementedException();
|
||||
|
||||
public bool Remove(string key) => throw new NotImplementedException();
|
||||
|
||||
public bool TryGetValue(string key, out SelectorBlock value) => throw new NotImplementedException();
|
||||
}
|
||||
|
||||
// Cardigann yaml classes
|
||||
public class CardigannDefinition
|
||||
{
|
||||
public string Id { get; set; }
|
||||
public List<SettingsField> Settings { get; set; }
|
||||
public string Name { get; set; }
|
||||
public string Description { get; set; }
|
||||
public string Type { get; set; }
|
||||
public string Language { get; set; }
|
||||
public string Encoding { get; set; }
|
||||
public List<string> Links { get; set; }
|
||||
public List<string> Legacylinks { get; set; }
|
||||
public bool Followredirect { get; set; } = false;
|
||||
public List<string> Certificates { get; set; }
|
||||
public CapabilitiesBlock Caps { get; set; }
|
||||
public LoginBlock Login { get; set; }
|
||||
public RatioBlock Ratio { get; set; }
|
||||
public SearchBlock Search { get; set; }
|
||||
public DownloadBlock Download { get; set; }
|
||||
}
|
||||
|
||||
public class SettingsField
|
||||
{
|
||||
public string Name { get; set; }
|
||||
public string Type { get; set; }
|
||||
public string Label { get; set; }
|
||||
public string Default { get; set; }
|
||||
public string[] Defaults { get; set; }
|
||||
public Dictionary<string, string> Options { get; set; }
|
||||
}
|
||||
|
||||
public class CategorymappingBlock
|
||||
{
|
||||
public string id { get; set; }
|
||||
public string cat { get; set; }
|
||||
public string desc { get; set; }
|
||||
public bool Default { get; set; }
|
||||
}
|
||||
|
||||
public class CapabilitiesBlock
|
||||
{
|
||||
public Dictionary<string, string> Categories { get; set; }
|
||||
public List<CategorymappingBlock> Categorymappings { get; set; }
|
||||
public Dictionary<string, List<string>> Modes { get; set; }
|
||||
}
|
||||
|
||||
public class CaptchaBlock
|
||||
{
|
||||
public string Type { get; set; }
|
||||
public string Selector { get; set; }
|
||||
public string Image { get => throw new Exception("Deprecated, please use Login.Captcha.Selector instead"); set => throw new Exception("Deprecated, please use login/captcha/selector instead of image"); }
|
||||
public string Input { get; set; }
|
||||
}
|
||||
|
||||
public class LoginBlock
|
||||
{
|
||||
public string Path { get; set; }
|
||||
public string Submitpath { get; set; }
|
||||
public List<string> Cookies { get; set; }
|
||||
public string Method { get; set; }
|
||||
public string Form { get; set; }
|
||||
public bool Selectors { get; set; } = false;
|
||||
public Dictionary<string, string> Inputs { get; set; }
|
||||
public Dictionary<string, SelectorBlock> Selectorinputs { get; set; }
|
||||
public Dictionary<string, SelectorBlock> Getselectorinputs { get; set; }
|
||||
public List<ErrorBlock> Error { get; set; }
|
||||
public PageTestBlock Test { get; set; }
|
||||
public CaptchaBlock Captcha { get; set; }
|
||||
}
|
||||
|
||||
public class ErrorBlock
|
||||
{
|
||||
public string Path { get; set; }
|
||||
public string Selector { get; set; }
|
||||
public SelectorBlock Message { get; set; }
|
||||
}
|
||||
|
||||
public class SelectorBlock
|
||||
{
|
||||
public string Selector { get; set; }
|
||||
public bool Optional { get; set; } = false;
|
||||
public string Text { get; set; }
|
||||
public string Attribute { get; set; }
|
||||
public string Remove { get; set; }
|
||||
public List<FilterBlock> Filters { get; set; }
|
||||
public Dictionary<string, string> Case { get; set; }
|
||||
}
|
||||
|
||||
public class FilterBlock
|
||||
{
|
||||
public string Name { get; set; }
|
||||
public dynamic Args { get; set; }
|
||||
}
|
||||
|
||||
public class PageTestBlock
|
||||
{
|
||||
public string Path { get; set; }
|
||||
public string Selector { get; set; }
|
||||
}
|
||||
|
||||
public class RatioBlock : SelectorBlock
|
||||
{
|
||||
public string Path { get; set; }
|
||||
}
|
||||
|
||||
public class SearchBlock
|
||||
{
|
||||
public string Path { get; set; }
|
||||
public List<SearchPathBlock> Paths { get; set; }
|
||||
public Dictionary<string, List<string>> Headers { get; set; }
|
||||
public List<FilterBlock> Keywordsfilters { get; set; }
|
||||
public Dictionary<string, string> Inputs { get; set; }
|
||||
public List<ErrorBlock> Error { get; set; }
|
||||
public List<FilterBlock> Preprocessingfilters { get; set; }
|
||||
public RowsBlock Rows { get; set; }
|
||||
public KeyValuePairList Fields { get; set; }
|
||||
}
|
||||
|
||||
public class RowsBlock : SelectorBlock
|
||||
{
|
||||
public int After { get; set; }
|
||||
public SelectorBlock Dateheaders { get; set; }
|
||||
}
|
||||
|
||||
public class SearchPathBlock : RequestBlock
|
||||
{
|
||||
public List<string> Categories { get; set; }
|
||||
public bool Inheritinputs { get; set; } = true;
|
||||
public bool Followredirect { get; set; } = false;
|
||||
}
|
||||
|
||||
public class RequestBlock
|
||||
{
|
||||
public string Path { get; set; }
|
||||
public string Method { get; set; }
|
||||
public Dictionary<string, string> Inputs { get; set; }
|
||||
public string Queryseparator { get; set; } = "&";
|
||||
}
|
||||
|
||||
public class DownloadBlock
|
||||
{
|
||||
public string Selector { get; set; }
|
||||
public string Attribute { get; set; }
|
||||
public List<FilterBlock> Filters { get; set; }
|
||||
public string Method { get; set; }
|
||||
public RequestBlock Before { get; set; }
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,102 @@
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public static class ParseUtil
|
||||
{
|
||||
private static readonly Regex InvalidXmlChars =
|
||||
new Regex(
|
||||
@"(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|[\x00-\x08\x0B\x0C\x0E-\x1F\x7F-\x9F\uFEFF\uFFFE\uFFFF]",
|
||||
RegexOptions.Compiled);
|
||||
private static readonly Regex ImdbId = new Regex(@"^(?:tt)?(\d{1,8})$", RegexOptions.Compiled);
|
||||
|
||||
public static string NormalizeSpace(string s) => s.Trim();
|
||||
|
||||
public static string NormalizeMultiSpaces(string s) =>
|
||||
new Regex(@"\s+").Replace(NormalizeSpace(s), " ");
|
||||
|
||||
public static string NormalizeNumber(string s) =>
|
||||
NormalizeSpace(s)
|
||||
.Replace("-", "0")
|
||||
.Replace(",", "");
|
||||
|
||||
public static string RemoveInvalidXmlChars(string text) => string.IsNullOrEmpty(text) ? "" : InvalidXmlChars.Replace(text, "");
|
||||
|
||||
public static double CoerceDouble(string str) => double.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
|
||||
|
||||
public static float CoerceFloat(string str) => float.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
|
||||
|
||||
public static int CoerceInt(string str) => int.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
|
||||
|
||||
public static long CoerceLong(string str) => long.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture);
|
||||
|
||||
public static bool TryCoerceDouble(string str, out double result) => double.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
|
||||
|
||||
public static bool TryCoerceFloat(string str, out float result) => float.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
|
||||
|
||||
public static bool TryCoerceInt(string str, out int result) => int.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
|
||||
|
||||
public static bool TryCoerceLong(string str, out long result) => long.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result);
|
||||
|
||||
/*
|
||||
public static string GetArgumentFromQueryString(string url, string argument)
|
||||
{
|
||||
if (url == null || argument == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var qsStr = url.Split(new char[] { '?' }, 2)[1];
|
||||
qsStr = qsStr.Split(new char[] { '#' }, 2)[0];
|
||||
var qs = QueryHelpers.ParseQuery(qsStr);
|
||||
return qs[argument].FirstOrDefault();
|
||||
}*/
|
||||
|
||||
public static long? GetLongFromString(string str)
|
||||
{
|
||||
if (str == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var idRegEx = new Regex(@"(\d+)", RegexOptions.Compiled);
|
||||
var idMatch = idRegEx.Match(str);
|
||||
if (!idMatch.Success)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var id = idMatch.Groups[1].Value;
|
||||
return CoerceLong(id);
|
||||
}
|
||||
|
||||
public static int? GetImdbID(string imdbstr)
|
||||
{
|
||||
if (imdbstr == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var match = ImdbId.Match(imdbstr);
|
||||
if (!match.Success)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return int.Parse(match.Groups[1].Value, NumberStyles.Any, CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
public static string GetFullImdbID(string imdbstr)
|
||||
{
|
||||
var imdbid = GetImdbID(imdbstr);
|
||||
if (imdbid == null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return "tt" + ((int)imdbid).ToString("D7");
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,30 @@
|
||||
using System.Net;
|
||||
using System.Text;
|
||||
|
||||
namespace NzbDrone.Core.Indexers.Cardigann
|
||||
{
|
||||
public static class WebUtilityHelpers
|
||||
{
|
||||
public static string UrlEncode(string searchString, Encoding encoding)
|
||||
{
|
||||
if (string.IsNullOrEmpty(searchString))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var bytes = encoding.GetBytes(searchString);
|
||||
return encoding.GetString(WebUtility.UrlEncodeToBytes(bytes, 0, bytes.Length));
|
||||
}
|
||||
|
||||
public static string UrlDecode(string searchString, Encoding encoding)
|
||||
{
|
||||
if (string.IsNullOrEmpty(searchString))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var inputBytes = encoding.GetBytes(searchString);
|
||||
return encoding.GetString(WebUtility.UrlDecodeToBytes(inputBytes, 0, inputBytes.Length));
|
||||
}
|
||||
}
|
||||
}
|
||||
Loading…
Reference in new issue