From d041b152d6e5d8a09abc853fa513b16f681735ff Mon Sep 17 00:00:00 2001 From: ta264 Date: Thu, 22 Oct 2020 21:43:29 +0100 Subject: [PATCH] Basic support for cardigann definitions --- .../Indexers/Indexers/AddIndexerItem.js | 5 +- .../Indexers/AddIndexerModalContent.js | 4 +- .../AddIndexerModalContentConnector.js | 2 +- frontend/src/Store/Actions/indexerActions.js | 33 +- .../DictionaryStringObjectConverter.cs | 105 +++ .../Converters/EmbeddedDocumentConverter.cs | 1 + .../Datastore/Migration/001_initial_setup.cs | 3 +- .../Definitions/Cardigann/Cardigann.cs | 108 +++ .../Definitions/Cardigann/CardigannBase.cs | 613 ++++++++++++++++++ .../Cardigann/CardigannDefinitionService.cs | 44 ++ .../Definitions/Cardigann/CardigannMetaDef.cs | 18 + .../Definitions/Cardigann/CardigannParser.cs | 493 ++++++++++++++ .../Cardigann/CardigannReleaseInfo.cs | 77 +++ .../Definitions/Cardigann/CardigannRequest.cs | 16 + .../Cardigann/CardigannRequestGenerator.cs | 216 ++++++ .../Cardigann/CardigannSettings.cs | 57 ++ .../Definitions/Cardigann/DateTimeUtil.cs | 325 ++++++++++ .../Cardigann/IndexerDefinition.cs | 176 +++++ .../Definitions/Cardigann/ParseUtil.cs | 102 +++ .../Definitions/Cardigann/StringUtil.cs | 229 +++++++ .../Cardigann/WebUtilityHelpers.cs | 30 + src/NzbDrone.Core/Indexers/IndexerBase.cs | 1 + .../Indexers/IndexerDefinition.cs | 4 + src/NzbDrone.Core/Indexers/IndexerFactory.cs | 58 +- src/NzbDrone.Core/Prowlarr.Core.csproj | 1 + .../ThingiProvider/ProviderFactory.cs | 8 +- .../ThingiProvider/ProviderRepository.cs | 1 + .../Indexers/IndexerResource.cs | 82 +++ src/Prowlarr.Api.V1/ProviderModuleBase.cs | 4 +- 29 files changed, 2801 insertions(+), 15 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Converters/DictionaryStringObjectConverter.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinitionService.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannReleaseInfo.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequest.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/DateTimeUtil.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinition.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/ParseUtil.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/StringUtil.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Cardigann/WebUtilityHelpers.cs diff --git a/frontend/src/Settings/Indexers/Indexers/AddIndexerItem.js b/frontend/src/Settings/Indexers/Indexers/AddIndexerItem.js index d0b02e45b..48efe2b6b 100644 --- a/frontend/src/Settings/Indexers/Indexers/AddIndexerItem.js +++ b/frontend/src/Settings/Indexers/Indexers/AddIndexerItem.js @@ -16,10 +16,11 @@ class AddIndexerItem extends Component { onIndexerSelect = () => { const { - implementation + implementation, + name } = this.props; - this.props.onIndexerSelect({ implementation }); + this.props.onIndexerSelect({ implementation, name }); } // diff --git a/frontend/src/Settings/Indexers/Indexers/AddIndexerModalContent.js b/frontend/src/Settings/Indexers/Indexers/AddIndexerModalContent.js index 6447246d4..6b6702912 100644 --- a/frontend/src/Settings/Indexers/Indexers/AddIndexerModalContent.js +++ b/frontend/src/Settings/Indexers/Indexers/AddIndexerModalContent.js @@ -67,7 +67,7 @@ class AddIndexerModalContent extends Component { usenetIndexers.map((indexer) => { return ( { return ( { - this.props.selectIndexerSchema({ implementation, presetName: name }); + this.props.selectIndexerSchema({ implementation, name }); this.props.onModalClose({ indexerSelected: true }); } diff --git a/frontend/src/Store/Actions/indexerActions.js b/frontend/src/Store/Actions/indexerActions.js index 86a720c90..17f1c4a36 100644 --- a/frontend/src/Store/Actions/indexerActions.js +++ b/frontend/src/Store/Actions/indexerActions.js @@ -1,3 +1,4 @@ +import _ from 'lodash'; import { createAction } from 'redux-actions'; import createFetchHandler from 'Store/Actions/Creators/createFetchHandler'; import createFetchSchemaHandler from 'Store/Actions/Creators/createFetchSchemaHandler'; @@ -9,7 +10,6 @@ import createSetProviderFieldValueReducer from 'Store/Actions/Creators/Reducers/ import createSetSettingValueReducer from 'Store/Actions/Creators/Reducers/createSetSettingValueReducer'; import { createThunk, handleThunks } from 'Store/thunks'; import getSectionState from 'Utilities/State/getSectionState'; -import selectProviderSchema from 'Utilities/State/selectProviderSchema'; import updateSectionState from 'Utilities/State/updateSectionState'; import createHandleActions from './Creators/createHandleActions'; @@ -86,6 +86,35 @@ export const setIndexerFieldValue = createAction(SET_INDEXER_FIELD_VALUE, (paylo // // Action Handlers +function applySchemaDefaults(selectedSchema, schemaDefaults) { + if (!schemaDefaults) { + return selectedSchema; + } else if (_.isFunction(schemaDefaults)) { + return schemaDefaults(selectedSchema); + } + + return Object.assign(selectedSchema, schemaDefaults); +} + +function selectSchema(state, payload, schemaDefaults) { + const newState = getSectionState(state, section); + + console.log(payload); + + const { + implementation, + name + } = payload; + + const selectedImplementation = _.find(newState.schema, { implementation, name }); + + console.log(selectedImplementation); + + newState.selectedSchema = applySchemaDefaults(_.cloneDeep(selectedImplementation), schemaDefaults); + + return updateSectionState(state, section, newState); +} + export const actionHandlers = handleThunks({ [FETCH_INDEXERS]: createFetchHandler(section, '/indexer'), [FETCH_INDEXER_SCHEMA]: createFetchSchemaHandler(section, '/indexer/schema'), @@ -106,7 +135,7 @@ export const reducers = createHandleActions({ [SET_INDEXER_FIELD_VALUE]: createSetProviderFieldValueReducer(section), [SELECT_INDEXER_SCHEMA]: (state, { payload }) => { - return selectProviderSchema(state, section, payload, (selectedSchema) => { + return selectSchema(state, payload, (selectedSchema) => { selectedSchema.enableRss = selectedSchema.supportsRss; selectedSchema.enableAutomaticSearch = selectedSchema.supportsSearch; selectedSchema.enableInteractiveSearch = selectedSchema.supportsSearch; diff --git a/src/NzbDrone.Core/Datastore/Converters/DictionaryStringObjectConverter.cs b/src/NzbDrone.Core/Datastore/Converters/DictionaryStringObjectConverter.cs new file mode 100644 index 000000000..936da358d --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Converters/DictionaryStringObjectConverter.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace NzbDrone.Core.Datastore.Converters +{ + public class DictionaryStringObjectJsonConverter : JsonConverter> + { + public override Dictionary Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.StartObject) + { + throw new JsonException($"JsonTokenType was of type {reader.TokenType}, only objects are supported"); + } + + var dictionary = new Dictionary(); + while (reader.Read()) + { + if (reader.TokenType == JsonTokenType.EndObject) + { + return dictionary; + } + + if (reader.TokenType != JsonTokenType.PropertyName) + { + throw new JsonException("JsonTokenType was not PropertyName"); + } + + var propertyName = reader.GetString(); + + if (string.IsNullOrWhiteSpace(propertyName)) + { + throw new JsonException("Failed to get property name"); + } + + reader.Read(); + + dictionary.Add(propertyName, ExtractValue(ref reader, options)); + } + + return dictionary; + } + + public override void Write(Utf8JsonWriter writer, Dictionary dictionary, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + foreach (var kvp in dictionary) + { + writer.WritePropertyName(kvp.Key.ToString()); + + JsonSerializer.Serialize(writer, kvp.Value, options); + } + + writer.WriteEndObject(); + } + + private object ExtractValue(ref Utf8JsonReader reader, JsonSerializerOptions options) + { + switch (reader.TokenType) + { + case JsonTokenType.String: + if (reader.TryGetDateTime(out var date)) + { + return date; + } + + return reader.GetString(); + + case JsonTokenType.False: + return false; + + case JsonTokenType.True: + return true; + + case JsonTokenType.Null: + return null; + + case JsonTokenType.Number: + if (reader.TryGetInt64(out var result)) + { + return result; + } + + return reader.GetDecimal(); + + case JsonTokenType.StartObject: + return Read(ref reader, null, options); + + case JsonTokenType.StartArray: + var list = new List(); + while (reader.Read() && reader.TokenType != JsonTokenType.EndArray) + { + list.Add(ExtractValue(ref reader, options)); + } + + return list; + + default: + throw new JsonException($"'{reader.TokenType}' is not supported"); + } + } + } +} diff --git a/src/NzbDrone.Core/Datastore/Converters/EmbeddedDocumentConverter.cs b/src/NzbDrone.Core/Datastore/Converters/EmbeddedDocumentConverter.cs index 6b6744110..d4073e1d1 100644 --- a/src/NzbDrone.Core/Datastore/Converters/EmbeddedDocumentConverter.cs +++ b/src/NzbDrone.Core/Datastore/Converters/EmbeddedDocumentConverter.cs @@ -24,6 +24,7 @@ namespace NzbDrone.Core.Datastore.Converters serializerSettings.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, true)); serializerSettings.Converters.Add(new TimeSpanConverter()); serializerSettings.Converters.Add(new UtcConverter()); + serializerSettings.Converters.Add(new DictionaryStringObjectJsonConverter()); SerializerSettings = serializerSettings; } diff --git a/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs b/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs index 3164dad6c..3c83367b0 100644 --- a/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs +++ b/src/NzbDrone.Core/Datastore/Migration/001_initial_setup.cs @@ -86,7 +86,8 @@ namespace NzbDrone.Core.Datastore.Migration .WithColumn("MostRecentFailure").AsDateTime().Nullable() .WithColumn("EscalationLevel").AsInt32().NotNullable() .WithColumn("DisabledTill").AsDateTime().Nullable() - .WithColumn("LastRssSyncReleaseInfo").AsString().Nullable(); + .WithColumn("LastRssSyncReleaseInfo").AsString().Nullable() + .WithColumn("Cookies").AsString().WithDefaultValue("{}"); Create.TableForModel("CustomFilters") .WithColumn("Type").AsString().NotNullable() diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs new file mode 100644 index 000000000..31c2b33d4 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -0,0 +1,108 @@ +using System.Collections.Generic; +using System.Linq; +using FluentValidation.Results; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public class Cardigann : HttpIndexerBase + { + private readonly ICardigannDefinitionService _definitionService; + + public override string Name => "Cardigann"; + + public override DownloadProtocol Protocol => DownloadProtocol.Torrent; + public override IndexerPrivacy Privacy => IndexerPrivacy.Private; + public override int PageSize => 100; + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new CardigannRequestGenerator(_definitionService.GetDefinition(Settings.DefinitionFile), + Settings, + _logger); + } + + public override IParseIndexerResponse GetParser() + { + return new CardigannParser(_definitionService.GetDefinition(Settings.DefinitionFile), + Settings, + _logger); + } + + public override IndexerCapabilities GetCapabilities() + { + // TODO: This uses indexer capabilities when called so we don't have to keep up with all of them + // however, this is not pulled on a all pull from UI, doing so will kill the UI load if an indexer is down + // should we just purge and manage + return new IndexerCapabilities(); + } + + public override IEnumerable DefaultDefinitions + { + get + { + foreach (var def in _definitionService.All()) + { + yield return GetDefinition(def); + } + } + } + + public Cardigann(ICardigannDefinitionService definitionService, + IHttpClient httpClient, + IIndexerStatusService indexerStatusService, + IConfigService configService, + Logger logger) + : base(httpClient, indexerStatusService, configService, logger) + { + _definitionService = definitionService; + } + + private IndexerDefinition GetDefinition(CardigannMetaDefinition definition) + { + return new IndexerDefinition + { + EnableRss = false, + EnableAutomaticSearch = false, + EnableInteractiveSearch = false, + Name = definition.Name, + Implementation = GetType().Name, + Settings = new CardigannSettings { DefinitionFile = definition.File }, + Protocol = DownloadProtocol.Torrent, + Privacy = definition.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public, + SupportsRss = SupportsRss, + SupportsSearch = SupportsSearch, + Capabilities = Capabilities, + ExtraFields = definition.Settings + }; + } + + protected override void Test(List failures) + { + base.Test(failures); + if (failures.HasErrors()) + { + return; + } + } + + protected static List CategoryIds(List categories) + { + var l = categories.Select(c => c.Id).ToList(); + + foreach (var category in categories) + { + if (category.SubCategories != null) + { + l.AddRange(CategoryIds(category.SubCategories)); + } + } + + return l; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs new file mode 100644 index 000000000..f0600660a --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs @@ -0,0 +1,613 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using Newtonsoft.Json.Linq; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Serializer; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public class CardigannBase + { + protected readonly CardigannDefinition _definition; + protected readonly CardigannSettings _settings; + protected readonly Logger _logger; + protected readonly Encoding _encoding; + + protected string SiteLink { get; private set; } + + /* protected readonly List categoryMapping = new List(); */ + + protected readonly string[] OptionalFields = new string[] { "imdb", "rageid", "tvdbid", "banner" }; + + protected static readonly string[] _SupportedLogicFunctions = + { + "and", + "or", + "eq", + "ne" + }; + + protected static readonly string[] _LogicFunctionsUsingStringLiterals = + { + "eq", + "ne" + }; + + // Matches a logic function above and 2 or more of (.varname) or .varname or "string literal" in any combination + protected static readonly Regex _LogicFunctionRegex = new Regex( + $@"\b({string.Join("|", _SupportedLogicFunctions.Select(Regex.Escape))})(?:\s+(\(?\.[^\)\s]+\)?|""[^""]+"")){{2,}}"); + + public CardigannBase(CardigannDefinition definition, + CardigannSettings settings, + Logger logger) + { + _definition = definition; + _settings = settings; + _encoding = Encoding.GetEncoding(definition.Encoding); + _logger = logger; + + SiteLink = definition.Links.First(); + } + + protected IElement QuerySelector(IElement element, string selector) + { + // AngleSharp doesn't support the :root pseudo selector, so we check for it manually + if (selector.StartsWith(":root")) + { + selector = selector.Substring(5); + while (element.ParentElement != null) + { + element = element.ParentElement; + } + } + + return element.QuerySelector(selector); + } + + protected string HandleSelector(SelectorBlock selector, IElement dom, Dictionary variables = null) + { + if (selector.Text != null) + { + return ApplyFilters(ApplyGoTemplateText(selector.Text, variables), selector.Filters, variables); + } + + var selection = dom; + string value = null; + + if (selector.Selector != null) + { + if (dom.Matches(selector.Selector)) + { + selection = dom; + } + else + { + selection = QuerySelector(dom, selector.Selector); + } + + if (selection == null) + { + throw new Exception(string.Format("Selector \"{0}\" didn't match {1}", selector.Selector, dom.ToHtmlPretty())); + } + } + + if (selector.Remove != null) + { + foreach (var i in selection.QuerySelectorAll(selector.Remove)) + { + i.Remove(); + } + } + + if (selector.Case != null) + { + foreach (var @case in selector.Case) + { + if (selection.Matches(@case.Key) || QuerySelector(selection, @case.Key) != null) + { + value = @case.Value; + break; + } + } + + if (value == null) + { + throw new Exception(string.Format("None of the case selectors \"{0}\" matched {1}", string.Join(",", selector.Case), selection.ToHtmlPretty())); + } + } + else if (selector.Attribute != null) + { + value = selection.GetAttribute(selector.Attribute); + if (value == null) + { + throw new Exception(string.Format("Attribute \"{0}\" is not set for element {1}", selector.Attribute, selection.ToHtmlPretty())); + } + } + else + { + value = selection.TextContent; + } + + return ApplyFilters(ParseUtil.NormalizeSpace(value), selector.Filters, variables); + } + + protected Dictionary GetBaseTemplateVariables() + { + var variables = new Dictionary + { + [".Config.sitelink"] = SiteLink, + [".True"] = "True", + [".False"] = null, + [".Today.Year"] = DateTime.Today.Year.ToString() + }; + + _logger.Debug("Populating config vars"); + + foreach (var setting in _definition.Settings) + { + var name = ".Config." + setting.Name; + var value = _settings.ExtraFieldData.GetValueOrDefault(setting.Name, setting.Default); + + _logger.Debug($"{name} got value {value.ToJson()}"); + + if (setting.Type == "text") + { + variables[name] = value; + } + else if (setting.Type == "checkbox") + { + variables[name] = ((bool)value) ? ".True" : ".False"; + } + else if (setting.Type == "select") + { + _logger.Debug($"setting options: {setting.Options.ToJson()}"); + var sorted = setting.Options.OrderBy(x => x.Key).ToList(); + var selected = sorted[(int)(long)value]; + + _logger.Debug($"selected option: {selected.ToJson()}"); + + variables[name] = selected.Value; + } + else + { + throw new NotSupportedException(); + } + + _logger.Debug($"Setting {setting.Name} to {variables[name]}"); + } + + return variables; + } + + /* + protected ICollection MapTrackerCatToNewznab(string input) + { + if (input == null) + return new List(); + + var cats = categoryMapping.Where(m => m.TrackerCategory != null && m.TrackerCategory.ToLowerInvariant() == input.ToLowerInvariant()).Select(c => c.NewzNabCategory).ToList(); + + // 1:1 category mapping + try + { + var trackerCategoryInt = int.Parse(input); + cats.Add(trackerCategoryInt + 100000); + } + catch (FormatException) + { + // input is not an integer, continue + } + + return cats; + }*/ + + protected delegate string TemplateTextModifier(string str); + + protected string ApplyGoTemplateText(string template, Dictionary variables = null, TemplateTextModifier modifier = null) + { + if (variables == null) + { + variables = GetBaseTemplateVariables(); + } + + // handle re_replace expression + // Example: {{ re_replace .Query.Keywords "[^a-zA-Z0-9]+" "%" }} + var reReplaceRegex = new Regex(@"{{\s*re_replace\s+(\..+?)\s+""(.*?)""\s+""(.*?)""\s*}}"); + var reReplaceRegexMatches = reReplaceRegex.Match(template); + + while (reReplaceRegexMatches.Success) + { + var all = reReplaceRegexMatches.Groups[0].Value; + var variable = reReplaceRegexMatches.Groups[1].Value; + var regexp = reReplaceRegexMatches.Groups[2].Value; + var newvalue = reReplaceRegexMatches.Groups[3].Value; + + var replaceRegex = new Regex(regexp); + var input = (string)variables[variable]; + var expanded = replaceRegex.Replace(input, newvalue); + + if (modifier != null) + { + expanded = modifier(expanded); + } + + template = template.Replace(all, expanded); + reReplaceRegexMatches = reReplaceRegexMatches.NextMatch(); + } + + // handle join expression + // Example: {{ join .Categories "," }} + var joinRegex = new Regex(@"{{\s*join\s+(\..+?)\s+""(.*?)""\s*}}"); + var joinMatches = joinRegex.Match(template); + + while (joinMatches.Success) + { + var all = joinMatches.Groups[0].Value; + var variable = joinMatches.Groups[1].Value; + var delimiter = joinMatches.Groups[2].Value; + + var input = (ICollection)variables[variable]; + var expanded = string.Join(delimiter, input); + + if (modifier != null) + { + expanded = modifier(expanded); + } + + template = template.Replace(all, expanded); + joinMatches = joinMatches.NextMatch(); + } + + var logicMatch = _LogicFunctionRegex.Match(template); + + while (logicMatch.Success) + { + var functionStartIndex = logicMatch.Groups[0].Index; + var functionLength = logicMatch.Groups[0].Length; + var functionName = logicMatch.Groups[1].Value; + + // Use Group.Captures to get each matching string in a repeating Match.Group + // Strip () around variable names here, as they are optional. Use quotes to differentiate variables and literals + var parameters = logicMatch.Groups[2].Captures.Cast().Select(c => c.Value.Trim('(', ')')).ToList(); + var functionResult = ""; + + // If the function can't use string literals, fail silently by removing the literals. + if (!_LogicFunctionsUsingStringLiterals.Contains(functionName)) + { + parameters.RemoveAll(param => param.StartsWith("\"")); + } + + switch (functionName) + { + case "and": // returns first null or empty, else last variable + case "or": // returns first not null or empty, else last variable + var isAnd = functionName == "and"; + foreach (var parameter in parameters) + { + functionResult = parameter; + + // (null as string) == null + // (if null or empty) break if and, continue if or + // (if neither null nor empty) continue if and, break if or + if (string.IsNullOrWhiteSpace(variables[parameter] as string) == isAnd) + { + break; + } + } + + break; + case "eq": // Returns .True if equal + case "ne": // Returns .False if equal + { + var wantEqual = functionName == "eq"; + + // eq/ne take exactly 2 params. Update the length to match + // This removes the whitespace between params 2 and 3. + // It shouldn't matter because the match starts at a word boundary + if (parameters.Count > 2) + { + functionLength = logicMatch.Groups[2].Captures[2].Index - functionStartIndex; + } + + // Take first two parameters, convert vars to values and strip quotes on string literals + // Counting distinct gives us 1 if equal and 2 if not. + var isEqual = + parameters.Take(2).Select(param => param.StartsWith("\"") ? param.Trim('"') : variables[param] as string) + .Distinct().Count() == 1; + + functionResult = isEqual == wantEqual ? ".True" : ".False"; + break; + } + } + + template = template.Remove(functionStartIndex, functionLength) + .Insert(functionStartIndex, functionResult); + + // Rerunning match instead of using nextMatch allows us to support nested functions + // like {{if and eq (.Var1) "string1" eq (.Var2) "string2"}} + // No performance is lost because Match/NextMatch are lazy evaluated and pause execution after first match + logicMatch = _LogicFunctionRegex.Match(template); + } + + // handle if ... else ... expression + var ifElseRegex = new Regex(@"{{\s*if\s*(.+?)\s*}}(.*?){{\s*else\s*}}(.*?){{\s*end\s*}}"); + var ifElseRegexMatches = ifElseRegex.Match(template); + + while (ifElseRegexMatches.Success) + { + string conditionResult = null; + + var all = ifElseRegexMatches.Groups[0].Value; + var condition = ifElseRegexMatches.Groups[1].Value; + var onTrue = ifElseRegexMatches.Groups[2].Value; + var onFalse = ifElseRegexMatches.Groups[3].Value; + + if (condition.StartsWith(".")) + { + var conditionResultState = false; + var value = variables[condition]; + + if (value == null) + { + conditionResultState = false; + } + else if (value is string) + { + conditionResultState = !string.IsNullOrWhiteSpace((string)value); + } + else if (value is ICollection) + { + conditionResultState = ((ICollection)value).Count > 0; + } + else + { + throw new Exception(string.Format("Unexpceted type for variable {0}: {1}", condition, value.GetType())); + } + + if (conditionResultState) + { + conditionResult = onTrue; + } + else + { + conditionResult = onFalse; + } + } + else + { + throw new NotImplementedException("CardigannIndexer: Condition operation '" + condition + "' not implemented"); + } + + template = template.Replace(all, conditionResult); + ifElseRegexMatches = ifElseRegexMatches.NextMatch(); + } + + // handle range expression + var rangeRegex = new Regex(@"{{\s*range\s*(.+?)\s*}}(.*?){{\.}}(.*?){{end}}"); + var rangeRegexMatches = rangeRegex.Match(template); + + while (rangeRegexMatches.Success) + { + var expanded = string.Empty; + + var all = rangeRegexMatches.Groups[0].Value; + var variable = rangeRegexMatches.Groups[1].Value; + var prefix = rangeRegexMatches.Groups[2].Value; + var postfix = rangeRegexMatches.Groups[3].Value; + + foreach (var value in (ICollection)variables[variable]) + { + var newvalue = value; + if (modifier != null) + { + newvalue = modifier(newvalue); + } + + expanded += prefix + newvalue + postfix; + } + + template = template.Replace(all, expanded); + rangeRegexMatches = rangeRegexMatches.NextMatch(); + } + + // handle simple variables + var variablesRegEx = new Regex(@"{{\s*(\..+?)\s*}}"); + var variablesRegExMatches = variablesRegEx.Match(template); + + while (variablesRegExMatches.Success) + { + var expanded = string.Empty; + + var all = variablesRegExMatches.Groups[0].Value; + var variable = variablesRegExMatches.Groups[1].Value; + + var value = (string)variables[variable]; + if (modifier != null) + { + value = modifier(value); + } + + template = template.Replace(all, value); + variablesRegExMatches = variablesRegExMatches.NextMatch(); + } + + return template; + } + + protected string ApplyFilters(string data, List filters, Dictionary variables = null) + { + if (filters == null) + { + return data; + } + + foreach (var filter in filters) + { + switch (filter.Name) + { + case "querystring": + var param = (string)filter.Args; + + // data = ParseUtil.GetArgumentFromQueryString(data, param); + break; + case "timeparse": + case "dateparse": + var layout = (string)filter.Args; + try + { + var date = DateTimeUtil.ParseDateTimeGoLang(data, layout); + data = date.ToString(DateTimeUtil.Rfc1123ZPattern); + } + catch (FormatException ex) + { + _logger.Debug(ex.Message); + } + + break; + case "regexp": + var pattern = (string)filter.Args; + var regexp = new Regex(pattern); + var match = regexp.Match(data); + data = match.Groups[1].Value; + break; + case "re_replace": + var regexpreplace_pattern = (string)filter.Args[0]; + var regexpreplace_replacement = (string)filter.Args[1]; + regexpreplace_replacement = ApplyGoTemplateText(regexpreplace_replacement, variables); + var regexpreplace_regex = new Regex(regexpreplace_pattern); + data = regexpreplace_regex.Replace(data, regexpreplace_replacement); + break; + case "split": + var sep = (string)filter.Args[0]; + var pos = (string)filter.Args[1]; + var posInt = int.Parse(pos); + var strParts = data.Split(sep[0]); + if (posInt < 0) + { + posInt += strParts.Length; + } + + data = strParts[posInt]; + break; + case "replace": + var from = (string)filter.Args[0]; + var to = (string)filter.Args[1]; + to = ApplyGoTemplateText(to, variables); + data = data.Replace(from, to); + break; + case "trim": + var cutset = (string)filter.Args; + if (cutset != null) + { + data = data.Trim(cutset[0]); + } + else + { + data = data.Trim(); + } + + break; + case "prepend": + var prependstr = (string)filter.Args; + data = ApplyGoTemplateText(prependstr, variables) + data; + break; + case "append": + var str = (string)filter.Args; + data += ApplyGoTemplateText(str, variables); + break; + case "tolower": + data = data.ToLower(); + break; + case "toupper": + data = data.ToUpper(); + break; + case "urldecode": + data = WebUtilityHelpers.UrlDecode(data, _encoding); + break; + case "urlencode": + data = WebUtilityHelpers.UrlEncode(data, _encoding); + break; + case "timeago": + case "reltime": + data = DateTimeUtil.FromTimeAgo(data).ToString(DateTimeUtil.Rfc1123ZPattern); + break; + case "fuzzytime": + data = DateTimeUtil.FromUnknown(data).ToString(DateTimeUtil.Rfc1123ZPattern); + break; + case "validfilename": + data = StringUtil.MakeValidFileName(data, '_', false); + break; + case "diacritics": + var diacriticsOp = (string)filter.Args; + if (diacriticsOp == "replace") + { + // Should replace diacritics charcaters with their base character + // It's not perfect, e.g. "ŠĐĆŽ - šđčćž" becomes "SĐCZ-sđccz" + var stFormD = data.Normalize(NormalizationForm.FormD); + var len = stFormD.Length; + var sb = new StringBuilder(); + for (var i = 0; i < len; i++) + { + var uc = System.Globalization.CharUnicodeInfo.GetUnicodeCategory(stFormD[i]); + if (uc != System.Globalization.UnicodeCategory.NonSpacingMark) + { + sb.Append(stFormD[i]); + } + } + + data = sb.ToString().Normalize(NormalizationForm.FormC); + } + else + { + throw new Exception("unsupported diacritics filter argument"); + } + + break; + case "jsonjoinarray": + var jsonjoinarrayJSONPath = (string)filter.Args[0]; + var jsonjoinarraySeparator = (string)filter.Args[1]; + var jsonjoinarrayO = JObject.Parse(data); + var jsonjoinarrayOResult = jsonjoinarrayO.SelectToken(jsonjoinarrayJSONPath); + var jsonjoinarrayOResultStrings = jsonjoinarrayOResult.Select(j => j.ToString()); + data = string.Join(jsonjoinarraySeparator, jsonjoinarrayOResultStrings); + break; + case "hexdump": + // this is mainly for debugging invisible special char related issues + var hexData = string.Join("", data.Select(c => c + "(" + ((int)c).ToString("X2") + ")")); + _logger.Debug(string.Format("CardigannIndexer ({0}): strdump: {1}", _definition.Id, hexData)); + break; + case "strdump": + // for debugging + var debugData = data.Replace("\r", "\\r").Replace("\n", "\\n").Replace("\xA0", "\\xA0"); + var strTag = (string)filter.Args; + if (strTag != null) + { + strTag = string.Format("({0}):", strTag); + } + else + { + strTag = ":"; + } + + _logger.Debug(string.Format("CardigannIndexer ({0}): strdump{1} {2}", _definition.Id, strTag, debugData)); + break; + default: + break; + } + } + + return data; + } + + protected Uri ResolvePath(string path, Uri currentUrl = null) + { + return new Uri(currentUrl ?? new Uri(SiteLink), path); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinitionService.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinitionService.cs new file mode 100644 index 000000000..beb5bd202 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannDefinitionService.cs @@ -0,0 +1,44 @@ +using System.Collections.Generic; +using NzbDrone.Common.Http; +using YamlDotNet.Serialization; +using YamlDotNet.Serialization.NamingConventions; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public interface ICardigannDefinitionService + { + List All(); + CardigannDefinition GetDefinition(string id); + } + + public class CardigannDefinitionService : ICardigannDefinitionService + { + private const int DEFINITION_VERSION = 1; + + private readonly IHttpClient _httpClient; + + private readonly IDeserializer _deserializer = new DeserializerBuilder() + .IgnoreUnmatchedProperties() + .WithNamingConvention(CamelCaseNamingConvention.Instance) + .Build(); + + public CardigannDefinitionService(IHttpClient httpClient) + { + _httpClient = httpClient; + } + + public List All() + { + var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}"); + var response = _httpClient.Get>(request); + return response.Resource; + } + + public CardigannDefinition GetDefinition(string id) + { + var req = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{id}"); + var response = _httpClient.Get(req); + return _deserializer.Deserialize(response.Content); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs new file mode 100644 index 000000000..a27e0c9dc --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public class CardigannMetaDefinition + { + public string Id { get; set; } + public string File { get; set; } + public string Name { get; set; } + public string Description { get; set; } + public string Type { get; set; } + public string Language { get; set; } + public string Encoding { get; set; } + public List Links { get; set; } + public List Legacylinks { get; set; } + public List Settings { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs new file mode 100644 index 000000000..fd023c0d2 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannParser.cs @@ -0,0 +1,493 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Text.RegularExpressions; +using AngleSharp.Dom; +using AngleSharp.Html.Parser; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public class CardigannParser : CardigannBase, IParseIndexerResponse + { + public Action, DateTime?> CookiesUpdater { get; set; } + + public CardigannParser(CardigannDefinition definition, + CardigannSettings settings, + Logger logger) + : base(definition, settings, logger) + { + } + + public IList ParseResponse(IndexerResponse indexerResponse) + { + var releases = new List(); + + _logger.Debug("Parsing"); + + if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + // Remove cookie cache + if (indexerResponse.HttpResponse.HasHttpRedirect && indexerResponse.HttpResponse.Headers["Location"] + .ContainsIgnoreCase("login.php")) + { + CookiesUpdater(null, null); + throw new IndexerException(indexerResponse, "We are being redirected to the PTP login page. Most likely your session expired or was killed. Try testing the indexer in the settings."); + } + + throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); + } + + var results = indexerResponse.Content; + var request = indexerResponse.Request as CardigannRequest; + var variables = request.Variables; + var search = _definition.Search; + + var searchUrlUri = new Uri(request.Url.FullUri); + + try + { + var searchResultParser = new HtmlParser(); + var searchResultDocument = searchResultParser.ParseDocument(results); + + /* checkForError(response, Definition.Search.Error); */ + + if (search.Preprocessingfilters != null) + { + results = ApplyFilters(results, search.Preprocessingfilters, variables); + searchResultDocument = searchResultParser.ParseDocument(results); + _logger.Debug(string.Format("CardigannIndexer ({0}): result after preprocessingfilters: {1}", _definition.Id, results)); + } + + var rowsSelector = ApplyGoTemplateText(search.Rows.Selector, variables); + var rowsDom = searchResultDocument.QuerySelectorAll(rowsSelector); + var rows = new List(); + foreach (var rowDom in rowsDom) + { + rows.Add(rowDom); + } + + // merge following rows for After selector + var after = search.Rows.After; + if (after > 0) + { + for (var i = 0; i < rows.Count; i += 1) + { + var currentRow = rows[i]; + for (var j = 0; j < after; j += 1) + { + var mergeRowIndex = i + j + 1; + var mergeRow = rows[mergeRowIndex]; + var mergeNodes = new List(); + foreach (var node in mergeRow.ChildNodes) + { + mergeNodes.Add(node); + } + + currentRow.Append(mergeNodes.ToArray()); + } + + rows.RemoveRange(i + 1, after); + } + } + + foreach (var row in rows) + { + try + { + var release = new CardigannReleaseInfo(); + + // Parse fields + foreach (var field in search.Fields) + { + var fieldParts = field.Key.Split('|'); + var fieldName = fieldParts[0]; + var fieldModifiers = new List(); + for (var i = 1; i < fieldParts.Length; i++) + { + fieldModifiers.Add(fieldParts[i]); + } + + string value = null; + var variablesKey = ".Result." + fieldName; + try + { + value = HandleSelector(field.Value, row, variables); + switch (fieldName) + { + case "download": + if (string.IsNullOrEmpty(value)) + { + value = null; + release.Link = null; + break; + } + + if (value.StartsWith("magnet:")) + { + release.MagnetUri = new Uri(value); + value = release.MagnetUri.ToString(); + } + else + { + release.Link = ResolvePath(value, searchUrlUri); + value = release.Link.ToString(); + } + + break; + case "magnet": + var magnetUri = new Uri(value); + release.MagnetUri = magnetUri; + value = magnetUri.ToString(); + if (release.Guid == null) + { + release.Guid = magnetUri; + } + + break; + case "details": + var url = ResolvePath(value, searchUrlUri); + release.Guid = url; + release.Comments = url; + if (release.Guid == null) + { + release.Guid = url; + } + + value = url.ToString(); + break; + case "comments": + var commentsUrl = ResolvePath(value, searchUrlUri); + if (release.Comments == null) + { + release.Comments = commentsUrl; + } + + if (release.Guid == null) + { + release.Guid = commentsUrl; + } + + value = commentsUrl.ToString(); + break; + case "title": + if (fieldModifiers.Contains("append")) + { + release.Title += value; + } + else + { + release.Title = value; + } + + value = release.Title; + break; + case "description": + if (fieldModifiers.Contains("append")) + { + release.Description += value; + } + else + { + release.Description = value; + } + + value = release.Description; + break; + case "category": + // var cats = MapTrackerCatToNewznab(value); + var cats = new List { 2000 }; + if (cats.Any()) + { + if (release.Category == null || fieldModifiers.Contains("noappend")) + { + release.Category = cats; + } + else + { + release.Category = release.Category.Union(cats).ToList(); + } + } + + value = release.Category.ToString(); + break; + case "size": + release.Size = CardigannReleaseInfo.GetBytes(value); + value = release.Size.ToString(); + break; + case "leechers": + var leechers = ParseUtil.CoerceLong(value); + leechers = leechers < 5000000L ? leechers : 0; // to fix #6558 + if (release.Peers == null) + { + release.Peers = leechers; + } + else + { + release.Peers += leechers; + } + + value = leechers.ToString(); + break; + case "seeders": + release.Seeders = ParseUtil.CoerceLong(value); + release.Seeders = release.Seeders < 5000000L ? release.Seeders : 0; // to fix #6558 + if (release.Peers == null) + { + release.Peers = release.Seeders; + } + else + { + release.Peers += release.Seeders; + } + + value = release.Seeders.ToString(); + break; + case "date": + release.PublishDate = DateTimeUtil.FromUnknown(value); + value = release.PublishDate.ToString(DateTimeUtil.Rfc1123ZPattern); + break; + case "files": + release.Files = ParseUtil.CoerceLong(value); + value = release.Files.ToString(); + break; + case "grabs": + release.Grabs = ParseUtil.CoerceLong(value); + value = release.Grabs.ToString(); + break; + case "downloadvolumefactor": + release.DownloadVolumeFactor = ParseUtil.CoerceDouble(value); + value = release.DownloadVolumeFactor.ToString(); + break; + case "uploadvolumefactor": + release.UploadVolumeFactor = ParseUtil.CoerceDouble(value); + value = release.UploadVolumeFactor.ToString(); + break; + case "minimumratio": + release.MinimumRatio = ParseUtil.CoerceDouble(value); + value = release.MinimumRatio.ToString(); + break; + case "minimumseedtime": + release.MinimumSeedTime = ParseUtil.CoerceLong(value); + value = release.MinimumSeedTime.ToString(); + break; + case "imdb": + release.Imdb = ParseUtil.GetLongFromString(value); + value = release.Imdb.ToString(); + break; + case "tmdbid": + var tmdbIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled); + var tmdbIDMatch = tmdbIDRegEx.Match(value); + var tmdbID = tmdbIDMatch.Groups[1].Value; + release.TMDb = ParseUtil.CoerceLong(tmdbID); + value = release.TMDb.ToString(); + break; + case "rageid": + var rageIDRegEx = new Regex(@"(\d+)", RegexOptions.Compiled); + var rageIDMatch = rageIDRegEx.Match(value); + var rageID = rageIDMatch.Groups[1].Value; + release.RageID = ParseUtil.CoerceLong(rageID); + value = release.RageID.ToString(); + break; + case "tvdbid": + var tvdbIdRegEx = new Regex(@"(\d+)", RegexOptions.Compiled); + var tvdbIdMatch = tvdbIdRegEx.Match(value); + var tvdbId = tvdbIdMatch.Groups[1].Value; + release.TVDBId = ParseUtil.CoerceLong(tvdbId); + value = release.TVDBId.ToString(); + break; + case "author": + release.Author = value; + break; + case "booktitle": + release.BookTitle = value; + break; + case "banner": + if (!string.IsNullOrWhiteSpace(value)) + { + var bannerurl = ResolvePath(value, searchUrlUri); + release.BannerUrl = bannerurl; + } + + value = release.BannerUrl.ToString(); + break; + default: + break; + } + + variables[variablesKey] = value; + } + catch (Exception ex) + { + if (!variables.ContainsKey(variablesKey)) + { + variables[variablesKey] = null; + } + + if (OptionalFields.Contains(field.Key) || fieldModifiers.Contains("optional") || field.Value.Optional) + { + variables[variablesKey] = null; + continue; + } + + _logger.Error("Error while parsing field={0}, selector={1}, value={2}: {3}", field.Key, field.Value.Selector, value == null ? "" : value, ex.Message); + throw; + } + } + + var filters = search.Rows.Filters; + var skipRelease = false; + if (filters != null) + { + foreach (var filter in filters) + { + switch (filter.Name) + { + case "andmatch": + var characterLimit = -1; + if (filter.Args != null) + { + characterLimit = int.Parse(filter.Args); + } + + /* + if (query.ImdbID != null && TorznabCaps.SupportsImdbMovieSearch) + { + break; // skip andmatch filter for imdb searches + } + + if (query.TmdbID != null && TorznabCaps.SupportsTmdbMovieSearch) + { + break; // skip andmatch filter for tmdb searches + } + + if (query.TvdbID != null && TorznabCaps.SupportsTvdbSearch) + { + break; // skip andmatch filter for tvdb searches + } + + var queryKeywords = variables[".Keywords"] as string; + + if (!query.MatchQueryStringAND(release.Title, characterLimit, queryKeywords)) + { + _logger.Debug(string.Format("CardigannIndexer ({0}): skipping {1} (andmatch filter)", _definition.Id, release.Title)); + skipRelease = true; + } + */ + + break; + case "strdump": + // for debugging + _logger.Debug(string.Format("CardigannIndexer ({0}): row strdump: {1}", _definition.Id, row.ToHtmlPretty())); + break; + default: + _logger.Error(string.Format("CardigannIndexer ({0}): Unsupported rows filter: {1}", _definition.Id, filter.Name)); + break; + } + } + } + + if (skipRelease) + { + continue; + } + + // if DateHeaders is set go through the previous rows and look for the header selector + var dateHeaders = _definition.Search.Rows.Dateheaders; + if (release.PublishDate == DateTime.MinValue && dateHeaders != null) + { + var prevRow = row.PreviousElementSibling; + string value = null; + if (prevRow == null) + { + // continue with parent + var parent = row.ParentElement; + if (parent != null) + { + prevRow = parent.PreviousElementSibling; + } + } + + while (prevRow != null) + { + var curRow = prevRow; + _logger.Debug(prevRow.OuterHtml); + try + { + value = HandleSelector(dateHeaders, curRow); + break; + } + catch (Exception) + { + // do nothing + } + + prevRow = curRow.PreviousElementSibling; + if (prevRow == null) + { + // continue with parent + var parent = curRow.ParentElement; + if (parent != null) + { + prevRow = parent.PreviousElementSibling; + } + } + } + + if (value == null && dateHeaders.Optional == false) + { + throw new Exception(string.Format("No date header row found for {0}", release.ToString())); + } + + if (value != null) + { + release.PublishDate = DateTimeUtil.FromUnknown(value); + } + } + + releases.Add(release); + } + catch (Exception ex) + { + _logger.Error(ex, "CardigannIndexer ({0}): Error while parsing row '{1}':\n\n{2}", _definition.Id, row.ToHtmlPretty()); + } + } + } + catch (Exception) + { + // OnParseError(results, ex); + throw; + } + + /* + if (query.Limit > 0) + { + releases = releases.Take(query.Limit).ToList(); + }*/ + + var result = new List(); + + result.AddRange(releases.Select(x => new TorrentInfo + { + Guid = x.Guid.ToString(), + Title = x.Title, + Size = x.Size.Value, + DownloadUrl = x.Link?.ToString(), + CommentUrl = x.Comments?.ToString(), + InfoUrl = x.Link?.ToString(), + MagnetUrl = x.MagnetUri?.ToString(), + InfoHash = x.InfoHash, + Seeders = (int?)x.Seeders, + Peers = (int?)x.Peers + })); + + _logger.Debug($"Got {result.Count} releases"); + + return result; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannReleaseInfo.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannReleaseInfo.cs new file mode 100644 index 000000000..e1cad172f --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannReleaseInfo.cs @@ -0,0 +1,77 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public class CardigannReleaseInfo + { + public string Title { get; set; } + public Uri Guid { get; set; } + public Uri Link { get; set; } + public Uri Comments { get; set; } + public DateTime PublishDate { get; set; } + public ICollection Category { get; set; } + public long? Size { get; set; } + public long? Files { get; set; } + public long? Grabs { get; set; } + public string Description { get; set; } + public long? RageID { get; set; } + public long? TVDBId { get; set; } + public long? Imdb { get; set; } + public long? TMDb { get; set; } + public string Author { get; set; } + public string BookTitle { get; set; } + public long? Seeders { get; set; } + public long? Peers { get; set; } + public Uri BannerUrl { get; set; } + public string InfoHash { get; set; } + public Uri MagnetUri { get; set; } + public double? MinimumRatio { get; set; } + public long? MinimumSeedTime { get; set; } + public double? DownloadVolumeFactor { get; set; } + public double? UploadVolumeFactor { get; set; } + + public static long GetBytes(string str) + { + var valStr = new string(str.Where(c => char.IsDigit(c) || c == '.').ToArray()); + var unit = new string(str.Where(char.IsLetter).ToArray()); + var val = ParseUtil.CoerceFloat(valStr); + return GetBytes(unit, val); + } + + public static long GetBytes(string unit, float value) + { + unit = unit.Replace("i", "").ToLowerInvariant(); + if (unit.Contains("kb")) + { + return BytesFromKB(value); + } + + if (unit.Contains("mb")) + { + return BytesFromMB(value); + } + + if (unit.Contains("gb")) + { + return BytesFromGB(value); + } + + if (unit.Contains("tb")) + { + return BytesFromTB(value); + } + + return (long)value; + } + + public static long BytesFromTB(float tb) => BytesFromGB(tb * 1024f); + + public static long BytesFromGB(float gb) => BytesFromMB(gb * 1024f); + + public static long BytesFromMB(float mb) => BytesFromKB(mb * 1024f); + + public static long BytesFromKB(float kb) => (long)(kb * 1024f); + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequest.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequest.cs new file mode 100644 index 000000000..77b07ffdc --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequest.cs @@ -0,0 +1,16 @@ +using System.Collections.Generic; +using NzbDrone.Common.Http; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public class CardigannRequest : IndexerRequest + { + public Dictionary Variables { get; private set; } + + public CardigannRequest(string url, HttpAccept httpAccept, Dictionary variables) + : base(url, httpAccept) + { + Variables = variables; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs new file mode 100644 index 000000000..8cff5dc80 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs @@ -0,0 +1,216 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.IndexerSearch.Definitions; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public class CardigannRequestGenerator : CardigannBase, IIndexerRequestGenerator + { + private List _defaultCategories = new List(); + + public CardigannRequestGenerator(CardigannDefinition definition, + CardigannSettings settings, + Logger logger) + : base(definition, settings, logger) + { + } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } + + public virtual IndexerPageableRequestChain GetRecentRequests() + { + _logger.Trace("Getting recent"); + + var pageableRequests = new IndexerPageableRequestChain(); + + pageableRequests.Add(GetRequest(null)); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) + { + _logger.Trace("Getting search"); + + var pageableRequests = new IndexerPageableRequestChain(); + + foreach (var queryTitle in searchCriteria.QueryTitles) + { + pageableRequests.Add(GetRequest(string.Format("{0}", queryTitle))); + } + + return pageableRequests; + } + + private IEnumerable GetRequest(string searchCriteria) + { + var search = _definition.Search; + + // init template context + var variables = GetBaseTemplateVariables(); + + variables[".Query.Type"] = null; + variables[".Query.Q"] = searchCriteria; + variables[".Query.Series"] = null; + variables[".Query.Ep"] = null; + variables[".Query.Season"] = null; + variables[".Query.Movie"] = null; + variables[".Query.Year"] = null; + variables[".Query.Limit"] = null; + variables[".Query.Offset"] = null; + variables[".Query.Extended"] = null; + variables[".Query.Categories"] = null; + variables[".Query.APIKey"] = null; + variables[".Query.TVDBID"] = null; + variables[".Query.TVRageID"] = null; + variables[".Query.IMDBID"] = null; + variables[".Query.IMDBIDShort"] = null; + variables[".Query.TMDBID"] = null; + variables[".Query.TVMazeID"] = null; + variables[".Query.TraktID"] = null; + variables[".Query.Album"] = null; + variables[".Query.Artist"] = null; + variables[".Query.Label"] = null; + variables[".Query.Track"] = null; + variables[".Query.Episode"] = null; + variables[".Query.Author"] = null; + variables[".Query.Title"] = null; + + /* + var mappedCategories = MapTorznabCapsToTrackers(query); + if (mappedCategories.Count == 0) + { + mappedCategories = _defaultCategories; + } + */ + + var mappedCategories = _defaultCategories; + + variables[".Categories"] = mappedCategories; + + var keywordTokens = new List(); + var keywordTokenKeys = new List { "Q", "Series", "Movie", "Year" }; + foreach (var key in keywordTokenKeys) + { + var value = (string)variables[".Query." + key]; + if (!string.IsNullOrWhiteSpace(value)) + { + keywordTokens.Add(value); + } + } + + if (!string.IsNullOrWhiteSpace((string)variables[".Query.Episode"])) + { + keywordTokens.Add((string)variables[".Query.Episode"]); + } + + variables[".Query.Keywords"] = string.Join(" ", keywordTokens); + variables[".Keywords"] = ApplyFilters((string)variables[".Query.Keywords"], search.Keywordsfilters); + + // TODO: prepare queries first and then send them parallel + var searchPaths = search.Paths; + foreach (var searchPath in searchPaths) + { + // skip path if categories don't match + if (searchPath.Categories != null && mappedCategories.Count > 0) + { + var invertMatch = searchPath.Categories[0] == "!"; + var hasIntersect = mappedCategories.Intersect(searchPath.Categories).Any(); + if (invertMatch) + { + hasIntersect = !hasIntersect; + } + + if (!hasIntersect) + { + continue; + } + } + + // build search URL + // HttpUtility.UrlPathEncode seems to only encode spaces, we use UrlEncode and replace + with %20 as a workaround + var searchUrl = ResolvePath(ApplyGoTemplateText(searchPath.Path, variables, WebUtility.UrlEncode).Replace("+", "%20")).AbsoluteUri; + var queryCollection = new List>(); + var method = HttpMethod.GET; + + if (string.Equals(searchPath.Method, "post", StringComparison.OrdinalIgnoreCase)) + { + method = HttpMethod.POST; + } + + var inputsList = new List>(); + if (searchPath.Inheritinputs) + { + inputsList.Add(search.Inputs); + } + + inputsList.Add(searchPath.Inputs); + + foreach (var inputs in inputsList) + { + if (inputs != null) + { + foreach (var input in inputs) + { + if (input.Key == "$raw") + { + var rawStr = ApplyGoTemplateText(input.Value, variables, WebUtility.UrlEncode); + foreach (var part in rawStr.Split('&')) + { + var parts = part.Split(new char[] { '=' }, 2); + var key = parts[0]; + if (key.Length == 0) + { + continue; + } + + var value = ""; + if (parts.Length == 2) + { + value = parts[1]; + } + + queryCollection.Add(key, value); + } + } + else + { + queryCollection.Add(input.Key, ApplyGoTemplateText(input.Value, variables)); + } + } + } + } + + if (method == HttpMethod.GET) + { + if (queryCollection.Count > 0) + { + searchUrl += "?" + queryCollection.GetQueryString(_encoding); + } + } + + _logger.Info($"Adding request: {searchUrl}"); + + var request = new CardigannRequest(searchUrl, HttpAccept.Html, variables); + + // send HTTP request + if (search.Headers != null) + { + foreach (var header in search.Headers) + { + request.HttpRequest.Headers.Add(header.Key, header.Value[0]); + } + } + + request.HttpRequest.Method = method; + + yield return request; + } + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs new file mode 100644 index 000000000..98250604c --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs @@ -0,0 +1,57 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using FluentValidation; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Languages; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public class CardigannSettingsValidator : AbstractValidator + { + public CardigannSettingsValidator() + { + RuleFor(c => c).Custom((c, context) => + { + if (c.Categories.Empty()) + { + context.AddFailure("'Categories' must be provided"); + } + }); + } + } + + public class CardigannSettings : IIndexerSettings + { + private static readonly CardigannSettingsValidator Validator = new CardigannSettingsValidator(); + + public CardigannSettings() + { + Categories = new[] { 2000, 2010, 2020, 2030, 2035, 2040, 2045, 2050, 2060 }; + MultiLanguages = new List(); + ExtraFieldData = new Dictionary(); + } + + [FieldDefinition(0, Hidden = HiddenType.Hidden)] + public string DefinitionFile { get; set; } + + public Dictionary ExtraFieldData { get; set; } + + public string BaseUrl { get; set; } + + [FieldDefinition(1000, Type = FieldType.Select, SelectOptions = typeof(LanguageFieldConverter), Label = "Multi Languages", HelpText = "What languages are normally in a multi release on this indexer?", Advanced = true)] + public IEnumerable MultiLanguages { get; set; } + + [FieldDefinition(1001, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable all categories", Advanced = true)] + public IEnumerable Categories { get; set; } + + // Field 8 is used by TorznabSettings MinimumSeeders + // If you need to add another field here, update TorznabSettings as well and this comment + public virtual NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/DateTimeUtil.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/DateTimeUtil.cs new file mode 100644 index 000000000..562375713 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/DateTimeUtil.cs @@ -0,0 +1,325 @@ +using System; +using System.Globalization; +using System.Text.RegularExpressions; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public static class DateTimeUtil + { + public const string Rfc1123ZPattern = "ddd, dd MMM yyyy HH':'mm':'ss z"; + + private static readonly Regex _TimeAgoRegexp = new Regex(@"(?i)\bago", RegexOptions.Compiled); + private static readonly Regex _TodayRegexp = new Regex(@"(?i)\btoday(?:[\s,]+(?:at){0,1}\s*|[\s,]*|$)", RegexOptions.Compiled); + private static readonly Regex _TomorrowRegexp = new Regex(@"(?i)\btomorrow(?:[\s,]+(?:at){0,1}\s*|[\s,]*|$)", RegexOptions.Compiled); + private static readonly Regex _YesterdayRegexp = new Regex(@"(?i)\byesterday(?:[\s,]+(?:at){0,1}\s*|[\s,]*|$)", RegexOptions.Compiled); + private static readonly Regex _DaysOfWeekRegexp = new Regex(@"(?i)\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday)\s+at\s+", RegexOptions.Compiled); + private static readonly Regex _MissingYearRegexp = new Regex(@"^(\d{1,2}-\d{1,2})(\s|$)", RegexOptions.Compiled); + private static readonly Regex _MissingYearRegexp2 = new Regex(@"^(\d{1,2}\s+\w{3})\s+(\d{1,2}\:\d{1,2}.*)$", RegexOptions.Compiled); // 1 Jan 10:30 + + public static DateTime UnixTimestampToDateTime(long unixTime) + { + var dt = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + dt = dt.AddSeconds(unixTime).ToLocalTime(); + return dt; + } + + public static DateTime UnixTimestampToDateTime(double unixTime) + { + var unixStart = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc); + var unixTimeStampInTicks = (long)(unixTime * TimeSpan.TicksPerSecond); + return new DateTime(unixStart.Ticks + unixTimeStampInTicks); + } + + public static double DateTimeToUnixTimestamp(DateTime dt) + { + var date = dt.ToUniversalTime(); + var ticks = date.Ticks - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc).Ticks; + var ts = ticks / TimeSpan.TicksPerSecond; + return ts; + } + + // ex: "2 hours 1 day" + public static DateTime FromTimeAgo(string str) + { + str = str.ToLowerInvariant(); + if (str.Contains("now")) + { + return DateTime.SpecifyKind(DateTime.Now, DateTimeKind.Local); + } + + str = str.Replace(",", ""); + str = str.Replace("ago", ""); + str = str.Replace("and", ""); + + var timeAgo = TimeSpan.Zero; + var timeagoRegex = new Regex(@"\s*?([\d\.]+)\s*?([^\d\s\.]+)\s*?"); + var timeagoMatches = timeagoRegex.Match(str); + + while (timeagoMatches.Success) + { + var val = ParseUtil.CoerceFloat(timeagoMatches.Groups[1].Value); + var unit = timeagoMatches.Groups[2].Value; + timeagoMatches = timeagoMatches.NextMatch(); + + if (unit.Contains("sec") || unit == "s") + { + timeAgo += TimeSpan.FromSeconds(val); + } + else if (unit.Contains("min") || unit == "m") + { + timeAgo += TimeSpan.FromMinutes(val); + } + else if (unit.Contains("hour") || unit.Contains("hr") || unit == "h") + { + timeAgo += TimeSpan.FromHours(val); + } + else if (unit.Contains("day") || unit == "d") + { + timeAgo += TimeSpan.FromDays(val); + } + else if (unit.Contains("week") || unit.Contains("wk") || unit == "w") + { + timeAgo += TimeSpan.FromDays(val * 7); + } + else if (unit.Contains("month") || unit == "mo") + { + timeAgo += TimeSpan.FromDays(val * 30); + } + else if (unit.Contains("year") || unit == "y") + { + timeAgo += TimeSpan.FromDays(val * 365); + } + else + { + throw new Exception("TimeAgo parsing failed, unknown unit: " + unit); + } + } + + return DateTime.SpecifyKind(DateTime.Now - timeAgo, DateTimeKind.Local); + } + + // Uses the DateTimeRoutines library to parse the date + // http://www.codeproject.com/Articles/33298/C-Date-Time-Parser + public static DateTime FromFuzzyTime(string str, string format = null) + { + /*var dtFormat = format == "UK" ? + DateTimeRoutines.DateTimeRoutines.DateTimeFormat.UkDate : + DateTimeRoutines.DateTimeRoutines.DateTimeFormat.UsaDate; + + if (DateTimeRoutines.DateTimeRoutines.TryParseDateOrTime( + str, dtFormat, out DateTimeRoutines.DateTimeRoutines.ParsedDateTime dt)) + return dt.DateTime;*/ + + throw new Exception("FromFuzzyTime parsing failed"); + } + + public static DateTime FromUnknown(string str, string format = null) + { + try + { + str = ParseUtil.NormalizeSpace(str); + if (str.ToLower().Contains("now")) + { + return DateTime.UtcNow; + } + + // ... ago + var match = _TimeAgoRegexp.Match(str); + if (match.Success) + { + var timeago = str; + return FromTimeAgo(timeago); + } + + // Today ... + match = _TodayRegexp.Match(str); + if (match.Success) + { + var time = str.Replace(match.Groups[0].Value, ""); + var dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified); + dt += ParseTimeSpan(time); + return dt; + } + + // Yesterday ... + match = _YesterdayRegexp.Match(str); + if (match.Success) + { + var time = str.Replace(match.Groups[0].Value, ""); + var dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified); + dt += ParseTimeSpan(time); + dt -= TimeSpan.FromDays(1); + return dt; + } + + // Tomorrow ... + match = _TomorrowRegexp.Match(str); + if (match.Success) + { + var time = str.Replace(match.Groups[0].Value, ""); + var dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified); + dt += ParseTimeSpan(time); + dt += TimeSpan.FromDays(1); + return dt; + } + + // [day of the week] at ... (eg: Saturday at 14:22) + match = _DaysOfWeekRegexp.Match(str); + if (match.Success) + { + var time = str.Replace(match.Groups[0].Value, ""); + var dt = DateTime.SpecifyKind(DateTime.UtcNow.Date, DateTimeKind.Unspecified); + dt += ParseTimeSpan(time); + + DayOfWeek dow; + var groupMatchLower = match.Groups[1].Value.ToLower(); + if (groupMatchLower.StartsWith("monday")) + { + dow = DayOfWeek.Monday; + } + else if (groupMatchLower.StartsWith("tuesday")) + { + dow = DayOfWeek.Tuesday; + } + else if (groupMatchLower.StartsWith("wednesday")) + { + dow = DayOfWeek.Wednesday; + } + else if (groupMatchLower.StartsWith("thursday")) + { + dow = DayOfWeek.Thursday; + } + else if (groupMatchLower.StartsWith("friday")) + { + dow = DayOfWeek.Friday; + } + else if (groupMatchLower.StartsWith("saturday")) + { + dow = DayOfWeek.Saturday; + } + else + { + dow = DayOfWeek.Sunday; + } + + while (dt.DayOfWeek != dow) + { + dt = dt.AddDays(-1); + } + + return dt; + } + + try + { + // try parsing the str as an unix timestamp + var unixTimeStamp = long.Parse(str); + return UnixTimestampToDateTime(unixTimeStamp); + } + catch (FormatException) + { + // it wasn't a timestamp, continue.... + } + + // add missing year + match = _MissingYearRegexp.Match(str); + if (match.Success) + { + var date = match.Groups[1].Value; + var newDate = DateTime.Now.Year + "-" + date; + str = str.Replace(date, newDate); + } + + // add missing year 2 + match = _MissingYearRegexp2.Match(str); + if (match.Success) + { + var date = match.Groups[1].Value; + var time = match.Groups[2].Value; + str = date + " " + DateTime.Now.Year + " " + time; + } + + return FromFuzzyTime(str, format); + } + catch (Exception ex) + { + throw new Exception($"DateTime parsing failed for \"{str}\": {ex}"); + } + } + + // converts a date/time string to a DateTime object using a GoLang layout + public static DateTime ParseDateTimeGoLang(string date, string layout) + { + date = ParseUtil.NormalizeSpace(date); + var pattern = layout; + + // year + pattern = pattern.Replace("2006", "yyyy"); + pattern = pattern.Replace("06", "yy"); + + // month + pattern = pattern.Replace("January", "MMMM"); + pattern = pattern.Replace("Jan", "MMM"); + pattern = pattern.Replace("01", "MM"); + + // day + pattern = pattern.Replace("Monday", "dddd"); + pattern = pattern.Replace("Mon", "ddd"); + pattern = pattern.Replace("02", "dd"); + pattern = pattern.Replace("2", "d"); + + // hours/minutes/seconds + pattern = pattern.Replace("05", "ss"); + + pattern = pattern.Replace("15", "HH"); + pattern = pattern.Replace("03", "hh"); + pattern = pattern.Replace("3", "h"); + + pattern = pattern.Replace("04", "mm"); + pattern = pattern.Replace("4", "m"); + + pattern = pattern.Replace("5", "s"); + + // month again + pattern = pattern.Replace("1", "M"); + + // fractional seconds + pattern = pattern.Replace(".0000", "ffff"); + pattern = pattern.Replace(".000", "fff"); + pattern = pattern.Replace(".00", "ff"); + pattern = pattern.Replace(".0", "f"); + + pattern = pattern.Replace(".9999", "FFFF"); + pattern = pattern.Replace(".999", "FFF"); + pattern = pattern.Replace(".99", "FF"); + pattern = pattern.Replace(".9", "F"); + + // AM/PM + pattern = pattern.Replace("PM", "tt"); + pattern = pattern.Replace("pm", "tt"); // not sure if this works + + // timezones + // these might need further tuning + pattern = pattern.Replace("Z07:00", "'Z'zzz"); + pattern = pattern.Replace("Z07", "'Z'zz"); + pattern = pattern.Replace("Z07:00", "'Z'zzz"); + pattern = pattern.Replace("Z07", "'Z'zz"); + pattern = pattern.Replace("-07:00", "zzz"); + pattern = pattern.Replace("-07", "zz"); + + try + { + return DateTime.ParseExact(date, pattern, CultureInfo.InvariantCulture); + } + catch (FormatException ex) + { + throw new FormatException($"Error while parsing DateTime \"{date}\", using layout \"{layout}\" ({pattern}): {ex.Message}"); + } + } + + private static TimeSpan ParseTimeSpan(string time) => + string.IsNullOrWhiteSpace(time) + ? TimeSpan.Zero + : DateTime.Parse(time).TimeOfDay; + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinition.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinition.cs new file mode 100644 index 000000000..4a69943c6 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/IndexerDefinition.cs @@ -0,0 +1,176 @@ +using System; +using System.Collections.Generic; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + // A Dictionary allowing the same key multiple times + public class KeyValuePairList : List>, IDictionary + { + public SelectorBlock this[string key] + { + get => throw new NotImplementedException(); + + set => Add(new KeyValuePair(key, value)); + } + + public ICollection Keys => throw new NotImplementedException(); + + public ICollection Values => throw new NotImplementedException(); + + public void Add(string key, SelectorBlock value) => Add(new KeyValuePair(key, value)); + + public bool ContainsKey(string key) => throw new NotImplementedException(); + + public bool Remove(string key) => throw new NotImplementedException(); + + public bool TryGetValue(string key, out SelectorBlock value) => throw new NotImplementedException(); + } + + // Cardigann yaml classes + public class CardigannDefinition + { + public string Id { get; set; } + public List Settings { get; set; } + public string Name { get; set; } + public string Description { get; set; } + public string Type { get; set; } + public string Language { get; set; } + public string Encoding { get; set; } + public List Links { get; set; } + public List Legacylinks { get; set; } + public bool Followredirect { get; set; } = false; + public List Certificates { get; set; } + public CapabilitiesBlock Caps { get; set; } + public LoginBlock Login { get; set; } + public RatioBlock Ratio { get; set; } + public SearchBlock Search { get; set; } + public DownloadBlock Download { get; set; } + } + + public class SettingsField + { + public string Name { get; set; } + public string Type { get; set; } + public string Label { get; set; } + public string Default { get; set; } + public string[] Defaults { get; set; } + public Dictionary Options { get; set; } + } + + public class CategorymappingBlock + { + public string id { get; set; } + public string cat { get; set; } + public string desc { get; set; } + public bool Default { get; set; } + } + + public class CapabilitiesBlock + { + public Dictionary Categories { get; set; } + public List Categorymappings { get; set; } + public Dictionary> Modes { get; set; } + } + + public class CaptchaBlock + { + public string Type { get; set; } + public string Selector { get; set; } + public string Image { get => throw new Exception("Deprecated, please use Login.Captcha.Selector instead"); set => throw new Exception("Deprecated, please use login/captcha/selector instead of image"); } + public string Input { get; set; } + } + + public class LoginBlock + { + public string Path { get; set; } + public string Submitpath { get; set; } + public List Cookies { get; set; } + public string Method { get; set; } + public string Form { get; set; } + public bool Selectors { get; set; } = false; + public Dictionary Inputs { get; set; } + public Dictionary Selectorinputs { get; set; } + public Dictionary Getselectorinputs { get; set; } + public List Error { get; set; } + public PageTestBlock Test { get; set; } + public CaptchaBlock Captcha { get; set; } + } + + public class ErrorBlock + { + public string Path { get; set; } + public string Selector { get; set; } + public SelectorBlock Message { get; set; } + } + + public class SelectorBlock + { + public string Selector { get; set; } + public bool Optional { get; set; } = false; + public string Text { get; set; } + public string Attribute { get; set; } + public string Remove { get; set; } + public List Filters { get; set; } + public Dictionary Case { get; set; } + } + + public class FilterBlock + { + public string Name { get; set; } + public dynamic Args { get; set; } + } + + public class PageTestBlock + { + public string Path { get; set; } + public string Selector { get; set; } + } + + public class RatioBlock : SelectorBlock + { + public string Path { get; set; } + } + + public class SearchBlock + { + public string Path { get; set; } + public List Paths { get; set; } + public Dictionary> Headers { get; set; } + public List Keywordsfilters { get; set; } + public Dictionary Inputs { get; set; } + public List Error { get; set; } + public List Preprocessingfilters { get; set; } + public RowsBlock Rows { get; set; } + public KeyValuePairList Fields { get; set; } + } + + public class RowsBlock : SelectorBlock + { + public int After { get; set; } + public SelectorBlock Dateheaders { get; set; } + } + + public class SearchPathBlock : RequestBlock + { + public List Categories { get; set; } + public bool Inheritinputs { get; set; } = true; + public bool Followredirect { get; set; } = false; + } + + public class RequestBlock + { + public string Path { get; set; } + public string Method { get; set; } + public Dictionary Inputs { get; set; } + public string Queryseparator { get; set; } = "&"; + } + + public class DownloadBlock + { + public string Selector { get; set; } + public string Attribute { get; set; } + public List Filters { get; set; } + public string Method { get; set; } + public RequestBlock Before { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/ParseUtil.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/ParseUtil.cs new file mode 100644 index 000000000..044c507c7 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/ParseUtil.cs @@ -0,0 +1,102 @@ +using System.Globalization; +using System.Linq; +using System.Text.RegularExpressions; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public static class ParseUtil + { + private static readonly Regex InvalidXmlChars = + new Regex( + @"(? s.Trim(); + + public static string NormalizeMultiSpaces(string s) => + new Regex(@"\s+").Replace(NormalizeSpace(s), " "); + + public static string NormalizeNumber(string s) => + NormalizeSpace(s) + .Replace("-", "0") + .Replace(",", ""); + + public static string RemoveInvalidXmlChars(string text) => string.IsNullOrEmpty(text) ? "" : InvalidXmlChars.Replace(text, ""); + + public static double CoerceDouble(string str) => double.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture); + + public static float CoerceFloat(string str) => float.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture); + + public static int CoerceInt(string str) => int.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture); + + public static long CoerceLong(string str) => long.Parse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture); + + public static bool TryCoerceDouble(string str, out double result) => double.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result); + + public static bool TryCoerceFloat(string str, out float result) => float.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result); + + public static bool TryCoerceInt(string str, out int result) => int.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result); + + public static bool TryCoerceLong(string str, out long result) => long.TryParse(NormalizeNumber(str), NumberStyles.Any, CultureInfo.InvariantCulture, out result); + + /* + public static string GetArgumentFromQueryString(string url, string argument) + { + if (url == null || argument == null) + { + return null; + } + + var qsStr = url.Split(new char[] { '?' }, 2)[1]; + qsStr = qsStr.Split(new char[] { '#' }, 2)[0]; + var qs = QueryHelpers.ParseQuery(qsStr); + return qs[argument].FirstOrDefault(); + }*/ + + public static long? GetLongFromString(string str) + { + if (str == null) + { + return null; + } + + var idRegEx = new Regex(@"(\d+)", RegexOptions.Compiled); + var idMatch = idRegEx.Match(str); + if (!idMatch.Success) + { + return null; + } + + var id = idMatch.Groups[1].Value; + return CoerceLong(id); + } + + public static int? GetImdbID(string imdbstr) + { + if (imdbstr == null) + { + return null; + } + + var match = ImdbId.Match(imdbstr); + if (!match.Success) + { + return null; + } + + return int.Parse(match.Groups[1].Value, NumberStyles.Any, CultureInfo.InvariantCulture); + } + + public static string GetFullImdbID(string imdbstr) + { + var imdbid = GetImdbID(imdbstr); + if (imdbid == null) + { + return null; + } + + return "tt" + ((int)imdbid).ToString("D7"); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/StringUtil.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/StringUtil.cs new file mode 100644 index 000000000..de0c3ff95 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/StringUtil.cs @@ -0,0 +1,229 @@ +using System.Collections.Generic; +using System.Collections.Specialized; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using AngleSharp.Dom; +using AngleSharp.Html; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public static class StringUtil + { + /* + public static string StripNonAlphaNumeric(this string str, string replacement = "") => + StripRegex(str, "[^a-zA-Z0-9 -]", replacement); + + public static string StripRegex(string str, string regex, string replacement = "") + { + var rgx = new Regex(regex); + str = rgx.Replace(str, replacement); + return str; + } + + // replaces culture specific characters with the corresponding base characters (e.g. è becomes e). + public static string RemoveDiacritics(string s) + { + var normalizedString = s.Normalize(NormalizationForm.FormD); + var stringBuilder = new StringBuilder(); + + for (var i = 0; i < normalizedString.Length; i++) + { + var c = normalizedString[i]; + if (CharUnicodeInfo.GetUnicodeCategory(c) != UnicodeCategory.NonSpacingMark) + stringBuilder.Append(c); + } + + return stringBuilder.ToString(); + } + + public static string FromBase64(string str) => + Encoding.UTF8.GetString(Convert.FromBase64String(str)); + + /// + /// Convert an array of bytes to a string of hex digits + /// + /// array of bytes + /// String of hex digits + public static string HexStringFromBytes(byte[] bytes) => + string.Join("", bytes.Select(b => b.ToString("X2"))); + + /// + /// Compute hash for string encoded as UTF8 + /// + /// String to be hashed + /// 40-character hex string + public static string HashSHA1(string s) + { + var sha1 = SHA1.Create(); + + var bytes = Encoding.UTF8.GetBytes(s); + var hashBytes = sha1.ComputeHash(bytes); + + return HexStringFromBytes(hashBytes); + } + + public static string Hash(string s) + { + // Use input string to calculate MD5 hash + var md5 = System.Security.Cryptography.MD5.Create(); + + var inputBytes = System.Text.Encoding.ASCII.GetBytes(s); + var hashBytes = md5.ComputeHash(inputBytes); + + return HexStringFromBytes(hashBytes); + } + + // Is never used + // remove in favor of Exception.ToString() ? + public static string GetExceptionDetails(this Exception exception) + { + var properties = exception.GetType() + .GetProperties(); + var fields = properties + .Select(property => new + { + Name = property.Name, + Value = property.GetValue(exception, null) + }) + .Select(x => string.Format( + "{0} = {1}", + x.Name, + x.Value != null ? x.Value.ToString() : string.Empty + )); + return string.Join("\n", fields); + } + */ + + private static char[] MakeValidFileName_invalids; + + /// Replaces characters in text that are not allowed in + /// file names with the specified replacement character. + /// Text to make into a valid filename. The same string is returned if it is valid already. + /// Replacement character, or null to simply remove bad characters. + /// Whether to replace quotes and slashes with the non-ASCII characters ” and ⁄. + /// A string that can be used as a filename. If the output string would otherwise be empty, returns "_". + public static string MakeValidFileName(string text, char? replacement = '_', bool fancy = true) + { + var sb = new StringBuilder(text.Length); + var invalids = MakeValidFileName_invalids ?? (MakeValidFileName_invalids = Path.GetInvalidFileNameChars()); + var changed = false; + for (var i = 0; i < text.Length; i++) + { + var c = text[i]; + if (invalids.Contains(c)) + { + changed = true; + var repl = replacement ?? '\0'; + if (fancy) + { + if (c == '"') + { + repl = '”'; // U+201D right double quotation mark + } + else if (c == '\'') + { + repl = '’'; // U+2019 right single quotation mark + } + else if (c == '/') + { + repl = '⁄'; // U+2044 fraction slash + } + } + + if (repl != '\0') + { + sb.Append(repl); + } + } + else + { + sb.Append(c); + } + } + + if (sb.Length == 0) + { + return "_"; + } + + return changed ? sb.ToString() : text; + } + + /// + /// Converts a NameValueCollection to an appropriately formatted query string. + /// Duplicate keys are allowed in a NameValueCollection, but are stored as a csv string in Value. + /// This function handles leaving the values together in the csv string or splitting the value into separate keys + /// + /// The NameValueCollection being converted + /// The Encoding to use in url encoding Value + /// Duplicate keys are handled as true => {"Key=Val1", "Key=Val2} or false => {"Key=Val1,Val2"} + /// The string used to separate each query value + /// A web encoded string of key=value parameters separated by the separator + public static string GetQueryString(this NameValueCollection collection, + Encoding encoding = null, + bool duplicateKeysIfMulti = false, + string separator = "&") => + collection.ToEnumerable(duplicateKeysIfMulti).GetQueryString(encoding, separator); + + public static string GetQueryString(this IEnumerable> collection, + Encoding encoding = null, + string separator = "&") => + string.Join(separator, + collection.Select(a => $"{a.Key}={WebUtilityHelpers.UrlEncode(a.Value, encoding ?? Encoding.UTF8)}")); + + public static void Add(this ICollection> collection, string key, string value) => collection.Add(new KeyValuePair(key, value)); + + public static IEnumerable> ToEnumerable( + this NameValueCollection collection, bool duplicateKeysIfMulti = false) + { + foreach (string key in collection.Keys) + { + var value = collection[key]; + if (duplicateKeysIfMulti) + { + foreach (var val in value.Split(',')) + { + yield return new KeyValuePair(key, val); + } + } + else + { + yield return new KeyValuePair(key, value); + } + } + } + + public static string ToHtmlPretty(this IElement element) + { + if (element == null) + { + return ""; + } + + var sb = new StringBuilder(); + var sw = new StringWriter(sb); + var formatter = new PrettyMarkupFormatter(); + element.ToHtml(sw, formatter); + return sb.ToString(); + } + + public static string GenerateRandom(int length) + { + var chars = "abcdefghijklmnopqrstuvwxyz0123456789"; + var randBytes = new byte[length]; + using (var rngCsp = new RNGCryptoServiceProvider()) + { + rngCsp.GetBytes(randBytes); + var key = ""; + foreach (var b in randBytes) + { + key += chars[b % chars.Length]; + } + + return key; + } + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/WebUtilityHelpers.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/WebUtilityHelpers.cs new file mode 100644 index 000000000..bc4daedf7 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/WebUtilityHelpers.cs @@ -0,0 +1,30 @@ +using System.Net; +using System.Text; + +namespace NzbDrone.Core.Indexers.Cardigann +{ + public static class WebUtilityHelpers + { + public static string UrlEncode(string searchString, Encoding encoding) + { + if (string.IsNullOrEmpty(searchString)) + { + return string.Empty; + } + + var bytes = encoding.GetBytes(searchString); + return encoding.GetString(WebUtility.UrlEncodeToBytes(bytes, 0, bytes.Length)); + } + + public static string UrlDecode(string searchString, Encoding encoding) + { + if (string.IsNullOrEmpty(searchString)) + { + return string.Empty; + } + + var inputBytes = encoding.GetBytes(searchString); + return encoding.GetString(WebUtility.UrlDecodeToBytes(inputBytes, 0, inputBytes.Length)); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 3f0b984e1..baeeff9b3 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -4,6 +4,7 @@ using System.Linq; using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; +using NzbDrone.Core.Annotations; using NzbDrone.Core.Configuration; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser.Model; diff --git a/src/NzbDrone.Core/Indexers/IndexerDefinition.cs b/src/NzbDrone.Core/Indexers/IndexerDefinition.cs index 33e070d75..e89d6db70 100644 --- a/src/NzbDrone.Core/Indexers/IndexerDefinition.cs +++ b/src/NzbDrone.Core/Indexers/IndexerDefinition.cs @@ -1,4 +1,6 @@ using System; +using System.Collections.Generic; +using NzbDrone.Core.Indexers.Cardigann; using NzbDrone.Core.ThingiProvider; namespace NzbDrone.Core.Indexers @@ -19,5 +21,7 @@ namespace NzbDrone.Core.Indexers public override bool Enable => EnableRss || EnableAutomaticSearch || EnableInteractiveSearch; public IndexerStatus Status { get; set; } + + public List ExtraFields { get; set; } = new List(); } } diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs index 8db47837b..2066dd9d9 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs @@ -4,6 +4,7 @@ using System.Linq; using FluentValidation.Results; using NLog; using NzbDrone.Common.Composition; +using NzbDrone.Core.Indexers.Cardigann; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; @@ -19,10 +20,12 @@ namespace NzbDrone.Core.Indexers public class IndexerFactory : ProviderFactory, IIndexerFactory { + private readonly ICardigannDefinitionService _definitionService; private readonly IIndexerStatusService _indexerStatusService; private readonly Logger _logger; - public IndexerFactory(IIndexerStatusService indexerStatusService, + public IndexerFactory(ICardigannDefinitionService definitionService, + IIndexerStatusService indexerStatusService, IIndexerRepository providerRepository, IEnumerable providers, IContainer container, @@ -30,15 +33,68 @@ namespace NzbDrone.Core.Indexers Logger logger) : base(providerRepository, providers, container, eventAggregator, logger) { + _definitionService = definitionService; _indexerStatusService = indexerStatusService; _logger = logger; } + public override List All() + { + var definitions = base.All(); + var metaDefs = _definitionService.All().ToDictionary(x => x.File); + + foreach (var definition in definitions) + { + if (definition.Implementation == typeof(Cardigann.Cardigann).Name) + { + var settings = (CardigannSettings)definition.Settings; + definition.ExtraFields = metaDefs[settings.DefinitionFile].Settings; + } + } + + return definitions; + } + + public override IndexerDefinition Get(int id) + { + var definition = base.Get(id); + var metaDefs = _definitionService.All().ToDictionary(x => x.File); + + if (definition.Implementation == typeof(Cardigann.Cardigann).Name) + { + var settings = (CardigannSettings)definition.Settings; + definition.ExtraFields = metaDefs[settings.DefinitionFile].Settings; + } + + return definition; + } + protected override List Active() { return base.Active().Where(c => c.Enable).ToList(); } + public override IEnumerable GetDefaultDefinitions() + { + foreach (var provider in _providers) + { + var definitions = provider.DefaultDefinitions + .Where(v => v.Name != null && v.Name != provider.GetType().Name) + .Take(10); + + foreach (IndexerDefinition definition in definitions) + { + SetProviderCharacteristics(provider, definition); + yield return definition; + } + } + } + + public override IEnumerable GetPresetDefinitions(IndexerDefinition providerDefinition) + { + return new List(); + } + public override void SetProviderCharacteristics(IIndexer provider, IndexerDefinition definition) { base.SetProviderCharacteristics(provider, definition); diff --git a/src/NzbDrone.Core/Prowlarr.Core.csproj b/src/NzbDrone.Core/Prowlarr.Core.csproj index f2be69e9d..046487d47 100644 --- a/src/NzbDrone.Core/Prowlarr.Core.csproj +++ b/src/NzbDrone.Core/Prowlarr.Core.csproj @@ -18,6 +18,7 @@ + diff --git a/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs b/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs index 60dbc8e99..a59fc5d96 100644 --- a/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs +++ b/src/NzbDrone.Core/ThingiProvider/ProviderFactory.cs @@ -34,12 +34,12 @@ namespace NzbDrone.Core.ThingiProvider _logger = logger; } - public List All() + public virtual List All() { return _providerRepository.All().ToList(); } - public IEnumerable GetDefaultDefinitions() + public virtual IEnumerable GetDefaultDefinitions() { foreach (var provider in _providers) { @@ -64,7 +64,7 @@ namespace NzbDrone.Core.ThingiProvider } } - public IEnumerable GetPresetDefinitions(TProviderDefinition providerDefinition) + public virtual IEnumerable GetPresetDefinitions(TProviderDefinition providerDefinition) { var provider = _providers.First(v => v.GetType().Name == providerDefinition.Implementation); @@ -91,7 +91,7 @@ namespace NzbDrone.Core.ThingiProvider return Active().Select(GetInstance).ToList(); } - public TProviderDefinition Get(int id) + public virtual TProviderDefinition Get(int id) { return _providerRepository.Get(id); } diff --git a/src/NzbDrone.Core/ThingiProvider/ProviderRepository.cs b/src/NzbDrone.Core/ThingiProvider/ProviderRepository.cs index 71e134ca8..36f058c7a 100644 --- a/src/NzbDrone.Core/ThingiProvider/ProviderRepository.cs +++ b/src/NzbDrone.Core/ThingiProvider/ProviderRepository.cs @@ -31,6 +31,7 @@ namespace NzbDrone.Core.ThingiProvider serializerSettings.Converters.Add(new JsonStringEnumConverter(JsonNamingPolicy.CamelCase, true)); serializerSettings.Converters.Add(new TimeSpanConverter()); serializerSettings.Converters.Add(new UtcConverter()); + serializerSettings.Converters.Add(new DictionaryStringObjectJsonConverter()); _serializerSettings = serializerSettings; } diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs b/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs index d53f7c81a..6c1818845 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs @@ -1,5 +1,9 @@ using System; +using System.Linq; +using NzbDrone.Core.Annotations; using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.Cardigann; +using Prowlarr.Http.ClientSchema; namespace Prowlarr.Api.V1.Indexers { @@ -28,6 +32,27 @@ namespace Prowlarr.Api.V1.Indexers var resource = base.ToResource(definition); + if (definition.Implementation == typeof(Cardigann).Name) + { + Console.WriteLine("mapping cardigann def"); + + var extraFields = definition.ExtraFields.Select((x, i) => MapField(x, i)).ToList(); + + resource.Fields.AddRange(extraFields); + + var settings = (CardigannSettings)definition.Settings; + Console.WriteLine($"Got {settings.ExtraFieldData.Count} fields"); + foreach (var setting in settings.ExtraFieldData) + { + var field = extraFields.FirstOrDefault(x => x.Name == setting.Key); + if (field != null) + { + Console.WriteLine($"setting {setting.Key} to {setting.Value}"); + field.Value = setting.Value; + } + } + } + resource.EnableRss = definition.EnableRss; resource.EnableAutomaticSearch = definition.EnableAutomaticSearch; resource.EnableInteractiveSearch = definition.EnableInteractiveSearch; @@ -51,6 +76,22 @@ namespace Prowlarr.Api.V1.Indexers var definition = base.ToModel(resource); + if (resource.Implementation == typeof(Cardigann).Name) + { + Console.WriteLine("mapping cardigann resource"); + + var standardFields = base.ToResource(definition).Fields.Select(x => x.Name).ToList(); + + var settings = (CardigannSettings)definition.Settings; + foreach (var field in resource.Fields) + { + if (!standardFields.Contains(field.Name)) + { + settings.ExtraFieldData[field.Name] = field.Value; + } + } + } + definition.EnableRss = resource.EnableRss; definition.EnableAutomaticSearch = resource.EnableAutomaticSearch; definition.EnableInteractiveSearch = resource.EnableInteractiveSearch; @@ -60,5 +101,46 @@ namespace Prowlarr.Api.V1.Indexers return definition; } + + private Field MapField(SettingsField fieldAttribute, int order) + { + Console.WriteLine($"Adding field {fieldAttribute.Name}"); + var field = new Field + { + Name = fieldAttribute.Name, + Label = fieldAttribute.Label, + Order = order, + Type = fieldAttribute.Type == "text" ? "textbox" : fieldAttribute.Type + }; + + if (fieldAttribute.Type == "select") + { + var sorted = fieldAttribute.Options.OrderBy(x => x.Key).ToList(); + field.SelectOptions = sorted.Select((x, i) => new SelectOption + { + Value = i, + Name = x.Value + }).ToList(); + + field.Value = sorted.Select(x => x.Key).ToList().IndexOf(fieldAttribute.Default); + } + else if (fieldAttribute.Type == "checkbox") + { + if (bool.TryParse(fieldAttribute.Default, out var value)) + { + field.Value = value; + } + else + { + field.Value = false; + } + } + else + { + field.Value = fieldAttribute.Default; + } + + return field; + } } } diff --git a/src/Prowlarr.Api.V1/ProviderModuleBase.cs b/src/Prowlarr.Api.V1/ProviderModuleBase.cs index 33fdabd23..3c49ebf7f 100644 --- a/src/Prowlarr.Api.V1/ProviderModuleBase.cs +++ b/src/Prowlarr.Api.V1/ProviderModuleBase.cs @@ -17,7 +17,7 @@ namespace Prowlarr.Api.V1 where TProviderResource : ProviderResource, new() { protected readonly IProviderFactory _providerFactory; - private readonly ProviderResourceMapper _resourceMapper; + protected readonly ProviderResourceMapper _resourceMapper; protected ProviderModuleBase(IProviderFactory providerFactory, string resource, ProviderResourceMapper resourceMapper) : base(resource) @@ -113,7 +113,7 @@ namespace Prowlarr.Api.V1 _providerFactory.Delete(id); } - private object GetTemplates() + protected virtual object GetTemplates() { var defaultDefinitions = _providerFactory.GetDefaultDefinitions().OrderBy(p => p.ImplementationName).ToList();