More Cardigann

pull/6/head
Qstick 4 years ago
parent b2b354a7a3
commit fef81171ba

@ -59,4 +59,13 @@ CapabilitiesLabel.propTypes = {
capabilities: PropTypes.object.isRequired
};
CapabilitiesLabel.defaultProps = {
capabilities: {
movieSearchAvailable: false,
tvSearchAvailable: false,
musicSearchAvailable: false,
bookSearchAvailable: false
}
};
export default CapabilitiesLabel;

@ -0,0 +1,9 @@
using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.IndexerVersions
{
public class IndexerDefinitionUpdateCommand : Command
{
public override bool SendUpdatesToClient => true;
}
}

@ -0,0 +1,157 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Core.Indexers.Cardigann;
using NzbDrone.Core.Messaging.Commands;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace NzbDrone.Core.IndexerVersions
{
public interface IIndexerDefinitionUpdateService
{
List<CardigannMetaDefinition> All();
CardigannDefinition GetDefinition(string fileKey);
}
public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, IExecute<IndexerDefinitionUpdateCommand>
{
private const int DEFINITION_VERSION = 1;
private readonly IHttpClient _httpClient;
private readonly IAppFolderInfo _appFolderInfo;
private readonly ICached<CardigannDefinition> _cache;
private readonly Logger _logger;
private readonly IDeserializer _deserializer = new DeserializerBuilder()
.IgnoreUnmatchedProperties()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
public IndexerDefinitionUpdateService(IHttpClient httpClient,
IAppFolderInfo appFolderInfo,
ICacheManager cacheManager,
Logger logger)
{
_appFolderInfo = appFolderInfo;
_cache = cacheManager.GetCache<CardigannDefinition>(typeof(CardigannDefinition), "definitions");
_httpClient = httpClient;
_logger = logger;
}
public List<CardigannMetaDefinition> All()
{
var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}");
var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
var remoteDefs = response.Resource.ToDictionary(x => x.File);
var startupFolder = _appFolderInfo.StartUpFolder;
var prefix = Path.Combine(startupFolder, "Definitions");
var directoryInfos = new List<DirectoryInfo> { new DirectoryInfo(prefix) };
var existingDirectories = directoryInfos.Where(d => d.Exists);
var files = existingDirectories.SelectMany(d => d.GetFiles("*.yml"));
var indexerList = new List<CardigannMetaDefinition>();
foreach (var file in files)
{
indexerList.AddIfNotNull(remoteDefs[Path.GetFileNameWithoutExtension(file.Name)]);
}
return indexerList;
}
public CardigannDefinition GetDefinition(string file)
{
if (string.IsNullOrEmpty(file))
{
throw new ArgumentNullException(nameof(file));
}
var definition = _cache.Get(file, () => LoadIndexerDef(file));
return definition;
}
private CardigannDefinition GetHttpDefinition(string id)
{
var req = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{id}");
var response = _httpClient.Get(req);
return _deserializer.Deserialize<CardigannDefinition>(response.Content);
}
private CardigannDefinition LoadIndexerDef(string fileKey)
{
if (string.IsNullOrEmpty(fileKey))
{
throw new ArgumentNullException(nameof(fileKey));
}
var definitionFolder = Path.Combine(_appFolderInfo.StartUpFolder, "Definitions");
var directoryInfo = new DirectoryInfo(definitionFolder);
if (directoryInfo.Exists)
{
var files = directoryInfo.GetFiles($"{fileKey}.yml");
if (files.Any())
{
var file = files.First();
_logger.Trace("Loading Cardigann definition " + file.FullName);
try
{
var definitionString = File.ReadAllText(file.FullName);
var definition = _deserializer.Deserialize<CardigannDefinition>(definitionString);
return definition;
}
catch (Exception e)
{
_logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}");
}
}
}
throw new ArgumentOutOfRangeException(nameof(fileKey));
}
public void Execute(IndexerDefinitionUpdateCommand message)
{
UpdateLocalDefinitions();
}
private void UpdateLocalDefinitions()
{
var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}");
var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
foreach (var def in response.Resource)
{
try
{
var startupFolder = _appFolderInfo.StartUpFolder;
var saveFile = Path.Combine(startupFolder, "Definitions", $"{def.File}.yml");
_httpClient.DownloadFile($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{def.File}", saveFile);
_cache.Remove(def.File);
_logger.Info("Updated definition: {0}", def.File);
}
catch (Exception ex)
{
_logger.Error("Definition download failed: {0}, {1}", def.File, ex.Message);
}
}
}
}
}

@ -4,6 +4,7 @@ using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.IndexerVersions;
using NzbDrone.Core.ThingiProvider;
using NzbDrone.Core.Validation;
@ -11,7 +12,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
{
public class Cardigann : HttpIndexerBase<CardigannSettings>
{
private readonly ICardigannDefinitionService _definitionService;
private readonly IIndexerDefinitionUpdateService _definitionService;
public override string Name => "Cardigann";
@ -33,14 +34,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
_logger);
}
public override IndexerCapabilities GetCapabilities()
{
// TODO: This uses indexer capabilities when called so we don't have to keep up with all of them
// however, this is not pulled on a all pull from UI, doing so will kill the UI load if an indexer is down
// should we just purge and manage
return new IndexerCapabilities();
}
public override IEnumerable<ProviderDefinition> DefaultDefinitions
{
get
@ -52,7 +45,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
}
}
public Cardigann(ICardigannDefinitionService definitionService,
public Cardigann(IIndexerDefinitionUpdateService definitionService,
IHttpClient httpClient,
IIndexerStatusService indexerStatusService,
IConfigService configService,
@ -76,7 +69,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
Privacy = definition.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public,
SupportsRss = SupportsRss,
SupportsSearch = SupportsSearch,
Capabilities = Capabilities,
Capabilities = new IndexerCapabilities(),
ExtraFields = definition.Settings
};
}

@ -1,44 +0,0 @@
using System.Collections.Generic;
using NzbDrone.Common.Http;
using YamlDotNet.Serialization;
using YamlDotNet.Serialization.NamingConventions;
namespace NzbDrone.Core.Indexers.Cardigann
{
public interface ICardigannDefinitionService
{
List<CardigannMetaDefinition> All();
CardigannDefinition GetDefinition(string id);
}
public class CardigannDefinitionService : ICardigannDefinitionService
{
private const int DEFINITION_VERSION = 1;
private readonly IHttpClient _httpClient;
private readonly IDeserializer _deserializer = new DeserializerBuilder()
.IgnoreUnmatchedProperties()
.WithNamingConvention(CamelCaseNamingConvention.Instance)
.Build();
public CardigannDefinitionService(IHttpClient httpClient)
{
_httpClient = httpClient;
}
public List<CardigannMetaDefinition> All()
{
var request = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}");
var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
return response.Resource;
}
public CardigannDefinition GetDefinition(string id)
{
var req = new HttpRequest($"https://indexers.prowlarr.com/master/{DEFINITION_VERSION}/{id}");
var response = _httpClient.Get(req);
return _deserializer.Deserialize<CardigannDefinition>(response.Content);
}
}
}

@ -14,5 +14,6 @@ namespace NzbDrone.Core.Indexers.Cardigann
public List<string> Links { get; set; }
public List<string> Legacylinks { get; set; }
public List<SettingsField> Settings { get; set; }
public string Sha { get; set; }
}
}

@ -0,0 +1,14 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using NzbDrone.Core.Messaging.Commands;
namespace NzbDrone.Core.Indexers.Definitions.Cardigann
{
public class IndexerDefinitionUpdateCommand : Command
{
public override bool SendUpdatesToClient => true;
}
}

@ -20,6 +20,8 @@ namespace NzbDrone.Core.Indexers.Newznab
public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
public override IndexerCapabilities Capabilities { get => new IndexerCapabilities(); protected set => base.Capabilities = value; }
public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings).LimitsDefault.Value;
public override IIndexerRequestGenerator GetRequestGenerator()
@ -36,14 +38,6 @@ namespace NzbDrone.Core.Indexers.Newznab
return new NewznabRssParser(Settings);
}
public override IndexerCapabilities GetCapabilities()
{
// TODO: This uses indexer capabilities when called so we don't have to keep up with all of them
// however, this is not pulled on a all pull from UI, doing so will kill the UI load if an indexer is down
// should we just purge and manage
return _capabilitiesProvider.GetCapabilities(Settings);
}
public override IEnumerable<ProviderDefinition> DefaultDefinitions
{
get

@ -23,7 +23,7 @@ namespace NzbDrone.Core.Indexers
public override bool SupportsRss => true;
public override bool SupportsSearch => true;
public override IndexerCapabilities Capabilities => new IndexerCapabilities();
public override IndexerCapabilities Capabilities { get; protected set; }
public bool SupportsPaging => PageSize > 0;
@ -86,11 +86,6 @@ namespace NzbDrone.Core.Indexers
return requests;
}
public override IndexerCapabilities GetCapabilities()
{
return Capabilities;
}
protected virtual IList<ReleaseInfo> FetchReleases(Func<IIndexerRequestGenerator, IndexerPageableRequestChain> pageableRequestChainSelector, bool isRecent = false)
{
var releases = new List<ReleaseInfo>();

@ -16,7 +16,5 @@ namespace NzbDrone.Core.Indexers
IList<ReleaseInfo> FetchRecent();
IList<ReleaseInfo> Fetch(MovieSearchCriteria searchCriteria);
IndexerCapabilities GetCapabilities();
}
}

@ -26,7 +26,7 @@ namespace NzbDrone.Core.Indexers
public abstract bool SupportsRss { get; }
public abstract bool SupportsSearch { get; }
public abstract IndexerCapabilities Capabilities { get; }
public abstract IndexerCapabilities Capabilities { get; protected set; }
public IndexerBase(IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger)
{
@ -69,8 +69,6 @@ namespace NzbDrone.Core.Indexers
public abstract IList<ReleaseInfo> FetchRecent();
public abstract IList<ReleaseInfo> Fetch(MovieSearchCriteria searchCriteria);
public abstract IndexerCapabilities GetCapabilities();
protected virtual IList<ReleaseInfo> CleanupReleases(IEnumerable<ReleaseInfo> releases)
{
var result = releases.DistinctBy(v => v.Guid).ToList();

@ -2,6 +2,8 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
using NzbDrone.Core.Indexers.Cardigann;
using NzbDrone.Core.Parser.Model;
namespace NzbDrone.Core.Indexers
{
@ -88,6 +90,47 @@ namespace NzbDrone.Core.Indexers
Categories = new List<IndexerCategory>();
}
public void ParseCardigannSearchModes(Dictionary<string, List<string>> modes)
{
if (modes == null || !modes.Any())
{
throw new Exception("At least one search mode is required");
}
if (!modes.ContainsKey("search"))
{
throw new Exception("The search mode 'search' is mandatory");
}
foreach (var entry in modes)
{
switch (entry.Key)
{
case "search":
if (entry.Value == null || entry.Value.Count != 1 || entry.Value[0] != "q")
{
throw new Exception("In search mode 'search' only 'q' parameter is supported and it's mandatory");
}
break;
case "tv-search":
ParseTvSearchParams(entry.Value);
break;
case "movie-search":
ParseMovieSearchParams(entry.Value);
break;
case "music-search":
ParseMusicSearchParams(entry.Value);
break;
case "book-search":
ParseBookSearchParams(entry.Value);
break;
default:
throw new Exception($"Unsupported search mode: {entry.Key}");
}
}
}
public void ParseTvSearchParams(IEnumerable<string> paramsList)
{
if (paramsList == null)
@ -169,6 +212,33 @@ namespace NzbDrone.Core.Indexers
}
}
private void ParseBookSearchParams(IEnumerable<string> paramsList)
{
if (paramsList == null)
{
return;
}
foreach (var paramStr in paramsList)
{
if (Enum.TryParse(paramStr, true, out BookSearchParam param))
{
if (!BookSearchParams.Contains(param))
{
BookSearchParams.Add(param);
}
else
{
throw new Exception($"Duplicate book-search param: {paramStr}");
}
}
else
{
throw new Exception($"Not supported book-search param: {paramStr}");
}
}
}
private string SupportedTvSearchParams()
{
var parameters = new List<string> { "q" }; // q is always enabled

@ -5,6 +5,7 @@ using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Composition;
using NzbDrone.Core.Indexers.Cardigann;
using NzbDrone.Core.IndexerVersions;
using NzbDrone.Core.Messaging.Events;
using NzbDrone.Core.ThingiProvider;
@ -20,11 +21,11 @@ namespace NzbDrone.Core.Indexers
public class IndexerFactory : ProviderFactory<IIndexer, IndexerDefinition>, IIndexerFactory
{
private readonly ICardigannDefinitionService _definitionService;
private readonly IIndexerDefinitionUpdateService _definitionService;
private readonly IIndexerStatusService _indexerStatusService;
private readonly Logger _logger;
public IndexerFactory(ICardigannDefinitionService definitionService,
public IndexerFactory(IIndexerDefinitionUpdateService definitionService,
IIndexerStatusService indexerStatusService,
IIndexerRepository providerRepository,
IEnumerable<IIndexer> providers,
@ -41,14 +42,17 @@ namespace NzbDrone.Core.Indexers
public override List<IndexerDefinition> All()
{
var definitions = base.All();
var metaDefs = _definitionService.All().ToDictionary(x => x.File);
foreach (var definition in definitions)
{
if (definition.Implementation == typeof(Cardigann.Cardigann).Name)
{
var settings = (CardigannSettings)definition.Settings;
definition.ExtraFields = metaDefs[settings.DefinitionFile].Settings;
var defFile = _definitionService.GetDefinition(settings.DefinitionFile);
definition.ExtraFields = defFile.Settings;
definition.Privacy = defFile.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public;
definition.Capabilities = new IndexerCapabilities();
definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes);
}
}
@ -58,12 +62,15 @@ namespace NzbDrone.Core.Indexers
public override IndexerDefinition Get(int id)
{
var definition = base.Get(id);
var metaDefs = _definitionService.All().ToDictionary(x => x.File);
if (definition.Implementation == typeof(Cardigann.Cardigann).Name)
{
var settings = (CardigannSettings)definition.Settings;
definition.ExtraFields = metaDefs[settings.DefinitionFile].Settings;
var defFile = _definitionService.GetDefinition(settings.DefinitionFile);
definition.ExtraFields = defFile.Settings;
definition.Privacy = defFile.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public;
definition.Capabilities = new IndexerCapabilities();
definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes);
}
return definition;
@ -79,7 +86,7 @@ namespace NzbDrone.Core.Indexers
foreach (var provider in _providers)
{
var definitions = provider.DefaultDefinitions
.Where(v => v.Name != null && v.Name != provider.GetType().Name);
.Where(v => v.Name != null && (v.Name != typeof(Cardigann.Cardigann).Name || v.Name != typeof(Newznab.Newznab).Name));
foreach (IndexerDefinition definition in definitions)
{
@ -99,10 +106,15 @@ namespace NzbDrone.Core.Indexers
base.SetProviderCharacteristics(provider, definition);
definition.Protocol = provider.Protocol;
definition.Privacy = provider.Privacy;
definition.SupportsRss = provider.SupportsRss;
definition.SupportsSearch = provider.SupportsSearch;
definition.Capabilities = provider.Capabilities;
//We want to use the definition Caps and Privacy for Cardigann instead of the provider.
if (definition.Implementation != typeof(Cardigann.Cardigann).Name)
{
definition.Privacy = provider.Privacy;
definition.Capabilities = provider.Capabilities;
}
}
public List<IIndexer> RssEnabled(bool filterBlockedIndexers = true)

@ -6,6 +6,7 @@ using NzbDrone.Core.Backup;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.HealthCheck;
using NzbDrone.Core.Housekeeping;
using NzbDrone.Core.IndexerVersions;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Commands;
using NzbDrone.Core.Messaging.Events;
@ -59,6 +60,7 @@ namespace NzbDrone.Core.Jobs
new ScheduledTask { Interval = 6 * 60, TypeName = typeof(ApplicationCheckUpdateCommand).FullName },
new ScheduledTask { Interval = 6 * 60, TypeName = typeof(CheckHealthCommand).FullName },
new ScheduledTask { Interval = 24 * 60, TypeName = typeof(HousekeepingCommand).FullName },
new ScheduledTask { Interval = 6 * 60, TypeName = typeof(IndexerDefinitionUpdateCommand).FullName },
new ScheduledTask
{

@ -59,7 +59,7 @@ namespace Prowlarr.Api.V1.Indexers
switch (requestType)
{
case "caps":
Response response = indexerInstance.GetCapabilities().ToXml();
Response response = indexer.Capabilities.ToXml();
response.ContentType = "application/rss+xml";
return response;
case "tvsearch":

Loading…
Cancel
Save