using System; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.Disk; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Http; using NzbDrone.Core.Indexers.Definitions.Cardigann; using NzbDrone.Core.Lifecycle; using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using YamlDotNet.Serialization; using YamlDotNet.Serialization.NamingConventions; namespace NzbDrone.Core.IndexerVersions { public interface IIndexerDefinitionUpdateService { List All(); CardigannDefinition GetCachedDefinition(string fileKey); List GetBlocklist(); } public class IndexerDefinitionUpdateService : IIndexerDefinitionUpdateService, IExecute, IHandle { /* Update Service will fall back if version # does not exist for an indexer per Ta */ private const string DEFINITION_BRANCH = "cardigann-contains"; private const int DEFINITION_VERSION = 10; // Used when moving yml to C# private readonly List _definitionBlocklist = new () { "aither", "animeworld", "audiobookbay", "beyond-hd-oneurl", "beyond-hd", "blutopia", "brsociety", "danishbytes", "datascene", "desitorrents", "hdbits", "lat-team", "reelflix", "shareisland", "skipthecommercials", "tellytorrent" }; private readonly IHttpClient _httpClient; private readonly IAppFolderInfo _appFolderInfo; private readonly IDiskProvider _diskProvider; private readonly IIndexerDefinitionVersionService _versionService; private readonly ICached _cache; private readonly Logger _logger; private readonly IDeserializer _deserializer = new DeserializerBuilder() .IgnoreUnmatchedProperties() .WithNamingConvention(CamelCaseNamingConvention.Instance) .Build(); public IndexerDefinitionUpdateService(IHttpClient httpClient, IAppFolderInfo appFolderInfo, IDiskProvider diskProvider, IIndexerDefinitionVersionService versionService, ICacheManager cacheManager, Logger logger) { _appFolderInfo = appFolderInfo; _diskProvider = diskProvider; _versionService = versionService; _cache = cacheManager.GetCache(typeof(CardigannDefinition), "definitions"); _httpClient = httpClient; _logger = logger; } public List All() { var indexerList = new List(); try { // Grab latest def list from server or fallback to disk try { var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}"); var response = _httpClient.Get>(request); indexerList = response.Resource.Where(i => !_definitionBlocklist.Contains(i.File)).ToList(); } catch { var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions"); indexerList = ReadDefinitionsFromDisk(indexerList, definitionFolder); } //Check for custom definitions var customDefinitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions", "Custom"); indexerList = ReadDefinitionsFromDisk(indexerList, customDefinitionFolder); } catch { _logger.Error("Failed to Connect to Indexer Definition Server for Indexer listing"); } return indexerList; } public CardigannDefinition GetCachedDefinition(string fileKey) { if (string.IsNullOrEmpty(fileKey)) { throw new ArgumentNullException(nameof(fileKey)); } var definition = _cache.Get(fileKey, () => GetUncachedDefinition(fileKey)); return definition; } public List GetBlocklist() { return _definitionBlocklist; } private List ReadDefinitionsFromDisk(List defs, string path, SearchOption options = SearchOption.TopDirectoryOnly) { var indexerList = defs; var directoryInfo = new DirectoryInfo(path); if (directoryInfo.Exists) { var files = directoryInfo.GetFiles($"*.yml", options); foreach (var file in files) { _logger.Debug("Loading definition " + file.FullName); try { var definitionString = File.ReadAllText(file.FullName); var definition = _deserializer.Deserialize(definitionString); definition.File = Path.GetFileNameWithoutExtension(file.Name); if (indexerList.Any(i => i.File == definition.File || i.Name == definition.Name)) { _logger.Warn("Definition {0} does not have unique file name or Indexer name", file.FullName); continue; } indexerList.Add(definition); } catch (Exception e) { _logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}"); } } } return indexerList; } private CardigannDefinition GetUncachedDefinition(string fileKey) { if (string.IsNullOrEmpty(fileKey)) { throw new ArgumentNullException(nameof(fileKey)); } EnsureDefinitionsFolder(); var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions"); var directoryInfo = new DirectoryInfo(definitionFolder); if (directoryInfo.Exists) { var files = directoryInfo.GetFiles($"{fileKey}.yml", SearchOption.AllDirectories); if (files.Any()) { var file = files.First(); _logger.Trace("Loading Cardigann definition " + file.FullName); try { var definitionString = File.ReadAllText(file.FullName); var definition = _deserializer.Deserialize(definitionString); return CleanIndexerDefinition(definition); } catch (Exception e) { _logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}"); } } } var dbDefs = _versionService.All(); //Check to ensure it's in versioned defs before we go to web if (dbDefs.Count > 0 && !dbDefs.Any(x => x.File == fileKey)) { throw new ArgumentNullException(nameof(fileKey)); } //No definition was returned locally, go to the web return GetHttpDefinition(fileKey); } private CardigannDefinition GetHttpDefinition(string id) { var req = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{id}"); var response = _httpClient.Get(req); var definition = _deserializer.Deserialize(response.Content); return CleanIndexerDefinition(definition); } private CardigannDefinition CleanIndexerDefinition(CardigannDefinition definition) { if (definition.Settings == null) { definition.Settings = new List { new () { Name = "username", Label = "Username", Type = "text" }, new () { Name = "password", Label = "Password", Type = "password" } }; } if (definition.Encoding == null) { definition.Encoding = "UTF-8"; } if (definition.Login != null && definition.Login.Method == null) { definition.Login.Method = "form"; } if (definition.Search.Paths == null) { definition.Search.Paths = new List(); } // convert definitions with a single search Path to a Paths entry if (definition.Search.Path != null) { definition.Search.Paths.Add(new SearchPathBlock { Path = definition.Search.Path, Inheritinputs = true }); } return definition; } public void Handle(ApplicationStartedEvent message) { // Sync indexers on app start UpdateLocalDefinitions(); } public void Execute(IndexerDefinitionUpdateCommand message) { UpdateLocalDefinitions(); } private void EnsureDefinitionsFolder() { var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions"); _diskProvider.CreateFolder(definitionFolder); } private void UpdateLocalDefinitions() { var startupFolder = _appFolderInfo.AppDataFolder; try { EnsureDefinitionsFolder(); var definitionsFolder = Path.Combine(startupFolder, "Definitions"); var saveFile = Path.Combine(definitionsFolder, $"indexers.zip"); _httpClient.DownloadFile($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/package.zip", saveFile); using (var archive = ZipFile.OpenRead(saveFile)) { archive.ExtractToDirectory(definitionsFolder, true); } _diskProvider.DeleteFile(saveFile); _cache.Clear(); _logger.Debug("Updated indexer definitions"); } catch (Exception ex) { _logger.Error(ex, "Definition update failed"); } } } }