Misc definition handling improvements

frontend/src/Indexer/Index/Table/IndexerIndexRow.js

@@ -244,12 +244,15 @@ class IndexerIndexRow extends Component {
                     onPress={this.onIndexerInfoPress}
                   />
 
-                  <IconButton
-                    className={styles.externalLink}
-                    name={icons.EXTERNAL_LINK}
-                    title={translate('Website')}
-                    to={indexerUrls[0].replace('api.', '')}
-                  />
+                  {
+                    indexerUrls ?
+                      <IconButton
+                        className={styles.externalLink}
+                        name={icons.EXTERNAL_LINK}
+                        title={translate('Website')}
+                        to={indexerUrls[0].replace('api.', '')}
+                      /> : null
+                  }
 
                   <IconButton
                     name={icons.EDIT}
@@ -289,7 +292,7 @@ class IndexerIndexRow extends Component {
 
 IndexerIndexRow.propTypes = {
   id: PropTypes.number.isRequired,
-  indexerUrls: PropTypes.arrayOf(PropTypes.string).isRequired,
+  indexerUrls: PropTypes.arrayOf(PropTypes.string),
   protocol: PropTypes.string.isRequired,
   privacy: PropTypes.string.isRequired,
   priority: PropTypes.number.isRequired,
@@ -298,7 +301,7 @@ IndexerIndexRow.propTypes = {
   redirect: PropTypes.bool.isRequired,
   appProfile: PropTypes.object.isRequired,
   status: PropTypes.object,
-  capabilities: PropTypes.object.isRequired,
+  capabilities: PropTypes.object,
   added: PropTypes.string.isRequired,
   tags: PropTypes.arrayOf(PropTypes.number).isRequired,
   columns: PropTypes.arrayOf(PropTypes.object).isRequired,

src/NzbDrone.Core/Datastore/Migration/015_indexer_versions.cs

@@ -0,0 +1,18 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(15)]
+    public class IndexerVersions : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Create.TableForModel("IndexerDefinitionVersions")
+                .WithColumn("DefinitionId").AsString().NotNullable().Unique()
+                .WithColumn("File").AsString().NotNullable().Unique()
+                .WithColumn("Sha").AsString().Nullable()
+                .WithColumn("LastUpdated").AsDateTime().Nullable();
+        }
+    }
+}

src/NzbDrone.Core/Datastore/TableMapping.cs

@@ -11,6 +11,7 @@ using NzbDrone.Core.Datastore.Converters;
 using NzbDrone.Core.Download;
 using NzbDrone.Core.IndexerProxies;
 using NzbDrone.Core.Indexers;
+using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.Instrumentation;
 using NzbDrone.Core.Jobs;
 using NzbDrone.Core.Languages;
@@ -94,6 +95,7 @@ namespace NzbDrone.Core.Datastore
             Mapper.Entity<UpdateHistory>("UpdateHistory").RegisterModel();
 
             Mapper.Entity<AppSyncProfile>("AppSyncProfiles").RegisterModel();
+            Mapper.Entity<IndexerDefinitionVersion>("IndexerDefinitionVersions").RegisterModel();
         }
 
         private static void RegisterMappers()

src/NzbDrone.Core/HealthCheck/Checks/NoDefinitionCheck.cs

@@ -0,0 +1,48 @@
+using System.Linq;
+using NLog;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Cardigann;
+using NzbDrone.Core.IndexerVersions;
+using NzbDrone.Core.Localization;
+using NzbDrone.Core.ThingiProvider.Events;
+
+namespace NzbDrone.Core.HealthCheck.Checks
+{
+    [CheckOn(typeof(ProviderDeletedEvent<IIndexer>))]
+    public class NoDefinitionCheck : HealthCheckBase
+    {
+        private readonly IIndexerDefinitionUpdateService _indexerDefinitionUpdateService;
+        private readonly IIndexerFactory _indexerFactory;
+
+        public NoDefinitionCheck(IIndexerDefinitionUpdateService indexerDefinitionUpdateService, IIndexerFactory indexerFactory, ILocalizationService localizationService)
+            : base(localizationService)
+        {
+            _indexerDefinitionUpdateService = indexerDefinitionUpdateService;
+            _indexerFactory = indexerFactory;
+        }
+
+        public override HealthCheck Check()
+        {
+            var currentDefs = _indexerDefinitionUpdateService.All();
+
+            var noDefIndexers = _indexerFactory.AllProviders(false)
+                    .Where(i => i.Definition.Implementation == "Cardigann" && !currentDefs.Any(d => d.File == ((CardigannSettings)i.Definition.Settings).DefinitionFile)).ToList();
+
+            if (noDefIndexers.Count == 0)
+            {
+                return new HealthCheck(GetType());
+            }
+
+            var healthType = HealthCheckResult.Error;
+            var healthMessage = string.Format(_localizationService.GetLocalizedString("IndexerNoDefCheckMessage"),
+                string.Join(", ", noDefIndexers.Select(v => v.Definition.Name)));
+
+            return new HealthCheck(GetType(),
+                healthType,
+                healthMessage,
+                "#indexers-have-no-definition");
+        }
+
+        public override bool CheckOnSchedule => false;
+    }
+}

src/NzbDrone.Core/HealthCheck/Checks/OutdatedDefinitionCheck.cs

@@ -35,10 +35,13 @@ namespace NzbDrone.Core.HealthCheck.Checks
                 return new HealthCheck(GetType());
             }
 
+            var healthType = HealthCheckResult.Warning;
+            var healthMessage = string.Format(_localizationService.GetLocalizedString("IndexerObsoleteCheckMessage"),
+                string.Join(", ", oldIndexers.Select(v => v.Definition.Name)));
+
             return new HealthCheck(GetType(),
-                HealthCheckResult.Warning,
-                string.Format(_localizationService.GetLocalizedString("IndexerObsoleteCheckMessage"),
-                string.Join(", ", oldIndexers.Select(v => v.Definition.Name))),
+                healthType,
+                healthMessage,
                 "#indexers-are-obsolete");
         }
 

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs

@@ -1,13 +1,11 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-using System.IO.Compression;
 using System.Linq;
 using NLog;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Disk;
 using NzbDrone.Common.EnvironmentInfo;
-using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Indexers.Cardigann;
 using NzbDrone.Core.Messaging.Commands;
@@ -19,7 +17,7 @@ namespace NzbDrone.Core.IndexerVersions
     public interface IIndexerDefinitionUpdateService
     {
         List<CardigannMetaDefinition> All();
-        CardigannDefinition GetDefinition(string fileKey);
+        CardigannDefinition GetCachedDefinition(string fileKey);
         List<string> GetBlocklist();
     }
 
@@ -29,6 +27,8 @@ namespace NzbDrone.Core.IndexerVersions
 
         private const string DEFINITION_BRANCH = "master";
         private const int DEFINITION_VERSION = 3;
+
+        //Used when moving yml to C#
         private readonly List<string> _defintionBlocklist = new List<string>()
         {
             "aither",
@@ -51,6 +51,7 @@ namespace NzbDrone.Core.IndexerVersions
         private readonly IHttpClient _httpClient;
         private readonly IAppFolderInfo _appFolderInfo;
         private readonly IDiskProvider _diskProvider;
+        private readonly IIndexerDefinitionVersionService _versionService;
         private readonly ICached<CardigannDefinition> _cache;
         private readonly Logger _logger;
 
@@ -62,11 +63,13 @@ namespace NzbDrone.Core.IndexerVersions
         public IndexerDefinitionUpdateService(IHttpClient httpClient,
                                           IAppFolderInfo appFolderInfo,
                                           IDiskProvider diskProvider,
+                                          IIndexerDefinitionVersionService versionService,
                                           ICacheManager cacheManager,
                                           Logger logger)
         {
             _appFolderInfo = appFolderInfo;
             _diskProvider = diskProvider;
+            _versionService = versionService;
             _cache = cacheManager.GetCache<CardigannDefinition>(typeof(CardigannDefinition), "definitions");
             _httpClient = httpClient;
             _logger = logger;
@@ -78,43 +81,24 @@ namespace NzbDrone.Core.IndexerVersions
 
             try
             {
-                var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
-                var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
-                indexerList = response.Resource.Where(i => !_defintionBlocklist.Contains(i.File)).ToList();
-
-                var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions", "Custom");
-
-                var directoryInfo = new DirectoryInfo(definitionFolder);
-
-                if (directoryInfo.Exists)
+                // Grab latest def list from server or fallback to disk
+                try
                 {
-                    var files = directoryInfo.GetFiles($"*.yml");
-
-                    foreach (var file in files)
-                    {
-                        _logger.Debug("Loading Custom Cardigann definition " + file.FullName);
-
-                        try
-                        {
-                            var definitionString = File.ReadAllText(file.FullName);
-                            var definition = _deserializer.Deserialize<CardigannMetaDefinition>(definitionString);
+                    var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
+                    var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
+                    indexerList = response.Resource.Where(i => !_defintionBlocklist.Contains(i.File)).ToList();
+                }
+                catch
+                {
+                    var definitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions");
 
-                            definition.File = Path.GetFileNameWithoutExtension(file.Name);
+                    indexerList = ReadDefinitionsFromDisk(indexerList, definitionFolder);
+                }
 
-                            if (indexerList.Any(i => i.File == definition.File || i.Name == definition.Name))
-                            {
-                                _logger.Warn("Custom Cardigann definition {0} does not have unique file name or Indexer name", file.FullName);
-                                continue;
-                            }
+                //Check for custom definitions
+                var customDefinitionFolder = Path.Combine(_appFolderInfo.AppDataFolder, "Definitions", "Custom");
 
-                            indexerList.Add(definition);
-                        }
-                        catch (Exception e)
-                        {
-                            _logger.Error($"Error while parsing custom Cardigann definition {file.FullName}\n{e}");
-                        }
-                    }
-                }
+                indexerList = ReadDefinitionsFromDisk(indexerList, customDefinitionFolder);
             }
             catch
             {
@@ -124,14 +108,14 @@ namespace NzbDrone.Core.IndexerVersions
             return indexerList;
         }
 
-        public CardigannDefinition GetDefinition(string file)
+        public CardigannDefinition GetCachedDefinition(string fileKey)
         {
-            if (string.IsNullOrEmpty(file))
+            if (string.IsNullOrEmpty(fileKey))
             {
-                throw new ArgumentNullException(nameof(file));
+                throw new ArgumentNullException(nameof(fileKey));
             }
 
-            var definition = _cache.Get(file, () => LoadIndexerDef(file));
+            var definition = _cache.Get(fileKey, () => GetUncachedDefinition(fileKey));
 
             return definition;
         }
@@ -141,15 +125,46 @@ namespace NzbDrone.Core.IndexerVersions
             return _defintionBlocklist;
         }
 
-        private CardigannDefinition GetHttpDefinition(string id)
+        private List<CardigannMetaDefinition> ReadDefinitionsFromDisk(List<CardigannMetaDefinition> defs, string path, SearchOption options = SearchOption.TopDirectoryOnly)
         {
-            var req = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{id}");
-            var response = _httpClient.Get(req);
-            var definition = _deserializer.Deserialize<CardigannDefinition>(response.Content);
-            return CleanIndexerDefinition(definition);
+            var indexerList = defs;
+
+            var directoryInfo = new DirectoryInfo(path);
+
+            if (directoryInfo.Exists)
+            {
+                var files = directoryInfo.GetFiles($"*.yml", options);
+
+                foreach (var file in files)
+                {
+                    _logger.Debug("Loading definition " + file.FullName);
+
+                    try
+                    {
+                        var definitionString = File.ReadAllText(file.FullName);
+                        var definition = _deserializer.Deserialize<CardigannMetaDefinition>(definitionString);
+
+                        definition.File = Path.GetFileNameWithoutExtension(file.Name);
+
+                        if (indexerList.Any(i => i.File == definition.File || i.Name == definition.Name))
+                        {
+                            _logger.Warn("Definition {0} does not have unique file name or Indexer name", file.FullName);
+                            continue;
+                        }
+
+                        indexerList.Add(definition);
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.Error($"Error while parsing Cardigann definition {file.FullName}\n{e}");
+                    }
+                }
+            }
+
+            return indexerList;
         }
 
-        private CardigannDefinition LoadIndexerDef(string fileKey)
+        private CardigannDefinition GetUncachedDefinition(string fileKey)
         {
             if (string.IsNullOrEmpty(fileKey))
             {
@@ -184,9 +199,24 @@ namespace NzbDrone.Core.IndexerVersions
                 }
             }
 
+            //Check to ensure it's in versioned defs before we go to web
+            if (!_versionService.All().Any(x => x.File == fileKey))
+            {
+                throw new ArgumentNullException(nameof(fileKey));
+            }
+
+            //No definition was returned locally, go to the web
             return GetHttpDefinition(fileKey);
         }
 
+        private CardigannDefinition GetHttpDefinition(string id)
+        {
+            var req = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{id}");
+            var response = _httpClient.Get(req);
+            var definition = _deserializer.Deserialize<CardigannDefinition>(response.Content);
+            return CleanIndexerDefinition(definition);
+        }
+
         private CardigannDefinition CleanIndexerDefinition(CardigannDefinition definition)
         {
             if (definition.Settings == null)
@@ -242,29 +272,44 @@ namespace NzbDrone.Core.IndexerVersions
         {
             var startupFolder = _appFolderInfo.AppDataFolder;
 
+            var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
+            var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
+
+            var currentDefs = _versionService.All().ToDictionary(x => x.DefinitionId, x => x.Sha);
+
             try
             {
                 EnsureDefinitionsFolder();
 
-                var definitionsFolder = Path.Combine(startupFolder, "Definitions");
-                var saveFile = Path.Combine(startupFolder, "Definitions", $"indexers.zip");
+                foreach (var def in response.Resource)
+                {
+                    try
+                    {
+                        var saveFile = Path.Combine(startupFolder, "Definitions", $"{def.File}.yml");
 
-                _httpClient.DownloadFile($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/package.zip", saveFile);
+                        if (currentDefs.TryGetValue(def.Id, out var defSha) && defSha == def.Sha)
+                        {
+                            _logger.Trace("Indexer already up to date: {0}", def.File);
 
-                using (ZipArchive archive = ZipFile.OpenRead(saveFile))
-                {
-                    archive.ExtractToDirectory(definitionsFolder, true);
-                }
+                            continue;
+                        }
 
-                _diskProvider.DeleteFile(saveFile);
+                        _httpClient.DownloadFile($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}/{def.File}", saveFile);
 
-                _cache.Clear();
+                        _versionService.Upsert(new IndexerDefinitionVersion { Sha = def.Sha, DefinitionId = def.Id, File = def.File, LastUpdated = DateTime.UtcNow });
 
-                _logger.Debug("Updated indexer definitions");
+                        _cache.Remove(def.File);
+                        _logger.Debug("Updated definition: {0}", def.File);
+                    }
+                    catch (Exception ex)
+                    {
+                        _logger.Error("Definition download failed: {0}, {1}", def.File, ex.Message);
+                    }
+                }
             }
             catch (Exception ex)
             {
-                _logger.Error(ex, "Definition update failed");
+                _logger.Error(ex, "Definition download failed, error creating definitions folder in {0}", startupFolder);
             }
         }
     }

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersion.cs

@@ -0,0 +1,13 @@
+using System;
+using NzbDrone.Core.Datastore;
+
+namespace NzbDrone.Core.IndexerVersions
+{
+    public class IndexerDefinitionVersion : ModelBase
+    {
+        public string File { get; set; }
+        public string Sha { get; set; }
+        public DateTime LastUpdated { get; set; }
+        public string DefinitionId { get; set; }
+    }
+}

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionRepository.cs

@@ -0,0 +1,25 @@
+using System;
+using System.Linq;
+using NzbDrone.Core.Datastore;
+using NzbDrone.Core.Messaging.Events;
+
+namespace NzbDrone.Core.IndexerVersions
+{
+    public interface IIndexerDefinitionVersionRepository : IBasicRepository<IndexerDefinitionVersion>
+    {
+        public IndexerDefinitionVersion GetByDefId(string defId);
+    }
+
+    public class IndexerDefinitionVersionRepository : BasicRepository<IndexerDefinitionVersion>, IIndexerDefinitionVersionRepository
+    {
+        public IndexerDefinitionVersionRepository(IMainDatabase database, IEventAggregator eventAggregator)
+            : base(database, eventAggregator)
+        {
+        }
+
+        public IndexerDefinitionVersion GetByDefId(string defId)
+        {
+            return Query(x => x.DefinitionId == defId).SingleOrDefault();
+        }
+    }
+}

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionVersionService.cs

@@ -0,0 +1,66 @@
+using System.Collections.Generic;
+using System.Linq;
+
+namespace NzbDrone.Core.IndexerVersions
+{
+    public interface IIndexerDefinitionVersionService
+    {
+        IndexerDefinitionVersion Get(int indexerVersionId);
+        IndexerDefinitionVersion GetByDefId(string defId);
+        List<IndexerDefinitionVersion> All();
+        IndexerDefinitionVersion Add(IndexerDefinitionVersion defVersion);
+        IndexerDefinitionVersion Upsert(IndexerDefinitionVersion defVersion);
+        void Delete(int indexerVersionId);
+    }
+
+    public class IndexerDefinitionVersionService : IIndexerDefinitionVersionService
+    {
+        private readonly IIndexerDefinitionVersionRepository _repo;
+
+        public IndexerDefinitionVersionService(IIndexerDefinitionVersionRepository repo)
+        {
+            _repo = repo;
+        }
+
+        public IndexerDefinitionVersion Get(int indexerVersionId)
+        {
+            return _repo.Get(indexerVersionId);
+        }
+
+        public IndexerDefinitionVersion GetByDefId(string defId)
+        {
+            return _repo.GetByDefId(defId);
+        }
+
+        public List<IndexerDefinitionVersion> All()
+        {
+            return _repo.All().ToList();
+        }
+
+        public IndexerDefinitionVersion Add(IndexerDefinitionVersion defVersion)
+        {
+            _repo.Insert(defVersion);
+
+            return defVersion;
+        }
+
+        public IndexerDefinitionVersion Upsert(IndexerDefinitionVersion defVersion)
+        {
+            var existing = _repo.GetByDefId(defVersion.DefinitionId);
+
+            if (existing != null)
+            {
+                defVersion.Id = existing.Id;
+            }
+
+            defVersion = _repo.Upsert(defVersion);
+
+            return defVersion;
+        }
+
+        public void Delete(int indexerVersionId)
+        {
+            _repo.Delete(indexerVersionId);
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -37,7 +37,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
         {
             var generator = _generatorCache.Get(Settings.DefinitionFile, () =>
                 new CardigannRequestGenerator(_configService,
-                    _definitionService.GetDefinition(Settings.DefinitionFile),
+                    _definitionService.GetCachedDefinition(Settings.DefinitionFile),
                     _logger)
                 {
                     HttpClient = _httpClient,
@@ -57,7 +57,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
         public override IParseIndexerResponse GetParser()
         {
             return new CardigannParser(_configService,
-                _definitionService.GetDefinition(Settings.DefinitionFile),
+                _definitionService.GetCachedDefinition(Settings.DefinitionFile),
                 _logger)
             {
                 Settings = Settings

src/NzbDrone.Core/Indexers/IndexerFactory.cs

@@ -59,7 +59,7 @@ namespace NzbDrone.Core.Indexers
                     catch
                     {
                         // Skip indexer if we fail in Cardigann mapping
-                        continue;
+                        _logger.Debug("Indexer {0} has no definition", definition.Name);
                     }
                 }
 
@@ -96,7 +96,7 @@ namespace NzbDrone.Core.Indexers
         private void MapCardigannDefinition(IndexerDefinition definition)
         {
             var settings = (CardigannSettings)definition.Settings;
-            var defFile = _definitionService.GetDefinition(settings.DefinitionFile);
+            var defFile = _definitionService.GetCachedDefinition(settings.DefinitionFile);
             definition.ExtraFields = defFile.Settings;
 
             if (defFile.Login?.Captcha != null && !definition.ExtraFields.Any(x => x.Type == "cardigannCaptcha"))

src/NzbDrone.Core/Localization/Core/en.json

@@ -178,6 +178,7 @@
   "IndexerLongTermStatusCheckAllClientMessage": "All indexers are unavailable due to failures for more than 6 hours",
   "IndexerLongTermStatusCheckSingleClientMessage": "Indexers unavailable due to failures for more than 6 hours: {0}",
   "IndexerObsoleteCheckMessage": "Indexers are obsolete or have been updated: {0}. Please remove and (or) re-add to Prowlarr",
+  "IndexerNoDefCheckMessage": "Indexers have no definition and will not work: {0}. Please remove and (or) re-add to Prowlarr",
   "IndexerPriority": "Indexer Priority",
   "IndexerPriorityHelpText": "Indexer Priority from 1 (Highest) to 50 (Lowest). Default: 25.",
   "IndexerProxies": "Indexer Proxies",

src/Prowlarr.Api.V1/Indexers/IndexerResource.cs

@@ -114,7 +114,7 @@ namespace Prowlarr.Api.V1.Indexers
 
                 var settings = (CardigannSettings)definition.Settings;
 
-                var cardigannDefinition = _definitionService.GetDefinition(settings.DefinitionFile);
+                var cardigannDefinition = _definitionService.GetCachedDefinition(settings.DefinitionFile);
 
                 foreach (var field in resource.Fields)
                 {