Newznab to Yml

src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs

@@ -15,14 +15,14 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
     [TestFixture]
     public class NewznabCapabilitiesProviderFixture : CoreTest<NewznabCapabilitiesProvider>
     {
-        private NewznabSettings _settings;
+        private GenericNewznabSettings _settings;
         private IndexerDefinition _definition;
         private string _caps;
 
         [SetUp]
         public void SetUp()
         {
-            _settings = new NewznabSettings()
+            _settings = new GenericNewznabSettings()
             {
                 BaseUrl = "http://indxer.local"
             };

src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs

@@ -34,7 +34,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
 
             _caps = new IndexerCapabilities();
             Mocker.GetMock<INewznabCapabilitiesProvider>()
-                .Setup(v => v.GetCapabilities(It.IsAny<NewznabSettings>(), It.IsAny<IndexerDefinition>()))
+                .Setup(v => v.GetCapabilities(It.IsAny<GenericNewznabSettings>(), It.IsAny<IndexerDefinition>()))
                 .Returns(_caps);
         }
 

src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs

@@ -10,7 +10,7 @@ using NzbDrone.Core.Test.Framework;
 
 namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
 {
-    public class NewznabRequestGeneratorFixture : CoreTest<NewznabRequestGenerator>
+    public class NewznabRequestGeneratorFixture : CoreTest<GenericNewznabRequestGenerator>
     {
         private MovieSearchCriteria _movieSearchCriteria;
         private TvSearchCriteria _tvSearchCriteria;
@@ -19,7 +19,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
         [SetUp]
         public void SetUp()
         {
-            Subject.Settings = new NewznabSettings()
+            Subject.Settings = new GenericNewznabSettings()
             {
                 BaseUrl = "http://127.0.0.1:1234/",
                 ApiKey = "abcd",
@@ -41,7 +41,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests
             _capabilities = new IndexerCapabilities();
 
             Mocker.GetMock<INewznabCapabilitiesProvider>()
-                .Setup(v => v.GetCapabilities(It.IsAny<NewznabSettings>(), It.IsAny<IndexerDefinition>()))
+                .Setup(v => v.GetCapabilities(It.IsAny<GenericNewznabSettings>(), It.IsAny<IndexerDefinition>()))
                 .Returns(_capabilities);
         }
 

src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs

@@ -35,7 +35,7 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests
 
             _caps = new IndexerCapabilities();
             Mocker.GetMock<INewznabCapabilitiesProvider>()
-                .Setup(v => v.GetCapabilities(It.IsAny<NewznabSettings>(), It.IsAny<IndexerDefinition>()))
+                .Setup(v => v.GetCapabilities(It.IsAny<GenericNewznabSettings>(), It.IsAny<IndexerDefinition>()))
                 .Returns(_caps);
         }
 

src/NzbDrone.Core/Datastore/Migration/024_newznab_yml.cs

@@ -0,0 +1,14 @@
+using FluentMigrator;
+using NzbDrone.Core.Datastore.Migration.Framework;
+
+namespace NzbDrone.Core.Datastore.Migration
+{
+    [Migration(24)]
+    public class newznab_yml : NzbDroneMigrationBase
+    {
+        protected override void MainDbUpgrade()
+        {
+            Update.Table("Indexers").Set(new { Implementation = "GenericNewznab", ConfigContract = "GenericNewznabSettings" }).Where(new { Implementation = "Newznab" });
+        }
+    }
+}

src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs

@@ -19,7 +19,8 @@ namespace NzbDrone.Core.IndexerVersions
 {
     public interface IIndexerDefinitionUpdateService
     {
-        List<CardigannMetaDefinition> All();
+        List<IndexerMetaDefinition> All();
+        List<IndexerMetaDefinition> AllForImplementation(string implementation);
         CardigannDefinition GetCachedDefinition(string fileKey);
         List<string> GetBlocklist();
     }
@@ -28,8 +29,8 @@ namespace NzbDrone.Core.IndexerVersions
     {
         /* Update Service will fall back if version # does not exist for an indexer  per Ta */
 
-        private const string DEFINITION_BRANCH = "master";
+        private const string DEFINITION_BRANCH = "newznab-yml";
-        private const int DEFINITION_VERSION = 7;
+        private const int DEFINITION_VERSION = 8;
 
         //Used when moving yml to C#
         private readonly List<string> _defintionBlocklist = new List<string>()
@@ -78,9 +79,9 @@ namespace NzbDrone.Core.IndexerVersions
             _logger = logger;
         }
 
-        public List<CardigannMetaDefinition> All()
+        public List<IndexerMetaDefinition> All()
         {
-            var indexerList = new List<CardigannMetaDefinition>();
+            var indexerList = new List<IndexerMetaDefinition>();
 
             try
             {
@@ -88,7 +89,7 @@ namespace NzbDrone.Core.IndexerVersions
                 try
                 {
                     var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}");
-                    var response = _httpClient.Get<List<CardigannMetaDefinition>>(request);
+                    var response = _httpClient.Get<List<IndexerMetaDefinition>>(request);
                     indexerList = response.Resource.Where(i => !_defintionBlocklist.Contains(i.File)).ToList();
                 }
                 catch
@@ -111,6 +112,11 @@ namespace NzbDrone.Core.IndexerVersions
             return indexerList;
         }
 
+        public List<IndexerMetaDefinition> AllForImplementation(string implementation)
+        {
+            return All().Where(d => d.Implementation == implementation.ToLower()).ToList();
+        }
+
         public CardigannDefinition GetCachedDefinition(string fileKey)
         {
             if (string.IsNullOrEmpty(fileKey))
@@ -128,7 +134,7 @@ namespace NzbDrone.Core.IndexerVersions
             return _defintionBlocklist;
         }
 
-        private List<CardigannMetaDefinition> ReadDefinitionsFromDisk(List<CardigannMetaDefinition> defs, string path, SearchOption options = SearchOption.TopDirectoryOnly)
+        private List<IndexerMetaDefinition> ReadDefinitionsFromDisk(List<IndexerMetaDefinition> defs, string path, SearchOption options = SearchOption.TopDirectoryOnly)
         {
             var indexerList = defs;
 
@@ -145,7 +151,7 @@ namespace NzbDrone.Core.IndexerVersions
                     try
                     {
                         var definitionString = File.ReadAllText(file.FullName);
-                        var definition = _deserializer.Deserialize<CardigannMetaDefinition>(definitionString);
+                        var definition = _deserializer.Deserialize<IndexerMetaDefinition>(definitionString);
 
                         definition.File = Path.GetFileNameWithoutExtension(file.Name);
 
@@ -243,6 +249,11 @@ namespace NzbDrone.Core.IndexerVersions
                 definition.Login.Method = "form";
             }
 
+            if (definition.Search == null)
+            {
+                definition.Search = new SearchBlock();
+            }
+
             if (definition.Search.Paths == null)
             {
                 definition.Search.Paths = new List<SearchPathBlock>();

src/NzbDrone.Core/IndexerVersions/IndexerMetaDefinition.cs

@@ -1,10 +1,11 @@
 using System.Collections.Generic;
+using NzbDrone.Core.Indexers.Cardigann;
 
-namespace NzbDrone.Core.Indexers.Cardigann
+namespace NzbDrone.Core.IndexerVersions
 {
-    public class CardigannMetaDefinition
+    public class IndexerMetaDefinition
     {
-        public CardigannMetaDefinition()
+        public IndexerMetaDefinition()
         {
             Legacylinks = new List<string>();
         }
@@ -13,6 +14,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
         public string File { get; set; }
         public string Name { get; set; }
         public string Description { get; set; }
+        public string Implementation { get; set; }
         public string Type { get; set; }
         public string Language { get; set; }
         public string Encoding { get; set; }

src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs

@@ -78,7 +78,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
         {
             get
             {
-                foreach (var def in _definitionService.All())
+                foreach (var def in _definitionService.AllForImplementation(GetType().Name))
                 {
                     yield return GetDefinition(def);
                 }
@@ -98,7 +98,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
             _generatorCache = cacheManager.GetRollingCache<CardigannRequestGenerator>(GetType(), "CardigannGeneratorCache", TimeSpan.FromMinutes(5));
         }
 
-        private IndexerDefinition GetDefinition(CardigannMetaDefinition definition)
+        private IndexerDefinition GetDefinition(IndexerMetaDefinition definition)
         {
             var defaultSettings = new List<SettingsField>
             {

src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs

@@ -13,7 +13,7 @@ namespace NzbDrone.Core.Indexers.Cardigann
         }
     }
 
-    public class CardigannSettings : NoAuthTorrentBaseSettings
+    public class CardigannSettings : NoAuthTorrentBaseSettings, IYmlIndexerSettings
     {
         private static readonly CardigannSettingsValidator Validator = new CardigannSettingsValidator();
 

src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznab.cs

@@ -0,0 +1,195 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading.Tasks;
+using FluentValidation.Results;
+using NLog;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using NzbDrone.Core.Download;
+using NzbDrone.Core.Messaging.Events;
+using NzbDrone.Core.ThingiProvider;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Newznab
+{
+    public class GenericNewznab : UsenetIndexerBase<GenericNewznabSettings>
+    {
+        private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
+
+        public override string Name => "Generic Newznab";
+        public override string[] IndexerUrls => GetBaseUrlFromSettings();
+        public override string Description => "Newznab is an API search specification for Usenet";
+        public override bool FollowRedirect => true;
+        public override bool SupportsRedirect => true;
+
+        public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
+        public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
+
+        public override IndexerCapabilities Capabilities { get => GetCapabilitiesFromSettings(); protected set => base.Capabilities = value; }
+
+        public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings, Definition).LimitsDefault.Value;
+
+        public override IIndexerRequestGenerator GetRequestGenerator()
+        {
+            return new GenericNewznabRequestGenerator(_capabilitiesProvider)
+            {
+                PageSize = PageSize,
+                Settings = Settings
+            };
+        }
+
+        public override IParseIndexerResponse GetParser()
+        {
+            return new GenericNewznabRssParser(Settings.Categories);
+        }
+
+        public string[] GetBaseUrlFromSettings()
+        {
+            var baseUrl = "";
+
+            if (Definition == null || Settings == null || Settings.Categories == null)
+            {
+                return new string[] { baseUrl };
+            }
+
+            return new string[] { Settings.BaseUrl };
+        }
+
+        protected override GenericNewznabSettings GetDefaultBaseUrl(GenericNewznabSettings settings)
+        {
+            return settings;
+        }
+
+        public IndexerCapabilities GetCapabilitiesFromSettings()
+        {
+            var caps = new IndexerCapabilities();
+
+            if (Definition == null || Settings == null || Settings.Categories == null)
+            {
+                return caps;
+            }
+
+            foreach (var category in Settings.Categories)
+            {
+                caps.Categories.AddCategoryMapping(category.Name, category);
+            }
+
+            return caps;
+        }
+
+        public override IndexerCapabilities GetCapabilities()
+        {
+            // Newznab uses different Caps per site, so we need to cache them to db on first indexer add to prevent issues with loading UI and pulling caps every time.
+            return _capabilitiesProvider.GetCapabilities(Settings, Definition);
+        }
+
+        public override IEnumerable<ProviderDefinition> DefaultDefinitions
+        {
+            get
+            {
+                yield return GetDefinition("Generic Newznab", GetSettings(""));
+            }
+        }
+
+        public GenericNewznab(INewznabCapabilitiesProvider capabilitiesProvider, IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)
+            : base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger)
+        {
+            _capabilitiesProvider = capabilitiesProvider;
+        }
+
+        private IndexerDefinition GetDefinition(string name, GenericNewznabSettings settings)
+        {
+            return new IndexerDefinition
+            {
+                Enable = true,
+                Name = name,
+                Implementation = GetType().Name,
+                Settings = settings,
+                Protocol = DownloadProtocol.Usenet,
+                Privacy = IndexerPrivacy.Private,
+                SupportsRss = SupportsRss,
+                SupportsSearch = SupportsSearch,
+                SupportsRedirect = SupportsRedirect,
+                Capabilities = Capabilities
+            };
+        }
+
+        private GenericNewznabSettings GetSettings(string url, string apiPath = null)
+        {
+            var settings = new GenericNewznabSettings { BaseUrl = url };
+
+            if (apiPath.IsNotNullOrWhiteSpace())
+            {
+                settings.ApiPath = apiPath;
+            }
+
+            return settings;
+        }
+
+        protected override async Task Test(List<ValidationFailure> failures)
+        {
+            await base.Test(failures);
+            if (failures.HasErrors())
+            {
+                return;
+            }
+
+            failures.AddIfNotNull(TestCapabilities());
+        }
+
+        protected static List<int> CategoryIds(IndexerCapabilitiesCategories categories)
+        {
+            var l = categories.GetTorznabCategoryTree().Select(c => c.Id).ToList();
+
+            return l;
+        }
+
+        protected virtual ValidationFailure TestCapabilities()
+        {
+            try
+            {
+                var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+
+                if (capabilities.SearchParams != null && capabilities.SearchParams.Contains(SearchParam.Q))
+                {
+                    return null;
+                }
+
+                if (capabilities.MovieSearchParams != null &&
+                    new[] { MovieSearchParam.Q, MovieSearchParam.ImdbId }.Any(v => capabilities.MovieSearchParams.Contains(v)))
+                {
+                    return null;
+                }
+
+                if (capabilities.TvSearchParams != null &&
+                    new[] { TvSearchParam.Q, TvSearchParam.TvdbId, TvSearchParam.TmdbId, TvSearchParam.RId }.Any(v => capabilities.TvSearchParams.Contains(v)) &&
+                    new[] { TvSearchParam.Season, TvSearchParam.Ep }.All(v => capabilities.TvSearchParams.Contains(v)))
+                {
+                    return null;
+                }
+
+                if (capabilities.MusicSearchParams != null &&
+                    new[] { MusicSearchParam.Q, MusicSearchParam.Artist, MusicSearchParam.Album }.Any(v => capabilities.MusicSearchParams.Contains(v)))
+                {
+                    return null;
+                }
+
+                if (capabilities.BookSearchParams != null &&
+                    new[] { BookSearchParam.Q, BookSearchParam.Author, BookSearchParam.Title }.Any(v => capabilities.BookSearchParams.Contains(v)))
+                {
+                    return null;
+                }
+
+                return new ValidationFailure(string.Empty, "This indexer does not support searching for tv, music, or movies :(. Tell your indexer staff to enable this or force add the indexer by disabling search, adding the indexer and then enabling it again.");
+            }
+            catch (Exception ex)
+            {
+                _logger.Warn(ex, "Unable to connect to indexer: " + ex.Message);
+
+                return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details");
+            }
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRequestGenerator.cs

@@ -0,0 +1,292 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.Linq;
+using DryIoc;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Parser;
+using NzbDrone.Core.ThingiProvider;
+
+namespace NzbDrone.Core.Indexers.Newznab
+{
+    public class GenericNewznabRequestGenerator : IIndexerRequestGenerator
+    {
+        private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
+        public int MaxPages { get; set; }
+        public int PageSize { get; set; }
+        public GenericNewznabSettings Settings { get; set; }
+        public ProviderDefinition Definition { get; set; }
+
+        public GenericNewznabRequestGenerator(INewznabCapabilitiesProvider capabilitiesProvider)
+        {
+            _capabilitiesProvider = capabilitiesProvider;
+
+            MaxPages = 30;
+            PageSize = 100;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
+        {
+            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+
+            var pageableRequests = new IndexerPageableRequestChain();
+            var parameters = new NameValueCollection();
+
+            if (searchCriteria.TmdbId.HasValue && capabilities.MovieSearchTmdbAvailable)
+            {
+                parameters.Add("tmdbid", searchCriteria.TmdbId.Value.ToString());
+            }
+
+            if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace() && capabilities.MovieSearchImdbAvailable)
+            {
+                parameters.Add("imdbid", searchCriteria.ImdbId);
+            }
+
+            if (searchCriteria.TraktId.HasValue && capabilities.MovieSearchTraktAvailable)
+            {
+                parameters.Add("traktid", searchCriteria.TraktId.ToString());
+            }
+
+            //Workaround issue with Sphinx search returning garbage results on some indexers. If we don't use id parameters, fallback to t=search
+            if (parameters.Count == 0)
+            {
+                searchCriteria.SearchType = "search";
+
+                if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable)
+                {
+                    parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+                }
+            }
+            else
+            {
+                if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.MovieSearchAvailable)
+                {
+                    parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+                }
+            }
+
+            pageableRequests.Add(GetPagedRequests(searchCriteria,
+                parameters));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
+        {
+            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+
+            var pageableRequests = new IndexerPageableRequestChain();
+            var parameters = new NameValueCollection();
+
+            if (searchCriteria.Artist.IsNotNullOrWhiteSpace() && capabilities.MusicSearchArtistAvailable)
+            {
+                parameters.Add("artist", searchCriteria.Artist);
+            }
+
+            if (searchCriteria.Album.IsNotNullOrWhiteSpace() && capabilities.MusicSearchAlbumAvailable)
+            {
+                parameters.Add("album", searchCriteria.Album);
+            }
+
+            //Workaround issue with Sphinx search returning garbage results on some indexers. If we don't use id parameters, fallback to t=search
+            if (parameters.Count == 0)
+            {
+                searchCriteria.SearchType = "search";
+
+                if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable)
+                {
+                    parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+                }
+            }
+            else
+            {
+                if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.MusicSearchAvailable)
+                {
+                    parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+                }
+            }
+
+            pageableRequests.Add(GetPagedRequests(searchCriteria,
+                parameters));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
+        {
+            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+
+            var pageableRequests = new IndexerPageableRequestChain();
+            var parameters = new NameValueCollection();
+
+            if (searchCriteria.TvdbId.HasValue && capabilities.TvSearchTvdbAvailable)
+            {
+                parameters.Add("tvdbid", searchCriteria.TvdbId.Value.ToString());
+            }
+
+            if (searchCriteria.TmdbId.HasValue && capabilities.TvSearchTvdbAvailable)
+            {
+                parameters.Add("tmdbid", searchCriteria.TvdbId.Value.ToString());
+            }
+
+            if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace() && capabilities.TvSearchImdbAvailable)
+            {
+                parameters.Add("imdbid", searchCriteria.ImdbId);
+            }
+
+            if (searchCriteria.TvMazeId.HasValue && capabilities.TvSearchTvMazeAvailable)
+            {
+                parameters.Add("tvmazeid", searchCriteria.TvMazeId.ToString());
+            }
+
+            if (searchCriteria.RId.HasValue && capabilities.TvSearchTvRageAvailable)
+            {
+                parameters.Add("rid", searchCriteria.RId.ToString());
+            }
+
+            if (searchCriteria.Season.HasValue && capabilities.TvSearchSeasonAvailable)
+            {
+                parameters.Add("season", NewznabifySeasonNumber(searchCriteria.Season.Value));
+            }
+
+            if (searchCriteria.Episode.IsNotNullOrWhiteSpace() && capabilities.TvSearchEpAvailable)
+            {
+                parameters.Add("ep", searchCriteria.Episode);
+            }
+
+            //Workaround issue with Sphinx search returning garbage results on some indexers. If we don't use id parameters, fallback to t=search
+            if (parameters.Count == 0)
+            {
+                searchCriteria.SearchType = "search";
+
+                if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable)
+                {
+                    parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+                }
+            }
+            else
+            {
+                if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.TvSearchAvailable)
+                {
+                    parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+                }
+            }
+
+            pageableRequests.Add(GetPagedRequests(searchCriteria,
+                parameters));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
+        {
+            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+
+            var pageableRequests = new IndexerPageableRequestChain();
+            var parameters = new NameValueCollection();
+
+            if (searchCriteria.Author.IsNotNullOrWhiteSpace() && capabilities.BookSearchAuthorAvailable)
+            {
+                parameters.Add("author", searchCriteria.Author);
+            }
+
+            if (searchCriteria.Title.IsNotNullOrWhiteSpace() && capabilities.BookSearchTitleAvailable)
+            {
+                parameters.Add("title", searchCriteria.Title);
+            }
+
+            //Workaround issue with Sphinx search returning garbage results on some indexers. If we don't use id parameters, fallback to t=search
+            if (parameters.Count == 0)
+            {
+                searchCriteria.SearchType = "search";
+
+                if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable)
+                {
+                    parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+                }
+            }
+            else
+            {
+                if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.BookSearchAvailable)
+                {
+                    parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+                }
+            }
+
+            pageableRequests.Add(GetPagedRequests(searchCriteria,
+                parameters));
+
+            return pageableRequests;
+        }
+
+        public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
+        {
+            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+            var pageableRequests = new IndexerPageableRequestChain();
+
+            var parameters = new NameValueCollection();
+
+            if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable)
+            {
+                parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
+            }
+
+            pageableRequests.Add(GetPagedRequests(searchCriteria, parameters));
+
+            return pageableRequests;
+        }
+
+        private IEnumerable<IndexerRequest> GetPagedRequests(SearchCriteriaBase searchCriteria, NameValueCollection parameters)
+        {
+            var baseUrl = string.Format("{0}{1}?t={2}&extended=1", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchCriteria.SearchType);
+            var categories = searchCriteria.Categories;
+
+            if (categories != null && categories.Any())
+            {
+                var categoriesQuery = string.Join(",", categories.Distinct());
+                baseUrl += string.Format("&cat={0}", categoriesQuery);
+            }
+
+            if (Settings.AdditionalParameters.IsNotNullOrWhiteSpace())
+            {
+                baseUrl += Settings.AdditionalParameters;
+            }
+
+            if (Settings.ApiKey.IsNotNullOrWhiteSpace())
+            {
+                baseUrl += "&apikey=" + Settings.ApiKey;
+            }
+
+            if (searchCriteria.Limit.HasValue)
+            {
+                parameters.Add("limit", searchCriteria.Limit.ToString());
+            }
+
+            if (searchCriteria.Offset.HasValue)
+            {
+                parameters.Add("offset", searchCriteria.Offset.ToString());
+            }
+
+            var request = new IndexerRequest(string.Format("{0}&{1}", baseUrl, parameters.GetQueryString()), HttpAccept.Rss);
+            request.HttpRequest.AllowAutoRedirect = true;
+
+            yield return request;
+        }
+
+        private static string NewsnabifyTitle(string title)
+        {
+            return title.Replace("+", "%20");
+        }
+
+        // Temporary workaround for NNTMux considering season=0 -> null. '00' should work on existing newznab indexers.
+        private static string NewznabifySeasonNumber(int seasonNumber)
+        {
+            return seasonNumber == 0 ? "00" : seasonNumber.ToString();
+        }
+
+        public Func<IDictionary<string, string>> GetCookies { get; set; }
+        public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRssParser.cs

@@ -8,17 +8,17 @@ using NzbDrone.Core.Parser.Model;
 
 namespace NzbDrone.Core.Indexers.Newznab
 {
-    public class NewznabRssParser : RssParser
+    public class GenericNewznabRssParser : RssParser
     {
         public const string ns = "{http://www.newznab.com/DTD/2010/feeds/attributes/}";
 
-        private readonly NewznabSettings _settings;
+        private readonly List<IndexerCategory> _categories;
 
-        public NewznabRssParser(NewznabSettings settings)
+        public GenericNewznabRssParser(List<IndexerCategory> categories)
         {
             PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes;
             UseEnclosureUrl = true;
-            _settings = settings;
+            _categories = categories;
         }
 
         public static void CheckError(XDocument xdoc, IndexerResponse indexerResponse)
@@ -125,7 +125,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             {
                 if (int.TryParse(cat, out var intCategory))
                 {
-                    var indexerCat = _settings.Categories?.FirstOrDefault(c => c.Id == intCategory) ?? null;
+                    var indexerCat = _categories?.FirstOrDefault(c => c.Id == intCategory) ?? null;
 
                     if (indexerCat != null)
                     {

src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabSettings.cs

@@ -0,0 +1,91 @@
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.RegularExpressions;
+using FluentValidation;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Validation;
+
+namespace NzbDrone.Core.Indexers.Newznab
+{
+    public class GenericNewznabSettingsValidator : AbstractValidator<GenericNewznabSettings>
+    {
+        private static readonly string[] ApiKeyWhiteList =
+        {
+            "nzbs.org",
+            "nzb.su",
+            "dognzb.cr",
+            "nzbplanet.net",
+            "nzbid.org",
+            "nzbndx.com",
+            "nzbindex.in"
+        };
+
+        private static bool ShouldHaveApiKey(GenericNewznabSettings settings)
+        {
+            if (settings.BaseUrl == null)
+            {
+                return false;
+            }
+
+            return ApiKeyWhiteList.Any(c => settings.BaseUrl.ToLowerInvariant().Contains(c));
+        }
+
+        private static readonly Regex AdditionalParametersRegex = new Regex(@"(&.+?\=.+?)+", RegexOptions.Compiled);
+
+        public GenericNewznabSettingsValidator()
+        {
+            RuleFor(c => c.BaseUrl).ValidRootUrl();
+            RuleFor(c => c.ApiPath).ValidUrlBase("/api");
+            RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
+            RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
+                                                .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());
+
+            RuleFor(c => c.VipExpiration).Must(c => c.IsValidDate())
+                                         .When(c => c.VipExpiration.IsNotNullOrWhiteSpace())
+                                         .WithMessage("Correctly formatted date is required");
+
+            RuleFor(c => c.VipExpiration).Must(c => c.IsFutureDate())
+                                         .When(c => c.VipExpiration.IsNotNullOrWhiteSpace())
+                                         .WithMessage("Must be a future date");
+        }
+    }
+
+    public class GenericNewznabSettings : IIndexerSettings
+    {
+        private static readonly GenericNewznabSettingsValidator Validator = new GenericNewznabSettingsValidator();
+
+        public GenericNewznabSettings()
+        {
+            ApiPath = "/api";
+            VipExpiration = "";
+        }
+
+        [FieldDefinition(0, Label = "URL")]
+        public string BaseUrl { get; set; }
+
+        [FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)]
+        public string ApiPath { get; set; }
+
+        [FieldDefinition(2, Label = "API Key", HelpText = "Site API Key", Privacy = PrivacyLevel.ApiKey)]
+        public string ApiKey { get; set; }
+
+        [FieldDefinition(5, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)]
+        public string AdditionalParameters { get; set; }
+
+        [FieldDefinition(6, Label = "VIP Expiration", HelpText = "Enter date (yyyy-mm-dd) for VIP Expiration or blank, Prowlarr will notify 1 week from expiration of VIP")]
+        public string VipExpiration { get; set; }
+
+        [FieldDefinition(7)]
+        public IndexerBaseSettings BaseSettings { get; set; } = new IndexerBaseSettings();
+
+        public List<IndexerCategory> Categories { get; set; }
+
+        // Field 8 is used by TorznabSettings MinimumSeeders
+        // If you need to add another field here, update TorznabSettings as well and this comment
+        public virtual NzbDroneValidationResult Validate()
+        {
+            return new NzbDroneValidationResult(Validator.Validate(this));
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs

@@ -5,9 +5,10 @@ using System.Threading.Tasks;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Extensions;
-using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Download;
+using NzbDrone.Core.Indexers.Cardigann;
+using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider;
 using NzbDrone.Core.Validation;
@@ -16,10 +17,10 @@ namespace NzbDrone.Core.Indexers.Newznab
 {
     public class Newznab : UsenetIndexerBase<NewznabSettings>
     {
-        private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
+        private readonly IIndexerDefinitionUpdateService _definitionService;
 
         public override string Name => "Newznab";
-        public override string[] IndexerUrls => GetBaseUrlFromSettings();
+        public override string[] IndexerUrls => new string[] { "" };
         public override string Description => "Newznab is an API search specification for Usenet";
         public override bool FollowRedirect => true;
         public override bool SupportsRedirect => true;
@@ -27,130 +28,72 @@ namespace NzbDrone.Core.Indexers.Newznab
         public override DownloadProtocol Protocol => DownloadProtocol.Usenet;
         public override IndexerPrivacy Privacy => IndexerPrivacy.Private;
 
-        public override IndexerCapabilities Capabilities { get => GetCapabilitiesFromSettings(); protected set => base.Capabilities = value; }
-
-        public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings, Definition).LimitsDefault.Value;
-
         public override IIndexerRequestGenerator GetRequestGenerator()
         {
-            return new NewznabRequestGenerator(_capabilitiesProvider)
+            var defFile = _definitionService.GetCachedDefinition(Settings.DefinitionFile);
+
+            return new NewznabRequestGenerator()
             {
                 PageSize = PageSize,
-                Settings = Settings
+                Settings = Settings,
+                Definition = defFile
             };
         }
 
         public override IParseIndexerResponse GetParser()
         {
-            return new NewznabRssParser(Settings);
+            var defFile = _definitionService.GetCachedDefinition(Settings.DefinitionFile);
-        }
+            var capabilities = new IndexerCapabilities();
-
+            capabilities.ParseYmlSearchModes(defFile.Caps.Modes);
-        public string[] GetBaseUrlFromSettings()
+            capabilities.SupportsRawSearch = defFile.Caps.Allowrawsearch;
-        {
+            capabilities.MapYmlCategories(defFile);
-            var baseUrl = "";
-
-            if (Definition == null || Settings == null || Settings.Categories == null)
-            {
-                return new string[] { baseUrl };
-            }
-
-            return new string[] { Settings.BaseUrl };
-        }
-
-        protected override NewznabSettings GetDefaultBaseUrl(NewznabSettings settings)
-        {
-            return settings;
-        }
-
-        public IndexerCapabilities GetCapabilitiesFromSettings()
-        {
-            var caps = new IndexerCapabilities();
-
-            if (Definition == null || Settings == null || Settings.Categories == null)
-            {
-                return caps;
-            }
-
-            foreach (var category in Settings.Categories)
-            {
-                caps.Categories.AddCategoryMapping(category.Name, category);
-            }
-
-            return caps;
-        }
 
-        public override IndexerCapabilities GetCapabilities()
+            return new GenericNewznabRssParser(capabilities.Categories.GetTorznabCategoryList());
-        {
-            // Newznab uses different Caps per site, so we need to cache them to db on first indexer add to prevent issues with loading UI and pulling caps every time.
-            return _capabilitiesProvider.GetCapabilities(Settings, Definition);
         }
 
         public override IEnumerable<ProviderDefinition> DefaultDefinitions
         {
             get
             {
-                yield return GetDefinition("abNZB", GetSettings("https://abnzb.com"));
+                foreach (var def in _definitionService.AllForImplementation(GetType().Name))
-                yield return GetDefinition("altHUB", GetSettings("https://api.althub.co.za"));
+                {
-                yield return GetDefinition("AnimeTosho (Usenet)", GetSettings("https://feed.animetosho.org"));
+                    yield return GetDefinition(def);
-                yield return GetDefinition("DOGnzb", GetSettings("https://api.dognzb.cr"));
+                }
-                yield return GetDefinition("DrunkenSlug", GetSettings("https://drunkenslug.com"));
-                yield return GetDefinition("GingaDADDY", GetSettings("https://www.gingadaddy.com"));
-                yield return GetDefinition("Miatrix", GetSettings("https://www.miatrix.com"));
-                yield return GetDefinition("Newz-Complex", GetSettings("https://newz-complex.org/www"));
-                yield return GetDefinition("Newz69", GetSettings("https://newz69.keagaming.com"));
-                yield return GetDefinition("NinjaCentral", GetSettings("https://ninjacentral.co.za"));
-                yield return GetDefinition("Nzb.su", GetSettings("https://api.nzb.su"));
-                yield return GetDefinition("NZBCat", GetSettings("https://nzb.cat"));
-                yield return GetDefinition("NZBFinder", GetSettings("https://nzbfinder.ws"));
-                yield return GetDefinition("NZBgeek", GetSettings("https://api.nzbgeek.info"));
-                yield return GetDefinition("NzbNoob", GetSettings("https://www.nzbnoob.com"));
-                yield return GetDefinition("NZBNDX", GetSettings("https://www.nzbndx.com"));
-                yield return GetDefinition("NzbPlanet", GetSettings("https://api.nzbplanet.net"));
-                yield return GetDefinition("NZBStars", GetSettings("https://nzbstars.com"));
-                yield return GetDefinition("OZnzb", GetSettings("https://api.oznzb.com"));
-                yield return GetDefinition("SimplyNZBs", GetSettings("https://simplynzbs.com"));
-                yield return GetDefinition("SpotNZB", GetSettings("https://spotnzb.xyz"));
-                yield return GetDefinition("Tabula Rasa", GetSettings("https://www.tabula-rasa.pw", apiPath: @"/api/v1/api"));
-                yield return GetDefinition("Usenet Crawler", GetSettings("https://www.usenet-crawler.com"));
-                yield return GetDefinition("Generic Newznab", GetSettings(""));
             }
         }
 
-        public Newznab(INewznabCapabilitiesProvider capabilitiesProvider, IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)
+        public Newznab(IIndexerDefinitionUpdateService definitionService, IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger)
             : base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger)
         {
-            _capabilitiesProvider = capabilitiesProvider;
+            _definitionService = definitionService;
         }
 
-        private IndexerDefinition GetDefinition(string name, NewznabSettings settings)
+        private IndexerDefinition GetDefinition(IndexerMetaDefinition definition)
         {
             return new IndexerDefinition
             {
                 Enable = true,
-                Name = name,
+                Name = definition.Name,
+                Language = definition.Language,
+                Description = definition.Description,
                 Implementation = GetType().Name,
-                Settings = settings,
+                IndexerUrls = definition.Links.ToArray(),
+                LegacyUrls = definition.Legacylinks.ToArray(),
+                Settings = new NewznabSettings { DefinitionFile = definition.File },
                 Protocol = DownloadProtocol.Usenet,
-                Privacy = IndexerPrivacy.Private,
+                Privacy = definition.Type switch
+                {
+                    "private" => IndexerPrivacy.Private,
+                    "public" => IndexerPrivacy.Public,
+                    _ => IndexerPrivacy.SemiPrivate
+                },
                 SupportsRss = SupportsRss,
                 SupportsSearch = SupportsSearch,
                 SupportsRedirect = SupportsRedirect,
-                Capabilities = Capabilities
+                Capabilities = new IndexerCapabilities()
             };
         }
 
-        private NewznabSettings GetSettings(string url, string apiPath = null)
-        {
-            var settings = new NewznabSettings { BaseUrl = url };
-
-            if (apiPath.IsNotNullOrWhiteSpace())
-            {
-                settings.ApiPath = apiPath;
-            }
-
-            return settings;
-        }
-
         protected override async Task Test(List<ValidationFailure> failures)
         {
             await base.Test(failures);
@@ -158,61 +101,21 @@ namespace NzbDrone.Core.Indexers.Newznab
             {
                 return;
             }
-
-            failures.AddIfNotNull(TestCapabilities());
         }
 
-        protected static List<int> CategoryIds(IndexerCapabilitiesCategories categories)
+        public override object RequestAction(string action, IDictionary<string, string> query)
         {
-            var l = categories.GetTorznabCategoryTree().Select(c => c.Id).ToList();
+            if (action == "getUrls")
-
-            return l;
-        }
-
-        protected virtual ValidationFailure TestCapabilities()
-        {
-            try
             {
-                var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+                var devices = ((IndexerDefinition)Definition).IndexerUrls;
-
-                if (capabilities.SearchParams != null && capabilities.SearchParams.Contains(SearchParam.Q))
-                {
-                    return null;
-                }
-
-                if (capabilities.MovieSearchParams != null &&
-                    new[] { MovieSearchParam.Q, MovieSearchParam.ImdbId }.Any(v => capabilities.MovieSearchParams.Contains(v)))
-                {
-                    return null;
-                }
-
-                if (capabilities.TvSearchParams != null &&
-                    new[] { TvSearchParam.Q, TvSearchParam.TvdbId, TvSearchParam.TmdbId, TvSearchParam.RId }.Any(v => capabilities.TvSearchParams.Contains(v)) &&
-                    new[] { TvSearchParam.Season, TvSearchParam.Ep }.All(v => capabilities.TvSearchParams.Contains(v)))
-                {
-                    return null;
-                }
 
-                if (capabilities.MusicSearchParams != null &&
+                return new
-                    new[] { MusicSearchParam.Q, MusicSearchParam.Artist, MusicSearchParam.Album }.Any(v => capabilities.MusicSearchParams.Contains(v)))
                 {
-                    return null;
+                    options = devices.Select(d => new { Value = d, Name = d })
-                }
+                };
-
-                if (capabilities.BookSearchParams != null &&
-                    new[] { BookSearchParam.Q, BookSearchParam.Author, BookSearchParam.Title }.Any(v => capabilities.BookSearchParams.Contains(v)))
-                {
-                    return null;
-                }
-
-                return new ValidationFailure(string.Empty, "This indexer does not support searching for tv, music, or movies :(. Tell your indexer staff to enable this or force add the indexer by disabling search, adding the indexer and then enabling it again.");
             }
-            catch (Exception ex)
-            {
-                _logger.Warn(ex, "Unable to connect to indexer: " + ex.Message);
 
-                return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details");
+            return null;
-            }
         }
     }
 }

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs

@@ -14,7 +14,7 @@ namespace NzbDrone.Core.Indexers.Newznab
 {
     public interface INewznabCapabilitiesProvider
     {
-        IndexerCapabilities GetCapabilities(NewznabSettings settings, ProviderDefinition definition);
+        IndexerCapabilities GetCapabilities(GenericNewznabSettings settings, ProviderDefinition definition);
     }
 
     public class NewznabCapabilitiesProvider : INewznabCapabilitiesProvider
@@ -30,7 +30,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             _logger = logger;
         }
 
-        public IndexerCapabilities GetCapabilities(NewznabSettings indexerSettings, ProviderDefinition definition)
+        public IndexerCapabilities GetCapabilities(GenericNewznabSettings indexerSettings, ProviderDefinition definition)
         {
             var key = indexerSettings.ToJson();
             var capabilities = _capabilitiesCache.Get(key, () => FetchCapabilities(indexerSettings, definition), TimeSpan.FromDays(7));
@@ -38,7 +38,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             return capabilities;
         }
 
-        private IndexerCapabilities FetchCapabilities(NewznabSettings indexerSettings, ProviderDefinition definition)
+        private IndexerCapabilities FetchCapabilities(GenericNewznabSettings indexerSettings, ProviderDefinition definition)
         {
             var capabilities = new IndexerCapabilities();
 
@@ -96,7 +96,7 @@ namespace NzbDrone.Core.Indexers.Newznab
                 throw new XmlException("Invalid XML").WithData(response);
             }
 
-            NewznabRssParser.CheckError(xDoc, new IndexerResponse(new IndexerRequest(response.Request), response));
+            GenericNewznabRssParser.CheckError(xDoc, new IndexerResponse(new IndexerRequest(response.Request), response));
 
             var xmlRoot = xDoc.Element("caps");
 

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs

@@ -5,6 +5,7 @@ using System.Linq;
 using DryIoc;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
+using NzbDrone.Core.Indexers.Cardigann;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Parser;
 using NzbDrone.Core.ThingiProvider;
@@ -13,23 +14,20 @@ namespace NzbDrone.Core.Indexers.Newznab
 {
     public class NewznabRequestGenerator : IIndexerRequestGenerator
     {
-        private readonly INewznabCapabilitiesProvider _capabilitiesProvider;
         public int MaxPages { get; set; }
         public int PageSize { get; set; }
         public NewznabSettings Settings { get; set; }
-        public ProviderDefinition Definition { get; set; }
+        public CardigannDefinition Definition { get; set; }
 
-        public NewznabRequestGenerator(INewznabCapabilitiesProvider capabilitiesProvider)
+        public NewznabRequestGenerator()
         {
-            _capabilitiesProvider = capabilitiesProvider;
-
             MaxPages = 30;
             PageSize = 100;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria)
         {
-            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+            var capabilities = GetCapabilities();
 
             var pageableRequests = new IndexerPageableRequestChain();
             var parameters = new NameValueCollection();
@@ -67,15 +65,14 @@ namespace NzbDrone.Core.Indexers.Newznab
                 }
             }
 
-            pageableRequests.Add(GetPagedRequests(searchCriteria,
+            pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters));
-                parameters));
 
             return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria)
         {
-            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+            var capabilities = GetCapabilities();
 
             var pageableRequests = new IndexerPageableRequestChain();
             var parameters = new NameValueCollection();
@@ -108,15 +105,14 @@ namespace NzbDrone.Core.Indexers.Newznab
                 }
             }
 
-            pageableRequests.Add(GetPagedRequests(searchCriteria,
+            pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters));
-                parameters));
 
             return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria)
         {
-            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+            var capabilities = GetCapabilities();
 
             var pageableRequests = new IndexerPageableRequestChain();
             var parameters = new NameValueCollection();
@@ -174,15 +170,14 @@ namespace NzbDrone.Core.Indexers.Newznab
                 }
             }
 
-            pageableRequests.Add(GetPagedRequests(searchCriteria,
+            pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters));
-                parameters));
 
             return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria)
         {
-            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+            var capabilities = GetCapabilities();
 
             var pageableRequests = new IndexerPageableRequestChain();
             var parameters = new NameValueCollection();
@@ -215,15 +210,15 @@ namespace NzbDrone.Core.Indexers.Newznab
                 }
             }
 
-            pageableRequests.Add(GetPagedRequests(searchCriteria,
+            pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters));
-                parameters));
 
             return pageableRequests;
         }
 
         public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria)
         {
-            var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition);
+            var capabilities = GetCapabilities();
+
             var pageableRequests = new IndexerPageableRequestChain();
 
             var parameters = new NameValueCollection();
@@ -233,15 +228,15 @@ namespace NzbDrone.Core.Indexers.Newznab
                 parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm));
             }
 
-            pageableRequests.Add(GetPagedRequests(searchCriteria, parameters));
+            pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters));
 
             return pageableRequests;
         }
 
-        private IEnumerable<IndexerRequest> GetPagedRequests(SearchCriteriaBase searchCriteria, NameValueCollection parameters)
+        private IEnumerable<IndexerRequest> GetPagedRequests(SearchCriteriaBase searchCriteria, IndexerCapabilities capabilities, NameValueCollection parameters)
         {
-            var baseUrl = string.Format("{0}{1}?t={2}&extended=1", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchCriteria.SearchType);
+            var baseUrl = string.Format("{0}{1}?t={2}&extended=1", ResolveSiteLink().TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchCriteria.SearchType);
-            var categories = searchCriteria.Categories;
+            var categories = capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories);
 
             if (categories != null && categories.Any())
             {
@@ -286,6 +281,34 @@ namespace NzbDrone.Core.Indexers.Newznab
             return seasonNumber == 0 ? "00" : seasonNumber.ToString();
         }
 
+        protected string ResolveSiteLink()
+        {
+            var settingsBaseUrl = Settings?.BaseUrl;
+            var defaultLink = Definition.Links.First();
+
+            if (settingsBaseUrl == null)
+            {
+                return defaultLink;
+            }
+
+            if (Definition?.Legacylinks?.Contains(settingsBaseUrl) ?? false)
+            {
+                return defaultLink;
+            }
+
+            return settingsBaseUrl;
+        }
+
+        private IndexerCapabilities GetCapabilities()
+        {
+            var capabilities = new IndexerCapabilities();
+
+            capabilities.ParseYmlSearchModes(Definition.Caps.Modes);
+            capabilities.MapYmlCategories(Definition);
+
+            return capabilities;
+        }
+
         public Func<IDictionary<string, string>> GetCookies { get; set; }
         public Action<IDictionary<string, string>, DateTime?> CookiesUpdater { get; set; }
     }

src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs

@@ -4,40 +4,18 @@ using System.Text.RegularExpressions;
 using FluentValidation;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Annotations;
+using NzbDrone.Core.Indexers.Settings;
 using NzbDrone.Core.Validation;
 
 namespace NzbDrone.Core.Indexers.Newznab
 {
     public class NewznabSettingsValidator : AbstractValidator<NewznabSettings>
     {
-        private static readonly string[] ApiKeyWhiteList =
-        {
-            "nzbs.org",
-            "nzb.su",
-            "dognzb.cr",
-            "nzbplanet.net",
-            "nzbid.org",
-            "nzbndx.com",
-            "nzbindex.in"
-        };
-
-        private static bool ShouldHaveApiKey(NewznabSettings settings)
-        {
-            if (settings.BaseUrl == null)
-            {
-                return false;
-            }
-
-            return ApiKeyWhiteList.Any(c => settings.BaseUrl.ToLowerInvariant().Contains(c));
-        }
-
         private static readonly Regex AdditionalParametersRegex = new Regex(@"(&.+?\=.+?)+", RegexOptions.Compiled);
 
         public NewznabSettingsValidator()
         {
-            RuleFor(c => c.BaseUrl).ValidRootUrl();
             RuleFor(c => c.ApiPath).ValidUrlBase("/api");
-            RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey);
             RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex)
                                                 .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace());
 
@@ -51,7 +29,7 @@ namespace NzbDrone.Core.Indexers.Newznab
         }
     }
 
-    public class NewznabSettings : IIndexerSettings
+    public class NewznabSettings : IYmlIndexerSettings
     {
         private static readonly NewznabSettingsValidator Validator = new NewznabSettingsValidator();
 
@@ -61,7 +39,7 @@ namespace NzbDrone.Core.Indexers.Newznab
             VipExpiration = "";
         }
 
-        [FieldDefinition(0, Label = "URL")]
+        [FieldDefinition(0, Label = "Base Url", Type = FieldType.Select, SelectOptionsProviderAction = "getUrls", HelpText = "Select which baseurl Prowlarr will use for requests to the site")]
         public string BaseUrl { get; set; }
 
         [FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)]
@@ -76,6 +54,9 @@ namespace NzbDrone.Core.Indexers.Newznab
         [FieldDefinition(6, Label = "VIP Expiration", HelpText = "Enter date (yyyy-mm-dd) for VIP Expiration or blank, Prowlarr will notify 1 week from expiration of VIP")]
         public string VipExpiration { get; set; }
 
+        [FieldDefinition(0, Hidden = HiddenType.Hidden)]
+        public string DefinitionFile { get; set; }
+
         [FieldDefinition(7)]
         public IndexerBaseSettings BaseSettings { get; set; } = new IndexerBaseSettings();
 

src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs

@@ -32,7 +32,7 @@ namespace NzbDrone.Core.Indexers.Torznab
 
         public override IIndexerRequestGenerator GetRequestGenerator()
         {
-            return new NewznabRequestGenerator(_capabilitiesProvider)
+            return new GenericNewznabRequestGenerator(_capabilitiesProvider)
             {
                 PageSize = PageSize,
                 Settings = Settings

src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabSettings.cs

@@ -37,7 +37,7 @@ namespace NzbDrone.Core.Indexers.Torznab
         }
     }
 
-    public class TorznabSettings : NewznabSettings, ITorrentIndexerSettings
+    public class TorznabSettings : GenericNewznabSettings, ITorrentIndexerSettings
     {
         private static readonly TorznabSettingsValidator Validator = new TorznabSettingsValidator();
 

src/NzbDrone.Core/Indexers/IndexerCapabilities.cs

@@ -2,6 +2,8 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Xml.Linq;
+using DryIoc.ImTools;
+using NzbDrone.Core.Indexers.Cardigann;
 
 namespace NzbDrone.Core.Indexers
 {
@@ -127,7 +129,7 @@ namespace NzbDrone.Core.Indexers
             LimitsMax = 100;
         }
 
-        public void ParseCardigannSearchModes(Dictionary<string, List<string>> modes)
+        public void ParseYmlSearchModes(Dictionary<string, List<string>> modes)
         {
             if (modes == null || !modes.Any())
             {
@@ -169,6 +171,48 @@ namespace NzbDrone.Core.Indexers
             }
         }
 
+        public void MapYmlCategories(CardigannDefinition defFile)
+        {
+            if (defFile.Caps.Categories != null)
+            {
+                foreach (var category in defFile.Caps.Categories)
+                {
+                    var cat = NewznabStandardCategory.GetCatByName(category.Value);
+
+                    if (cat == null)
+                    {
+                        continue;
+                    }
+
+                    Categories.AddCategoryMapping(category.Key, cat);
+                }
+            }
+
+            if (defFile.Caps.Categorymappings != null)
+            {
+                foreach (var categorymapping in defFile.Caps.Categorymappings)
+                {
+                    IndexerCategory torznabCat = null;
+
+                    if (categorymapping.cat != null)
+                    {
+                        torznabCat = NewznabStandardCategory.GetCatByName(categorymapping.cat);
+                        if (torznabCat == null)
+                        {
+                            continue;
+                        }
+                    }
+
+                    Categories.AddCategoryMapping(categorymapping.id, torznabCat, categorymapping.desc);
+
+                    //if (categorymapping.Default)
+                    //{
+                    //    DefaultCategories.Add(categorymapping.id);
+                    //}
+                }
+            }
+        }
+
         public void ParseTvSearchParams(IEnumerable<string> paramsList)
         {
             if (paramsList == null)

src/NzbDrone.Core/Indexers/IndexerFactory.cs

@@ -7,6 +7,7 @@ using NLog;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Indexers.Cardigann;
 using NzbDrone.Core.Indexers.Newznab;
+using NzbDrone.Core.Indexers.Settings;
 using NzbDrone.Core.IndexerVersions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider;
@@ -50,11 +51,11 @@ namespace NzbDrone.Core.Indexers
 
             foreach (var definition in definitions)
             {
-                if (definition.Implementation == typeof(Cardigann.Cardigann).Name)
+                if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) != null)
                 {
                     try
                     {
-                        MapCardigannDefinition(definition);
+                        MapYmlDefinition(definition);
                     }
                     catch
                     {
@@ -73,11 +74,11 @@ namespace NzbDrone.Core.Indexers
         {
             var definition = base.Get(id);
 
-            if (definition.Implementation == typeof(Cardigann.Cardigann).Name)
+            if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) != null)
             {
                 try
                 {
-                    MapCardigannDefinition(definition);
+                    MapYmlDefinition(definition);
                 }
                 catch
                 {
@@ -93,9 +94,9 @@ namespace NzbDrone.Core.Indexers
             return base.Active().Where(c => c.Enable).ToList();
         }
 
-        private void MapCardigannDefinition(IndexerDefinition definition)
+        private void MapYmlDefinition(IndexerDefinition definition)
         {
-            var settings = (CardigannSettings)definition.Settings;
+            var settings = (IYmlIndexerSettings)definition.Settings;
             var defFile = _definitionService.GetCachedDefinition(settings.DefinitionFile);
             definition.ExtraFields = defFile.Settings;
 
@@ -121,51 +122,9 @@ namespace NzbDrone.Core.Indexers
                 _ => IndexerPrivacy.SemiPrivate
             };
             definition.Capabilities = new IndexerCapabilities();
-            definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes);
+            definition.Capabilities.ParseYmlSearchModes(defFile.Caps.Modes);
             definition.Capabilities.SupportsRawSearch = defFile.Caps.Allowrawsearch;
-            MapCardigannCategories(definition, defFile);
+            definition.Capabilities.MapYmlCategories(defFile);
-        }
-
-        private void MapCardigannCategories(IndexerDefinition def, CardigannDefinition defFile)
-        {
-            if (defFile.Caps.Categories != null)
-            {
-                foreach (var category in defFile.Caps.Categories)
-                {
-                    var cat = NewznabStandardCategory.GetCatByName(category.Value);
-
-                    if (cat == null)
-                    {
-                        continue;
-                    }
-
-                    def.Capabilities.Categories.AddCategoryMapping(category.Key, cat);
-                }
-            }
-
-            if (defFile.Caps.Categorymappings != null)
-            {
-                foreach (var categorymapping in defFile.Caps.Categorymappings)
-                {
-                    IndexerCategory torznabCat = null;
-
-                    if (categorymapping.cat != null)
-                    {
-                        torznabCat = NewznabStandardCategory.GetCatByName(categorymapping.cat);
-                        if (torznabCat == null)
-                        {
-                            continue;
-                        }
-                    }
-
-                    def.Capabilities.Categories.AddCategoryMapping(categorymapping.id, torznabCat, categorymapping.desc);
-
-                    //if (categorymapping.Default)
-                    //{
-                    //    DefaultCategories.Add(categorymapping.id);
-                    //}
-                }
-            }
         }
 
         public override IEnumerable<IndexerDefinition> GetDefaultDefinitions()
@@ -178,7 +137,7 @@ namespace NzbDrone.Core.Indexers
                 }
 
                 var definitions = provider.DefaultDefinitions
-                    .Where(v => v.Name != null && (v.Name != typeof(Cardigann.Cardigann).Name || v.Name != typeof(Newznab.Newznab).Name || v.Name != typeof(Torznab.Torznab).Name));
+                    .Where(v => v.Name != null && (v.Name != typeof(Cardigann.Cardigann).Name || v.Name != typeof(Newznab.Newznab).Name || v.Name != typeof(Newznab.GenericNewznab).Name || v.Name != typeof(Torznab.Torznab).Name));
 
                 foreach (IndexerDefinition definition in definitions)
                 {
@@ -203,7 +162,7 @@ namespace NzbDrone.Core.Indexers
             definition.SupportsRedirect = provider.SupportsRedirect;
 
             //We want to use the definition Caps and Privacy for Cardigann instead of the provider.
-            if (definition.Implementation != typeof(Cardigann.Cardigann).Name)
+            if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) == null)
             {
                 definition.IndexerUrls = provider.IndexerUrls;
                 definition.LegacyUrls = provider.LegacyUrls;
@@ -288,15 +247,15 @@ namespace NzbDrone.Core.Indexers
 
             SetProviderCharacteristics(provider, definition);
 
-            if (definition.Implementation == typeof(Newznab.Newznab).Name || definition.Implementation == typeof(Torznab.Torznab).Name)
+            if (definition.Implementation == typeof(Newznab.GenericNewznab).Name || definition.Implementation == typeof(Torznab.Torznab).Name)
             {
-                var settings = (NewznabSettings)definition.Settings;
+                var settings = (GenericNewznabSettings)definition.Settings;
                 settings.Categories = _newznabCapabilitiesProvider.GetCapabilities(settings, definition)?.Categories.GetTorznabCategoryList() ?? null;
             }
 
-            if (definition.Implementation == typeof(Cardigann.Cardigann).Name)
+            if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) != null)
             {
-                MapCardigannDefinition(definition);
+                MapYmlDefinition(definition);
             }
 
             return base.Create(definition);
@@ -310,13 +269,13 @@ namespace NzbDrone.Core.Indexers
 
             if (definition.Enable && (definition.Implementation == typeof(Newznab.Newznab).Name || definition.Implementation == typeof(Torznab.Torznab).Name))
             {
-                var settings = (NewznabSettings)definition.Settings;
+                var settings = (GenericNewznabSettings)definition.Settings;
                 settings.Categories = _newznabCapabilitiesProvider.GetCapabilities(settings, definition)?.Categories.GetTorznabCategoryList() ?? null;
             }
 
-            if (definition.Implementation == typeof(Cardigann.Cardigann).Name)
+            if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) != null)
             {
-                MapCardigannDefinition(definition);
+                MapYmlDefinition(definition);
             }
 
             base.Update(definition);

src/NzbDrone.Core/Indexers/Settings/IYmlIndexerSettings.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NzbDrone.Core.Indexers.Settings
+{
+    public interface IYmlIndexerSettings : IIndexerSettings
+    {
+        public string DefinitionFile { get; set; }
+    }
+}