From 0c45eb68fa860faa415ea62514be0739b078e573 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sat, 17 Dec 2022 17:53:37 -0600 Subject: [PATCH] Newznab to Yml --- .../NewznabCapabilitiesProviderFixture.cs | 4 +- .../NewznabTests/NewznabFixture.cs | 2 +- .../NewznabRequestGeneratorFixture.cs | 6 +- .../TorznabTests/TorznabFixture.cs | 2 +- .../Datastore/Migration/024_newznab_yml.cs | 14 + .../IndexerDefinitionUpdateService.cs | 27 +- .../IndexerMetaDefinition.cs} | 8 +- .../Definitions/Cardigann/Cardigann.cs | 4 +- .../Cardigann/CardigannSettings.cs | 2 +- .../Definitions/Newznab/GenericNewznab.cs | 195 ++++++++++++ .../Newznab/GenericNewznabRequestGenerator.cs | 292 ++++++++++++++++++ ...ssParser.cs => GenericNewznabRssParser.cs} | 10 +- .../Newznab/GenericNewznabSettings.cs | 91 ++++++ .../Indexers/Definitions/Newznab/Newznab.cs | 181 +++-------- .../Newznab/NewznabCapabilitiesProvider.cs | 8 +- .../Newznab/NewznabRequestGenerator.cs | 67 ++-- .../Definitions/Newznab/NewznabSettings.cs | 31 +- .../Indexers/Definitions/Torznab/Torznab.cs | 2 +- .../Definitions/Torznab/TorznabSettings.cs | 2 +- .../Indexers/IndexerCapabilities.cs | 46 ++- src/NzbDrone.Core/Indexers/IndexerFactory.cs | 77 ++--- .../Indexers/Settings/IYmlIndexerSettings.cs | 13 + 22 files changed, 806 insertions(+), 278 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/024_newznab_yml.cs rename src/NzbDrone.Core/{Indexers/Definitions/Cardigann/CardigannMetaDef.cs => IndexerVersions/IndexerMetaDefinition.cs} (76%) create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznab.cs create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRequestGenerator.cs rename src/NzbDrone.Core/Indexers/Definitions/Newznab/{NewznabRssParser.cs => GenericNewznabRssParser.cs} (96%) create mode 100644 src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabSettings.cs create mode 100644 src/NzbDrone.Core/Indexers/Settings/IYmlIndexerSettings.cs diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs index a5a7c7c14..8d3ed3597 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabCapabilitiesProviderFixture.cs @@ -15,14 +15,14 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests [TestFixture] public class NewznabCapabilitiesProviderFixture : CoreTest { - private NewznabSettings _settings; + private GenericNewznabSettings _settings; private IndexerDefinition _definition; private string _caps; [SetUp] public void SetUp() { - _settings = new NewznabSettings() + _settings = new GenericNewznabSettings() { BaseUrl = "http://indxer.local" }; diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs index 231c4bb56..2a3bc6383 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs @@ -34,7 +34,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests _caps = new IndexerCapabilities(); Mocker.GetMock() - .Setup(v => v.GetCapabilities(It.IsAny(), It.IsAny())) + .Setup(v => v.GetCapabilities(It.IsAny(), It.IsAny())) .Returns(_caps); } diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs index ead7db168..2c0f7a9a2 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabRequestGeneratorFixture.cs @@ -10,7 +10,7 @@ using NzbDrone.Core.Test.Framework; namespace NzbDrone.Core.Test.IndexerTests.NewznabTests { - public class NewznabRequestGeneratorFixture : CoreTest + public class NewznabRequestGeneratorFixture : CoreTest { private MovieSearchCriteria _movieSearchCriteria; private TvSearchCriteria _tvSearchCriteria; @@ -19,7 +19,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests [SetUp] public void SetUp() { - Subject.Settings = new NewznabSettings() + Subject.Settings = new GenericNewznabSettings() { BaseUrl = "http://127.0.0.1:1234/", ApiKey = "abcd", @@ -41,7 +41,7 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests _capabilities = new IndexerCapabilities(); Mocker.GetMock() - .Setup(v => v.GetCapabilities(It.IsAny(), It.IsAny())) + .Setup(v => v.GetCapabilities(It.IsAny(), It.IsAny())) .Returns(_capabilities); } diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs index 2c96236fc..2c8b335c9 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs @@ -35,7 +35,7 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests _caps = new IndexerCapabilities(); Mocker.GetMock() - .Setup(v => v.GetCapabilities(It.IsAny(), It.IsAny())) + .Setup(v => v.GetCapabilities(It.IsAny(), It.IsAny())) .Returns(_caps); } diff --git a/src/NzbDrone.Core/Datastore/Migration/024_newznab_yml.cs b/src/NzbDrone.Core/Datastore/Migration/024_newznab_yml.cs new file mode 100644 index 000000000..251fd6321 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/024_newznab_yml.cs @@ -0,0 +1,14 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(24)] + public class newznab_yml : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Update.Table("Indexers").Set(new { Implementation = "GenericNewznab", ConfigContract = "GenericNewznabSettings" }).Where(new { Implementation = "Newznab" }); + } + } +} diff --git a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs index e8e799ca9..08820b0f0 100644 --- a/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs +++ b/src/NzbDrone.Core/IndexerVersions/IndexerDefinitionUpdateService.cs @@ -19,7 +19,8 @@ namespace NzbDrone.Core.IndexerVersions { public interface IIndexerDefinitionUpdateService { - List All(); + List All(); + List AllForImplementation(string implementation); CardigannDefinition GetCachedDefinition(string fileKey); List GetBlocklist(); } @@ -28,8 +29,8 @@ namespace NzbDrone.Core.IndexerVersions { /* Update Service will fall back if version # does not exist for an indexer per Ta */ - private const string DEFINITION_BRANCH = "master"; - private const int DEFINITION_VERSION = 7; + private const string DEFINITION_BRANCH = "newznab-yml"; + private const int DEFINITION_VERSION = 8; //Used when moving yml to C# private readonly List _defintionBlocklist = new List() @@ -78,9 +79,9 @@ namespace NzbDrone.Core.IndexerVersions _logger = logger; } - public List All() + public List All() { - var indexerList = new List(); + var indexerList = new List(); try { @@ -88,7 +89,7 @@ namespace NzbDrone.Core.IndexerVersions try { var request = new HttpRequest($"https://indexers.prowlarr.com/{DEFINITION_BRANCH}/{DEFINITION_VERSION}"); - var response = _httpClient.Get>(request); + var response = _httpClient.Get>(request); indexerList = response.Resource.Where(i => !_defintionBlocklist.Contains(i.File)).ToList(); } catch @@ -111,6 +112,11 @@ namespace NzbDrone.Core.IndexerVersions return indexerList; } + public List AllForImplementation(string implementation) + { + return All().Where(d => d.Implementation == implementation.ToLower()).ToList(); + } + public CardigannDefinition GetCachedDefinition(string fileKey) { if (string.IsNullOrEmpty(fileKey)) @@ -128,7 +134,7 @@ namespace NzbDrone.Core.IndexerVersions return _defintionBlocklist; } - private List ReadDefinitionsFromDisk(List defs, string path, SearchOption options = SearchOption.TopDirectoryOnly) + private List ReadDefinitionsFromDisk(List defs, string path, SearchOption options = SearchOption.TopDirectoryOnly) { var indexerList = defs; @@ -145,7 +151,7 @@ namespace NzbDrone.Core.IndexerVersions try { var definitionString = File.ReadAllText(file.FullName); - var definition = _deserializer.Deserialize(definitionString); + var definition = _deserializer.Deserialize(definitionString); definition.File = Path.GetFileNameWithoutExtension(file.Name); @@ -243,6 +249,11 @@ namespace NzbDrone.Core.IndexerVersions definition.Login.Method = "form"; } + if (definition.Search == null) + { + definition.Search = new SearchBlock(); + } + if (definition.Search.Paths == null) { definition.Search.Paths = new List(); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs b/src/NzbDrone.Core/IndexerVersions/IndexerMetaDefinition.cs similarity index 76% rename from src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs rename to src/NzbDrone.Core/IndexerVersions/IndexerMetaDefinition.cs index 32ca9ee67..b9744fd00 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannMetaDef.cs +++ b/src/NzbDrone.Core/IndexerVersions/IndexerMetaDefinition.cs @@ -1,10 +1,11 @@ using System.Collections.Generic; +using NzbDrone.Core.Indexers.Cardigann; -namespace NzbDrone.Core.Indexers.Cardigann +namespace NzbDrone.Core.IndexerVersions { - public class CardigannMetaDefinition + public class IndexerMetaDefinition { - public CardigannMetaDefinition() + public IndexerMetaDefinition() { Legacylinks = new List(); } @@ -13,6 +14,7 @@ namespace NzbDrone.Core.Indexers.Cardigann public string File { get; set; } public string Name { get; set; } public string Description { get; set; } + public string Implementation { get; set; } public string Type { get; set; } public string Language { get; set; } public string Encoding { get; set; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index 34d0f8a99..ff8fb3c20 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -78,7 +78,7 @@ namespace NzbDrone.Core.Indexers.Cardigann { get { - foreach (var def in _definitionService.All()) + foreach (var def in _definitionService.AllForImplementation(GetType().Name)) { yield return GetDefinition(def); } @@ -98,7 +98,7 @@ namespace NzbDrone.Core.Indexers.Cardigann _generatorCache = cacheManager.GetRollingCache(GetType(), "CardigannGeneratorCache", TimeSpan.FromMinutes(5)); } - private IndexerDefinition GetDefinition(CardigannMetaDefinition definition) + private IndexerDefinition GetDefinition(IndexerMetaDefinition definition) { var defaultSettings = new List { diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs index 9b1ef4e15..14ff77ef5 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannSettings.cs @@ -13,7 +13,7 @@ namespace NzbDrone.Core.Indexers.Cardigann } } - public class CardigannSettings : NoAuthTorrentBaseSettings + public class CardigannSettings : NoAuthTorrentBaseSettings, IYmlIndexerSettings { private static readonly CardigannSettingsValidator Validator = new CardigannSettingsValidator(); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznab.cs new file mode 100644 index 000000000..8363d638b --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznab.cs @@ -0,0 +1,195 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using FluentValidation.Results; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Download; +using NzbDrone.Core.Messaging.Events; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.Newznab +{ + public class GenericNewznab : UsenetIndexerBase + { + private readonly INewznabCapabilitiesProvider _capabilitiesProvider; + + public override string Name => "Generic Newznab"; + public override string[] IndexerUrls => GetBaseUrlFromSettings(); + public override string Description => "Newznab is an API search specification for Usenet"; + public override bool FollowRedirect => true; + public override bool SupportsRedirect => true; + + public override DownloadProtocol Protocol => DownloadProtocol.Usenet; + public override IndexerPrivacy Privacy => IndexerPrivacy.Private; + + public override IndexerCapabilities Capabilities { get => GetCapabilitiesFromSettings(); protected set => base.Capabilities = value; } + + public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings, Definition).LimitsDefault.Value; + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new GenericNewznabRequestGenerator(_capabilitiesProvider) + { + PageSize = PageSize, + Settings = Settings + }; + } + + public override IParseIndexerResponse GetParser() + { + return new GenericNewznabRssParser(Settings.Categories); + } + + public string[] GetBaseUrlFromSettings() + { + var baseUrl = ""; + + if (Definition == null || Settings == null || Settings.Categories == null) + { + return new string[] { baseUrl }; + } + + return new string[] { Settings.BaseUrl }; + } + + protected override GenericNewznabSettings GetDefaultBaseUrl(GenericNewznabSettings settings) + { + return settings; + } + + public IndexerCapabilities GetCapabilitiesFromSettings() + { + var caps = new IndexerCapabilities(); + + if (Definition == null || Settings == null || Settings.Categories == null) + { + return caps; + } + + foreach (var category in Settings.Categories) + { + caps.Categories.AddCategoryMapping(category.Name, category); + } + + return caps; + } + + public override IndexerCapabilities GetCapabilities() + { + // Newznab uses different Caps per site, so we need to cache them to db on first indexer add to prevent issues with loading UI and pulling caps every time. + return _capabilitiesProvider.GetCapabilities(Settings, Definition); + } + + public override IEnumerable DefaultDefinitions + { + get + { + yield return GetDefinition("Generic Newznab", GetSettings("")); + } + } + + public GenericNewznab(INewznabCapabilitiesProvider capabilitiesProvider, IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger) + : base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger) + { + _capabilitiesProvider = capabilitiesProvider; + } + + private IndexerDefinition GetDefinition(string name, GenericNewznabSettings settings) + { + return new IndexerDefinition + { + Enable = true, + Name = name, + Implementation = GetType().Name, + Settings = settings, + Protocol = DownloadProtocol.Usenet, + Privacy = IndexerPrivacy.Private, + SupportsRss = SupportsRss, + SupportsSearch = SupportsSearch, + SupportsRedirect = SupportsRedirect, + Capabilities = Capabilities + }; + } + + private GenericNewznabSettings GetSettings(string url, string apiPath = null) + { + var settings = new GenericNewznabSettings { BaseUrl = url }; + + if (apiPath.IsNotNullOrWhiteSpace()) + { + settings.ApiPath = apiPath; + } + + return settings; + } + + protected override async Task Test(List failures) + { + await base.Test(failures); + if (failures.HasErrors()) + { + return; + } + + failures.AddIfNotNull(TestCapabilities()); + } + + protected static List CategoryIds(IndexerCapabilitiesCategories categories) + { + var l = categories.GetTorznabCategoryTree().Select(c => c.Id).ToList(); + + return l; + } + + protected virtual ValidationFailure TestCapabilities() + { + try + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + + if (capabilities.SearchParams != null && capabilities.SearchParams.Contains(SearchParam.Q)) + { + return null; + } + + if (capabilities.MovieSearchParams != null && + new[] { MovieSearchParam.Q, MovieSearchParam.ImdbId }.Any(v => capabilities.MovieSearchParams.Contains(v))) + { + return null; + } + + if (capabilities.TvSearchParams != null && + new[] { TvSearchParam.Q, TvSearchParam.TvdbId, TvSearchParam.TmdbId, TvSearchParam.RId }.Any(v => capabilities.TvSearchParams.Contains(v)) && + new[] { TvSearchParam.Season, TvSearchParam.Ep }.All(v => capabilities.TvSearchParams.Contains(v))) + { + return null; + } + + if (capabilities.MusicSearchParams != null && + new[] { MusicSearchParam.Q, MusicSearchParam.Artist, MusicSearchParam.Album }.Any(v => capabilities.MusicSearchParams.Contains(v))) + { + return null; + } + + if (capabilities.BookSearchParams != null && + new[] { BookSearchParam.Q, BookSearchParam.Author, BookSearchParam.Title }.Any(v => capabilities.BookSearchParams.Contains(v))) + { + return null; + } + + return new ValidationFailure(string.Empty, "This indexer does not support searching for tv, music, or movies :(. Tell your indexer staff to enable this or force add the indexer by disabling search, adding the indexer and then enabling it again."); + } + catch (Exception ex) + { + _logger.Warn(ex, "Unable to connect to indexer: " + ex.Message); + + return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details"); + } + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRequestGenerator.cs new file mode 100644 index 000000000..02b4f00bf --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRequestGenerator.cs @@ -0,0 +1,292 @@ +using System; +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Linq; +using DryIoc; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Core.Parser; +using NzbDrone.Core.ThingiProvider; + +namespace NzbDrone.Core.Indexers.Newznab +{ + public class GenericNewznabRequestGenerator : IIndexerRequestGenerator + { + private readonly INewznabCapabilitiesProvider _capabilitiesProvider; + public int MaxPages { get; set; } + public int PageSize { get; set; } + public GenericNewznabSettings Settings { get; set; } + public ProviderDefinition Definition { get; set; } + + public GenericNewznabRequestGenerator(INewznabCapabilitiesProvider capabilitiesProvider) + { + _capabilitiesProvider = capabilitiesProvider; + + MaxPages = 30; + PageSize = 100; + } + + public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + + var pageableRequests = new IndexerPageableRequestChain(); + var parameters = new NameValueCollection(); + + if (searchCriteria.TmdbId.HasValue && capabilities.MovieSearchTmdbAvailable) + { + parameters.Add("tmdbid", searchCriteria.TmdbId.Value.ToString()); + } + + if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace() && capabilities.MovieSearchImdbAvailable) + { + parameters.Add("imdbid", searchCriteria.ImdbId); + } + + if (searchCriteria.TraktId.HasValue && capabilities.MovieSearchTraktAvailable) + { + parameters.Add("traktid", searchCriteria.TraktId.ToString()); + } + + //Workaround issue with Sphinx search returning garbage results on some indexers. If we don't use id parameters, fallback to t=search + if (parameters.Count == 0) + { + searchCriteria.SearchType = "search"; + + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + } + else + { + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.MovieSearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + } + + pageableRequests.Add(GetPagedRequests(searchCriteria, + parameters)); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria) + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + + var pageableRequests = new IndexerPageableRequestChain(); + var parameters = new NameValueCollection(); + + if (searchCriteria.Artist.IsNotNullOrWhiteSpace() && capabilities.MusicSearchArtistAvailable) + { + parameters.Add("artist", searchCriteria.Artist); + } + + if (searchCriteria.Album.IsNotNullOrWhiteSpace() && capabilities.MusicSearchAlbumAvailable) + { + parameters.Add("album", searchCriteria.Album); + } + + //Workaround issue with Sphinx search returning garbage results on some indexers. If we don't use id parameters, fallback to t=search + if (parameters.Count == 0) + { + searchCriteria.SearchType = "search"; + + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + } + else + { + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.MusicSearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + } + + pageableRequests.Add(GetPagedRequests(searchCriteria, + parameters)); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria) + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + + var pageableRequests = new IndexerPageableRequestChain(); + var parameters = new NameValueCollection(); + + if (searchCriteria.TvdbId.HasValue && capabilities.TvSearchTvdbAvailable) + { + parameters.Add("tvdbid", searchCriteria.TvdbId.Value.ToString()); + } + + if (searchCriteria.TmdbId.HasValue && capabilities.TvSearchTvdbAvailable) + { + parameters.Add("tmdbid", searchCriteria.TvdbId.Value.ToString()); + } + + if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace() && capabilities.TvSearchImdbAvailable) + { + parameters.Add("imdbid", searchCriteria.ImdbId); + } + + if (searchCriteria.TvMazeId.HasValue && capabilities.TvSearchTvMazeAvailable) + { + parameters.Add("tvmazeid", searchCriteria.TvMazeId.ToString()); + } + + if (searchCriteria.RId.HasValue && capabilities.TvSearchTvRageAvailable) + { + parameters.Add("rid", searchCriteria.RId.ToString()); + } + + if (searchCriteria.Season.HasValue && capabilities.TvSearchSeasonAvailable) + { + parameters.Add("season", NewznabifySeasonNumber(searchCriteria.Season.Value)); + } + + if (searchCriteria.Episode.IsNotNullOrWhiteSpace() && capabilities.TvSearchEpAvailable) + { + parameters.Add("ep", searchCriteria.Episode); + } + + //Workaround issue with Sphinx search returning garbage results on some indexers. If we don't use id parameters, fallback to t=search + if (parameters.Count == 0) + { + searchCriteria.SearchType = "search"; + + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + } + else + { + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.TvSearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + } + + pageableRequests.Add(GetPagedRequests(searchCriteria, + parameters)); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria) + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + + var pageableRequests = new IndexerPageableRequestChain(); + var parameters = new NameValueCollection(); + + if (searchCriteria.Author.IsNotNullOrWhiteSpace() && capabilities.BookSearchAuthorAvailable) + { + parameters.Add("author", searchCriteria.Author); + } + + if (searchCriteria.Title.IsNotNullOrWhiteSpace() && capabilities.BookSearchTitleAvailable) + { + parameters.Add("title", searchCriteria.Title); + } + + //Workaround issue with Sphinx search returning garbage results on some indexers. If we don't use id parameters, fallback to t=search + if (parameters.Count == 0) + { + searchCriteria.SearchType = "search"; + + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + } + else + { + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.BookSearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + } + + pageableRequests.Add(GetPagedRequests(searchCriteria, + parameters)); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria) + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + var pageableRequests = new IndexerPageableRequestChain(); + + var parameters = new NameValueCollection(); + + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace() && capabilities.SearchAvailable) + { + parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); + } + + pageableRequests.Add(GetPagedRequests(searchCriteria, parameters)); + + return pageableRequests; + } + + private IEnumerable GetPagedRequests(SearchCriteriaBase searchCriteria, NameValueCollection parameters) + { + var baseUrl = string.Format("{0}{1}?t={2}&extended=1", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchCriteria.SearchType); + var categories = searchCriteria.Categories; + + if (categories != null && categories.Any()) + { + var categoriesQuery = string.Join(",", categories.Distinct()); + baseUrl += string.Format("&cat={0}", categoriesQuery); + } + + if (Settings.AdditionalParameters.IsNotNullOrWhiteSpace()) + { + baseUrl += Settings.AdditionalParameters; + } + + if (Settings.ApiKey.IsNotNullOrWhiteSpace()) + { + baseUrl += "&apikey=" + Settings.ApiKey; + } + + if (searchCriteria.Limit.HasValue) + { + parameters.Add("limit", searchCriteria.Limit.ToString()); + } + + if (searchCriteria.Offset.HasValue) + { + parameters.Add("offset", searchCriteria.Offset.ToString()); + } + + var request = new IndexerRequest(string.Format("{0}&{1}", baseUrl, parameters.GetQueryString()), HttpAccept.Rss); + request.HttpRequest.AllowAutoRedirect = true; + + yield return request; + } + + private static string NewsnabifyTitle(string title) + { + return title.Replace("+", "%20"); + } + + // Temporary workaround for NNTMux considering season=0 -> null. '00' should work on existing newznab indexers. + private static string NewznabifySeasonNumber(int seasonNumber) + { + return seasonNumber == 0 ? "00" : seasonNumber.ToString(); + } + + public Func> GetCookies { get; set; } + public Action, DateTime?> CookiesUpdater { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRssParser.cs similarity index 96% rename from src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs rename to src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRssParser.cs index 851951356..2e3caeeaf 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabRssParser.cs @@ -8,17 +8,17 @@ using NzbDrone.Core.Parser.Model; namespace NzbDrone.Core.Indexers.Newznab { - public class NewznabRssParser : RssParser + public class GenericNewznabRssParser : RssParser { public const string ns = "{http://www.newznab.com/DTD/2010/feeds/attributes/}"; - private readonly NewznabSettings _settings; + private readonly List _categories; - public NewznabRssParser(NewznabSettings settings) + public GenericNewznabRssParser(List categories) { PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes; UseEnclosureUrl = true; - _settings = settings; + _categories = categories; } public static void CheckError(XDocument xdoc, IndexerResponse indexerResponse) @@ -125,7 +125,7 @@ namespace NzbDrone.Core.Indexers.Newznab { if (int.TryParse(cat, out var intCategory)) { - var indexerCat = _settings.Categories?.FirstOrDefault(c => c.Id == intCategory) ?? null; + var indexerCat = _categories?.FirstOrDefault(c => c.Id == intCategory) ?? null; if (indexerCat != null) { diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabSettings.cs new file mode 100644 index 000000000..7d59bc71e --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/GenericNewznabSettings.cs @@ -0,0 +1,91 @@ +using System.Collections.Generic; +using System.Linq; +using System.Text.RegularExpressions; +using FluentValidation; +using NzbDrone.Common.Extensions; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.Newznab +{ + public class GenericNewznabSettingsValidator : AbstractValidator + { + private static readonly string[] ApiKeyWhiteList = + { + "nzbs.org", + "nzb.su", + "dognzb.cr", + "nzbplanet.net", + "nzbid.org", + "nzbndx.com", + "nzbindex.in" + }; + + private static bool ShouldHaveApiKey(GenericNewznabSettings settings) + { + if (settings.BaseUrl == null) + { + return false; + } + + return ApiKeyWhiteList.Any(c => settings.BaseUrl.ToLowerInvariant().Contains(c)); + } + + private static readonly Regex AdditionalParametersRegex = new Regex(@"(&.+?\=.+?)+", RegexOptions.Compiled); + + public GenericNewznabSettingsValidator() + { + RuleFor(c => c.BaseUrl).ValidRootUrl(); + RuleFor(c => c.ApiPath).ValidUrlBase("/api"); + RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey); + RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex) + .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace()); + + RuleFor(c => c.VipExpiration).Must(c => c.IsValidDate()) + .When(c => c.VipExpiration.IsNotNullOrWhiteSpace()) + .WithMessage("Correctly formatted date is required"); + + RuleFor(c => c.VipExpiration).Must(c => c.IsFutureDate()) + .When(c => c.VipExpiration.IsNotNullOrWhiteSpace()) + .WithMessage("Must be a future date"); + } + } + + public class GenericNewznabSettings : IIndexerSettings + { + private static readonly GenericNewznabSettingsValidator Validator = new GenericNewznabSettingsValidator(); + + public GenericNewznabSettings() + { + ApiPath = "/api"; + VipExpiration = ""; + } + + [FieldDefinition(0, Label = "URL")] + public string BaseUrl { get; set; } + + [FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)] + public string ApiPath { get; set; } + + [FieldDefinition(2, Label = "API Key", HelpText = "Site API Key", Privacy = PrivacyLevel.ApiKey)] + public string ApiKey { get; set; } + + [FieldDefinition(5, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)] + public string AdditionalParameters { get; set; } + + [FieldDefinition(6, Label = "VIP Expiration", HelpText = "Enter date (yyyy-mm-dd) for VIP Expiration or blank, Prowlarr will notify 1 week from expiration of VIP")] + public string VipExpiration { get; set; } + + [FieldDefinition(7)] + public IndexerBaseSettings BaseSettings { get; set; } = new IndexerBaseSettings(); + + public List Categories { get; set; } + + // Field 8 is used by TorznabSettings MinimumSeeders + // If you need to add another field here, update TorznabSettings as well and this comment + public virtual NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs index 9660c9391..228443df9 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs @@ -5,9 +5,10 @@ using System.Threading.Tasks; using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; -using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Download; +using NzbDrone.Core.Indexers.Cardigann; +using NzbDrone.Core.IndexerVersions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; using NzbDrone.Core.Validation; @@ -16,10 +17,10 @@ namespace NzbDrone.Core.Indexers.Newznab { public class Newznab : UsenetIndexerBase { - private readonly INewznabCapabilitiesProvider _capabilitiesProvider; + private readonly IIndexerDefinitionUpdateService _definitionService; public override string Name => "Newznab"; - public override string[] IndexerUrls => GetBaseUrlFromSettings(); + public override string[] IndexerUrls => new string[] { "" }; public override string Description => "Newznab is an API search specification for Usenet"; public override bool FollowRedirect => true; public override bool SupportsRedirect => true; @@ -27,130 +28,72 @@ namespace NzbDrone.Core.Indexers.Newznab public override DownloadProtocol Protocol => DownloadProtocol.Usenet; public override IndexerPrivacy Privacy => IndexerPrivacy.Private; - public override IndexerCapabilities Capabilities { get => GetCapabilitiesFromSettings(); protected set => base.Capabilities = value; } - - public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings, Definition).LimitsDefault.Value; - public override IIndexerRequestGenerator GetRequestGenerator() { - return new NewznabRequestGenerator(_capabilitiesProvider) + var defFile = _definitionService.GetCachedDefinition(Settings.DefinitionFile); + + return new NewznabRequestGenerator() { PageSize = PageSize, - Settings = Settings + Settings = Settings, + Definition = defFile }; } public override IParseIndexerResponse GetParser() { - return new NewznabRssParser(Settings); - } - - public string[] GetBaseUrlFromSettings() - { - var baseUrl = ""; - - if (Definition == null || Settings == null || Settings.Categories == null) - { - return new string[] { baseUrl }; - } - - return new string[] { Settings.BaseUrl }; - } - - protected override NewznabSettings GetDefaultBaseUrl(NewznabSettings settings) - { - return settings; - } - - public IndexerCapabilities GetCapabilitiesFromSettings() - { - var caps = new IndexerCapabilities(); - - if (Definition == null || Settings == null || Settings.Categories == null) - { - return caps; - } - - foreach (var category in Settings.Categories) - { - caps.Categories.AddCategoryMapping(category.Name, category); - } - - return caps; - } + var defFile = _definitionService.GetCachedDefinition(Settings.DefinitionFile); + var capabilities = new IndexerCapabilities(); + capabilities.ParseYmlSearchModes(defFile.Caps.Modes); + capabilities.SupportsRawSearch = defFile.Caps.Allowrawsearch; + capabilities.MapYmlCategories(defFile); - public override IndexerCapabilities GetCapabilities() - { - // Newznab uses different Caps per site, so we need to cache them to db on first indexer add to prevent issues with loading UI and pulling caps every time. - return _capabilitiesProvider.GetCapabilities(Settings, Definition); + return new GenericNewznabRssParser(capabilities.Categories.GetTorznabCategoryList()); } public override IEnumerable DefaultDefinitions { get { - yield return GetDefinition("abNZB", GetSettings("https://abnzb.com")); - yield return GetDefinition("altHUB", GetSettings("https://api.althub.co.za")); - yield return GetDefinition("AnimeTosho (Usenet)", GetSettings("https://feed.animetosho.org")); - yield return GetDefinition("DOGnzb", GetSettings("https://api.dognzb.cr")); - yield return GetDefinition("DrunkenSlug", GetSettings("https://drunkenslug.com")); - yield return GetDefinition("GingaDADDY", GetSettings("https://www.gingadaddy.com")); - yield return GetDefinition("Miatrix", GetSettings("https://www.miatrix.com")); - yield return GetDefinition("Newz-Complex", GetSettings("https://newz-complex.org/www")); - yield return GetDefinition("Newz69", GetSettings("https://newz69.keagaming.com")); - yield return GetDefinition("NinjaCentral", GetSettings("https://ninjacentral.co.za")); - yield return GetDefinition("Nzb.su", GetSettings("https://api.nzb.su")); - yield return GetDefinition("NZBCat", GetSettings("https://nzb.cat")); - yield return GetDefinition("NZBFinder", GetSettings("https://nzbfinder.ws")); - yield return GetDefinition("NZBgeek", GetSettings("https://api.nzbgeek.info")); - yield return GetDefinition("NzbNoob", GetSettings("https://www.nzbnoob.com")); - yield return GetDefinition("NZBNDX", GetSettings("https://www.nzbndx.com")); - yield return GetDefinition("NzbPlanet", GetSettings("https://api.nzbplanet.net")); - yield return GetDefinition("NZBStars", GetSettings("https://nzbstars.com")); - yield return GetDefinition("OZnzb", GetSettings("https://api.oznzb.com")); - yield return GetDefinition("SimplyNZBs", GetSettings("https://simplynzbs.com")); - yield return GetDefinition("SpotNZB", GetSettings("https://spotnzb.xyz")); - yield return GetDefinition("Tabula Rasa", GetSettings("https://www.tabula-rasa.pw", apiPath: @"/api/v1/api")); - yield return GetDefinition("Usenet Crawler", GetSettings("https://www.usenet-crawler.com")); - yield return GetDefinition("Generic Newznab", GetSettings("")); + foreach (var def in _definitionService.AllForImplementation(GetType().Name)) + { + yield return GetDefinition(def); + } } } - public Newznab(INewznabCapabilitiesProvider capabilitiesProvider, IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger) + public Newznab(IIndexerDefinitionUpdateService definitionService, IIndexerHttpClient httpClient, IEventAggregator eventAggregator, IIndexerStatusService indexerStatusService, IConfigService configService, IValidateNzbs nzbValidationService, Logger logger) : base(httpClient, eventAggregator, indexerStatusService, configService, nzbValidationService, logger) { - _capabilitiesProvider = capabilitiesProvider; + _definitionService = definitionService; } - private IndexerDefinition GetDefinition(string name, NewznabSettings settings) + private IndexerDefinition GetDefinition(IndexerMetaDefinition definition) { return new IndexerDefinition { Enable = true, - Name = name, + Name = definition.Name, + Language = definition.Language, + Description = definition.Description, Implementation = GetType().Name, - Settings = settings, + IndexerUrls = definition.Links.ToArray(), + LegacyUrls = definition.Legacylinks.ToArray(), + Settings = new NewznabSettings { DefinitionFile = definition.File }, Protocol = DownloadProtocol.Usenet, - Privacy = IndexerPrivacy.Private, + Privacy = definition.Type switch + { + "private" => IndexerPrivacy.Private, + "public" => IndexerPrivacy.Public, + _ => IndexerPrivacy.SemiPrivate + }, SupportsRss = SupportsRss, SupportsSearch = SupportsSearch, SupportsRedirect = SupportsRedirect, - Capabilities = Capabilities + Capabilities = new IndexerCapabilities() }; } - private NewznabSettings GetSettings(string url, string apiPath = null) - { - var settings = new NewznabSettings { BaseUrl = url }; - - if (apiPath.IsNotNullOrWhiteSpace()) - { - settings.ApiPath = apiPath; - } - - return settings; - } - protected override async Task Test(List failures) { await base.Test(failures); @@ -158,61 +101,21 @@ namespace NzbDrone.Core.Indexers.Newznab { return; } - - failures.AddIfNotNull(TestCapabilities()); } - protected static List CategoryIds(IndexerCapabilitiesCategories categories) + public override object RequestAction(string action, IDictionary query) { - var l = categories.GetTorznabCategoryTree().Select(c => c.Id).ToList(); - - return l; - } - - protected virtual ValidationFailure TestCapabilities() - { - try + if (action == "getUrls") { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); - - if (capabilities.SearchParams != null && capabilities.SearchParams.Contains(SearchParam.Q)) - { - return null; - } - - if (capabilities.MovieSearchParams != null && - new[] { MovieSearchParam.Q, MovieSearchParam.ImdbId }.Any(v => capabilities.MovieSearchParams.Contains(v))) - { - return null; - } - - if (capabilities.TvSearchParams != null && - new[] { TvSearchParam.Q, TvSearchParam.TvdbId, TvSearchParam.TmdbId, TvSearchParam.RId }.Any(v => capabilities.TvSearchParams.Contains(v)) && - new[] { TvSearchParam.Season, TvSearchParam.Ep }.All(v => capabilities.TvSearchParams.Contains(v))) - { - return null; - } + var devices = ((IndexerDefinition)Definition).IndexerUrls; - if (capabilities.MusicSearchParams != null && - new[] { MusicSearchParam.Q, MusicSearchParam.Artist, MusicSearchParam.Album }.Any(v => capabilities.MusicSearchParams.Contains(v))) + return new { - return null; - } - - if (capabilities.BookSearchParams != null && - new[] { BookSearchParam.Q, BookSearchParam.Author, BookSearchParam.Title }.Any(v => capabilities.BookSearchParams.Contains(v))) - { - return null; - } - - return new ValidationFailure(string.Empty, "This indexer does not support searching for tv, music, or movies :(. Tell your indexer staff to enable this or force add the indexer by disabling search, adding the indexer and then enabling it again."); + options = devices.Select(d => new { Value = d, Name = d }) + }; } - catch (Exception ex) - { - _logger.Warn(ex, "Unable to connect to indexer: " + ex.Message); - return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details"); - } + return null; } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs index 155ec309a..bbc135939 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs @@ -14,7 +14,7 @@ namespace NzbDrone.Core.Indexers.Newznab { public interface INewznabCapabilitiesProvider { - IndexerCapabilities GetCapabilities(NewznabSettings settings, ProviderDefinition definition); + IndexerCapabilities GetCapabilities(GenericNewznabSettings settings, ProviderDefinition definition); } public class NewznabCapabilitiesProvider : INewznabCapabilitiesProvider @@ -30,7 +30,7 @@ namespace NzbDrone.Core.Indexers.Newznab _logger = logger; } - public IndexerCapabilities GetCapabilities(NewznabSettings indexerSettings, ProviderDefinition definition) + public IndexerCapabilities GetCapabilities(GenericNewznabSettings indexerSettings, ProviderDefinition definition) { var key = indexerSettings.ToJson(); var capabilities = _capabilitiesCache.Get(key, () => FetchCapabilities(indexerSettings, definition), TimeSpan.FromDays(7)); @@ -38,7 +38,7 @@ namespace NzbDrone.Core.Indexers.Newznab return capabilities; } - private IndexerCapabilities FetchCapabilities(NewznabSettings indexerSettings, ProviderDefinition definition) + private IndexerCapabilities FetchCapabilities(GenericNewznabSettings indexerSettings, ProviderDefinition definition) { var capabilities = new IndexerCapabilities(); @@ -96,7 +96,7 @@ namespace NzbDrone.Core.Indexers.Newznab throw new XmlException("Invalid XML").WithData(response); } - NewznabRssParser.CheckError(xDoc, new IndexerResponse(new IndexerRequest(response.Request), response)); + GenericNewznabRssParser.CheckError(xDoc, new IndexerResponse(new IndexerRequest(response.Request), response)); var xmlRoot = xDoc.Element("caps"); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs index 4ca28f18e..23c1ca004 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs @@ -5,6 +5,7 @@ using System.Linq; using DryIoc; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers.Cardigann; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser; using NzbDrone.Core.ThingiProvider; @@ -13,23 +14,20 @@ namespace NzbDrone.Core.Indexers.Newznab { public class NewznabRequestGenerator : IIndexerRequestGenerator { - private readonly INewznabCapabilitiesProvider _capabilitiesProvider; public int MaxPages { get; set; } public int PageSize { get; set; } public NewznabSettings Settings { get; set; } - public ProviderDefinition Definition { get; set; } + public CardigannDefinition Definition { get; set; } - public NewznabRequestGenerator(INewznabCapabilitiesProvider capabilitiesProvider) + public NewznabRequestGenerator() { - _capabilitiesProvider = capabilitiesProvider; - MaxPages = 30; PageSize = 100; } public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + var capabilities = GetCapabilities(); var pageableRequests = new IndexerPageableRequestChain(); var parameters = new NameValueCollection(); @@ -67,15 +65,14 @@ namespace NzbDrone.Core.Indexers.Newznab } } - pageableRequests.Add(GetPagedRequests(searchCriteria, - parameters)); + pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters)); return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(MusicSearchCriteria searchCriteria) { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + var capabilities = GetCapabilities(); var pageableRequests = new IndexerPageableRequestChain(); var parameters = new NameValueCollection(); @@ -108,15 +105,14 @@ namespace NzbDrone.Core.Indexers.Newznab } } - pageableRequests.Add(GetPagedRequests(searchCriteria, - parameters)); + pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters)); return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria) { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + var capabilities = GetCapabilities(); var pageableRequests = new IndexerPageableRequestChain(); var parameters = new NameValueCollection(); @@ -174,15 +170,14 @@ namespace NzbDrone.Core.Indexers.Newznab } } - pageableRequests.Add(GetPagedRequests(searchCriteria, - parameters)); + pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters)); return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria) { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + var capabilities = GetCapabilities(); var pageableRequests = new IndexerPageableRequestChain(); var parameters = new NameValueCollection(); @@ -215,15 +210,15 @@ namespace NzbDrone.Core.Indexers.Newznab } } - pageableRequests.Add(GetPagedRequests(searchCriteria, - parameters)); + pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters)); return pageableRequests; } public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria) { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings, Definition); + var capabilities = GetCapabilities(); + var pageableRequests = new IndexerPageableRequestChain(); var parameters = new NameValueCollection(); @@ -233,15 +228,15 @@ namespace NzbDrone.Core.Indexers.Newznab parameters.Add("q", NewsnabifyTitle(searchCriteria.SearchTerm)); } - pageableRequests.Add(GetPagedRequests(searchCriteria, parameters)); + pageableRequests.Add(GetPagedRequests(searchCriteria, capabilities, parameters)); return pageableRequests; } - private IEnumerable GetPagedRequests(SearchCriteriaBase searchCriteria, NameValueCollection parameters) + private IEnumerable GetPagedRequests(SearchCriteriaBase searchCriteria, IndexerCapabilities capabilities, NameValueCollection parameters) { - var baseUrl = string.Format("{0}{1}?t={2}&extended=1", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchCriteria.SearchType); - var categories = searchCriteria.Categories; + var baseUrl = string.Format("{0}{1}?t={2}&extended=1", ResolveSiteLink().TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchCriteria.SearchType); + var categories = capabilities.Categories.MapTorznabCapsToTrackers(searchCriteria.Categories); if (categories != null && categories.Any()) { @@ -286,6 +281,34 @@ namespace NzbDrone.Core.Indexers.Newznab return seasonNumber == 0 ? "00" : seasonNumber.ToString(); } + protected string ResolveSiteLink() + { + var settingsBaseUrl = Settings?.BaseUrl; + var defaultLink = Definition.Links.First(); + + if (settingsBaseUrl == null) + { + return defaultLink; + } + + if (Definition?.Legacylinks?.Contains(settingsBaseUrl) ?? false) + { + return defaultLink; + } + + return settingsBaseUrl; + } + + private IndexerCapabilities GetCapabilities() + { + var capabilities = new IndexerCapabilities(); + + capabilities.ParseYmlSearchModes(Definition.Caps.Modes); + capabilities.MapYmlCategories(Definition); + + return capabilities; + } + public Func> GetCookies { get; set; } public Action, DateTime?> CookiesUpdater { get; set; } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs index 7dc8c4d84..c77e2ea98 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs @@ -4,40 +4,18 @@ using System.Text.RegularExpressions; using FluentValidation; using NzbDrone.Common.Extensions; using NzbDrone.Core.Annotations; +using NzbDrone.Core.Indexers.Settings; using NzbDrone.Core.Validation; namespace NzbDrone.Core.Indexers.Newznab { public class NewznabSettingsValidator : AbstractValidator { - private static readonly string[] ApiKeyWhiteList = - { - "nzbs.org", - "nzb.su", - "dognzb.cr", - "nzbplanet.net", - "nzbid.org", - "nzbndx.com", - "nzbindex.in" - }; - - private static bool ShouldHaveApiKey(NewznabSettings settings) - { - if (settings.BaseUrl == null) - { - return false; - } - - return ApiKeyWhiteList.Any(c => settings.BaseUrl.ToLowerInvariant().Contains(c)); - } - private static readonly Regex AdditionalParametersRegex = new Regex(@"(&.+?\=.+?)+", RegexOptions.Compiled); public NewznabSettingsValidator() { - RuleFor(c => c.BaseUrl).ValidRootUrl(); RuleFor(c => c.ApiPath).ValidUrlBase("/api"); - RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey); RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex) .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace()); @@ -51,7 +29,7 @@ namespace NzbDrone.Core.Indexers.Newznab } } - public class NewznabSettings : IIndexerSettings + public class NewznabSettings : IYmlIndexerSettings { private static readonly NewznabSettingsValidator Validator = new NewznabSettingsValidator(); @@ -61,7 +39,7 @@ namespace NzbDrone.Core.Indexers.Newznab VipExpiration = ""; } - [FieldDefinition(0, Label = "URL")] + [FieldDefinition(0, Label = "Base Url", Type = FieldType.Select, SelectOptionsProviderAction = "getUrls", HelpText = "Select which baseurl Prowlarr will use for requests to the site")] public string BaseUrl { get; set; } [FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)] @@ -76,6 +54,9 @@ namespace NzbDrone.Core.Indexers.Newznab [FieldDefinition(6, Label = "VIP Expiration", HelpText = "Enter date (yyyy-mm-dd) for VIP Expiration or blank, Prowlarr will notify 1 week from expiration of VIP")] public string VipExpiration { get; set; } + [FieldDefinition(0, Hidden = HiddenType.Hidden)] + public string DefinitionFile { get; set; } + [FieldDefinition(7)] public IndexerBaseSettings BaseSettings { get; set; } = new IndexerBaseSettings(); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs index 27e5ee39a..dc5fc3b04 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs @@ -32,7 +32,7 @@ namespace NzbDrone.Core.Indexers.Torznab public override IIndexerRequestGenerator GetRequestGenerator() { - return new NewznabRequestGenerator(_capabilitiesProvider) + return new GenericNewznabRequestGenerator(_capabilitiesProvider) { PageSize = PageSize, Settings = Settings diff --git a/src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabSettings.cs index c1ea89cc9..5e9b43554 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Torznab/TorznabSettings.cs @@ -37,7 +37,7 @@ namespace NzbDrone.Core.Indexers.Torznab } } - public class TorznabSettings : NewznabSettings, ITorrentIndexerSettings + public class TorznabSettings : GenericNewznabSettings, ITorrentIndexerSettings { private static readonly TorznabSettingsValidator Validator = new TorznabSettingsValidator(); diff --git a/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs b/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs index cec5f59db..656315b2f 100644 --- a/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs +++ b/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs @@ -2,6 +2,8 @@ using System; using System.Collections.Generic; using System.Linq; using System.Xml.Linq; +using DryIoc.ImTools; +using NzbDrone.Core.Indexers.Cardigann; namespace NzbDrone.Core.Indexers { @@ -127,7 +129,7 @@ namespace NzbDrone.Core.Indexers LimitsMax = 100; } - public void ParseCardigannSearchModes(Dictionary> modes) + public void ParseYmlSearchModes(Dictionary> modes) { if (modes == null || !modes.Any()) { @@ -169,6 +171,48 @@ namespace NzbDrone.Core.Indexers } } + public void MapYmlCategories(CardigannDefinition defFile) + { + if (defFile.Caps.Categories != null) + { + foreach (var category in defFile.Caps.Categories) + { + var cat = NewznabStandardCategory.GetCatByName(category.Value); + + if (cat == null) + { + continue; + } + + Categories.AddCategoryMapping(category.Key, cat); + } + } + + if (defFile.Caps.Categorymappings != null) + { + foreach (var categorymapping in defFile.Caps.Categorymappings) + { + IndexerCategory torznabCat = null; + + if (categorymapping.cat != null) + { + torznabCat = NewznabStandardCategory.GetCatByName(categorymapping.cat); + if (torznabCat == null) + { + continue; + } + } + + Categories.AddCategoryMapping(categorymapping.id, torznabCat, categorymapping.desc); + + //if (categorymapping.Default) + //{ + // DefaultCategories.Add(categorymapping.id); + //} + } + } + } + public void ParseTvSearchParams(IEnumerable paramsList) { if (paramsList == null) diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs index 022b203b4..1053a96e6 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs @@ -7,6 +7,7 @@ using NLog; using NzbDrone.Core.Datastore; using NzbDrone.Core.Indexers.Cardigann; using NzbDrone.Core.Indexers.Newznab; +using NzbDrone.Core.Indexers.Settings; using NzbDrone.Core.IndexerVersions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; @@ -50,11 +51,11 @@ namespace NzbDrone.Core.Indexers foreach (var definition in definitions) { - if (definition.Implementation == typeof(Cardigann.Cardigann).Name) + if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) != null) { try { - MapCardigannDefinition(definition); + MapYmlDefinition(definition); } catch { @@ -73,11 +74,11 @@ namespace NzbDrone.Core.Indexers { var definition = base.Get(id); - if (definition.Implementation == typeof(Cardigann.Cardigann).Name) + if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) != null) { try { - MapCardigannDefinition(definition); + MapYmlDefinition(definition); } catch { @@ -93,9 +94,9 @@ namespace NzbDrone.Core.Indexers return base.Active().Where(c => c.Enable).ToList(); } - private void MapCardigannDefinition(IndexerDefinition definition) + private void MapYmlDefinition(IndexerDefinition definition) { - var settings = (CardigannSettings)definition.Settings; + var settings = (IYmlIndexerSettings)definition.Settings; var defFile = _definitionService.GetCachedDefinition(settings.DefinitionFile); definition.ExtraFields = defFile.Settings; @@ -121,51 +122,9 @@ namespace NzbDrone.Core.Indexers _ => IndexerPrivacy.SemiPrivate }; definition.Capabilities = new IndexerCapabilities(); - definition.Capabilities.ParseCardigannSearchModes(defFile.Caps.Modes); + definition.Capabilities.ParseYmlSearchModes(defFile.Caps.Modes); definition.Capabilities.SupportsRawSearch = defFile.Caps.Allowrawsearch; - MapCardigannCategories(definition, defFile); - } - - private void MapCardigannCategories(IndexerDefinition def, CardigannDefinition defFile) - { - if (defFile.Caps.Categories != null) - { - foreach (var category in defFile.Caps.Categories) - { - var cat = NewznabStandardCategory.GetCatByName(category.Value); - - if (cat == null) - { - continue; - } - - def.Capabilities.Categories.AddCategoryMapping(category.Key, cat); - } - } - - if (defFile.Caps.Categorymappings != null) - { - foreach (var categorymapping in defFile.Caps.Categorymappings) - { - IndexerCategory torznabCat = null; - - if (categorymapping.cat != null) - { - torznabCat = NewznabStandardCategory.GetCatByName(categorymapping.cat); - if (torznabCat == null) - { - continue; - } - } - - def.Capabilities.Categories.AddCategoryMapping(categorymapping.id, torznabCat, categorymapping.desc); - - //if (categorymapping.Default) - //{ - // DefaultCategories.Add(categorymapping.id); - //} - } - } + definition.Capabilities.MapYmlCategories(defFile); } public override IEnumerable GetDefaultDefinitions() @@ -178,7 +137,7 @@ namespace NzbDrone.Core.Indexers } var definitions = provider.DefaultDefinitions - .Where(v => v.Name != null && (v.Name != typeof(Cardigann.Cardigann).Name || v.Name != typeof(Newznab.Newznab).Name || v.Name != typeof(Torznab.Torznab).Name)); + .Where(v => v.Name != null && (v.Name != typeof(Cardigann.Cardigann).Name || v.Name != typeof(Newznab.Newznab).Name || v.Name != typeof(Newznab.GenericNewznab).Name || v.Name != typeof(Torznab.Torznab).Name)); foreach (IndexerDefinition definition in definitions) { @@ -203,7 +162,7 @@ namespace NzbDrone.Core.Indexers definition.SupportsRedirect = provider.SupportsRedirect; //We want to use the definition Caps and Privacy for Cardigann instead of the provider. - if (definition.Implementation != typeof(Cardigann.Cardigann).Name) + if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) == null) { definition.IndexerUrls = provider.IndexerUrls; definition.LegacyUrls = provider.LegacyUrls; @@ -288,15 +247,15 @@ namespace NzbDrone.Core.Indexers SetProviderCharacteristics(provider, definition); - if (definition.Implementation == typeof(Newznab.Newznab).Name || definition.Implementation == typeof(Torznab.Torznab).Name) + if (definition.Implementation == typeof(Newznab.GenericNewznab).Name || definition.Implementation == typeof(Torznab.Torznab).Name) { - var settings = (NewznabSettings)definition.Settings; + var settings = (GenericNewznabSettings)definition.Settings; settings.Categories = _newznabCapabilitiesProvider.GetCapabilities(settings, definition)?.Categories.GetTorznabCategoryList() ?? null; } - if (definition.Implementation == typeof(Cardigann.Cardigann).Name) + if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) != null) { - MapCardigannDefinition(definition); + MapYmlDefinition(definition); } return base.Create(definition); @@ -310,13 +269,13 @@ namespace NzbDrone.Core.Indexers if (definition.Enable && (definition.Implementation == typeof(Newznab.Newznab).Name || definition.Implementation == typeof(Torznab.Torznab).Name)) { - var settings = (NewznabSettings)definition.Settings; + var settings = (GenericNewznabSettings)definition.Settings; settings.Categories = _newznabCapabilitiesProvider.GetCapabilities(settings, definition)?.Categories.GetTorznabCategoryList() ?? null; } - if (definition.Implementation == typeof(Cardigann.Cardigann).Name) + if (definition.Settings.GetType().GetInterface(nameof(IYmlIndexerSettings)) != null) { - MapCardigannDefinition(definition); + MapYmlDefinition(definition); } base.Update(definition); diff --git a/src/NzbDrone.Core/Indexers/Settings/IYmlIndexerSettings.cs b/src/NzbDrone.Core/Indexers/Settings/IYmlIndexerSettings.cs new file mode 100644 index 000000000..02afb491c --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Settings/IYmlIndexerSettings.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace NzbDrone.Core.Indexers.Settings +{ + public interface IYmlIndexerSettings : IIndexerSettings + { + public string DefinitionFile { get; set; } + } +}