From b586b6ac1c11171ecc914832622b0d940b4a30c2 Mon Sep 17 00:00:00 2001 From: Qstick Date: Fri, 20 Nov 2020 02:42:36 -0500 Subject: [PATCH] Newznab Work --- frontend/src/Search/Table/CategoryLabel.js | 20 +- .../src/Search/Table/SearchIndexHeader.css | 7 +- frontend/src/Search/Table/SearchIndexRow.css | 7 +- .../Applications/ApplicationSettings.js | 20 +- .../ApplicationIndexerSyncCommand.cs | 16 ++ .../Applications/ApplicationService.cs | 25 ++- .../Definitions/MovieSearchCriteria.cs | 1 - .../IndexerSearch/NzbSearchService.cs | 1 - .../Definitions/Cardigann/CardigannBase.cs | 6 +- .../Cardigann/CardigannRequestGenerator.cs | 4 +- .../Indexers/Definitions/Newznab/Newznab.cs | 25 ++- .../Newznab/NewznabCapabilitiesProvider.cs | 7 +- .../Newznab/NewznabRequestGenerator.cs | 176 ++++++++---------- .../Definitions/Newznab/NewznabRssParser.cs | 38 ++++ .../Definitions/Newznab/NewznabSettings.cs | 2 + src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 6 + src/NzbDrone.Core/Indexers/IIndexer.cs | 2 + src/NzbDrone.Core/Indexers/IndexerBase.cs | 2 + .../Indexers/IndexerCapabilities.cs | 4 +- .../Indexers/IndexerCapabilitiesCategories.cs | 36 +++- src/NzbDrone.Core/Indexers/IndexerFactory.cs | 16 +- ...rCatType.cs => NewznabStandardCategory.cs} | 4 +- src/NzbDrone.Core/Indexers/RssParser.cs | 8 +- src/NzbDrone.Core/Jobs/TaskManager.cs | 2 + src/NzbDrone.Core/Localization/Core/en.json | 22 ++- src/Prowlarr.Api.V1/Indexers/IndexerModule.cs | 2 +- 26 files changed, 312 insertions(+), 147 deletions(-) create mode 100644 src/NzbDrone.Core/Applications/ApplicationIndexerSyncCommand.cs rename src/NzbDrone.Core/Indexers/{IndexerCatType.cs => NewznabStandardCategory.cs} (99%) diff --git a/frontend/src/Search/Table/CategoryLabel.js b/frontend/src/Search/Table/CategoryLabel.js index e43c32899..bdcbc0038 100644 --- a/frontend/src/Search/Table/CategoryLabel.js +++ b/frontend/src/Search/Table/CategoryLabel.js @@ -3,16 +3,20 @@ import React from 'react'; import Label from 'Components/Label'; function CategoryLabel({ categories }) { - let catName = ''; - - if (categories && categories.length > 0) { - catName = categories[0].name; - } + const sortedCategories = categories.sort((c) => c.id); return ( - + + { + sortedCategories.map((category) => { + return ( + + ); + }) + } + ); } diff --git a/frontend/src/Search/Table/SearchIndexHeader.css b/frontend/src/Search/Table/SearchIndexHeader.css index 284d8f597..099e4ba81 100644 --- a/frontend/src/Search/Table/SearchIndexHeader.css +++ b/frontend/src/Search/Table/SearchIndexHeader.css @@ -10,7 +10,12 @@ flex: 4 0 110px; } -.category, +.category { + composes: headerCell from '~Components/Table/VirtualTableHeaderCell.css'; + + flex: 0 0 110px; +} + .indexer { composes: headerCell from '~Components/Table/VirtualTableHeaderCell.css'; diff --git a/frontend/src/Search/Table/SearchIndexRow.css b/frontend/src/Search/Table/SearchIndexRow.css index a5f8415a7..7f6d96a96 100644 --- a/frontend/src/Search/Table/SearchIndexRow.css +++ b/frontend/src/Search/Table/SearchIndexRow.css @@ -17,7 +17,12 @@ flex: 4 0 110px; } -.category, +.category { + composes: cell; + + flex: 0 0 110px; +} + .indexer { composes: cell; diff --git a/frontend/src/Settings/Applications/ApplicationSettings.js b/frontend/src/Settings/Applications/ApplicationSettings.js index e75a257c3..a0a0c3d03 100644 --- a/frontend/src/Settings/Applications/ApplicationSettings.js +++ b/frontend/src/Settings/Applications/ApplicationSettings.js @@ -1,6 +1,9 @@ -import React from 'react'; +import React, { Fragment } from 'react'; import PageContent from 'Components/Page/PageContent'; import PageContentBody from 'Components/Page/PageContentBody'; +import PageToolbarButton from 'Components/Page/Toolbar/PageToolbarButton'; +import PageToolbarSeparator from 'Components/Page/Toolbar/PageToolbarSeparator'; +import { icons } from 'Helpers/Props'; import SettingsToolbarConnector from 'Settings/SettingsToolbarConnector'; import translate from 'Utilities/String/translate'; import ApplicationsConnector from './Applications/ApplicationsConnector'; @@ -10,6 +13,21 @@ function ApplicationSettings() { + + + + + + + } /> diff --git a/src/NzbDrone.Core/Applications/ApplicationIndexerSyncCommand.cs b/src/NzbDrone.Core/Applications/ApplicationIndexerSyncCommand.cs new file mode 100644 index 000000000..891a4c539 --- /dev/null +++ b/src/NzbDrone.Core/Applications/ApplicationIndexerSyncCommand.cs @@ -0,0 +1,16 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NzbDrone.Core.Messaging.Commands; + +namespace NzbDrone.Core.Applications +{ + public class ApplicationIndexerSyncCommand : Command + { + public override bool SendUpdatesToClient => true; + + public override string CompletionMessage => null; + } +} diff --git a/src/NzbDrone.Core/Applications/ApplicationService.cs b/src/NzbDrone.Core/Applications/ApplicationService.cs index 275c0ee32..af6809c54 100644 --- a/src/NzbDrone.Core/Applications/ApplicationService.cs +++ b/src/NzbDrone.Core/Applications/ApplicationService.cs @@ -1,11 +1,16 @@ using NLog; using NzbDrone.Core.Indexers; +using NzbDrone.Core.Messaging.Commands; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider.Events; namespace NzbDrone.Core.Applications { - public class ApplicationService : IHandle>, IHandle>, IHandle>, IHandle> + public class ApplicationService : IHandleAsync>, + IHandleAsync>, + IHandleAsync>, + IHandleAsync>, + IExecute { private readonly IApplicationFactory _applicationsFactory; private readonly Logger _logger; @@ -17,7 +22,7 @@ namespace NzbDrone.Core.Applications } // Sync Indexers on App Add if Sync Enabled - public void Handle(ProviderAddedEvent message) + public void HandleAsync(ProviderAddedEvent message) { var appDefinition = (ApplicationDefinition)message.Definition; @@ -29,7 +34,7 @@ namespace NzbDrone.Core.Applications } } - public void Handle(ProviderAddedEvent message) + public void HandleAsync(ProviderAddedEvent message) { var enabledApps = _applicationsFactory.GetAvailableProviders(); @@ -40,7 +45,7 @@ namespace NzbDrone.Core.Applications } } - public void Handle(ProviderDeletedEvent message) + public void HandleAsync(ProviderDeletedEvent message) { var enabledApps = _applicationsFactory.GetAvailableProviders(); @@ -51,7 +56,7 @@ namespace NzbDrone.Core.Applications } } - public void Handle(ProviderUpdatedEvent message) + public void HandleAsync(ProviderUpdatedEvent message) { var enabledApps = _applicationsFactory.GetAvailableProviders(); @@ -61,5 +66,15 @@ namespace NzbDrone.Core.Applications app.UpdateIndexer((IndexerDefinition)message.Definition); } } + + public void Execute(ApplicationIndexerSyncCommand message) + { + var enabledApps = _applicationsFactory.GetAvailableProviders(); + + foreach (var app in enabledApps) + { + app.SyncIndexers(); + } + } } } diff --git a/src/NzbDrone.Core/IndexerSearch/Definitions/MovieSearchCriteria.cs b/src/NzbDrone.Core/IndexerSearch/Definitions/MovieSearchCriteria.cs index 275118213..6fd9ffd09 100644 --- a/src/NzbDrone.Core/IndexerSearch/Definitions/MovieSearchCriteria.cs +++ b/src/NzbDrone.Core/IndexerSearch/Definitions/MovieSearchCriteria.cs @@ -4,7 +4,6 @@ namespace NzbDrone.Core.IndexerSearch.Definitions { public string ImdbId { get; set; } public int? TmdbId { get; set; } - public int? Year { get; set; } public int? TraktId { get; set; } } } diff --git a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs index 579a6053c..c70a52817 100644 --- a/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs +++ b/src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs @@ -60,7 +60,6 @@ namespace NzbDrone.Core.IndexerSearch searchSpec.ImdbId = request.imdbid; searchSpec.TmdbId = request.tmdbid; searchSpec.TraktId = request.traktid; - searchSpec.Year = request.year; return new NewznabResults { Releases = Dispatch(indexer => indexer.Fetch(searchSpec), searchSpec) }; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs index 2621202cc..8078a2196 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs @@ -60,7 +60,7 @@ namespace NzbDrone.Core.Indexers.Cardigann { foreach (var category in _definition.Caps.Categories) { - var cat = TorznabCatType.GetCatByName(category.Value); + var cat = NewznabStandardCategory.GetCatByName(category.Value); if (cat == null) { _logger.Error(string.Format("CardigannIndexer ({0}): invalid Torznab category for id {1}: {2}", _definition.Id, category.Key, category.Value)); @@ -79,7 +79,7 @@ namespace NzbDrone.Core.Indexers.Cardigann if (categorymapping.cat != null) { - torznabCat = TorznabCatType.GetCatByName(categorymapping.cat); + torznabCat = NewznabStandardCategory.GetCatByName(categorymapping.cat); if (torznabCat == null) { _logger.Error(string.Format("CardigannIndexer ({0}): invalid Torznab category for id {1}: {2}", _definition.Id, categorymapping.id, categorymapping.cat)); @@ -262,7 +262,7 @@ namespace NzbDrone.Core.Indexers.Cardigann .Where(m => !string.IsNullOrWhiteSpace(m.TrackerCategory) && string.Equals(m.TrackerCategory, input, StringComparison.InvariantCultureIgnoreCase)) - .Select(c => TorznabCatType.AllCats.FirstOrDefault(n => n.Id == c.NewzNabCategory) ?? new IndexerCategory { Id = c.NewzNabCategory }) + .Select(c => NewznabStandardCategory.AllCats.FirstOrDefault(n => n.Id == c.NewzNabCategory) ?? new IndexerCategory { Id = c.NewzNabCategory }) .ToList(); return cats; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs index 6f74ae742..344ba0548 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannRequestGenerator.cs @@ -29,9 +29,9 @@ namespace NzbDrone.Core.Indexers.Cardigann var variables = GetQueryVariableDefaults(searchCriteria); variables[".Query.Movie"] = null; - variables[".Query.Year"] = searchCriteria.Year; + variables[".Query.Year"] = null; variables[".Query.IMDBID"] = searchCriteria.ImdbId; - variables[".Query.IMDBIDShort"] = searchCriteria.ImdbId.Replace("tt", ""); + variables[".Query.IMDBIDShort"] = searchCriteria.ImdbId?.TrimStart('t') ?? null; variables[".Query.TMDBID"] = searchCriteria.TmdbId; variables[".Query.TraktID"] = searchCriteria.TraktId; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs index 368921410..16f7c5b7c 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs @@ -20,7 +20,7 @@ namespace NzbDrone.Core.Indexers.Newznab public override DownloadProtocol Protocol => DownloadProtocol.Usenet; public override IndexerPrivacy Privacy => IndexerPrivacy.Private; - public override IndexerCapabilities Capabilities { get => new IndexerCapabilities(); protected set => base.Capabilities = value; } + public override IndexerCapabilities Capabilities { get => GetCapabilitiesFromSettings(); protected set => base.Capabilities = value; } public override int PageSize => _capabilitiesProvider.GetCapabilities(Settings).LimitsDefault.Value; @@ -38,6 +38,29 @@ namespace NzbDrone.Core.Indexers.Newznab return new NewznabRssParser(Settings); } + public IndexerCapabilities GetCapabilitiesFromSettings() + { + var caps = new IndexerCapabilities(); + + if (Definition == null || Settings == null || Settings.Categories == null) + { + return caps; + } + + foreach (var category in Settings.Categories) + { + caps.Categories.AddCategoryMapping(category.Name, category); + } + + return caps; + } + + public override IndexerCapabilities GetCapabilities() + { + // Newznab uses different Caps per site, so we need to cache them to db on first indexer add to prevent issues with loading UI and pulling caps every time. + return _capabilitiesProvider.GetCapabilities(Settings); + } + public override IEnumerable DefaultDefinitions { get diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs index 524fa9be9..a94ff72ff 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabCapabilitiesProvider.cs @@ -206,12 +206,15 @@ namespace NzbDrone.Core.Indexers.Newznab foreach (var xmlSubcat in xmlCategory.Elements("subcat")) { - cat.SubCategories.Add(new IndexerCategory + var subCat = new IndexerCategory { Id = int.Parse(xmlSubcat.Attribute("id").Value), Name = xmlSubcat.Attribute("name").Value, Description = xmlSubcat.Attribute("description") != null ? xmlSubcat.Attribute("description").Value : string.Empty - }); + }; + + cat.SubCategories.Add(subCat); + capabilities.Categories.AddCategoryMapping(subCat.Name, subCat); } capabilities.Categories.AddCategoryMapping(cat.Name, cat); diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs index e26a25e50..1ca43027a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRequestGenerator.cs @@ -22,55 +22,35 @@ namespace NzbDrone.Core.Indexers.Newznab PageSize = 100; } - private bool SupportsSearch + public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) { - get - { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); + var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - return capabilities.SearchParams != null && - capabilities.SearchParams.Contains(SearchParam.Q); - } - } + var pageableRequests = new IndexerPageableRequestChain(); + var parameters = string.Empty; - private bool SupportsImdbSearch - { - get + if (searchCriteria.TmdbId.HasValue && capabilities.MovieSearchTmdbAvailable) { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - return capabilities.MovieSearchParams != null && - capabilities.MovieSearchParams.Contains(MovieSearchParam.ImdbId); + parameters += string.Format("&tmdbid={0}", searchCriteria.TmdbId.Value); } - } - private bool SupportsTmdbSearch - { - get + if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace() && capabilities.MovieSearchImdbAvailable) { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - return capabilities.MovieSearchParams != null && - capabilities.MovieSearchParams.Contains(MovieSearchParam.TmdbId); + parameters += string.Format("&imdbid={0}", searchCriteria.ImdbId); } - } - private bool SupportsAggregatedIdSearch - { - get + if (searchCriteria.TraktId.HasValue && capabilities.MovieSearchTraktAvailable) { - var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - - // TODO: Fix this, return capabilities.SupportsAggregateIdSearch; - return true; + parameters += string.Format("&traktid={0}", searchCriteria.ImdbId); } - } - public IndexerPageableRequestChain GetSearchRequests(MovieSearchCriteria searchCriteria) - { - var pageableRequests = new IndexerPageableRequestChain(); + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) + { + parameters += string.Format("&q={0}", searchCriteria.SearchTerm); + } - AddMovieIdPageableRequests(pageableRequests, MaxPages, searchCriteria.Categories, searchCriteria); + pageableRequests.Add(GetPagedRequests(searchCriteria, + parameters)); return pageableRequests; } @@ -82,86 +62,73 @@ namespace NzbDrone.Core.Indexers.Newznab public IndexerPageableRequestChain GetSearchRequests(TvSearchCriteria searchCriteria) { - return new IndexerPageableRequestChain(); - } - - public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria) - { - return new IndexerPageableRequestChain(); - } + var capabilities = _capabilitiesProvider.GetCapabilities(Settings); - public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria) - { var pageableRequests = new IndexerPageableRequestChain(); + var parameters = string.Empty; - pageableRequests.Add(GetPagedRequests(MaxPages, - searchCriteria.Categories, - "search", - string.Format("&q={0}", NewsnabifyTitle(searchCriteria.SearchTerm)))); - - return pageableRequests; - } - - private void AddMovieIdPageableRequests(IndexerPageableRequestChain chain, int maxPages, IEnumerable categories, MovieSearchCriteria searchCriteria) - { - var includeTmdbSearch = SupportsTmdbSearch && searchCriteria.TmdbId > 0; - var includeImdbSearch = SupportsImdbSearch && searchCriteria.ImdbId.IsNotNullOrWhiteSpace(); + if (searchCriteria.TvdbId.HasValue && capabilities.TvSearchTvdbAvailable) + { + parameters += string.Format("&tvdbid={0}", searchCriteria.TvdbId.Value); + } - if (SupportsAggregatedIdSearch && (includeTmdbSearch || includeImdbSearch)) + if (searchCriteria.ImdbId.IsNotNullOrWhiteSpace() && capabilities.TvSearchImdbAvailable) { - var ids = ""; + parameters += string.Format("&imdbid={0}", searchCriteria.ImdbId); + } - if (includeTmdbSearch) - { - ids += "&tmdbid=" + searchCriteria.TmdbId; - } + if (searchCriteria.TvMazeId.HasValue && capabilities.TvSearchTvMazeAvailable) + { + parameters += string.Format("&tvmazeid={0}", searchCriteria.TvMazeId); + } - if (includeImdbSearch) - { - ids += "&imdbid=" + searchCriteria.ImdbId.Substring(2); - } + if (searchCriteria.RId.HasValue && capabilities.TvSearchTvRageAvailable) + { + parameters += string.Format("&rid={0}", searchCriteria.RId); + } - chain.Add(GetPagedRequests(maxPages, categories, "movie", ids)); + if (searchCriteria.Season.HasValue && capabilities.TvSearchSeasonAvailable) + { + parameters += string.Format("&season={0}", searchCriteria.Season); } - else + + if (searchCriteria.Ep.HasValue && capabilities.TvSearchEpAvailable) { - if (includeTmdbSearch) - { - chain.Add(GetPagedRequests(maxPages, - categories, - "movie", - string.Format("&tmdbid={0}", searchCriteria.TmdbId))); - } - else if (includeImdbSearch) - { - chain.Add(GetPagedRequests(maxPages, - categories, - "movie", - string.Format("&imdbid={0}", searchCriteria.ImdbId.Substring(2)))); - } + parameters += string.Format("&ep={0}", searchCriteria.Ep); } - if (SupportsSearch) + if (searchCriteria.SearchTerm.IsNotNullOrWhiteSpace()) { - chain.AddTier(); + parameters += string.Format("&q={0}", searchCriteria.SearchTerm); + } - var searchQuery = searchCriteria.SearchTerm; + pageableRequests.Add(GetPagedRequests(searchCriteria, + parameters)); - if (!Settings.RemoveYear) - { - searchQuery = string.Format("{0}", searchQuery); - } + return pageableRequests; + } - chain.Add(GetPagedRequests(MaxPages, - categories, - "movie", - string.Format("&q={0}", NewsnabifyTitle(searchQuery)))); - } + public IndexerPageableRequestChain GetSearchRequests(BookSearchCriteria searchCriteria) + { + return new IndexerPageableRequestChain(); + } + + public IndexerPageableRequestChain GetSearchRequests(BasicSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + + var searchQuery = searchCriteria.SearchTerm; + + pageableRequests.Add(GetPagedRequests(searchCriteria, + searchQuery.IsNotNullOrWhiteSpace() ? string.Format("&q={0}", NewsnabifyTitle(searchCriteria.SearchTerm)) : string.Empty)); + + return pageableRequests; } - private IEnumerable GetPagedRequests(int maxPages, IEnumerable categories, string searchType, string parameters) + private IEnumerable GetPagedRequests(SearchCriteriaBase searchCriteria, string parameters) { - var baseUrl = string.Format("{0}{1}?t={2}&extended=1", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchType); + var baseUrl = string.Format("{0}{1}?t={2}&extended=1", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchCriteria.SearchType); + var categories = searchCriteria.Categories; if (categories != null && categories.Any()) { @@ -174,16 +141,23 @@ namespace NzbDrone.Core.Indexers.Newznab baseUrl += "&apikey=" + Settings.ApiKey; } + if (searchCriteria.Limit.HasValue) + { + parameters += string.Format("&limit={0}", searchCriteria.Limit); + } + + if (searchCriteria.Offset.HasValue) + { + parameters += string.Format("&offset={0}", searchCriteria.Offset); + } + if (PageSize == 0) { yield return new IndexerRequest(string.Format("{0}{1}", baseUrl, parameters), HttpAccept.Rss); } else { - for (var page = 0; page < maxPages; page++) - { - yield return new IndexerRequest(string.Format("{0}&offset={1}&limit={2}{3}", baseUrl, page * PageSize, PageSize, parameters), HttpAccept.Rss); - } + yield return new IndexerRequest(string.Format("{0}&offset={1}&limit={2}{3}", baseUrl, searchCriteria.Offset, searchCriteria.Limit, parameters), HttpAccept.Rss); } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs index e6f6e0f3b..446e4f238 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabRssParser.cs @@ -96,6 +96,27 @@ namespace NzbDrone.Core.Indexers.Newznab return ParseUrl(item.TryGetValue("comments")); } + protected override ICollection GetCategory(XElement item) + { + var cats = TryGetMultipleNewznabAttributes(item, "category"); + var results = new List(); + + foreach (var cat in cats) + { + if (int.TryParse(cat, out var intCategory)) + { + var indexerCat = _settings.Categories.FirstOrDefault(c => c.Id == intCategory); + + if (indexerCat != null) + { + results.Add(indexerCat); + } + } + } + + return results; + } + protected override long GetSize(XElement item) { long size; @@ -174,5 +195,22 @@ namespace NzbDrone.Core.Indexers.Newznab return defaultValue; } + + protected List TryGetMultipleNewznabAttributes(XElement item, string key) + { + var attrElements = item.Elements(ns + "attr").Where(e => e.Attribute("name").Value.Equals(key, StringComparison.OrdinalIgnoreCase)); + var results = new List(); + + foreach (var element in attrElements) + { + var attrValue = element.Attribute("value"); + if (attrValue != null) + { + results.Add(attrValue.Value); + } + } + + return results; + } } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs index 961ea92c7..49ec406e6 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/NewznabSettings.cs @@ -76,6 +76,8 @@ namespace NzbDrone.Core.Indexers.Newznab Type = FieldType.Checkbox)] public bool RemoveYear { get; set; } + public List Categories { get; set; } + // Field 8 is used by TorznabSettings MinimumSeeders // If you need to add another field here, update TorznabSettings as well and this comment public virtual NzbDroneValidationResult Validate() diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index afad08fa7..b9d6ed555 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -11,6 +11,7 @@ using NzbDrone.Core.Http.CloudFlare; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.IndexerSearch.Definitions; using NzbDrone.Core.Parser.Model; +using NzbDrone.Core.ThingiProvider; namespace NzbDrone.Core.Indexers { @@ -292,6 +293,11 @@ namespace NzbDrone.Core.Indexers return CleanupReleases(releases); } + public override IndexerCapabilities GetCapabilities() + { + return Capabilities ?? ((IndexerDefinition)Definition).Capabilities; + } + protected virtual bool IsValidRelease(ReleaseInfo release) { if (release.DownloadUrl == null) diff --git a/src/NzbDrone.Core/Indexers/IIndexer.cs b/src/NzbDrone.Core/Indexers/IIndexer.cs index 27e44d6a2..89e58e520 100644 --- a/src/NzbDrone.Core/Indexers/IIndexer.cs +++ b/src/NzbDrone.Core/Indexers/IIndexer.cs @@ -19,5 +19,7 @@ namespace NzbDrone.Core.Indexers IList Fetch(TvSearchCriteria searchCriteria); IList Fetch(BookSearchCriteria searchCriteria); IList Fetch(BasicSearchCriteria searchCriteria); + + IndexerCapabilities GetCapabilities(); } } diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 115723a30..361b864f7 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -71,6 +71,8 @@ namespace NzbDrone.Core.Indexers public abstract IList Fetch(BookSearchCriteria searchCriteria); public abstract IList Fetch(BasicSearchCriteria searchCriteria); + public abstract IndexerCapabilities GetCapabilities(); + protected virtual IList CleanupReleases(IEnumerable releases) { var result = releases.DistinctBy(v => v.Guid).ToList(); diff --git a/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs b/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs index 99f9234c9..c00d798b2 100644 --- a/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs +++ b/src/NzbDrone.Core/Indexers/IndexerCapabilities.cs @@ -22,7 +22,8 @@ namespace NzbDrone.Core.Indexers ImdbId, TmdbId, ImdbTitle, - ImdbYear + ImdbYear, + TraktId, } public enum MusicSearchParam @@ -67,6 +68,7 @@ namespace NzbDrone.Core.Indexers public bool MovieSearchAvailable => MovieSearchParams.Count > 0; public bool MovieSearchImdbAvailable => MovieSearchParams.Contains(MovieSearchParam.ImdbId); public bool MovieSearchTmdbAvailable => MovieSearchParams.Contains(MovieSearchParam.TmdbId); + public bool MovieSearchTraktAvailable => MovieSearchParams.Contains(MovieSearchParam.TraktId); public List MusicSearchParams; public bool MusicSearchAvailable => MusicSearchParams.Count > 0; diff --git a/src/NzbDrone.Core/Indexers/IndexerCapabilitiesCategories.cs b/src/NzbDrone.Core/Indexers/IndexerCapabilitiesCategories.cs index bb3cf079b..4bd6a21a3 100644 --- a/src/NzbDrone.Core/Indexers/IndexerCapabilitiesCategories.cs +++ b/src/NzbDrone.Core/Indexers/IndexerCapabilitiesCategories.cs @@ -168,7 +168,7 @@ namespace NzbDrone.Core.Indexers private void AddTorznabCategoryTree(IndexerCategory torznabCategory) { // build the category tree - if (TorznabCatType.ParentCats.Contains(torznabCategory)) + if (NewznabStandardCategory.ParentCats.Contains(torznabCategory)) { // parent cat if (!_torznabCategoryTree.Contains(torznabCategory)) @@ -179,7 +179,7 @@ namespace NzbDrone.Core.Indexers else { // child or custom cat - var parentCat = TorznabCatType.ParentCats.FirstOrDefault(c => c.Contains(torznabCategory)); + var parentCat = NewznabStandardCategory.ParentCats.FirstOrDefault(c => c.Contains(torznabCategory)); if (parentCat != null) { // child cat @@ -203,7 +203,37 @@ namespace NzbDrone.Core.Indexers else { // custom cat - _torznabCategoryTree.Add(torznabCategory); + if (torznabCategory.Id > 1000 && torznabCategory.Id < 10000) + { + var potentialParent = NewznabStandardCategory.ParentCats.FirstOrDefault(c => (c.Id / 1000) == (torznabCategory.Id / 1000)); + if (potentialParent != null) + { + var nodeCat = _torznabCategoryTree.FirstOrDefault(c => c.Equals(potentialParent)); + if (nodeCat != null) + { + // parent cat already exists + if (!nodeCat.Contains(torznabCategory)) + { + nodeCat.SubCategories.Add(torznabCategory); + } + } + else + { + // create parent cat and add child + nodeCat = potentialParent.CopyWithoutSubCategories(); + nodeCat.SubCategories.Add(torznabCategory); + _torznabCategoryTree.Add(nodeCat); + } + } + else + { + _torznabCategoryTree.Add(torznabCategory); + } + } + else + { + _torznabCategoryTree.Add(torznabCategory); + } } } } diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs index 65056caf4..de5fb1608 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs @@ -5,6 +5,7 @@ using FluentValidation.Results; using NLog; using NzbDrone.Common.Composition; using NzbDrone.Core.Indexers.Cardigann; +using NzbDrone.Core.Indexers.Newznab; using NzbDrone.Core.IndexerVersions; using NzbDrone.Core.Messaging.Events; using NzbDrone.Core.ThingiProvider; @@ -22,10 +23,12 @@ namespace NzbDrone.Core.Indexers public class IndexerFactory : ProviderFactory, IIndexerFactory { private readonly IIndexerDefinitionUpdateService _definitionService; + private readonly INewznabCapabilitiesProvider _newznabCapabilitiesProvider; private readonly IIndexerStatusService _indexerStatusService; private readonly Logger _logger; public IndexerFactory(IIndexerDefinitionUpdateService definitionService, + INewznabCapabilitiesProvider newznabCapabilitiesProvider, IIndexerStatusService indexerStatusService, IIndexerRepository providerRepository, IEnumerable providers, @@ -36,6 +39,7 @@ namespace NzbDrone.Core.Indexers { _definitionService = definitionService; _indexerStatusService = indexerStatusService; + _newznabCapabilitiesProvider = newznabCapabilitiesProvider; _logger = logger; } @@ -89,7 +93,7 @@ namespace NzbDrone.Core.Indexers { foreach (var category in defFile.Caps.Categories) { - var cat = TorznabCatType.GetCatByName(category.Value); + var cat = NewznabStandardCategory.GetCatByName(category.Value); if (cat == null) { @@ -108,7 +112,7 @@ namespace NzbDrone.Core.Indexers if (categorymapping.cat != null) { - torznabCat = TorznabCatType.GetCatByName(categorymapping.cat); + torznabCat = NewznabStandardCategory.GetCatByName(categorymapping.cat); if (torznabCat == null) { continue; @@ -242,6 +246,14 @@ namespace NzbDrone.Core.Indexers { definition.Added = DateTime.UtcNow; + var provider = _providers.First(v => v.GetType().Name == definition.Implementation); + + if (definition.Implementation == typeof(Newznab.Newznab).Name) + { + var settings = (NewznabSettings)definition.Settings; + settings.Categories = _newznabCapabilitiesProvider.GetCapabilities(settings).Categories.GetTorznabCategoryList(); + } + return base.Create(definition); } } diff --git a/src/NzbDrone.Core/Indexers/IndexerCatType.cs b/src/NzbDrone.Core/Indexers/NewznabStandardCategory.cs similarity index 99% rename from src/NzbDrone.Core/Indexers/IndexerCatType.cs rename to src/NzbDrone.Core/Indexers/NewznabStandardCategory.cs index c8c7e81cd..9eb8c6302 100644 --- a/src/NzbDrone.Core/Indexers/IndexerCatType.cs +++ b/src/NzbDrone.Core/Indexers/NewznabStandardCategory.cs @@ -3,7 +3,7 @@ using System.Linq; namespace NzbDrone.Core.Indexers { - public static class TorznabCatType + public static class NewznabStandardCategory { public static readonly IndexerCategory Console = new IndexerCategory(1000, "Console"); public static readonly IndexerCategory ConsoleNDS = new IndexerCategory(1010, "Console/NDS"); @@ -171,7 +171,7 @@ namespace NzbDrone.Core.Indexers OtherHashed }; - static TorznabCatType() + static NewznabStandardCategory() { Console.SubCategories.AddRange( new List diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index 8ae83a624..1811b1b10 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.IO; @@ -160,6 +160,7 @@ namespace NzbDrone.Core.Indexers releaseInfo.DownloadUrl = GetDownloadUrl(item); releaseInfo.InfoUrl = GetInfoUrl(item); releaseInfo.CommentUrl = GetCommentUrl(item); + releaseInfo.Category = GetCategory(item); try { @@ -188,6 +189,11 @@ namespace NzbDrone.Core.Indexers return item.TryGetValue("title", "Unknown"); } + protected virtual ICollection GetCategory(XElement item) + { + return new List { NewznabStandardCategory.Other }; + } + protected virtual DateTime GetPublishDate(XElement item) { var dateString = item.TryGetValue("pubDate"); diff --git a/src/NzbDrone.Core/Jobs/TaskManager.cs b/src/NzbDrone.Core/Jobs/TaskManager.cs index ef93adee4..82531fe37 100644 --- a/src/NzbDrone.Core/Jobs/TaskManager.cs +++ b/src/NzbDrone.Core/Jobs/TaskManager.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; using System.Linq; using NLog; +using NzbDrone.Core.Applications; using NzbDrone.Core.Backup; using NzbDrone.Core.Configuration; using NzbDrone.Core.HealthCheck; @@ -61,6 +62,7 @@ namespace NzbDrone.Core.Jobs new ScheduledTask { Interval = 6 * 60, TypeName = typeof(CheckHealthCommand).FullName }, new ScheduledTask { Interval = 24 * 60, TypeName = typeof(HousekeepingCommand).FullName }, new ScheduledTask { Interval = 6 * 60, TypeName = typeof(IndexerDefinitionUpdateCommand).FullName }, + new ScheduledTask { Interval = 6 * 60, TypeName = typeof(ApplicationIndexerSyncCommand).FullName }, new ScheduledTask { diff --git a/src/NzbDrone.Core/Localization/Core/en.json b/src/NzbDrone.Core/Localization/Core/en.json index 5c499fd07..095d9c12b 100644 --- a/src/NzbDrone.Core/Localization/Core/en.json +++ b/src/NzbDrone.Core/Localization/Core/en.json @@ -14,8 +14,8 @@ "AddMovies": "Add Movies", "AddMoviesMonitored": "Add Movies Monitored", "AddNew": "Add New", - "AddNewMessage": "It's easy to add a new movie, just start typing the name of the movie you want to add", "AddNewIndexer": "Add New Indexer", + "AddNewMessage": "It's easy to add a new movie, just start typing the name of the movie you want to add", "AddNewTmdbIdMessage": "You can also search using TMDb Id of a movie. eg. tmdb:71663", "AddRemotePathMapping": "Add Remote Path Mapping", "AddRestriction": "Add Restriction", @@ -332,6 +332,8 @@ "IncludeUnmonitored": "Include Unmonitored", "Indexer": "Indexer", "IndexerFlags": "Indexer Flags", + "IndexerLongTermStatusCheckAllClientMessage": "All indexers are unavailable due to failures for more than 6 hours", + "IndexerLongTermStatusCheckSingleClientMessage": "Indexers unavailable due to failures for more than 6 hours: {0}", "IndexerPriority": "Indexer Priority", "IndexerPriorityHelpText": "Indexer Priority from 1 (Highest) to 50 (Lowest). Default: 25.", "IndexerRssHealthCheckNoAvailableIndexers": "All rss-capable indexers are temporarily unavailable due to recent indexer errors", @@ -341,11 +343,10 @@ "IndexerSearchCheckNoAvailableIndexersMessage": "All search-capable indexers are temporarily unavailable due to recent indexer errors", "IndexerSearchCheckNoInteractiveMessage": "No indexers available with Interactive Search enabled, Prowlarr will not provide any interactive search results", "IndexerSettings": "Indexer Settings", + "IndexersSelectedInterp": "{0} Indexer(s) Selected", "IndexersSettingsSummary": "Indexers and release restrictions", "IndexerStatusCheckAllClientMessage": "All indexers are unavailable due to failures", "IndexerStatusCheckSingleClientMessage": "Indexers unavailable due to failures: {0}", - "IndexerLongTermStatusCheckAllClientMessage": "All indexers are unavailable due to failures for more than 6 hours", - "IndexerLongTermStatusCheckSingleClientMessage": "Indexers unavailable due to failures for more than 6 hours: {0}", "Info": "Info", "InteractiveImport": "Interactive Import", "InteractiveSearch": "Interactive Search", @@ -452,7 +453,6 @@ "MovieIsUnmonitored": "Movie is unmonitored", "MovieNaming": "Movie Naming", "Movies": "Movies", - "IndexersSelectedInterp": "{0} Indexer(s) Selected", "MovieTitle": "Movie Title", "MovieTitleHelpText": "The title of the movie to exclude (can be anything meaningful)", "MovieYear": "Movie Year", @@ -530,6 +530,12 @@ "Proper": "Proper", "Protocol": "Protocol", "ProtocolHelpText": "Choose which protocol(s) to use and which one is preferred when choosing between otherwise equal releases", + "Prowlarr": "Prowlarr", + "ProwlarrSupportsAnyDownloadClient": "Prowlarr supports any download client that uses the Newznab standard, as well as other download clients listed below.", + "ProwlarrSupportsAnyIndexer": "Prowlarr supports any indexer that uses the Newznab standard, as well as other indexers listed below.", + "ProwlarrSupportsAnyRSSMovieListsAsWellAsTheOneStatedBelow": "Prowlarr supports any RSS movie lists as well as the one stated below.", + "ProwlarrSupportsCustomConditionsAgainstTheReleasePropertiesBelow": "Prowlarr supports custom conditions against the release properties below.", + "ProwlarrTags": "Prowlarr Tags", "Proxy": "Proxy", "ProxyBypassFilterHelpText": "Use ',' as a separator, and '*.' as a wildcard for subdomains", "ProxyCheckBadRequestMessage": "Failed to test proxy. StatusCode: {0}", @@ -551,12 +557,6 @@ "Queue": "Queue", "Queued": "Queued", "QuickImport": "Quick Import", - "Prowlarr": "Prowlarr", - "ProwlarrSupportsAnyDownloadClient": "Prowlarr supports any download client that uses the Newznab standard, as well as other download clients listed below.", - "ProwlarrSupportsAnyIndexer": "Prowlarr supports any indexer that uses the Newznab standard, as well as other indexers listed below.", - "ProwlarrSupportsAnyRSSMovieListsAsWellAsTheOneStatedBelow": "Prowlarr supports any RSS movie lists as well as the one stated below.", - "ProwlarrSupportsCustomConditionsAgainstTheReleasePropertiesBelow": "Prowlarr supports custom conditions against the release properties below.", - "ProwlarrTags": "Prowlarr Tags", "Ratings": "Ratings", "ReadTheWikiForMoreInformation": "Read the Wiki for more information", "Real": "Real", @@ -727,6 +727,7 @@ "Style": "Style", "SubfolderWillBeCreatedAutomaticallyInterp": "'{0}' subfolder will be created automatically", "SuggestTranslationChange": "Suggest translation change", + "SyncAppIndexers": "Sync App Indexers", "System": "System", "SystemTimeCheckMessage": "System time is off by more than 1 day. Scheduled tasks may not run correctly until the time is corrected", "Table": "Table", @@ -740,6 +741,7 @@ "Tasks": "Tasks", "Test": "Test", "TestAll": "Test All", + "TestAllApps": "Test All Apps", "TestAllClients": "Test All Clients", "TestAllIndexers": "Test All Indexers", "TestAllLists": "Test All Lists", diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs b/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs index f24bad915..df3c9a063 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs @@ -59,7 +59,7 @@ namespace Prowlarr.Api.V1.Indexers switch (requestType) { case "caps": - Response response = indexer.Capabilities.ToXml(); + Response response = indexerInstance.GetCapabilities().ToXml(); response.ContentType = "application/rss+xml"; return response; case "tvsearch":