From 55412968e0d8ed7f0b295fc58c953223add0c66b Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sun, 19 Jul 2015 20:39:07 +0200 Subject: [PATCH] New: Added auto-detection of indexer capabilities to torznab. --- .../TorznabTests/TorznabFixture.cs | 4 + .../TorznabRequestGeneratorFixture.cs | 46 ++++++- src/NzbDrone.Core/Indexers/Torznab/Torznab.cs | 49 ++++++- .../Indexers/Torznab/TorznabCapabilities.cs | 28 ++++ .../Torznab/TorznabCapabilitiesProvider.cs | 126 ++++++++++++++++++ .../Torznab/TorznabRequestGenerator.cs | 122 ++++++++++++----- .../Indexers/Torznab/TorznabSettings.cs | 1 + src/NzbDrone.Core/NzbDrone.Core.csproj | 2 + 8 files changed, 339 insertions(+), 39 deletions(-) create mode 100644 src/NzbDrone.Core/Indexers/Torznab/TorznabCapabilities.cs create mode 100644 src/NzbDrone.Core/Indexers/Torznab/TorznabCapabilitiesProvider.cs diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs index 7ec161d90..a971b65f6 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs @@ -26,6 +26,10 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests Categories = new Int32[] { 1 } } }; + + Mocker.GetMock() + .Setup(v => v.GetCapabilities(It.IsAny())) + .Returns(new TorznabCapabilities()); } [Test] diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabRequestGeneratorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabRequestGeneratorFixture.cs index 27dc4392e..03943179a 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabRequestGeneratorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabRequestGeneratorFixture.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Linq; using FluentAssertions; +using Moq; using NUnit.Framework; using NzbDrone.Core.Indexers.Torznab; using NzbDrone.Core.IndexerSearch.Definitions; @@ -11,7 +12,9 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests { public class TorznabRequestGeneratorFixture : CoreTest { - AnimeEpisodeSearchCriteria _animeSearchCriteria; + private SingleEpisodeSearchCriteria _singleEpisodeSearchCriteria; + private AnimeEpisodeSearchCriteria _animeSearchCriteria; + private TorznabCapabilities _capabilities; [SetUp] public void SetUp() @@ -22,6 +25,15 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests Categories = new [] { 1, 2 }, AnimeCategories = new [] { 3, 4 }, ApiKey = "abcd", + EnableRageIDLookup = true + }; + + _singleEpisodeSearchCriteria = new SingleEpisodeSearchCriteria + { + Series = new Tv.Series { TvRageId = 10 }, + SceneTitles = new List { "Monkey Island" }, + SeasonNumber = 1, + EpisodeNumber = 2 }; _animeSearchCriteria = new AnimeEpisodeSearchCriteria() @@ -29,6 +41,12 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests SceneTitles = new List() { "Monkey+Island" }, AbsoluteEpisodeNumber = 100 }; + + _capabilities = new TorznabCapabilities(); + + Mocker.GetMock() + .Setup(v => v.GetCapabilities(It.IsAny())) + .Returns(_capabilities); } [Test] @@ -118,5 +136,31 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests pages.Count.Should().BeLessThan(500); } + + [Test] + public void should_not_search_by_rid_if_not_supported() + { + _capabilities.SupportedTvSearchParameters = new[] { "q", "season", "ep" }; + + var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); + + results.Should().HaveCount(1); + + var page = results.First().First(); + + page.Url.Query.Should().NotContain("rid=10"); + page.Url.Query.Should().Contain("q=Monkey"); + } + + [Test] + public void should_search_by_rid_if_supported() + { + var results = Subject.GetSearchRequests(_singleEpisodeSearchCriteria); + results.Should().HaveCount(1); + + var page = results.First().First(); + + page.Url.Query.Should().Contain("rid=10"); + } } } diff --git a/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs b/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs index 555762ee4..0ea1b1c87 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/Torznab.cs @@ -1,7 +1,9 @@ using System; using System.Collections.Generic; using System.Linq; +using FluentValidation.Results; using NLog; +using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Parser; @@ -11,6 +13,8 @@ namespace NzbDrone.Core.Indexers.Torznab { public class Torznab : HttpIndexerBase { + private readonly ITorznabCapabilitiesProvider _torznabCapabilitiesProvider; + public override string Name { get @@ -24,9 +28,9 @@ namespace NzbDrone.Core.Indexers.Torznab public override IIndexerRequestGenerator GetRequestGenerator() { - return new TorznabRequestGenerator() + return new TorznabRequestGenerator(_torznabCapabilitiesProvider) { - PageSize = PageSize, + PageSize = PageSize, Settings = Settings }; } @@ -44,10 +48,10 @@ namespace NzbDrone.Core.Indexers.Torznab } } - public Torznab(IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) + public Torznab(ITorznabCapabilitiesProvider torznabCapabilitiesProvider, IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, configService, parsingService, logger) { - + _torznabCapabilitiesProvider = torznabCapabilitiesProvider; } private IndexerDefinition GetDefinition(String name, TorznabSettings settings) @@ -76,5 +80,42 @@ namespace NzbDrone.Core.Indexers.Torznab return settings; } + + protected override void Test(List failures) + { + base.Test(failures); + + failures.AddIfNotNull(TestCapabilities()); + } + + protected virtual ValidationFailure TestCapabilities() + { + try + { + var capabilities = _torznabCapabilitiesProvider.GetCapabilities(Settings); + + if (capabilities.SupportedSearchParameters != null && capabilities.SupportedSearchParameters.Contains("q")) + { + return null; + } + + if (capabilities.SupportedTvSearchParameters != null && + (capabilities.SupportedSearchParameters.Contains("q") || capabilities.SupportedSearchParameters.Contains("rid")) && + capabilities.SupportedTvSearchParameters.Contains("season") && capabilities.SupportedTvSearchParameters.Contains("ep")) + { + return null; + } + + return new ValidationFailure(string.Empty, "Indexer does not support required search parameters"); + } + catch (Exception ex) + { + _logger.WarnException("Unable to connect to indexer: " + ex.Message, ex); + + return new ValidationFailure(string.Empty, "Unable to connect to indexer, check the log for more details"); + } + + return null; + } } } diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabCapabilities.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabCapabilities.cs new file mode 100644 index 000000000..f09c36477 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabCapabilities.cs @@ -0,0 +1,28 @@ +using System; +using System.Collections.Generic; + +namespace NzbDrone.Core.Indexers.Torznab +{ + public class TorznabCapabilities + { + public string[] SupportedSearchParameters { get; set; } + public string[] SupportedTvSearchParameters { get; set; } + public List Categories { get; set; } + + public TorznabCapabilities() + { + SupportedSearchParameters = new[] { "q", "offset", "limit" }; + SupportedTvSearchParameters = new[] { "q", "rid", "season", "ep", "offset", "limit" }; + Categories = new List(); + } + } + + public class TorznabCategory + { + public int Id { get; set; } + public string Name { get; set; } + public string Description { get; set; } + + public List Subcategories { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabCapabilitiesProvider.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabCapabilitiesProvider.cs new file mode 100644 index 000000000..3d5cef1ca --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabCapabilitiesProvider.cs @@ -0,0 +1,126 @@ +using System; +using System.Collections.Generic; +using System.Xml.Linq; +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.Serializer; + +namespace NzbDrone.Core.Indexers.Torznab +{ + public interface ITorznabCapabilitiesProvider + { + TorznabCapabilities GetCapabilities(TorznabSettings settings); + } + + public class TorznabCapabilitiesProvider : ITorznabCapabilitiesProvider + { + private readonly ICached _capabilitiesCache; + private readonly IHttpClient _httpClient; + private readonly Logger _logger; + + public TorznabCapabilitiesProvider(ICacheManager cacheManager, IHttpClient httpClient, Logger logger) + { + _capabilitiesCache = cacheManager.GetCache(GetType()); + _httpClient = httpClient; + _logger = logger; + } + + public TorznabCapabilities GetCapabilities(TorznabSettings indexerSettings) + { + var key = indexerSettings.ToJson(); + var capabilities = _capabilitiesCache.Get(key, () => FetchCapabilities(indexerSettings), TimeSpan.FromDays(7)); + + return capabilities; + } + + private TorznabCapabilities FetchCapabilities(TorznabSettings indexerSettings) + { + var capabilities = new TorznabCapabilities(); + + var url = string.Format("{0}/api?t=caps", indexerSettings.Url.TrimEnd('/')); + + if (indexerSettings.ApiKey.IsNotNullOrWhiteSpace()) + { + url += "&apikey=" + indexerSettings.ApiKey; + } + + var request = new HttpRequest(url, HttpAccept.Rss); + + try + { + var response = _httpClient.Get(request); + + capabilities = ParseCapabilities(response); + } + catch (Exception ex) + { + _logger.DebugException(string.Format("Failed to get capabilities from {0}: {1}", indexerSettings.Url, ex.Message), ex); + } + + return capabilities; + } + + private TorznabCapabilities ParseCapabilities(HttpResponse response) + { + var capabilities = new TorznabCapabilities(); + + var xmlRoot = XDocument.Parse(response.Content).Element("caps"); + + var xmlSearching = xmlRoot.Element("searching"); + if (xmlSearching != null) + { + var xmlBasicSearch = xmlSearching.Element("search"); + if (xmlBasicSearch == null || xmlBasicSearch.Attribute("available").Value != "yes") + { + capabilities.SupportedSearchParameters = null; + } + else if (xmlBasicSearch.Attribute("supportedParams") != null) + { + capabilities.SupportedSearchParameters = xmlBasicSearch.Attribute("supportedParams").Value.Split(','); + } + + var xmlTvSearch = xmlSearching.Element("tv-search"); + if (xmlTvSearch == null || xmlTvSearch.Attribute("available").Value != "yes") + { + capabilities.SupportedTvSearchParameters = null; + } + else if (xmlTvSearch.Attribute("supportedParams") != null) + { + capabilities.SupportedTvSearchParameters = xmlTvSearch.Attribute("supportedParams").Value.Split(','); + } + } + + var xmlCategories = xmlRoot.Element("categories"); + if (xmlCategories != null) + { + foreach (var xmlCategory in xmlCategories.Elements("category")) + { + var cat = new TorznabCategory + { + Id = int.Parse(xmlCategory.Attribute("id").Value), + Name = xmlCategory.Attribute("name").Value, + Description = xmlCategory.Attribute("description") != null ? xmlCategory.Attribute("description").Value : string.Empty, + Subcategories = new List() + }; + + foreach (var xmlSubcat in xmlCategory.Elements("subcat")) + { + cat.Subcategories.Add(new TorznabCategory + { + Id = int.Parse(xmlSubcat.Attribute("id").Value), + Name = xmlSubcat.Attribute("name").Value, + Description = xmlSubcat.Attribute("description") != null ? xmlCategory.Attribute("description").Value : string.Empty + + }); + } + + capabilities.Categories.Add(cat); + } + } + + return capabilities; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabRequestGenerator.cs index 0eb075efa..abc36ca11 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabRequestGenerator.cs @@ -9,22 +9,70 @@ namespace NzbDrone.Core.Indexers.Torznab { public class TorznabRequestGenerator : IIndexerRequestGenerator { - public Int32 MaxPages { get; set; } - public Int32 PageSize { get; set; } + private readonly ITorznabCapabilitiesProvider _capabilitiesProvider; + + public int MaxPages { get; set; } + public int PageSize { get; set; } + public TorznabSettings Settings { get; set; } - public TorznabRequestGenerator() + public TorznabRequestGenerator(ITorznabCapabilitiesProvider capabilitiesProvider) { + _capabilitiesProvider = capabilitiesProvider; + MaxPages = 30; PageSize = 100; } + private bool SupportsSearch + { + get + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings); + + return capabilities.SupportedSearchParameters != null + && capabilities.SupportedSearchParameters.Contains("q"); + } + } + + private bool SupportsTvSearch + { + get + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings); + + return capabilities.SupportedTvSearchParameters != null + && capabilities.SupportedTvSearchParameters.Contains("q") + && capabilities.SupportedTvSearchParameters.Contains("season") + && capabilities.SupportedTvSearchParameters.Contains("ep"); + } + } + + private bool SupportsTvRageSearch + { + get + { + var capabilities = _capabilitiesProvider.GetCapabilities(Settings); + + return capabilities.SupportedTvSearchParameters != null + && capabilities.SupportedTvSearchParameters.Contains("rid") + && capabilities.SupportedTvSearchParameters.Contains("season") + && capabilities.SupportedTvSearchParameters.Contains("ep") + && Settings.EnableRageIDLookup; + } + } + public virtual IList> GetRecentRequests() { var pageableRequests = new List>(); - // TODO: We might consider getting multiple pages in the future, but atm we limit it to 1 page. - pageableRequests.AddIfNotNull(GetPagedRequests(1, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); + var capabilities = _capabilitiesProvider.GetCapabilities(Settings); + + if (capabilities.SupportedTvSearchParameters != null) + { + // TODO: We might consider getting multiple pages in the future, but atm we limit it to 1 page. + pageableRequests.AddIfNotNull(GetPagedRequests(1, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); + } return pageableRequests; } @@ -33,20 +81,20 @@ namespace NzbDrone.Core.Indexers.Torznab { var pageableRequests = new List>(); - if (searchCriteria.Series.TvRageId > 0 && Settings.EnableRageIDLookup) + if (searchCriteria.Series.TvRageId > 0 && SupportsTvRageSearch) { pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - String.Format("&rid={0}&season={1}&ep={2}", + string.Format("&rid={0}&season={1}&ep={2}", searchCriteria.Series.TvRageId, searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber))); } - else + else if (SupportsTvSearch) { foreach (var queryTitle in searchCriteria.QueryTitles) { pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - String.Format("&q={0}&season={1}&ep={2}", + string.Format("&q={0}&season={1}&ep={2}", NewsnabifyTitle(queryTitle), searchCriteria.SeasonNumber, searchCriteria.EpisodeNumber))); @@ -60,19 +108,19 @@ namespace NzbDrone.Core.Indexers.Torznab { var pageableRequests = new List>(); - if (searchCriteria.Series.TvRageId > 0 && Settings.EnableRageIDLookup) + if (searchCriteria.Series.TvRageId > 0 && SupportsTvRageSearch) { pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - String.Format("&rid={0}&season={1}", + string.Format("&rid={0}&season={1}", searchCriteria.Series.TvRageId, searchCriteria.SeasonNumber))); } - else + else if (SupportsTvSearch) { foreach (var queryTitle in searchCriteria.QueryTitles) { pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - String.Format("&q={0}&season={1}", + string.Format("&q={0}&season={1}", NewsnabifyTitle(queryTitle), searchCriteria.SeasonNumber))); } @@ -85,19 +133,19 @@ namespace NzbDrone.Core.Indexers.Torznab { var pageableRequests = new List>(); - if (searchCriteria.Series.TvRageId > 0 && Settings.EnableRageIDLookup) + if (searchCriteria.Series.TvRageId > 0 && SupportsTvRageSearch) { pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - String.Format("&rid={0}&season={1:yyyy}&ep={1:MM}/{1:dd}", + string.Format("&rid={0}&season={1:yyyy}&ep={1:MM}/{1:dd}", searchCriteria.Series.TvRageId, searchCriteria.AirDate))); } - else + else if (SupportsTvSearch) { foreach (var queryTitle in searchCriteria.QueryTitles) { pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories, "tvsearch", - String.Format("&q={0}&season={1:yyyy}&ep={1:MM}/{1:dd}", + string.Format("&q={0}&season={1:yyyy}&ep={1:MM}/{1:dd}", NewsnabifyTitle(queryTitle), searchCriteria.AirDate))); } @@ -110,12 +158,15 @@ namespace NzbDrone.Core.Indexers.Torznab { var pageableRequests = new List>(); - foreach (var queryTitle in searchCriteria.QueryTitles) + if (SupportsSearch) { - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.AnimeCategories, "search", - String.Format("&q={0}+{1:00}", - NewsnabifyTitle(queryTitle), - searchCriteria.AbsoluteEpisodeNumber))); + foreach (var queryTitle in searchCriteria.QueryTitles) + { + pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.AnimeCategories, "search", + string.Format("&q={0}+{1:00}", + NewsnabifyTitle(queryTitle), + searchCriteria.AbsoluteEpisodeNumber))); + } } return pageableRequests; @@ -125,29 +176,32 @@ namespace NzbDrone.Core.Indexers.Torznab { var pageableRequests = new List>(); - foreach (var queryTitle in searchCriteria.EpisodeQueryTitles) + if (SupportsSearch) { - var query = queryTitle.Replace('+', ' '); - query = System.Web.HttpUtility.UrlEncode(query); + foreach (var queryTitle in searchCriteria.EpisodeQueryTitles) + { + var query = queryTitle.Replace('+', ' '); + query = System.Web.HttpUtility.UrlEncode(query); - pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "search", - String.Format("&q={0}", - query))); + pageableRequests.AddIfNotNull(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "search", + string.Format("&q={0}", + query))); + } } return pageableRequests; } - private IEnumerable GetPagedRequests(Int32 maxPages, IEnumerable categories, String searchType, String parameters) + private IEnumerable GetPagedRequests(int maxPages, IEnumerable categories, string searchType, string parameters) { if (categories.Empty()) { yield break; } - var categoriesQuery = String.Join(",", categories.Distinct()); + var categoriesQuery = string.Join(",", categories.Distinct()); - var baseUrl = String.Format("{0}/api?t={1}&cat={2}&extended=1{3}", Settings.Url.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters); + var baseUrl = string.Format("{0}/api?t={1}&cat={2}&extended=1{3}", Settings.Url.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters); if (Settings.ApiKey.IsNotNullOrWhiteSpace()) { @@ -156,18 +210,18 @@ namespace NzbDrone.Core.Indexers.Torznab if (PageSize == 0) { - yield return new IndexerRequest(String.Format("{0}{1}", baseUrl, parameters), HttpAccept.Rss); + yield return new IndexerRequest(string.Format("{0}{1}", baseUrl, parameters), HttpAccept.Rss); } else { for (var page = 0; page < maxPages; page++) { - yield return new IndexerRequest(String.Format("{0}&offset={1}&limit={2}{3}", baseUrl, page * PageSize, PageSize, parameters), HttpAccept.Rss); + yield return new IndexerRequest(string.Format("{0}&offset={1}&limit={2}{3}", baseUrl, page * PageSize, PageSize, parameters), HttpAccept.Rss); } } } - private static String NewsnabifyTitle(String title) + private static string NewsnabifyTitle(string title) { return title.Replace("+", "%20"); } diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs index 0745b0af9..55be40bed 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs @@ -68,6 +68,7 @@ namespace NzbDrone.Core.Indexers.Torznab [FieldDefinition(4, Label = "Additional Parameters", HelpText = "Additional Torznab parameters", Advanced = true)] public string AdditionalParameters { get; set; } + // TODO: To be removed in the next version. [FieldDefinition(5, Type = FieldType.Checkbox, Label = "Enable RageID Lookup", HelpText = "Disable this if your tracker doesn't have tvrage ids, Sonarr will then use (more expensive) title queries.", Advanced = true)] public bool EnableRageIDLookup { get; set; } diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 5aff47579..faa22abc5 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -538,6 +538,8 @@ + +