From 4fbc481780ca260cd180ad5c48a43520a958880a Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Sun, 21 May 2017 17:23:28 +0200 Subject: [PATCH] Fixed: Processing of mixed newznab/torznab api such as the experimental animetosho api. Ref #1384 --- .../Indexers/Torznab/torznab_animetosho.xml | 60 ++++++++++++++++ .../NewznabTests/NewznabFixture.cs | 30 ++++++++ .../TorrentRssSettingsDetectorFixture.cs | 2 +- .../TorznabTests/TorznabFixture.cs | 31 ++++++++ .../NzbDrone.Core.Test.csproj | 3 + src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 12 +++- src/NzbDrone.Core/Indexers/IndexerBase.cs | 1 + .../Newznab/NewznabCapabilitiesProvider.cs | 2 +- .../Newznab/NewznabRequestGenerator.cs | 6 +- .../Indexers/Newznab/NewznabRssParser.cs | 45 ++++++------ .../Indexers/Newznab/NewznabSettings.cs | 15 ++-- src/NzbDrone.Core/Indexers/RssEnclosure.cs | 14 ++++ src/NzbDrone.Core/Indexers/RssParser.cs | 72 ++++++++++++++----- .../Indexers/TorrentRssParser.cs | 2 +- .../Indexers/Torznab/TorznabRssParser.cs | 36 ++++++---- .../Indexers/Torznab/TorznabSettings.cs | 3 +- src/NzbDrone.Core/NzbDrone.Core.csproj | 1 + .../Validation/RuleBuilderExtensions.cs | 6 +- 18 files changed, 274 insertions(+), 67 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Files/Indexers/Torznab/torznab_animetosho.xml create mode 100644 src/NzbDrone.Core/Indexers/RssEnclosure.cs diff --git a/src/NzbDrone.Core.Test/Files/Indexers/Torznab/torznab_animetosho.xml b/src/NzbDrone.Core.Test/Files/Indexers/Torznab/torznab_animetosho.xml new file mode 100644 index 000000000..94505a443 --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/Torznab/torznab_animetosho.xml @@ -0,0 +1,60 @@ + + + + + Anime Tosho + https://localhost/ + Latest releases feed + en-gb + 30 + Wed, 17 May 2017 20:36:06 +0000 + + + [finFAGs]_Frame_Arms_Girl_07_(1280x720_TV_AAC)_[1262B6F7].mkv + Wed, 17 May 2017 20:36:06 +0000 + https://localhost/view/123451 + Anime + Total Size: 301.8 MB
]]> + https://localhost/view/finfags-_frame_arms_girl_07_-1280x720_tv_aac-_-1262b6f7-mkv.123451 + https://localhost/view/finfags-_frame_arms_girl_07_-1280x720_tv_aac-_-1262b6f7-mkv.123451 + + TokyoTosho + + + + + + + + + + + + + + + [HorribleSubs] Frame Arms Girl - 07 [720p].mkv + Mon, 15 May 2017 19:15:56 +0000 + https://localhost/view/123452 + Anime + Total Size: 452.0 MB
]]> + https://localhost/view/horriblesubs-frame-arms-girl-07-720p-mkv.123452 + https://localhost/view/horriblesubs-frame-arms-girl-07-720p-mkv.123452 + + + TokyoTosho + + + + + + + + + + + + + + + diff --git a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs index 7f92e6637..f38cdbf8a 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NewznabTests/NewznabFixture.cs @@ -7,6 +7,7 @@ using NUnit.Framework; using NzbDrone.Common.Http; using NzbDrone.Core.Indexers; using NzbDrone.Core.Indexers.Newznab; +using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Test.Framework; using NzbDrone.Test.Common; @@ -63,6 +64,35 @@ namespace NzbDrone.Core.Test.IndexerTests.NewznabTests releaseInfo.Size.Should().Be(1183105773); } + + [Test] + public void should_parse_recent_feed_from_newznab_animetosho() + { + var recentFeed = ReadAllText(@"Files/Indexers/Torznab/torznab_animetosho.xml"); + + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) + .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(1); + + releases.First().Should().BeOfType(); + var releaseInfo = releases.First() as ReleaseInfo; + + releaseInfo.Title.Should().Be("[HorribleSubs] Frame Arms Girl - 07 [720p].mkv"); + releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Usenet); + releaseInfo.DownloadUrl.Should().Be("http://storage.localhost/nzb/123452.nzb"); + releaseInfo.InfoUrl.Should().Be("https://localhost/view/horriblesubs-frame-arms-girl-07-720p-mkv.123452"); + releaseInfo.CommentUrl.Should().Be("https://localhost/view/horriblesubs-frame-arms-girl-07-720p-mkv.123452"); + releaseInfo.Indexer.Should().Be(Subject.Definition.Name); + releaseInfo.PublishDate.Should().Be(DateTime.Parse("Mon, 15 May 2017 19:15:56 +0000").ToUniversalTime()); + releaseInfo.Size.Should().Be(473987489); + releaseInfo.TvdbId.Should().Be(0); + releaseInfo.TvRageId.Should().Be(0); + } + [Test] public void should_use_pagesize_reported_by_caps() { diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs index 74934b160..d316cea85 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs @@ -256,11 +256,11 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests [TestCase("BitMeTv/BitMeTv.xml")] [TestCase("Fanzub/fanzub.xml")] [TestCase("IPTorrents/IPTorrents.xml")] - [TestCase("Newznab/newznab_nzb_su.xml")] [TestCase("Nyaa/Nyaa.xml")] [TestCase("Omgwtfnzbs/Omgwtfnzbs.xml")] [TestCase("Torznab/torznab_hdaccess_net.xml")] [TestCase("Torznab/torznab_tpb.xml")] + [TestCase("Torznab/torznab_animetosho.xml")] public void should_detect_recent_feed(string rssXmlFile) { GivenRecentFeedResponse(rssXmlFile); diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs index e5467d4ab..e2d5a9ea7 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorznabTests/TorznabFixture.cs @@ -97,6 +97,37 @@ namespace NzbDrone.Core.Test.IndexerTests.TorznabTests releaseInfo.Peers.Should().Be(36724); } + [Test] + public void should_parse_recent_feed_from_torznab_animetosho() + { + var recentFeed = ReadAllText(@"Files/Indexers/Torznab/torznab_animetosho.xml"); + + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) + .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(2); + + releases.First().Should().BeOfType(); + var releaseInfo = releases.First() as TorrentInfo; + + releaseInfo.Title.Should().Be("[finFAGs]_Frame_Arms_Girl_07_(1280x720_TV_AAC)_[1262B6F7].mkv"); + releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + releaseInfo.DownloadUrl.Should().Be("http://storage.localhost/torrents/123451.torrent"); + releaseInfo.InfoUrl.Should().Be("https://localhost/view/finfags-_frame_arms_girl_07_-1280x720_tv_aac-_-1262b6f7-mkv.123451"); + releaseInfo.CommentUrl.Should().Be("https://localhost/view/finfags-_frame_arms_girl_07_-1280x720_tv_aac-_-1262b6f7-mkv.123451"); + releaseInfo.Indexer.Should().Be(Subject.Definition.Name); + releaseInfo.PublishDate.Should().Be(DateTime.Parse("Wed, 17 May 2017 20:36:06 +0000").ToUniversalTime()); + releaseInfo.Size.Should().Be(316477946); + releaseInfo.TvdbId.Should().Be(0); + releaseInfo.TvRageId.Should().Be(0); + releaseInfo.InfoHash.Should().Be("2d69a861bef5a9f2cdf791b7328e37b7953205e1"); + releaseInfo.Seeders.Should().BeNull(); + releaseInfo.Peers.Should().BeNull(); + } + [Test] public void should_use_pagesize_reported_by_caps() { diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index de84c2cb9..ffd18cfd2 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -433,6 +433,9 @@ Always + + Always + Always diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index 973afe6b1..481d55056 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -166,7 +166,7 @@ namespace NzbDrone.Core.Indexers } } - releases.AddRange(pagedReleases); + releases.AddRange(pagedReleases.Where(IsValidRelease)); } if (releases.Any()) @@ -268,6 +268,16 @@ namespace NzbDrone.Core.Indexers return CleanupReleases(releases); } + protected virtual bool IsValidRelease(ReleaseInfo release) + { + if (release.DownloadUrl.IsNullOrWhiteSpace()) + { + return false; + } + + return true; + } + protected virtual bool IsFullPage(IList page) { return PageSize != 0 && page.Count >= PageSize; diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index 8d8419095..29d782f88 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -74,6 +74,7 @@ namespace NzbDrone.Core.Indexers result.ForEach(c => { + c.Guid = string.Concat(Definition.Id, "_", c.Guid); c.IndexerId = Definition.Id; c.Indexer = Definition.Name; c.DownloadProtocol = Protocol; diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs index cb883b851..1831d8341 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabCapabilitiesProvider.cs @@ -41,7 +41,7 @@ namespace NzbDrone.Core.Indexers.Newznab { var capabilities = new NewznabCapabilities(); - var url = string.Format("{0}/api?t=caps", indexerSettings.BaseUrl.TrimEnd('/')); + var url = string.Format("{0}{1}?t=caps", indexerSettings.BaseUrl.TrimEnd('/'), indexerSettings.ApiPath.TrimEnd('/')); if (indexerSettings.ApiKey.IsNotNullOrWhiteSpace()) { diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs index 450027233..10d1fb9c2 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRequestGenerator.cs @@ -104,6 +104,10 @@ namespace NzbDrone.Core.Indexers.Newznab { pageableRequests.Add(GetPagedRequests(MaxPages, Settings.Categories.Concat(Settings.AnimeCategories), "tvsearch", "")); } + else if (capabilities.SupportedSearchParameters != null) + { + pageableRequests.Add(GetPagedRequests(MaxPages, Settings.AnimeCategories, "search", "")); + } return pageableRequests; } @@ -249,7 +253,7 @@ namespace NzbDrone.Core.Indexers.Newznab var categoriesQuery = string.Join(",", categories.Distinct()); - var baseUrl = string.Format("{0}/api?t={1}&cat={2}&extended=1{3}", Settings.BaseUrl.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters); + var baseUrl = string.Format("{0}{1}?t={2}&cat={3}&extended=1{4}", Settings.BaseUrl.TrimEnd('/'), Settings.ApiPath.TrimEnd('/'), searchType, categoriesQuery, Settings.AdditionalParameters); if (Settings.ApiKey.IsNotNullOrWhiteSpace()) { diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs index 16c4dea9b..c2b733dfc 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Xml.Linq; using NzbDrone.Common.Extensions; @@ -13,7 +14,8 @@ namespace NzbDrone.Core.Indexers.Newznab public NewznabRssParser() { - PreferredEnclosureMimeType = "application/x-nzb"; + PreferredEnclosureMimeTypes = UsenetEnclosureMimeTypes; + UseEnclosureUrl = true; } protected override bool PreProcess(IndexerResponse indexerResponse) @@ -45,6 +47,24 @@ namespace NzbDrone.Core.Indexers.Newznab throw new NewznabException(indexerResponse, errorMessage); } + protected override bool PostProcess(IndexerResponse indexerResponse, List items, List releases) + { + var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray(); + if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty()) + { + if (enclosureTypes.Intersect(TorrentEnclosureMimeTypes).Any()) + { + _logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Torznab indexer?", NzbEnclosureMimeType, enclosureTypes[0]); + } + else + { + _logger.Warn("Feed does not contain {0}, found {1}.", NzbEnclosureMimeType, enclosureTypes[0]); + } + } + + return true; + } + protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo) { releaseInfo = base.ProcessItem(item, releaseInfo); @@ -55,17 +75,6 @@ namespace NzbDrone.Core.Indexers.Newznab return releaseInfo; } - protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) - { - var enclosureType = GetEnclosure(item).Attribute("type").Value; - if (enclosureType.Contains("application/x-bittorrent")) - { - throw new UnsupportedFeedException("Feed contains {0}, did you intend to add a Torznab indexer?", enclosureType); - } - - return base.PostProcess(item, releaseInfo); - } - protected override string GetInfoUrl(XElement item) { return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments")); @@ -102,18 +111,6 @@ namespace NzbDrone.Core.Indexers.Newznab return base.GetPublishDate(item); } - protected override string GetDownloadUrl(XElement item) - { - var url = base.GetDownloadUrl(item); - - if (!Uri.IsWellFormedUriString(url, UriKind.Absolute)) - { - url = ParseUrl((string)item.Element("enclosure").Attribute("url")); - } - - return url; - } - protected virtual int GetTvdbId(XElement item) { var tvdbIdString = TryGetNewznabAttribute(item, "tvdbid"); diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs index ead8d1e62..ee7bd3d9d 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabSettings.cs @@ -48,6 +48,7 @@ namespace NzbDrone.Core.Indexers.Newznab }); RuleFor(c => c.BaseUrl).ValidRootUrl(); + RuleFor(c => c.ApiPath).ValidUrlBase("/api"); RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey); RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex) .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace()); @@ -60,6 +61,7 @@ namespace NzbDrone.Core.Indexers.Newznab public NewznabSettings() { + ApiPath = "/api"; Categories = new[] { 5030, 5040 }; AnimeCategories = Enumerable.Empty(); } @@ -67,19 +69,22 @@ namespace NzbDrone.Core.Indexers.Newznab [FieldDefinition(0, Label = "URL")] public string BaseUrl { get; set; } - [FieldDefinition(1, Label = "API Key")] + [FieldDefinition(1, Label = "API Path", HelpText = "Path to the api, usually /api", Advanced = true)] + public string ApiPath { get; set; } + + [FieldDefinition(2, Label = "API Key")] public string ApiKey { get; set; } - [FieldDefinition(2, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows", Advanced = true)] + [FieldDefinition(3, Label = "Categories", HelpText = "Comma Separated list, leave blank to disable standard/daily shows")] public IEnumerable Categories { get; set; } - [FieldDefinition(3, Label = "Anime Categories", HelpText = "Comma Separated list, leave blank to disable anime", Advanced = true)] + [FieldDefinition(4, Label = "Anime Categories", HelpText = "Comma Separated list, leave blank to disable anime")] public IEnumerable AnimeCategories { get; set; } - [FieldDefinition(4, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)] + [FieldDefinition(5, Label = "Additional Parameters", HelpText = "Additional Newznab parameters", Advanced = true)] public string AdditionalParameters { get; set; } - // Field 5 is used by TorznabSettings MinimumSeeders + // Field 6 is used by TorznabSettings MinimumSeeders // If you need to add another field here, update TorznabSettings as well and this comment public virtual NzbDroneValidationResult Validate() diff --git a/src/NzbDrone.Core/Indexers/RssEnclosure.cs b/src/NzbDrone.Core/Indexers/RssEnclosure.cs new file mode 100644 index 000000000..de46e8d14 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/RssEnclosure.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.Indexers +{ + public class RssEnclosure + { + public string Url { get; set; } + public string Type { get; set; } + public long Length { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index 93304f2fd..5687adb51 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -19,6 +19,11 @@ namespace NzbDrone.Core.Indexers public class RssParser : IParseIndexerResponse { private static readonly Regex ReplaceEntities = new Regex("&[a-z]+;", RegexOptions.Compiled | RegexOptions.IgnoreCase); + public const string NzbEnclosureMimeType = "application/x-nzb"; + public const string TorrentEnclosureMimeType = "application/x-bittorrent"; + public const string MagnetEnclosureMimeType = "application/x-bittorrent;x-scheme-handler/magnet"; + public static readonly string[] UsenetEnclosureMimeTypes = new[] { NzbEnclosureMimeType }; + public static readonly string[] TorrentEnclosureMimeTypes = new[] { TorrentEnclosureMimeType, MagnetEnclosureMimeType }; protected readonly Logger _logger; @@ -32,7 +37,7 @@ namespace NzbDrone.Core.Indexers // Parse "Size: 1.3 GB" or "1.3 GB" parts in the description element and use that as Size. public bool ParseSizeInDescription { get; set; } - public string PreferredEnclosureMimeType { get; set; } + public string[] PreferredEnclosureMimeTypes { get; set; } private IndexerResponse _indexerResponse; @@ -53,7 +58,7 @@ namespace NzbDrone.Core.Indexers } var document = LoadXmlDocument(indexerResponse); - var items = GetItems(document); + var items = GetItems(document).ToList(); foreach (var item in items) { @@ -77,6 +82,11 @@ namespace NzbDrone.Core.Indexers } } + if (!PostProcess(indexerResponse, items, releases)) + { + return new List(); + } + return releases; } @@ -124,6 +134,11 @@ namespace NzbDrone.Core.Indexers return true; } + protected virtual bool PostProcess(IndexerResponse indexerResponse, List elements, List releases) + { + return true; + } + protected ReleaseInfo ProcessItem(XElement item) { var releaseInfo = CreateNewReleaseInfo(); @@ -132,7 +147,7 @@ namespace NzbDrone.Core.Indexers _logger.Trace("Parsed: {0}", releaseInfo.Title); - return PostProcess(item, releaseInfo); + return PostProcessItem(item, releaseInfo); } protected virtual ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo) @@ -156,7 +171,7 @@ namespace NzbDrone.Core.Indexers return releaseInfo; } - protected virtual ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) + protected virtual ReleaseInfo PostProcessItem(XElement item, ReleaseInfo releaseInfo) { return releaseInfo; } @@ -187,7 +202,8 @@ namespace NzbDrone.Core.Indexers { if (UseEnclosureUrl) { - return ParseUrl((string)GetEnclosure(item).Attribute("url")); + var enclosure = GetEnclosure(item); + return enclosure != null ? ParseUrl(enclosure.Url) : null; } return ParseUrl((string)item.Element("link")); @@ -228,37 +244,59 @@ namespace NzbDrone.Core.Indexers if (enclosure != null) { - return (long)enclosure.Attribute("length"); + return enclosure.Length; } return 0; } - protected virtual XElement GetEnclosure(XElement item) + protected virtual RssEnclosure[] GetEnclosures(XElement item) { - var enclosures = item.Elements("enclosure").ToArray(); + var enclosures = item.Elements("enclosure") + .Select(v => new RssEnclosure + { + Url = v.Attribute("url").Value, + Type = v.Attribute("type").Value, + Length = (long)v.Attribute("length") + }) + .ToArray(); + + return enclosures; + } + protected RssEnclosure GetEnclosure(XElement item, bool enforceMimeType = true) + { + var enclosures = GetEnclosures(item); + + return GetEnclosure(enclosures, enforceMimeType); + } + + protected virtual RssEnclosure GetEnclosure(RssEnclosure[] enclosures, bool enforceMimeType = true) + { if (enclosures.Length == 0) { return null; } - if (enclosures.Length == 1) + if (PreferredEnclosureMimeTypes != null) { - return enclosures.First(); - } + foreach (var preferredEnclosureType in PreferredEnclosureMimeTypes) + { + var preferredEnclosure = enclosures.FirstOrDefault(v => v.Type == preferredEnclosureType); - if (PreferredEnclosureMimeType != null) - { - var preferredEnclosure = enclosures.FirstOrDefault(v => v.Attribute("type").Value == PreferredEnclosureMimeType); + if (preferredEnclosure != null) + { + return preferredEnclosure; + } + } - if (preferredEnclosure != null) + if (enforceMimeType) { - return preferredEnclosure; + return null; } } - return item.Elements("enclosure").SingleOrDefault(); + return enclosures.SingleOrDefault(); } protected IEnumerable GetItems(XDocument document) diff --git a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs index b77022540..339618294 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs @@ -16,7 +16,7 @@ namespace NzbDrone.Core.Indexers public TorrentRssParser() { - PreferredEnclosureMimeType = "application/x-bittorrent"; + PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes; } public IEnumerable GetItems(IndexerResponse indexerResponse) diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs index 253386963..8eaaf851a 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabRssParser.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Linq; using System.Xml.Linq; using NzbDrone.Common.Extensions; @@ -11,6 +12,11 @@ namespace NzbDrone.Core.Indexers.Torznab { public const string ns = "{http://torznab.com/schemas/2015/feed}"; + public TorznabRssParser() + { + UseEnclosureUrl = true; + } + protected override bool PreProcess(IndexerResponse indexerResponse) { var xdoc = LoadXmlDocument(indexerResponse); @@ -36,6 +42,24 @@ namespace NzbDrone.Core.Indexers.Torznab throw new TorznabException("Torznab error detected: {0}", errorMessage); } + protected override bool PostProcess(IndexerResponse indexerResponse, List items, List releases) + { + var enclosureTypes = items.SelectMany(GetEnclosures).Select(v => v.Type).Distinct().ToArray(); + if (enclosureTypes.Any() && enclosureTypes.Intersect(PreferredEnclosureMimeTypes).Empty()) + { + if (enclosureTypes.Intersect(UsenetEnclosureMimeTypes).Any()) + { + _logger.Warn("Feed does not contain {0}, found {1}, did you intend to add a Newznab indexer?", TorrentEnclosureMimeType, enclosureTypes[0]); + } + else + { + _logger.Warn("Feed does not contain {0}, found {1}.", TorrentEnclosureMimeType, enclosureTypes[0]); + } + } + + return true; + } + protected override ReleaseInfo ProcessItem(XElement item, ReleaseInfo releaseInfo) { var torrentInfo = base.ProcessItem(item, releaseInfo) as TorrentInfo; @@ -46,18 +70,6 @@ namespace NzbDrone.Core.Indexers.Torznab return torrentInfo; } - protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) - { - var enclosureType = item.Element("enclosure").Attribute("type").Value; - if (!enclosureType.Contains("application/x-bittorrent")) - { - throw new UnsupportedFeedException("Feed contains {0} instead of application/x-bittorrent", enclosureType); - } - - return base.PostProcess(item, releaseInfo); - } - - protected override string GetInfoUrl(XElement item) { return ParseUrl(item.TryGetValue("comments").TrimEnd("#comments")); diff --git a/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs b/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs index 7e847c294..bbbfcfce5 100644 --- a/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs +++ b/src/NzbDrone.Core/Indexers/Torznab/TorznabSettings.cs @@ -41,6 +41,7 @@ namespace NzbDrone.Core.Indexers.Torznab }); RuleFor(c => c.BaseUrl).ValidRootUrl(); + RuleFor(c => c.ApiPath).ValidUrlBase("/api"); RuleFor(c => c.ApiKey).NotEmpty().When(ShouldHaveApiKey); RuleFor(c => c.AdditionalParameters).Matches(AdditionalParametersRegex) .When(c => !c.AdditionalParameters.IsNullOrWhiteSpace()); @@ -56,7 +57,7 @@ namespace NzbDrone.Core.Indexers.Torznab MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS; } - [FieldDefinition(5, Type = FieldType.Textbox, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)] + [FieldDefinition(6, Type = FieldType.Textbox, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)] public int MinimumSeeders { get; set; } public override NzbDroneValidationResult Validate() diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 25b4ece55..215aca9de 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -669,6 +669,7 @@ + diff --git a/src/NzbDrone.Core/Validation/RuleBuilderExtensions.cs b/src/NzbDrone.Core/Validation/RuleBuilderExtensions.cs index df1d8056b..2fae03d6c 100644 --- a/src/NzbDrone.Core/Validation/RuleBuilderExtensions.cs +++ b/src/NzbDrone.Core/Validation/RuleBuilderExtensions.cs @@ -34,9 +34,9 @@ namespace NzbDrone.Core.Validation return ruleBuilder.SetValidator(new RegularExpressionValidator("^https?://[-_a-z0-9.]+", RegexOptions.IgnoreCase)).WithMessage("must be valid URL that starts with http(s)://"); } - public static IRuleBuilderOptions ValidUrlBase(this IRuleBuilder ruleBuilder) + public static IRuleBuilderOptions ValidUrlBase(this IRuleBuilder ruleBuilder, string example = "/sonarr") { - return ruleBuilder.SetValidator(new RegularExpressionValidator(@"^(?!\/?https?://[-_a-z0-9.]+)", RegexOptions.IgnoreCase)).WithMessage("Must be a valid URL path (ie: '/sonarr')"); + return ruleBuilder.SetValidator(new RegularExpressionValidator(@"^(?!\/?https?://[-_a-z0-9.]+)", RegexOptions.IgnoreCase)).WithMessage($"Must be a valid URL path (ie: '{example}')"); } public static IRuleBuilderOptions ValidPort(this IRuleBuilder ruleBuilder) @@ -68,4 +68,4 @@ namespace NzbDrone.Core.Validation return ruleBuilder.WithState(v => NzbDroneValidationState.Warning); } } -} \ No newline at end of file +}