From 2b1c97ffa4e00d9309d2e7a9e15869ef7d1f541a Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Tue, 2 Aug 2016 17:41:00 +0200 Subject: [PATCH] Fixed: AnimoTosho RSS feed size parsing. Also added handling for multiple enclosure elements. ref #1384 --- .../Indexers/TorrentRss/AnimeTosho_NoSize.xml | 41 ++++++++++++++++++ .../TorrentRssIndexerFixture.cs | 43 +++++++++++++++++++ .../TorrentRssSettingsDetectorFixture.cs | 20 +++++++++ .../NzbDrone.Core.Test.csproj | 3 ++ .../Indexers/Newznab/NewznabRssParser.cs | 7 ++- src/NzbDrone.Core/Indexers/RssParser.cs | 35 +++++++++++++-- .../TorrentRss/TorrentRssSettingsDetector.cs | 6 ++- .../Indexers/TorrentRssParser.cs | 2 +- 8 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Files/Indexers/TorrentRss/AnimeTosho_NoSize.xml diff --git a/src/NzbDrone.Core.Test/Files/Indexers/TorrentRss/AnimeTosho_NoSize.xml b/src/NzbDrone.Core.Test/Files/Indexers/TorrentRss/AnimeTosho_NoSize.xml new file mode 100644 index 000000000..4befe325f --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/TorrentRss/AnimeTosho_NoSize.xml @@ -0,0 +1,41 @@ + + + + + + + Anime Tosho + https://animetosho.org/ + Latest releases feed + + en-us + 30 + Tue, 02 Aug 2016 13:48:04 +0000 + + + [FFF] Ore Monogatari!! - Vol.01 [BD][720p-AAC] + + Total Size: 1.366 GB
Download Links: 6 file(s)]]> + https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077 + https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077 + + + TokyoTosho + Tue, 02 Aug 2016 13:48:04 +0000 + https://animetosho.org/view/1009077 + + + DAYS - 05 (1280x720 HEVC2 AAC).mkv + + Total Size: 158.1 MB
Download Links: Go4Up | Jheberg | MultiUp | OpenLoad | Sendspace | SolidFiles | UsersCloud]]> + https://animetosho.org/view/days-05-1280x720-hevc2-aac-mkv.1009055 + https://animetosho.org/view/days-05-1280x720-hevc2-aac-mkv.1009055 + + + TokyoTosho + Tue, 02 Aug 2016 12:50:06 +0000 + https://animetosho.org/view/1009055 + + + + diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs index 01f689d91..77de77ab3 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs @@ -175,5 +175,48 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests torrentInfo.Peers.Should().NotHaveValue(); torrentInfo.Seeders.Should().NotHaveValue(); } + + [Test] + public void should_parse_recent_feed_from_AnimeTosho_without_size() + { + GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml"); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(2); + releases.First().Should().BeOfType(); + + var torrentInfo = releases.First() as TorrentInfo; + + torrentInfo.Title.Should().Be("[FFF] Ore Monogatari!! - Vol.01 [BD][720p-AAC]"); + torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + torrentInfo.DownloadUrl.Should().Be("http://storage.animetosho.org/torrents/85a570f25067f69b3c83b901ce6c00c491345288/%5BFFF%5D%20Ore%20Monogatari%21%21%20-%20Vol.01%20%5BBD%5D%5B720p-AAC%5D.torrent"); + torrentInfo.InfoUrl.Should().BeNullOrEmpty(); + torrentInfo.CommentUrl.Should().Be("https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077"); + torrentInfo.Indexer.Should().Be(Subject.Definition.Name); + torrentInfo.PublishDate.Should().Be(DateTime.Parse("Tue, 02 Aug 2016 13:48:04 +0000").ToUniversalTime()); + torrentInfo.Size.Should().Be((long)Math.Round((double)1.366m * 1024L * 1024L * 1024L)); + torrentInfo.InfoHash.Should().BeNull(); + torrentInfo.MagnetUrl.Should().BeNull(); + torrentInfo.Peers.Should().NotHaveValue(); + torrentInfo.Seeders.Should().NotHaveValue(); + } + + [Test] + public void should_parse_multi_enclosure_from_AnimeTosho() + { + GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml"); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(2); + releases.Last().Should().BeOfType(); + + var torrentInfo = releases.Last() as TorrentInfo; + + torrentInfo.Title.Should().Be("DAYS - 05 (1280x720 HEVC2 AAC).mkv"); + torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + torrentInfo.DownloadUrl.Should().Be("http://storage.animetosho.org/torrents/4b58360143d59a55cbd922397a3eaa378165f3ff/DAYS%20-%2005%20%281280x720%20HEVC2%20AAC%29.torrent"); + } } } diff --git a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs index 349f257bb..cf734c24d 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs @@ -196,6 +196,26 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests settings.Should().BeNull(); } + [Test] + public void should_detect_rss_settings_for_AnimeTosho_without_size() + { + _indexerSettings.AllowZeroSize = true; + + GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml"); + + var settings = Subject.Detect(_indexerSettings); + + settings.ShouldBeEquivalentTo(new TorrentRssIndexerParserSettings + { + UseEZTVFormat = false, + UseEnclosureUrl = true, + UseEnclosureLength = false, + ParseSizeInDescription = true, + ParseSeedersInDescription = false, + SizeElementName = null + }); + } + [TestCase("BitMeTv/BitMeTv.xml")] [TestCase("Fanzub/fanzub.xml")] [TestCase("KickassTorrents/KickassTorrents.xml")] diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index f4e45987d..8edd74b9d 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -426,6 +426,9 @@ Always + + Always + Always diff --git a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs index 1ebe8945d..16c4dea9b 100644 --- a/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs +++ b/src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs @@ -11,6 +11,11 @@ namespace NzbDrone.Core.Indexers.Newznab { public const string ns = "{http://www.newznab.com/DTD/2010/feeds/attributes/}"; + public NewznabRssParser() + { + PreferredEnclosureMimeType = "application/x-nzb"; + } + protected override bool PreProcess(IndexerResponse indexerResponse) { var xdoc = LoadXmlDocument(indexerResponse); @@ -52,7 +57,7 @@ namespace NzbDrone.Core.Indexers.Newznab protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo) { - var enclosureType = item.Element("enclosure").Attribute("type").Value; + var enclosureType = GetEnclosure(item).Attribute("type").Value; if (enclosureType.Contains("application/x-bittorrent")) { throw new UnsupportedFeedException("Feed contains {0}, did you intend to add a Torznab indexer?", enclosureType); diff --git a/src/NzbDrone.Core/Indexers/RssParser.cs b/src/NzbDrone.Core/Indexers/RssParser.cs index 271c5da6b..d2c03cfa4 100644 --- a/src/NzbDrone.Core/Indexers/RssParser.cs +++ b/src/NzbDrone.Core/Indexers/RssParser.cs @@ -32,6 +32,8 @@ namespace NzbDrone.Core.Indexers // Parse "Size: 1.3 GB" or "1.3 GB" parts in the description element and use that as Size. public bool ParseSizeInDescription { get; set; } + public string PreferredEnclosureMimeType { get; set; } + private IndexerResponse _indexerResponse; public RssParser() @@ -192,7 +194,7 @@ namespace NzbDrone.Core.Indexers { if (UseEnclosureUrl) { - return ParseUrl((string)item.Element("enclosure").Attribute("url")); + return ParseUrl((string)GetEnclosure(item).Attribute("url")); } return ParseUrl((string)item.Element("link")); @@ -229,7 +231,7 @@ namespace NzbDrone.Core.Indexers protected virtual long GetEnclosureLength(XElement item) { - var enclosure = item.Element("enclosure"); + var enclosure = GetEnclosure(item); if (enclosure != null) { @@ -239,6 +241,33 @@ namespace NzbDrone.Core.Indexers return 0; } + protected virtual XElement GetEnclosure(XElement item) + { + var enclosures = item.Elements("enclosure").ToArray(); + + if (enclosures.Length == 0) + { + return null; + } + + if (enclosures.Length == 1) + { + return enclosures.First(); + } + + if (PreferredEnclosureMimeType != null) + { + var preferredEnclosure = enclosures.FirstOrDefault(v => v.Attribute("type").Value == PreferredEnclosureMimeType); + + if (preferredEnclosure != null) + { + return preferredEnclosure; + } + } + + return item.Elements("enclosure").SingleOrDefault(); + } + protected IEnumerable GetItems(XDocument document) { var root = document.Root; @@ -278,7 +307,7 @@ namespace NzbDrone.Core.Indexers } } - private static readonly Regex ParseSizeRegex = new Regex(@"(?(?:\d+,)*\d+(?:\.\d{1,2})?)\W?(?[KMG]i?B)(?![\w/])", + private static readonly Regex ParseSizeRegex = new Regex(@"(?(?[KMG]i?B)(?![\w/])", RegexOptions.IgnoreCase | RegexOptions.Compiled); public static long ParseSize(string sizeString, bool defaultToBinaryPrefix) diff --git a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs index 919254abe..0c07e2a51 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs @@ -169,8 +169,12 @@ namespace NzbDrone.Core.Indexers.TorrentRss releases = ParseResponse(parser, response); ValidateReleases(releases, indexerSettings); - if (!releases.Any(r => r.Size < ValidSizeThreshold)) + if (releases.Count(r => r.Size >= ValidSizeThreshold) > releases.Count() / 2) { + if (releases.Any(r => r.Size < ValidSizeThreshold)) + { + _logger.Debug("Feed {0} contains very small releases.", response.Request.Url); + } _logger.Trace("Feed has valid size in description."); return settings; } diff --git a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs index af94ac174..e5451de86 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs @@ -17,7 +17,7 @@ namespace NzbDrone.Core.Indexers public TorrentRssParser() { - + PreferredEnclosureMimeType = "application/x-bittorrent"; } public IEnumerable GetItems(IndexerResponse indexerResponse)