open-source-mirrors
/
Prowlarr
mirror of https://github.com/Prowlarr/Prowlarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fixed: AnimoTosho RSS feed size parsing.

Browse Source 
Also added handling for multiple enclosure elements.

ref #1384
pull/2/head
Taloth Saldono 9 years ago
parent b80d6c74ad
commit 2b1c97ffa4
8 changed files with 151 additions and 6 deletions
Show Stats Download Patch File Download Diff File

41
src/NzbDrone.Core.Test/Files/Indexers/TorrentRss/AnimeTosho_NoSize.xml
Unescape View File

@ -0,0 +1,41 @@
<?xml version="1.0" encoding="utf-8"?>
<rss version="2.0" xmlns:atom="http://www.w3.org/2005/Atom">
<!-- xmlns:content="http://purl.org/rss/1.0/modules/content/" xmlns:dc="http://purl.org/dc/elements/1.1/" -->
<channel>
<atom:link href="https://animetosho.org/feed/rss2" rel="self" type="application/rss+xml" />
<title>Anime Tosho</title>
<link>https://animetosho.org/</link>
<description>Latest releases feed</description>
<language>en-us</language>
<ttl>30</ttl>
<lastBuildDate>Tue, 02 Aug 2016 13:48:04 +0000</lastBuildDate>
<item>
<title>[FFF] Ore Monogatari!! - Vol.01 [BD][720p-AAC]</title>
<!-- <category></category> -->
<description><![CDATA[<strong>Total Size</strong>: 1.366 GB<br /><strong>Download Links</strong>: <a href="https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077">6 file(s)</a>]]></description>
<link>https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077</link>
<comments>https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077</comments>
<enclosure url="http://storage.animetosho.org/torrents/85a570f25067f69b3c83b901ce6c00c491345288/%5BFFF%5D%20Ore%20Monogatari%21%21%20-%20Vol.01%20%5BBD%5D%5B720p-AAC%5D.torrent" type="application/x-bittorrent" length="0" />
<enclosure url="http://storage.animetosho.org/nzb/0002ae5b/%5BFFF%5D%20Ore%20Monogatari%21%21%20-%20Vol.01%20%5BBD%5D%5B720p-AAC%5D.nzb" type="application/x-nzb" length="0" />
<source url="http://www.tokyotosho.info/details.php?id=1009077">TokyoTosho</source>
<pubDate>Tue, 02 Aug 2016 13:48:04 +0000</pubDate>
<guid>https://animetosho.org/view/1009077</guid>
</item>
<item>
<title>DAYS - 05 (1280x720 HEVC2 AAC).mkv</title>
<!-- <category></category> -->
<description><![CDATA[<strong>Total Size</strong>: 158.1 MB<br /><strong>Download Links</strong>: <a href="http://go4up.com/dl/3296dcd8cbd458">Go4Up</a> | <a href="http://jheberg.net/download/days-05-1280x720-hevc2-aac/">Jheberg</a> | <a href="http://www.multiup.org/download/0cde8680f9e9449b0c11be3621872e63/DAYS_-_05__1280x720_HEVC2_AAC_.mkv">MultiUp</a> | <a href="http://openload.co/f/OHnuQ-APk9w/file_334464.7z">OpenLoad</a> | <a href="https://www.sendspace.com/file/ni9s83">Sendspace</a> | <a href="https://www.solidfiles.com/v/53d26VqjmVk6N">SolidFiles</a> | <a href="http://userscloud.com/h3firag6dah6">UsersCloud</a>]]></description>
<link>https://animetosho.org/view/days-05-1280x720-hevc2-aac-mkv.1009055</link>
<comments>https://animetosho.org/view/days-05-1280x720-hevc2-aac-mkv.1009055</comments>
<enclosure url="http://storage.animetosho.org/nzb/0002ae58/DAYS%20-%2005%20%281280x720%20HEVC2%20AAC%29.nzb" type="application/x-nzb" length="0" />
<enclosure url="http://storage.animetosho.org/torrents/4b58360143d59a55cbd922397a3eaa378165f3ff/DAYS%20-%2005%20%281280x720%20HEVC2%20AAC%29.torrent" type="application/x-bittorrent" length="0" />
<source url="http://www.tokyotosho.info/details.php?id=1009055">TokyoTosho</source>
<pubDate>Tue, 02 Aug 2016 12:50:06 +0000</pubDate>
<guid>https://animetosho.org/view/1009055</guid>
</item>
</channel>
</rss>

43
src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssIndexerFixture.cs
Unescape View File

@ -175,5 +175,48 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests
torrentInfo.Peers.Should().NotHaveValue();
torrentInfo.Seeders.Should().NotHaveValue();
}
[Test]
public void should_parse_recent_feed_from_AnimeTosho_without_size()
{
GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml");
var releases = Subject.FetchRecent();
releases.Should().HaveCount(2);
releases.First().Should().BeOfType<TorrentInfo>();
var torrentInfo = releases.First() as TorrentInfo;
torrentInfo.Title.Should().Be("[FFF] Ore Monogatari!! - Vol.01 [BD][720p-AAC]");
torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent);
torrentInfo.DownloadUrl.Should().Be("http://storage.animetosho.org/torrents/85a570f25067f69b3c83b901ce6c00c491345288/%5BFFF%5D%20Ore%20Monogatari%21%21%20-%20Vol.01%20%5BBD%5D%5B720p-AAC%5D.torrent");
torrentInfo.InfoUrl.Should().BeNullOrEmpty();
torrentInfo.CommentUrl.Should().Be("https://animetosho.org/view/fff-ore-monogatari-vol-01-bd-720p-aac.1009077");
torrentInfo.Indexer.Should().Be(Subject.Definition.Name);
torrentInfo.PublishDate.Should().Be(DateTime.Parse("Tue, 02 Aug 2016 13:48:04 +0000").ToUniversalTime());
torrentInfo.Size.Should().Be((long)Math.Round((double)1.366m * 1024L * 1024L * 1024L));
torrentInfo.InfoHash.Should().BeNull();
torrentInfo.MagnetUrl.Should().BeNull();
torrentInfo.Peers.Should().NotHaveValue();
torrentInfo.Seeders.Should().NotHaveValue();
}
[Test]
public void should_parse_multi_enclosure_from_AnimeTosho()
{
GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml");
var releases = Subject.FetchRecent();
releases.Should().HaveCount(2);
releases.Last().Should().BeOfType<TorrentInfo>();
var torrentInfo = releases.Last() as TorrentInfo;
torrentInfo.Title.Should().Be("DAYS - 05 (1280x720 HEVC2 AAC).mkv");
torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent);
torrentInfo.DownloadUrl.Should().Be("http://storage.animetosho.org/torrents/4b58360143d59a55cbd922397a3eaa378165f3ff/DAYS%20-%2005%20%281280x720%20HEVC2%20AAC%29.torrent");
}
}
}

20
src/NzbDrone.Core.Test/IndexerTests/TorrentRssIndexerTests/TorrentRssSettingsDetectorFixture.cs
Unescape View File

@ -196,6 +196,26 @@ namespace NzbDrone.Core.Test.IndexerTests.TorrentRssIndexerTests
settings.Should().BeNull();
}
[Test]
public void should_detect_rss_settings_for_AnimeTosho_without_size()
{
_indexerSettings.AllowZeroSize = true;
GivenRecentFeedResponse("TorrentRss/AnimeTosho_NoSize.xml");
var settings = Subject.Detect(_indexerSettings);
settings.ShouldBeEquivalentTo(new TorrentRssIndexerParserSettings
{
UseEZTVFormat = false,
UseEnclosureUrl = true,
UseEnclosureLength = false,
ParseSizeInDescription = true,
ParseSeedersInDescription = false,
SizeElementName = null
});
}
[TestCase("BitMeTv/BitMeTv.xml")]
[TestCase("Fanzub/fanzub.xml")]
[TestCase("KickassTorrents/KickassTorrents.xml")]

3
src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj
Unescape View File

@ -426,6 +426,9 @@
<Content Include="Files\Indexers\relative_urls.xml">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Content Include="Files\Indexers\TorrentRss\AnimeTosho_NoSize.xml">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Content Include="Files\Indexers\TorrentRss\ExtraTorrents.xml">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>

7
src/NzbDrone.Core/Indexers/Newznab/NewznabRssParser.cs
Unescape View File

@ -11,6 +11,11 @@ namespace NzbDrone.Core.Indexers.Newznab
{
public const string ns = "{http://www.newznab.com/DTD/2010/feeds/attributes/}";
public NewznabRssParser()
{
PreferredEnclosureMimeType = "application/x-nzb";
}
protected override bool PreProcess(IndexerResponse indexerResponse)
{
var xdoc = LoadXmlDocument(indexerResponse);
@ -52,7 +57,7 @@ namespace NzbDrone.Core.Indexers.Newznab
protected override ReleaseInfo PostProcess(XElement item, ReleaseInfo releaseInfo)
{
var enclosureType = item.Element("enclosure").Attribute("type").Value;
var enclosureType = GetEnclosure(item).Attribute("type").Value;
if (enclosureType.Contains("application/x-bittorrent"))
{
throw new UnsupportedFeedException("Feed contains {0}, did you intend to add a Torznab indexer?", enclosureType);

35
src/NzbDrone.Core/Indexers/RssParser.cs
Unescape View File

@ -32,6 +32,8 @@ namespace NzbDrone.Core.Indexers
// Parse "Size: 1.3 GB" or "1.3 GB" parts in the description element and use that as Size.
public bool ParseSizeInDescription { get; set; }
public string PreferredEnclosureMimeType { get; set; }
private IndexerResponse _indexerResponse;
public RssParser()
@ -192,7 +194,7 @@ namespace NzbDrone.Core.Indexers
{
if (UseEnclosureUrl)
{
return ParseUrl((string)item.Element("enclosure").Attribute("url"));
return ParseUrl((string)GetEnclosure(item).Attribute("url"));
}
return ParseUrl((string)item.Element("link"));
@ -229,7 +231,7 @@ namespace NzbDrone.Core.Indexers
protected virtual long GetEnclosureLength(XElement item)
{
var enclosure = item.Element("enclosure");
var enclosure = GetEnclosure(item);
if (enclosure != null)
{
@ -239,6 +241,33 @@ namespace NzbDrone.Core.Indexers
return 0;
}
protected virtual XElement GetEnclosure(XElement item)
{
var enclosures = item.Elements("enclosure").ToArray();
if (enclosures.Length == 0)
{
return null;
}
if (enclosures.Length == 1)
{
return enclosures.First();
}
if (PreferredEnclosureMimeType != null)
{
var preferredEnclosure = enclosures.FirstOrDefault(v => v.Attribute("type").Value == PreferredEnclosureMimeType);
if (preferredEnclosure != null)
{
return preferredEnclosure;
}
}
return item.Elements("enclosure").SingleOrDefault();
}
protected IEnumerable<XElement> GetItems(XDocument document)
{
var root = document.Root;
@ -278,7 +307,7 @@ namespace NzbDrone.Core.Indexers
}
}
private static readonly Regex ParseSizeRegex = new Regex(@"(?<value>(?:\d+,)*\d+(?:\.\d{1,2})?)\W?(?<unit>[KMG]i?B)(?![\w/])",
private static readonly Regex ParseSizeRegex = new Regex(@"(?<value>(?<!\.\d*)(?:\d+,)*\d+(?:\.\d{1,3})?)\W?(?<unit>[KMG]i?B)(?![\w/])",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static long ParseSize(string sizeString, bool defaultToBinaryPrefix)

6
src/NzbDrone.Core/Indexers/TorrentRss/TorrentRssSettingsDetector.cs
Unescape View File

@ -169,8 +169,12 @@ namespace NzbDrone.Core.Indexers.TorrentRss
releases = ParseResponse(parser, response);
ValidateReleases(releases, indexerSettings);
if (!releases.Any(r => r.Size < ValidSizeThreshold))
if (releases.Count(r => r.Size >= ValidSizeThreshold) > releases.Count() / 2)
{
if (releases.Any(r => r.Size < ValidSizeThreshold))
{
_logger.Debug("Feed {0} contains very small releases.", response.Request.Url);
}
_logger.Trace("Feed has valid size in description.");
return settings;
}

2
src/NzbDrone.Core/Indexers/TorrentRssParser.cs
Unescape View File

@ -17,7 +17,7 @@ namespace NzbDrone.Core.Indexers
public TorrentRssParser()
{
PreferredEnclosureMimeType = "application/x-bittorrent";
}
public IEnumerable<XElement> GetItems(IndexerResponse indexerResponse)

Write Preview
Loading…
Cancel
Save