open-source-mirrors
/
Lidarr
mirror of https://github.com/Lidarr/Lidarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

New: Support for new Nyaa RSS Feed format

Browse Source 
(cherry picked from commit 40ecdbc12de8b320a4d650aea65a36e8edea77d8)

Closes #2738
pull/4209/head
bakerboy448 3 years ago committed by Bogdan
parent 0ec7d030bc
commit f35173e670
5 changed files with 172 additions and 62 deletions
Show Stats Download Patch File Download Diff File

66
src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml
Unescape View File

@ -0,0 +1,66 @@
<rss xmlns:atom="http://www.w3.org/2005/Atom"
xmlns:nyaa="https://nyaa.si/xmlns/nyaa" version="2.0">
<channel>
<title>Nyaa - Home - Torrent File RSS</title>
<description>RSS Feed for Home</description>
<link>https://nyaa.si/</link>
<atom:link href="https://nyaa.si/?page=rss" rel="self" type="application/rss+xml"/>
<item>
<title>[Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv</title>
<link>https://nyaa.si/download/1424896.torrent</link>
<guid isPermaLink="true">https://nyaa.si/view/1424896</guid>
<pubDate>Tue, 24 Aug 2021 22:18:46 -0000</pubDate>
<nyaa:seeders>4</nyaa:seeders>
<nyaa:leechers>3</nyaa:leechers>
<nyaa:downloads>2</nyaa:downloads>
<nyaa:infoHash>e8ca5e20eca876339f41c3d9e95ea66c1d7caaee</nyaa:infoHash>
<nyaa:categoryId>1_3</nyaa:categoryId>
<nyaa:category>Anime - Non-English-translated</nyaa:category>
<nyaa:size>609.6 MiB</nyaa:size>
<nyaa:comments>0</nyaa:comments>
<nyaa:trusted>No</nyaa:trusted>
<nyaa:remake>No</nyaa:remake>
<description>
<![CDATA[ <a href="https://nyaa.si/view/1424896">#1424896 | [Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv</a> | 609.6 MiB | Anime - Non-English-translated | E8CA5E20ECA876339F41C3D9E95EA66C1D7CAAEE ]]>
</description>
</item>
<item>
<title>Macross Zero (BDRip 1920x1080p x265 HEVC TrueHD, FLAC 5.1+2.0)[sxales]</title>
<link>https://nyaa.si/download/1424895.torrent</link>
<guid isPermaLink="true">https://nyaa.si/view/1424895</guid>
<pubDate>Tue, 24 Aug 2021 22:03:11 -0000</pubDate>
<nyaa:seeders>23</nyaa:seeders>
<nyaa:leechers>32</nyaa:leechers>
<nyaa:downloads>17</nyaa:downloads>
<nyaa:infoHash>26f37f26d5b3475b41a98dc575fabfa6f8d32a76</nyaa:infoHash>
<nyaa:categoryId>1_2</nyaa:categoryId>
<nyaa:category>Anime - English-translated</nyaa:category>
<nyaa:size>5.7 GiB</nyaa:size>
<nyaa:comments>2</nyaa:comments>
<nyaa:trusted>No</nyaa:trusted>
<nyaa:remake>No</nyaa:remake>
<description>
<![CDATA[ <a href="https://nyaa.si/view/1424895">#1424895 | Macross Zero (BDRip 1920x1080p x265 HEVC TrueHD, FLAC 5.1+2.0)[sxales]</a> | 5.7 GiB | Anime - English-translated | 26F37F26D5B3475B41A98DC575FABFA6F8D32A76 ]]>
</description>
</item>
<item>
<title>Fumetsu no Anata e - 19 [WEBDL 1080p] Ukr DVO</title>
<link>https://nyaa.si/download/1424887.torrent</link>
<guid isPermaLink="true">https://nyaa.si/view/1424887</guid>
<pubDate>Tue, 24 Aug 2021 21:23:06 -0000</pubDate>
<nyaa:seeders>5</nyaa:seeders>
<nyaa:leechers>4</nyaa:leechers>
<nyaa:downloads>4</nyaa:downloads>
<nyaa:infoHash>3e4300e24b39983802162877755aab4380bd137a</nyaa:infoHash>
<nyaa:categoryId>1_3</nyaa:categoryId>
<nyaa:category>Anime - Non-English-translated</nyaa:category>
<nyaa:size>1.4 GiB</nyaa:size>
<nyaa:comments>0</nyaa:comments>
<nyaa:trusted>No</nyaa:trusted>
<nyaa:remake>No</nyaa:remake>
<description>
<![CDATA[ <a href="https://nyaa.si/view/1424887">#1424887 | Fumetsu no Anata e - 19 [WEBDL 1080p] Ukr DVO</a> | 1.4 GiB | Anime - Non-English-translated | 3E4300E24B39983802162877755AAB4380BD137A ]]>
</description>
</item>
</channel>
</rss>

43
src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs
Unescape View File

@ -26,16 +26,18 @@ namespace NzbDrone.Core.Test.IndexerTests.NyaaTests
};
}
[Test]
public async Task should_parse_recent_feed_from_Nyaa()
/* [Test]
// Legacy Nyaa feed test
public void should_parse_recent_feed_from_Nyaa()
{
var recentFeed = ReadAllText(@"Files/Indexers/Nyaa/Nyaa.xml");
Mocker.GetMock<IHttpClient>()
.Setup(o => o.ExecuteAsync(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => Task.FromResult(new HttpResponse(r, new HttpHeader(), recentFeed)));
.Setup(o => o.Execute(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => new HttpResponse(r, new HttpHeader(), recentFeed));
var releases = await Subject.FetchRecent();
var releases = Subject.FetchRecent();
releases.Should().HaveCount(4);
releases.First().Should().BeOfType<TorrentInfo>();
@ -49,11 +51,40 @@ namespace NzbDrone.Core.Test.IndexerTests.NyaaTests
torrentInfo.CommentUrl.Should().BeNullOrEmpty();
torrentInfo.Indexer.Should().Be(Subject.Definition.Name);
torrentInfo.PublishDate.Should().Be(DateTime.Parse("2014/08/14 18:10:36"));
torrentInfo.Size.Should().Be(2523293286); // 2.35 GiB
torrentInfo.Size.Should().Be(2523293286); //2.35 GiB
torrentInfo.InfoHash.Should().Be(null);
torrentInfo.MagnetUrl.Should().Be(null);
torrentInfo.Peers.Should().Be(2 + 1);
torrentInfo.Seeders.Should().Be(1);
}*/
[Test]
public async Task should_parse_2021_recent_feed_from_Nyaa()
{
var recentFeed = ReadAllText(@"Files/Indexers/Nyaa/Nyaa2021.xml");
Mocker.GetMock<IHttpClient>()
.Setup(o => o.ExecuteAsync(It.Is<HttpRequest>(v => v.Method == HttpMethod.Get)))
.Returns<HttpRequest>(r => Task.FromResult(new HttpResponse(r, new HttpHeader(), recentFeed)));
var releases = await Subject.FetchRecent();
releases.Should().HaveCount(3);
releases.First().Should().BeOfType<TorrentInfo>();
var torrentInfo = releases.First() as TorrentInfo;
torrentInfo.Title.Should().Be("[Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv");
torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent);
torrentInfo.DownloadUrl.Should().Be("https://nyaa.si/download/1424896.torrent");
torrentInfo.InfoUrl.Should().Be("https://nyaa.si/view/1424896");
torrentInfo.CommentUrl.Should().BeNullOrEmpty();
torrentInfo.Indexer.Should().Be(Subject.Definition.Name);
torrentInfo.PublishDate.Should().Be(DateTime.Parse("Tue, 24 Aug 2021 22:18:46"));
torrentInfo.Size.Should().Be(639211930); // 609.6 MiB
torrentInfo.MagnetUrl.Should().Be(null);
torrentInfo.Seeders.Should().Be(4);
torrentInfo.Peers.Should().Be(3 + 4);
}
}
}

52
src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs
Unescape View File

@ -1,6 +1,4 @@
using System.Linq;
using System.Xml.Linq;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.Indexers.Exceptions;
namespace NzbDrone.Core.Indexers
@ -12,6 +10,10 @@ namespace NzbDrone.Core.Indexers
UseGuidInfoUrl = true;
UseEnclosureLength = false;
UseEnclosureUrl = true;
SeedsElementName = "seeds";
InfoHashElementName = "infoHash";
SizeElementName = "contentLength";
MagnetElementName = "magnetURI";
}
protected override bool PreProcess(IndexerResponse indexerResponse)
@ -28,51 +30,5 @@ namespace NzbDrone.Core.Indexers
return true;
}
protected override long GetSize(XElement item)
{
var contentLength = item.FindDecendants("contentLength").SingleOrDefault();
if (contentLength != null)
{
return (long)contentLength;
}
return base.GetSize(item);
}
protected override string GetInfoHash(XElement item)
{
var infoHash = item.FindDecendants("infoHash").SingleOrDefault();
return (string)infoHash;
}
protected override string GetMagnetUrl(XElement item)
{
var magnetURI = item.FindDecendants("magnetURI").SingleOrDefault();
return (string)magnetURI;
}
protected override int? GetSeeders(XElement item)
{
var seeds = item.FindDecendants("seeds").SingleOrDefault();
if (seeds != null)
{
return (int)seeds;
}
return base.GetSeeders(item);
}
protected override int? GetPeers(XElement item)
{
var peers = item.FindDecendants("peers").SingleOrDefault();
if (peers != null)
{
return (int)peers;
}
return base.GetPeers(item);
}
}
}

2
src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs
Unescape View File

@ -24,7 +24,7 @@ namespace NzbDrone.Core.Indexers.Nyaa
public override IParseIndexerResponse GetParser()
{
return new TorrentRssParser() { UseGuidInfoUrl = true, ParseSizeInDescription = true, ParseSeedersInDescription = true };
return new TorrentRssParser() { UseGuidInfoUrl = true, SizeElementName = "size", InfoHashElementName = "infoHash", PeersElementName = "leechers", CalculatePeersAsSum = true, SeedsElementName = "seeders" };
}
}
}

71
src/NzbDrone.Core/Indexers/TorrentRssParser.cs
Unescape View File

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Xml.Linq;
using MonoTorrent;
@ -9,12 +10,27 @@ namespace NzbDrone.Core.Indexers
{
public class TorrentRssParser : RssParser
{
// Use to sum/calculate Peers as Leechers+Seeders
public bool CalculatePeersAsSum { get; set; }
// Use the specified element name to determine the Infohash
public string InfoHashElementName { get; set; }
// Parse various seeder/leecher/peers formats in the description element to determine number of seeders.
public bool ParseSeedersInDescription { get; set; }
// Use the specified element name to determine the size
// Use the specified element name to determine the Peers
public string PeersElementName { get; set; }
// Use the specified element name to determine the Seeds
public string SeedsElementName { get; set; }
// Use the specified element name to determine the Size
public string SizeElementName { get; set; }
// Use the specified element name to determine the Magnet link
public string MagnetElementName { get; set; }
public TorrentRssParser()
{
PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes;
@ -40,14 +56,28 @@ namespace NzbDrone.Core.Indexers
result.InfoHash = GetInfoHash(item);
result.MagnetUrl = GetMagnetUrl(item);
result.Seeders = GetSeeders(item);
result.Peers = GetPeers(item);
if (CalculatePeersAsSum)
{
result.Peers = GetPeers(item) + result.Seeders;
}
else
{
result.Peers = GetPeers(item);
}
return result;
}
protected virtual string GetInfoHash(XElement item)
{
if (InfoHashElementName.IsNotNullOrWhiteSpace())
{
return item.FindDecendants(InfoHashElementName).FirstOrDefault().Value;
}
var magnetUrl = GetMagnetUrl(item);
if (magnetUrl.IsNotNullOrWhiteSpace())
{
try
@ -64,10 +94,21 @@ namespace NzbDrone.Core.Indexers
protected virtual string GetMagnetUrl(XElement item)
{
var downloadUrl = GetDownloadUrl(item);
if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:"))
if (MagnetElementName.IsNotNullOrWhiteSpace())
{
return downloadUrl;
var magnetURL = item.FindDecendants(MagnetElementName).FirstOrDefault().Value;
if (magnetURL.IsNotNullOrWhiteSpace() && magnetURL.StartsWith("magnet:"))
{
return magnetURL;
}
}
else
{
var downloadUrl = GetDownloadUrl(item);
if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:"))
{
return downloadUrl;
}
}
return null;
@ -75,6 +116,9 @@ namespace NzbDrone.Core.Indexers
protected virtual int? GetSeeders(XElement item)
{
// safe to always use the element if it's present (and valid)
// fall back to description if ParseSeedersInDescription is enabled
if (ParseSeedersInDescription && item.Element("description") != null)
{
var matchSeeders = ParseSeedersRegex.Match(item.Element("description").Value);
@ -93,6 +137,12 @@ namespace NzbDrone.Core.Indexers
}
}
var seeds = item.FindDecendants(SeedsElementName).SingleOrDefault();
if (seeds != null)
{
return (int)seeds;
}
return null;
}
@ -116,6 +166,12 @@ namespace NzbDrone.Core.Indexers
}
}
if (PeersElementName.IsNotNullOrWhiteSpace())
{
var itempeers = item.FindDecendants(PeersElementName).SingleOrDefault();
return int.Parse(itempeers.Value);
}
return null;
}
@ -124,9 +180,10 @@ namespace NzbDrone.Core.Indexers
var size = base.GetSize(item);
if (size == 0 && SizeElementName.IsNotNullOrWhiteSpace())
{
if (item.Element(SizeElementName) != null)
var itemsize = item.FindDecendants(SizeElementName).SingleOrDefault();
if (itemsize != null)
{
size = ParseSize(item.Element(SizeElementName).Value, true);
size = ParseSize(itemsize.Value, true);
}
}

Write Preview
Loading…
Cancel
Save