From f35173e670fa33a68f05ae1d8d3f63bc427e51fd Mon Sep 17 00:00:00 2001 From: bakerboy448 <55419169+bakerboy448@users.noreply.github.com> Date: Tue, 5 Apr 2022 19:10:26 -0500 Subject: [PATCH] New: Support for new Nyaa RSS Feed format (cherry picked from commit 40ecdbc12de8b320a4d650aea65a36e8edea77d8) Closes #2738 --- .../Files/Indexers/Nyaa/Nyaa2021.xml | 66 +++++++++++++++++ .../IndexerTests/NyaaTests/NyaaFixture.cs | 43 +++++++++-- .../Indexers/EzrssTorrentRssParser.cs | 52 ++------------ src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs | 2 +- .../Indexers/TorrentRssParser.cs | 71 +++++++++++++++++-- 5 files changed, 172 insertions(+), 62 deletions(-) create mode 100644 src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml diff --git a/src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml b/src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml new file mode 100644 index 000000000..0d63ce046 --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/Nyaa/Nyaa2021.xml @@ -0,0 +1,66 @@ + + + Nyaa - Home - Torrent File RSS + RSS Feed for Home + https://nyaa.si/ + + + [Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv + https://nyaa.si/download/1424896.torrent + https://nyaa.si/view/1424896 + Tue, 24 Aug 2021 22:18:46 -0000 + 4 + 3 + 2 + e8ca5e20eca876339f41c3d9e95ea66c1d7caaee + 1_3 + Anime - Non-English-translated + 609.6 MiB + 0 + No + No + + #1424896 | [Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv | 609.6 MiB | Anime - Non-English-translated | E8CA5E20ECA876339F41C3D9E95EA66C1D7CAAEE ]]> + + + + Macross Zero (BDRip 1920x1080p x265 HEVC TrueHD, FLAC 5.1+2.0)[sxales] + https://nyaa.si/download/1424895.torrent + https://nyaa.si/view/1424895 + Tue, 24 Aug 2021 22:03:11 -0000 + 23 + 32 + 17 + 26f37f26d5b3475b41a98dc575fabfa6f8d32a76 + 1_2 + Anime - English-translated + 5.7 GiB + 2 + No + No + + #1424895 | Macross Zero (BDRip 1920x1080p x265 HEVC TrueHD, FLAC 5.1+2.0)[sxales] | 5.7 GiB | Anime - English-translated | 26F37F26D5B3475B41A98DC575FABFA6F8D32A76 ]]> + + + + Fumetsu no Anata e - 19 [WEBDL 1080p] Ukr DVO + https://nyaa.si/download/1424887.torrent + https://nyaa.si/view/1424887 + Tue, 24 Aug 2021 21:23:06 -0000 + 5 + 4 + 4 + 3e4300e24b39983802162877755aab4380bd137a + 1_3 + Anime - Non-English-translated + 1.4 GiB + 0 + No + No + + #1424887 | Fumetsu no Anata e - 19 [WEBDL 1080p] Ukr DVO | 1.4 GiB | Anime - Non-English-translated | 3E4300E24B39983802162877755AAB4380BD137A ]]> + + + + \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs index c98c9126d..5f42b5012 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/NyaaTests/NyaaFixture.cs @@ -26,16 +26,18 @@ namespace NzbDrone.Core.Test.IndexerTests.NyaaTests }; } - [Test] - public async Task should_parse_recent_feed_from_Nyaa() +/* [Test] + // Legacy Nyaa feed test + + public void should_parse_recent_feed_from_Nyaa() { var recentFeed = ReadAllText(@"Files/Indexers/Nyaa/Nyaa.xml"); Mocker.GetMock() - .Setup(o => o.ExecuteAsync(It.Is(v => v.Method == HttpMethod.Get))) - .Returns(r => Task.FromResult(new HttpResponse(r, new HttpHeader(), recentFeed))); + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.Get))) + .Returns(r => new HttpResponse(r, new HttpHeader(), recentFeed)); - var releases = await Subject.FetchRecent(); + var releases = Subject.FetchRecent(); releases.Should().HaveCount(4); releases.First().Should().BeOfType(); @@ -49,11 +51,40 @@ namespace NzbDrone.Core.Test.IndexerTests.NyaaTests torrentInfo.CommentUrl.Should().BeNullOrEmpty(); torrentInfo.Indexer.Should().Be(Subject.Definition.Name); torrentInfo.PublishDate.Should().Be(DateTime.Parse("2014/08/14 18:10:36")); - torrentInfo.Size.Should().Be(2523293286); // 2.35 GiB + torrentInfo.Size.Should().Be(2523293286); //2.35 GiB torrentInfo.InfoHash.Should().Be(null); torrentInfo.MagnetUrl.Should().Be(null); torrentInfo.Peers.Should().Be(2 + 1); torrentInfo.Seeders.Should().Be(1); + }*/ + + [Test] + public async Task should_parse_2021_recent_feed_from_Nyaa() + { + var recentFeed = ReadAllText(@"Files/Indexers/Nyaa/Nyaa2021.xml"); + + Mocker.GetMock() + .Setup(o => o.ExecuteAsync(It.Is(v => v.Method == HttpMethod.Get))) + .Returns(r => Task.FromResult(new HttpResponse(r, new HttpHeader(), recentFeed))); + + var releases = await Subject.FetchRecent(); + + releases.Should().HaveCount(3); + releases.First().Should().BeOfType(); + + var torrentInfo = releases.First() as TorrentInfo; + + torrentInfo.Title.Should().Be("[Foxy-Subs] Mahouka Koukou no Yuutousei - 08 [720p] [3194D881].mkv"); + torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + torrentInfo.DownloadUrl.Should().Be("https://nyaa.si/download/1424896.torrent"); + torrentInfo.InfoUrl.Should().Be("https://nyaa.si/view/1424896"); + torrentInfo.CommentUrl.Should().BeNullOrEmpty(); + torrentInfo.Indexer.Should().Be(Subject.Definition.Name); + torrentInfo.PublishDate.Should().Be(DateTime.Parse("Tue, 24 Aug 2021 22:18:46")); + torrentInfo.Size.Should().Be(639211930); // 609.6 MiB + torrentInfo.MagnetUrl.Should().Be(null); + torrentInfo.Seeders.Should().Be(4); + torrentInfo.Peers.Should().Be(3 + 4); } } } diff --git a/src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs b/src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs index dd5a2b3e2..b174b8573 100644 --- a/src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/EzrssTorrentRssParser.cs @@ -1,6 +1,4 @@ using System.Linq; -using System.Xml.Linq; -using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers.Exceptions; namespace NzbDrone.Core.Indexers @@ -12,6 +10,10 @@ namespace NzbDrone.Core.Indexers UseGuidInfoUrl = true; UseEnclosureLength = false; UseEnclosureUrl = true; + SeedsElementName = "seeds"; + InfoHashElementName = "infoHash"; + SizeElementName = "contentLength"; + MagnetElementName = "magnetURI"; } protected override bool PreProcess(IndexerResponse indexerResponse) @@ -28,51 +30,5 @@ namespace NzbDrone.Core.Indexers return true; } - - protected override long GetSize(XElement item) - { - var contentLength = item.FindDecendants("contentLength").SingleOrDefault(); - - if (contentLength != null) - { - return (long)contentLength; - } - - return base.GetSize(item); - } - - protected override string GetInfoHash(XElement item) - { - var infoHash = item.FindDecendants("infoHash").SingleOrDefault(); - return (string)infoHash; - } - - protected override string GetMagnetUrl(XElement item) - { - var magnetURI = item.FindDecendants("magnetURI").SingleOrDefault(); - return (string)magnetURI; - } - - protected override int? GetSeeders(XElement item) - { - var seeds = item.FindDecendants("seeds").SingleOrDefault(); - if (seeds != null) - { - return (int)seeds; - } - - return base.GetSeeders(item); - } - - protected override int? GetPeers(XElement item) - { - var peers = item.FindDecendants("peers").SingleOrDefault(); - if (peers != null) - { - return (int)peers; - } - - return base.GetPeers(item); - } } } diff --git a/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs b/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs index 89ecaadb3..670baecc9 100644 --- a/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs +++ b/src/NzbDrone.Core/Indexers/Nyaa/Nyaa.cs @@ -24,7 +24,7 @@ namespace NzbDrone.Core.Indexers.Nyaa public override IParseIndexerResponse GetParser() { - return new TorrentRssParser() { UseGuidInfoUrl = true, ParseSizeInDescription = true, ParseSeedersInDescription = true }; + return new TorrentRssParser() { UseGuidInfoUrl = true, SizeElementName = "size", InfoHashElementName = "infoHash", PeersElementName = "leechers", CalculatePeersAsSum = true, SeedsElementName = "seeders" }; } } } diff --git a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs index 5efe5fd64..712c23d39 100644 --- a/src/NzbDrone.Core/Indexers/TorrentRssParser.cs +++ b/src/NzbDrone.Core/Indexers/TorrentRssParser.cs @@ -1,4 +1,5 @@ using System.Collections.Generic; +using System.Linq; using System.Text.RegularExpressions; using System.Xml.Linq; using MonoTorrent; @@ -9,12 +10,27 @@ namespace NzbDrone.Core.Indexers { public class TorrentRssParser : RssParser { + // Use to sum/calculate Peers as Leechers+Seeders + public bool CalculatePeersAsSum { get; set; } + + // Use the specified element name to determine the Infohash + public string InfoHashElementName { get; set; } + // Parse various seeder/leecher/peers formats in the description element to determine number of seeders. public bool ParseSeedersInDescription { get; set; } - // Use the specified element name to determine the size + // Use the specified element name to determine the Peers + public string PeersElementName { get; set; } + + // Use the specified element name to determine the Seeds + public string SeedsElementName { get; set; } + + // Use the specified element name to determine the Size public string SizeElementName { get; set; } + // Use the specified element name to determine the Magnet link + public string MagnetElementName { get; set; } + public TorrentRssParser() { PreferredEnclosureMimeTypes = TorrentEnclosureMimeTypes; @@ -40,14 +56,28 @@ namespace NzbDrone.Core.Indexers result.InfoHash = GetInfoHash(item); result.MagnetUrl = GetMagnetUrl(item); result.Seeders = GetSeeders(item); - result.Peers = GetPeers(item); + + if (CalculatePeersAsSum) + { + result.Peers = GetPeers(item) + result.Seeders; + } + else + { + result.Peers = GetPeers(item); + } return result; } protected virtual string GetInfoHash(XElement item) { + if (InfoHashElementName.IsNotNullOrWhiteSpace()) + { + return item.FindDecendants(InfoHashElementName).FirstOrDefault().Value; + } + var magnetUrl = GetMagnetUrl(item); + if (magnetUrl.IsNotNullOrWhiteSpace()) { try @@ -64,10 +94,21 @@ namespace NzbDrone.Core.Indexers protected virtual string GetMagnetUrl(XElement item) { - var downloadUrl = GetDownloadUrl(item); - if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:")) + if (MagnetElementName.IsNotNullOrWhiteSpace()) { - return downloadUrl; + var magnetURL = item.FindDecendants(MagnetElementName).FirstOrDefault().Value; + if (magnetURL.IsNotNullOrWhiteSpace() && magnetURL.StartsWith("magnet:")) + { + return magnetURL; + } + } + else + { + var downloadUrl = GetDownloadUrl(item); + if (downloadUrl.IsNotNullOrWhiteSpace() && downloadUrl.StartsWith("magnet:")) + { + return downloadUrl; + } } return null; @@ -75,6 +116,9 @@ namespace NzbDrone.Core.Indexers protected virtual int? GetSeeders(XElement item) { + // safe to always use the element if it's present (and valid) + // fall back to description if ParseSeedersInDescription is enabled + if (ParseSeedersInDescription && item.Element("description") != null) { var matchSeeders = ParseSeedersRegex.Match(item.Element("description").Value); @@ -93,6 +137,12 @@ namespace NzbDrone.Core.Indexers } } + var seeds = item.FindDecendants(SeedsElementName).SingleOrDefault(); + if (seeds != null) + { + return (int)seeds; + } + return null; } @@ -116,6 +166,12 @@ namespace NzbDrone.Core.Indexers } } + if (PeersElementName.IsNotNullOrWhiteSpace()) + { + var itempeers = item.FindDecendants(PeersElementName).SingleOrDefault(); + return int.Parse(itempeers.Value); + } + return null; } @@ -124,9 +180,10 @@ namespace NzbDrone.Core.Indexers var size = base.GetSize(item); if (size == 0 && SizeElementName.IsNotNullOrWhiteSpace()) { - if (item.Element(SizeElementName) != null) + var itemsize = item.FindDecendants(SizeElementName).SingleOrDefault(); + if (itemsize != null) { - size = ParseSize(item.Element(SizeElementName).Value, true); + size = ParseSize(itemsize.Value, true); } }