From 569ef09f39cd8e9abed9ca49a7b3cd9960185e06 Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Fri, 5 Jun 2015 23:31:51 +0200 Subject: [PATCH] Updated Rarbg to api v2. fixes #752 ref rarbg/torrentapi#1 --- .../{RecentFeed.json => RecentFeed_v1.json} | 0 .../Files/Indexers/Rarbg/RecentFeed_v2.json | 84 +++++++++++++++++++ .../IndexerTests/RarbgTests/RarbgFixture.cs | 45 ++++++++-- .../NzbDrone.Core.Test.csproj | 5 +- src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs | 3 +- .../Indexers/Rarbg/RarbgParser.cs | 40 +++++---- .../Indexers/Rarbg/RarbgRequestGenerator.cs | 4 +- .../Indexers/Rarbg/RarbgResponse.cs | 33 ++++++++ .../Indexers/Rarbg/RarbgSettings.cs | 2 +- .../Indexers/Rarbg/RarbgTokenProvider.cs | 2 +- .../Indexers/Rarbg/RarbgTorrent.cs | 24 ------ src/NzbDrone.Core/NzbDrone.Core.csproj | 2 +- 12 files changed, 188 insertions(+), 56 deletions(-) rename src/NzbDrone.Core.Test/Files/Indexers/Rarbg/{RecentFeed.json => RecentFeed_v1.json} (100%) create mode 100644 src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed_v2.json create mode 100644 src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs delete mode 100644 src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs diff --git a/src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed.json b/src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed_v1.json similarity index 100% rename from src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed.json rename to src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed_v1.json diff --git a/src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed_v2.json b/src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed_v2.json new file mode 100644 index 000000000..94ed8092a --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/Rarbg/RecentFeed_v2.json @@ -0,0 +1,84 @@ +{ + "torrent_results": [ + { + "title": "Sense8.S01E01.WEBRip.x264-FGT", + "category": "TV Episodes", + "download": "magnet:?xt=urn:btih:d8bde635f573acb390c7d7e7efc1556965fdc802&dn=Sense8.S01E01.WEBRip.x264-FGT&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce", + "seeders": 304, + "leechers": 200, + "size": 564198371, + "pubdate": "2015-06-05 16:58:11 +0000", + "episode_info": { + "imdb": "tt2431438", + "tvrage": "35197", + "tvdb": "268156", + "airdate": "2015-06-05", + "epnum": "01", + "seasonnum": "1", + "title": "Limbic Resonance" + }, + "ranked": 1, + "info_page": "https:\/\/torrentapi.org\/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_5_6__d8bde635f5" + }, + { + "title": "Sense8.S01E02.WEBRip.x264-FGT", + "category": "TV Episodes", + "download": "magnet:?xt=urn:btih:e5ab5f398d929c791ac4f1d5bb2fba0997372a91&dn=Sense8.S01E02.WEBRip.x264-FGT&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce", + "seeders": 299, + "leechers": 209, + "size": 486918696, + "pubdate": "2015-06-05 16:58:23 +0000", + "episode_info": { + "imdb": "tt2431438", + "tvrage": "35197", + "tvdb": "268156", + "airdate": "2015-06-05", + "epnum": "02", + "seasonnum": "1", + "title": "I Am Also A We" + }, + "ranked": 1, + "info_page": "https:\/\/torrentapi.org\/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_5_7__e5ab5f398d" + }, + { + "title": "Comedy.Bang.Bang.S04E20.HDTV.x264-YesTV[rartv]", + "category": "TV Episodes", + "download": "magnet:?xt=urn:btih:0ed8bd14206e211eef9d3d36a48b038f280ef20c&dn=Comedy.Bang.Bang.S04E20.HDTV.x264-YesTV%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce", + "seeders": 45, + "leechers": 15, + "size": 154208067, + "pubdate": "2015-06-05 17:33:37 +0000", + "episode_info": { + "imdb": "tt2176287", + "tvrage": "31483", + "tvdb": "258310", + "airdate": "2015-06-05", + "epnum": "20", + "seasonnum": "4", + "title": "Judd Apatow Wears a Polo and Blue Suede Shoes" + }, + "ranked": 1, + "info_page": "https:\/\/torrentapi.org\/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_6_7__0ed8bd1420" + }, + { + "title": "Comedy.Bang.Bang.S04E20.720p.HDTV.x264-YesTV[rartv]", + "category": "TV HD Episodes", + "download": "magnet:?xt=urn:btih:10257dee06327ba66cc2674e08d71b3bb2089b06&dn=Comedy.Bang.Bang.S04E20.720p.HDTV.x264-YesTV%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce", + "seeders": 22, + "leechers": 6, + "size": 514574549, + "pubdate": "2015-06-05 17:33:49 +0000", + "episode_info": { + "imdb": "tt2176287", + "tvrage": "31483", + "tvdb": "258310", + "airdate": "2015-06-05", + "epnum": "20", + "seasonnum": "4", + "title": "Judd Apatow Wears a Polo and Blue Suede Shoes" + }, + "ranked": 1, + "info_page": "https:\/\/torrentapi.org\/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_6_8__10257dee06" + } + ] +} diff --git a/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs index c930fbabe..441133d6c 100644 --- a/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs +++ b/src/NzbDrone.Core.Test/IndexerTests/RarbgTests/RarbgFixture.cs @@ -8,6 +8,7 @@ using NzbDrone.Core.Indexers; using NzbDrone.Core.Indexers.Rarbg; using NzbDrone.Core.Parser.Model; using NzbDrone.Core.Test.Framework; +using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.IndexerTests.RarbgTests { @@ -31,7 +32,7 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests [Test] public void should_parse_recent_feed_from_Rarbg() { - var recentFeed = ReadAllText(@"Files/Indexers/Rarbg/RecentFeed.json"); + var recentFeed = ReadAllText(@"Files/Indexers/Rarbg/RecentFeed_v2.json"); Mocker.GetMock() .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) @@ -39,21 +40,49 @@ namespace NzbDrone.Core.Test.IndexerTests.RarbgTests var releases = Subject.FetchRecent(); - releases.Should().HaveCount(3); + releases.Should().HaveCount(4); releases.First().Should().BeOfType(); var torrentInfo = releases.First() as TorrentInfo; - torrentInfo.Title.Should().Be("Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7[rartv]"); + torrentInfo.Title.Should().Be("Sense8.S01E01.WEBRip.x264-FGT"); torrentInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); - torrentInfo.DownloadUrl.Should().Be("magnet:?xt=urn:btih:ff4737b5230307836ec8abce6ab73727f1358bf3&dn=Thunderbirds.Are.Go.S01E09.Slingshot.1080p.WEB-DL.AAC2.0.H.264-Coo7%5Brartv%5D&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce"); + torrentInfo.DownloadUrl.Should().Be("magnet:?xt=urn:btih:d8bde635f573acb390c7d7e7efc1556965fdc802&dn=Sense8.S01E01.WEBRip.x264-FGT&tr=http%3A%2F%2Ftracker.trackerfix.com%3A80%2Fannounce&tr=udp%3A%2F%2F9.rarbg.me%3A2710&tr=udp%3A%2F%2F9.rarbg.to%3A2710&tr=udp%3A%2F%2Fopen.demonii.com%3A1337%2Fannounce"); + torrentInfo.InfoUrl.Should().Be("https://torrentapi.org/redirect_to_info.php?token=i5cx7b9agd&p=8_6_4_4_5_6__d8bde635f5"); torrentInfo.Indexer.Should().Be(Subject.Definition.Name); - torrentInfo.PublishDate.Should().Be(DateTime.Parse("2015-05-24 19:36:09")); - torrentInfo.Size.Should().Be(896238116); + torrentInfo.PublishDate.Should().Be(DateTime.Parse("2015-06-05 16:58:11 +0000").ToUniversalTime()); + torrentInfo.Size.Should().Be(564198371); torrentInfo.InfoHash.Should().BeNull(); torrentInfo.MagnetUrl.Should().BeNull(); - torrentInfo.Peers.Should().Be(44+19); - torrentInfo.Seeders.Should().Be(44); + torrentInfo.Peers.Should().Be(304 + 200); + torrentInfo.Seeders.Should().Be(304); + torrentInfo.TvRageId.Should().Be(35197); + } + + [Test] + public void should_parse_error_20_as_empty_results() + { + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) + .Returns(r => new HttpResponse(r, new HttpHeader(), "{ error_code: 20, error: \"some message\" }")); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(0); + } + + [Test] + public void should_warn_on_unknown_error() + { + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET))) + .Returns(r => new HttpResponse(r, new HttpHeader(), "{ error_code: 25, error: \"some message\" }")); + + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(0); + + ExceptionVerification.ExpectedWarns(1); } } } diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index 4cb8dd6e9..d79eda84f 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -168,7 +168,10 @@ - + + Always + + Always diff --git a/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs b/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs index 92b6b18f9..f84cf36e2 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/Rarbg.cs @@ -9,12 +9,11 @@ namespace NzbDrone.Core.Indexers.Rarbg public class Rarbg : HttpIndexerBase { private readonly IRarbgTokenProvider _tokenProvider; - private static DateTime _lastFetch; public override string Name { get { return "Rarbg"; } } public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } } - public override TimeSpan RateLimit { get { return TimeSpan.FromSeconds(10); } } + public override TimeSpan RateLimit { get { return TimeSpan.FromSeconds(2); } } public Rarbg(IRarbgTokenProvider tokenProvider, IHttpClient httpClient, IConfigService configService, IParsingService parsingService, Logger logger) : base(httpClient, configService, parsingService, logger) diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs index c3274adf8..425161368 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgParser.cs @@ -27,33 +27,41 @@ namespace NzbDrone.Core.Indexers.Rarbg break; } - var jsonResponse = new HttpResponse(indexerResponse.HttpResponse).Resource as JContainer; + var jsonResponse = new HttpResponse(indexerResponse.HttpResponse); - var errorResponse = jsonResponse as JObject; - if (errorResponse != null && errorResponse["error"] != null) + if (jsonResponse.Resource.error_code.HasValue) { - var error = errorResponse["error"].ToString(); - if (error == "No results found") + if (jsonResponse.Resource.error_code == 20) { + // No results found return results; } - throw new IndexerException(indexerResponse, "Indexer API call returned an error [{0}]", errorResponse["error"]); + throw new IndexerException(indexerResponse, "Indexer API call returned error {0}: {1}", jsonResponse.Resource.error_code, jsonResponse.Resource.error); } - var torrentResponse = jsonResponse.ToObject>(); + if (jsonResponse.Resource.torrent_results == null) + { + return results; + } - foreach (var torrent in torrentResponse) + foreach (var torrent in jsonResponse.Resource.torrent_results) { var torrentInfo = new TorrentInfo(); torrentInfo.Guid = GetGuid(torrent); - torrentInfo.Title = torrent.Title; - torrentInfo.Size = torrent.Size; - torrentInfo.DownloadUrl = torrent.DownloadUrl; - torrentInfo.PublishDate = torrent.PublishDate; - torrentInfo.Seeders = torrent.Seeders; - torrentInfo.Peers = torrent.Leechers + torrent.Seeders; + torrentInfo.Title = torrent.title; + torrentInfo.Size = torrent.size; + torrentInfo.DownloadUrl = torrent.download; + torrentInfo.InfoUrl = torrent.info_page; + torrentInfo.PublishDate = torrent.pubdate; + torrentInfo.Seeders = torrent.seeders; + torrentInfo.Peers = torrent.leechers + torrent.seeders; + + if (torrent.episode_info != null && torrent.episode_info.tvrage != null) + { + torrentInfo.TvRageId = torrent.episode_info.tvrage.Value; + } results.Add(torrentInfo); } @@ -63,7 +71,7 @@ namespace NzbDrone.Core.Indexers.Rarbg private string GetGuid(RarbgTorrent torrent) { - var match = RegexGuid.Match(torrent.DownloadUrl); + var match = RegexGuid.Match(torrent.download); if (match.Success) { @@ -71,7 +79,7 @@ namespace NzbDrone.Core.Indexers.Rarbg } else { - return string.Format("rarbg-{0}", torrent.DownloadUrl); + return string.Format("rarbg-{0}", torrent.download); } } diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs index 673a6bef1..3c27c8dd6 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgRequestGenerator.cs @@ -65,7 +65,7 @@ namespace NzbDrone.Core.Indexers.Rarbg private IEnumerable GetPagedRequests(string mode, int? tvdbId, string query, params object[] args) { - var httpRequest = new HttpRequest(Settings.BaseUrl + "/pubapi.php", HttpAccept.Json); + var httpRequest = new HttpRequest(Settings.BaseUrl + "/pubapi_v2.php", HttpAccept.Json); httpRequest.AddQueryParam("mode", mode); @@ -88,7 +88,7 @@ namespace NzbDrone.Core.Indexers.Rarbg httpRequest.AddQueryParam("limit", "100"); httpRequest.AddQueryParam("token", _tokenProvider.GetToken(Settings)); httpRequest.AddQueryParam("format", "json_extended"); - httpRequest.AddQueryParam("response_type", "json"); + httpRequest.AddQueryParam("app_id", "Sonarr"); yield return new IndexerRequest(httpRequest); } diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs new file mode 100644 index 000000000..51c2e8350 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgResponse.cs @@ -0,0 +1,33 @@ +using System; +using System.Collections.Generic; + +namespace NzbDrone.Core.Indexers.Rarbg +{ + public class RarbgResponse + { + public string error { get; set; } + public int? error_code { get; set; } + public List torrent_results { get; set; } + } + + public class RarbgTorrent + { + public string title { get; set; } + public string category { get; set; } + public string download { get; set; } + public int? seeders { get; set; } + public int? leechers { get; set; } + public long size { get; set; } + public DateTime pubdate { get; set; } + public RarbgTorrentInfo episode_info { get; set; } + public int? ranked { get; set; } + public string info_page { get; set; } + } + + public class RarbgTorrentInfo + { + public string imdb { get; set; } + public int? tvrage { get; set; } + public int? tvdb { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs index 41a6573aa..3d3f33e35 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgSettings.cs @@ -27,7 +27,7 @@ namespace NzbDrone.Core.Indexers.Rarbg [FieldDefinition(0, Label = "API URL", HelpText = "URL to Rarbg api, not the website.")] public string BaseUrl { get; set; } - [FieldDefinition(1, Label = "Ranked Only", HelpText = "Only include ranked results.")] + [FieldDefinition(1, Type = FieldType.Checkbox, Label = "Ranked Only", HelpText = "Only include ranked results.")] public bool RankedOnly { get; set; } public NzbDroneValidationResult Validate() diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs index 1dadcb83f..dd7634429 100644 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs +++ b/src/NzbDrone.Core/Indexers/Rarbg/RarbgTokenProvider.cs @@ -28,7 +28,7 @@ namespace NzbDrone.Core.Indexers.Rarbg { return _tokenCache.Get(settings.BaseUrl, () => { - var url = settings.BaseUrl.Trim('/') + "/pubapi.php?get_token=get_token&format=json&response_type=json"; + var url = settings.BaseUrl.Trim('/') + "/pubapi_v2.php?get_token=get_token"; var response = _httpClient.Get(new HttpRequest(url, HttpAccept.Json)); diff --git a/src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs b/src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs deleted file mode 100644 index 304ce3abe..000000000 --- a/src/NzbDrone.Core/Indexers/Rarbg/RarbgTorrent.cs +++ /dev/null @@ -1,24 +0,0 @@ -using System; -using System.Collections.Generic; -using Newtonsoft.Json; - -namespace NzbDrone.Core.Indexers.Rarbg -{ - public class RarbgTorrent - { - [JsonProperty("f")] - public string Title { get; set; } - [JsonProperty("c")] - public string Category { get; set; } - [JsonProperty("d")] - public string DownloadUrl { get; set; } - [JsonProperty("s")] - public int Seeders { get; set; } - [JsonProperty("l")] - public int Leechers { get; set; } - [JsonProperty("t")] - public long Size { get; set; } - [JsonProperty("u")] - public DateTime PublishDate { get; set; } - } -} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 56c632442..462b84e13 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -517,7 +517,7 @@ - +