From 3ae2883eb587f60fd7bf81edc0a343b884980c34 Mon Sep 17 00:00:00 2001 From: scherzo Date: Mon, 13 Apr 2015 21:30:02 -0700 Subject: [PATCH] New: Add support for the HDBits torrent tracker. The indexer implementation borrows heavily from the BroadcastTheNet implementation as HDBits also provides a JSON API that can be used to query both the recent torrents and the catalog. --- .../Files/Indexers/HdBits/RecentFeed.json | 57 ++++++++ .../IndexerTests/HdBitsTests/HdBitsFixture.cs | 77 ++++++++++ .../NzbDrone.Core.Test.csproj | 4 + src/NzbDrone.Core/Indexers/HDBits/HdBits.cs | 33 +++++ .../Indexers/HDBits/HdBitsApi.cs | 134 ++++++++++++++++++ .../Indexers/HDBits/HdBitsParser.cs | 105 ++++++++++++++ .../Indexers/HDBits/HdBitsRequestGenerator.cs | 133 +++++++++++++++++ .../Indexers/HDBits/HdBitsSettings.cs | 70 +++++++++ src/NzbDrone.Core/NzbDrone.Core.csproj | 5 + 9 files changed, 618 insertions(+) create mode 100644 src/NzbDrone.Core.Test/Files/Indexers/HdBits/RecentFeed.json create mode 100644 src/NzbDrone.Core.Test/IndexerTests/HdBitsTests/HdBitsFixture.cs create mode 100644 src/NzbDrone.Core/Indexers/HDBits/HdBits.cs create mode 100644 src/NzbDrone.Core/Indexers/HDBits/HdBitsApi.cs create mode 100644 src/NzbDrone.Core/Indexers/HDBits/HdBitsParser.cs create mode 100644 src/NzbDrone.Core/Indexers/HDBits/HdBitsRequestGenerator.cs create mode 100644 src/NzbDrone.Core/Indexers/HDBits/HdBitsSettings.cs diff --git a/src/NzbDrone.Core.Test/Files/Indexers/HdBits/RecentFeed.json b/src/NzbDrone.Core.Test/Files/Indexers/HdBits/RecentFeed.json new file mode 100644 index 000000000..9e4b114f7 --- /dev/null +++ b/src/NzbDrone.Core.Test/Files/Indexers/HdBits/RecentFeed.json @@ -0,0 +1,57 @@ +{ + "status": 0, + "data": [ + { + "id": 257142, + "hash": "EABC50AEF9F53CEDED84ADF14144D3368E586F3A", + "leechers": 1, + "seeders": 46, + "name": "Supernatural S10E17 1080p WEB-DL DD5.1 H.264-ECI", + "times_completed": 49, + "size": 1718009717, + "utadded": 1428179446, + "added": "2015-04-04T20:30:46+0000", + "comments": 0, + "numfiles": 1, + "filename": "Supernatural.S10E17.1080p.WEB-DL.DD5.1.H.264-ECI.torrent", + "freeleech": "no", + "type_category": 2, + "type_codec": 1, + "type_medium": 6, + "type_origin": 0, + "username": "abc", + "owner": 1107944, + "tvdb": { + "id": 78901, + "season": 10, + "episode": 17 + } + }, + { + "id": 257140, + "hash": "BE3BA5396B9A30544353B55FDD89EDE46C8FB72A", + "leechers": 0, + "seeders": 18, + "name": "Scandal S04E18 1080p WEB-DL DD5.1 H.264-ECI", + "times_completed": 19, + "size": 1789106197, + "utadded": 1428179128, + "added": "2015-04-04T20:25:28+0000", + "comments": 0, + "numfiles": 1, + "filename": "Scandal.2012.S04E18.1080p.WEB-DL.DD5.1.H.264-ECI.torrent", + "freeleech": "no", + "type_category": 2, + "type_codec": 1, + "type_medium": 6, + "type_origin": 0, + "username": "abc", + "owner": 1107944, + "tvdb": { + "id": 248841, + "season": 4, + "episode": 18 + } + } + ] +} \ No newline at end of file diff --git a/src/NzbDrone.Core.Test/IndexerTests/HdBitsTests/HdBitsFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/HdBitsTests/HdBitsFixture.cs new file mode 100644 index 000000000..73375d644 --- /dev/null +++ b/src/NzbDrone.Core.Test/IndexerTests/HdBitsTests/HdBitsFixture.cs @@ -0,0 +1,77 @@ +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.HDBits; +using NzbDrone.Core.Test.Framework; +using FluentAssertions; +using System.Linq; +using NzbDrone.Core.Parser.Model; +using System; +using System.Text; +using NzbDrone.Test.Common; + +namespace NzbDrone.Core.Test.IndexerTests.HdBitsTests +{ + [TestFixture] + public class HdBitsFixture : CoreTest + { + [SetUp] + public void Setup() + { + Subject.Definition = new IndexerDefinition() + { + Name = "HdBits", + Settings = new HdBitsSettings() { ApiKey = "fakekey" } + }; + } + + [Test] + public void TestSimpleResponse() + { + var responseJson = ReadAllText(@"Files/Indexers/HdBits/RecentFeed.json"); + + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.POST))) + .Returns(r => new HttpResponse(r, new HttpHeader(), responseJson)); + + var torrents = Subject.FetchRecent(); + + torrents.Should().HaveCount(2); + torrents.First().Should().BeOfType(); + + var first = torrents.First() as TorrentInfo; + + first.Guid.Should().Be("HDBits-257142"); + first.Title.Should().Be("Supernatural S10E17 1080p WEB-DL DD5.1 H.264-ECI"); + first.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + first.DownloadUrl.Should().Be("https://hdbits.org/download.php?id=257142&passkey=fakekey"); + first.InfoUrl.Should().Be("https://hdbits.org/details.php?id=257142"); + first.PublishDate.Should().Be(DateTime.Parse("2015-04-04T20:30:46+0000")); + first.Size.Should().Be(1718009717); + first.MagnetUrl.Should().BeNullOrEmpty(); + first.Peers.Should().Be(47); + first.Seeders.Should().Be(46); + } + + [Test] + public void TestBadPasskey() + { + var responseJson = @" +{ + ""status"": 5, + ""message"": ""Invalid authentication credentials"" +}"; + Mocker.GetMock() + .Setup(v => v.Execute(It.IsAny())) + .Returns(r => new HttpResponse(r, new HttpHeader(), + Encoding.UTF8.GetBytes(responseJson))); + + var torrents = Subject.FetchRecent(); + + torrents.Should().BeEmpty(); + + ExceptionVerification.ExpectedWarns(1); + } + } +} diff --git a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj index 236181bb6..de4f2a133 100644 --- a/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj +++ b/src/NzbDrone.Core.Test/NzbDrone.Core.Test.csproj @@ -204,6 +204,7 @@ + @@ -373,6 +374,9 @@ Always + + Always + PreserveNewest diff --git a/src/NzbDrone.Core/Indexers/HDBits/HdBits.cs b/src/NzbDrone.Core/Indexers/HDBits/HdBits.cs new file mode 100644 index 000000000..fa5af64a4 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/HDBits/HdBits.cs @@ -0,0 +1,33 @@ +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; +using System; + +namespace NzbDrone.Core.Indexers.HDBits +{ + public class HdBits : HttpIndexerBase + { + public override DownloadProtocol Protocol { get { return DownloadProtocol.Torrent; } } + public override bool SupportsRss { get { return true; } } + public override bool SupportsSearch { get { return true; } } + public override int PageSize { get { return 30; } } + + public HdBits(IHttpClient httpClient, + IConfigService configService, + IParsingService parsingService, + Logger logger) + : base(httpClient, configService, parsingService, logger) + { } + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new HdBitsRequestGenerator() { Settings = Settings }; + } + + public override IParseIndexerResponse GetParser() + { + return new HdBitsParser(Settings); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/HDBits/HdBitsApi.cs b/src/NzbDrone.Core/Indexers/HDBits/HdBitsApi.cs new file mode 100644 index 000000000..307983908 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/HDBits/HdBitsApi.cs @@ -0,0 +1,134 @@ +using Newtonsoft.Json; +using System; + +namespace NzbDrone.Core.Indexers.HDBits +{ + public class TorrentQuery + { + [JsonProperty(Required = Required.Always)] + public string Username { get; set; } + [JsonProperty(Required = Required.Always)] + public string Passkey { get; set; } + + public int? Id { get; set; } + + public string Hash { get; set; } + + public string Search { get; set; } + + public int[] Category { get; set; } + + public int[] Codec { get; set; } + + public int[] Medium { get; set; } + + public int[] Origin { get; set; } + + [JsonProperty(PropertyName = "imdb")] + public ImdbInfo ImdbInfo { get; set; } + + [JsonProperty(PropertyName = "tvdb")] + public TvdbInfo TvdbInfo { get; set; } + + [JsonProperty(PropertyName = "file_in_torrent")] + public string FileInTorrent { get; set; } + + [JsonProperty(PropertyName = "snatched_only")] + public bool? SnatchedOnly { get; set; } + public int? Limit { get; set; } + public int? Page { get; set; } + + public TorrentQuery Clone() + { + return MemberwiseClone() as TorrentQuery; + } + } + + public class HdBitsResponse + { + [JsonProperty(Required = Required.Always)] + public StatusCode Status { get; set; } + public string Message { get; set; } + public object Data { get; set; } + } + + public class TorrentQueryResponse + { + public long Id { get; set; } + public string Hash { get; set; } + public int Leechers { get; set; } + public int Seeders { get; set; } + public string Name { get; set; } + + [JsonProperty(PropertyName = "times_completed")] + + public uint TimesCompleted { get; set; } + + public long Size { get; set; } + + [JsonProperty(PropertyName = "utadded")] + public long UtAdded { get; set; } + + public DateTime Added { get; set; } + + public uint Comments { get; set; } + + [JsonProperty(PropertyName = "numfiles")] + public uint NumFiles { get; set; } + + [JsonProperty(PropertyName = "filename")] + public string FileName { get; set; } + + [JsonProperty(PropertyName = "freeleech")] + public string FreeLeech { get; set; } + + [JsonProperty(PropertyName = "type_category")] + public int TypeCategory { get; set; } + + [JsonProperty(PropertyName = "type_codec")] + public int TypeCodec { get; set; } + + [JsonProperty(PropertyName = "type_medium")] + public int TypeMedium { get; set; } + + [JsonProperty(PropertyName = "type_origin")] + public int TypeOrigin { get; set; } + + [JsonProperty(PropertyName = "imdb")] + public ImdbInfo ImdbInfo { get; set; } + + [JsonProperty(PropertyName = "tvdb")] + public TvdbInfo TvdbInfo { get; set; } + } + + public class ImdbInfo + { + public int? Id { get; set; } + public string EnglishTitle { get; set; } + public string OriginalTitle { get; set; } + public int? Year { get; set; } + public string[] Genres { get; set; } + public float? Rating { get; set; } + } + + public class TvdbInfo + { + public int? Id { get; set; } + public int? Season { get; set; } + public int? Episode { get; set; } + } + + public enum StatusCode + { + Success = 0, + Failure = 1, + SslRequired = 2, + JsonMalformed = 3, + AuthDataMissing = 4, + AuthFailed = 5, + MissingRequiredParameters = 6, + InvalidParameter = 7, + ImdbImportFail = 8, + ImdbTvNotAllowed = 9 + } +} diff --git a/src/NzbDrone.Core/Indexers/HDBits/HdBitsParser.cs b/src/NzbDrone.Core/Indexers/HDBits/HdBitsParser.cs new file mode 100644 index 000000000..6d9b05a0f --- /dev/null +++ b/src/NzbDrone.Core/Indexers/HDBits/HdBitsParser.cs @@ -0,0 +1,105 @@ +using System; +using System.Collections.Generic; +using NzbDrone.Core.Parser.Model; +using System.Net; +using NzbDrone.Core.Indexers.Exceptions; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Web; +using System.Collections.Specialized; + +namespace NzbDrone.Core.Indexers.HDBits +{ + public class HdBitsParser : IParseIndexerResponse + { + private readonly HdBitsSettings _settings; + + public HdBitsParser(HdBitsSettings settings) + { + _settings = settings; + } + + public IList ParseResponse(IndexerResponse indexerResponse) + { + var torrentInfos = new List(); + + if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new IndexerException( + indexerResponse, + "Unexpected response status {0} code from API request", + indexerResponse.HttpResponse.StatusCode); + } + + var jsonResponse = JsonConvert.DeserializeObject(indexerResponse.Content); + + if (jsonResponse.Status != StatusCode.Success) + { + throw new IndexerException( + indexerResponse, + @"HDBits API request returned status code {0} with message ""{1}""", + jsonResponse.Status, + jsonResponse.Message ?? ""); + } + + var responseData = jsonResponse.Data as JArray; + if (responseData == null) + { + throw new IndexerException( + indexerResponse, + "Indexer API call response missing result data"); + } + + var queryResults = responseData.ToObject(); + + foreach (var result in queryResults) + { + var id = result.Id; + torrentInfos.Add(new TorrentInfo() + { + Guid = string.Format("HDBits-{0}", id), + Title = result.Name, + Size = result.Size, + DownloadUrl = GetDownloadUrl(id), + InfoUrl = GetInfoUrl(id), + Seeders = result.Seeders, + Peers = result.Leechers + result.Seeders, + PublishDate = result.Added + }); + } + + return torrentInfos.ToArray(); + } + + private string GetDownloadUrl(long torrentId) + { + var args = new NameValueCollection(2); + args["id"] = torrentId.ToString(); + args["passkey"] = _settings.ApiKey; + + return BuildUrl("/download.php", args); + } + + private string GetInfoUrl(long torrentId) + { + var args = new NameValueCollection(1); + args["id"] = torrentId.ToString(); + + return BuildUrl("/details.php", args); + + } + + private string BuildUrl(string path, NameValueCollection args) + { + var builder = new UriBuilder(_settings.BaseUrl); + builder.Path = path; + var queryString = HttpUtility.ParseQueryString(""); + + queryString.Add(args); + + builder.Query = queryString.ToString(); + + return builder.Uri.ToString(); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/HDBits/HdBitsRequestGenerator.cs b/src/NzbDrone.Core/Indexers/HDBits/HdBitsRequestGenerator.cs new file mode 100644 index 000000000..0ee22b146 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/HDBits/HdBitsRequestGenerator.cs @@ -0,0 +1,133 @@ +using System; +using System.Collections.Generic; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.Serializer; +using System.Linq; + +namespace NzbDrone.Core.Indexers.HDBits +{ + public class HdBitsRequestGenerator : IIndexerRequestGenerator + { + public HdBitsSettings Settings { get; set; } + + public IList> GetRecentRequests() + { + var pageableRequests = new List>(); + + pageableRequests.Add(GetRequest(new TorrentQuery())); + + return pageableRequests; + } + + public IList> GetSearchRequests(AnimeEpisodeSearchCriteria searchCriteria) + { + var requests = new List>(); + + var queryBase = new TorrentQuery(); + if (TryAddSearchParameters(queryBase, searchCriteria)) + { + foreach (var episode in searchCriteria.Episodes) + { + var query = queryBase.Clone(); + + query.TvdbInfo.Season = episode.SeasonNumber; + query.TvdbInfo.Episode = episode.EpisodeNumber; + } + } + + return requests; + } + + public IList> GetSearchRequests(SpecialEpisodeSearchCriteria searchCriteria) + { + return new List>(); + } + + public IList> GetSearchRequests(DailyEpisodeSearchCriteria searchCriteria) + { + var requests = new List>(); + + var query = new TorrentQuery(); + if (TryAddSearchParameters(query, searchCriteria)) + { + query.Search = String.Format("{0:yyyy}-{0:MM}-{0:dd}", searchCriteria.AirDate); + + requests.Add(GetRequest(query)); + } + + return requests; + } + + public IList> GetSearchRequests(SeasonSearchCriteria searchCriteria) + { + var requests = new List>(); + + var queryBase = new TorrentQuery(); + if (TryAddSearchParameters(queryBase, searchCriteria)) + { + foreach (var seasonNumber in searchCriteria.Episodes.Select(e => e.SeasonNumber).Distinct()) + { + var query = queryBase.Clone(); + + query.TvdbInfo.Season = seasonNumber; + + requests.Add(GetRequest(query)); + } + } + + return requests; + } + + public IList> GetSearchRequests(SingleEpisodeSearchCriteria searchCriteria) + { + var requests = new List>(); + + var queryBase = new TorrentQuery(); + if (TryAddSearchParameters(queryBase, searchCriteria)) + { + foreach (var episode in searchCriteria.Episodes) + { + var query = queryBase.Clone(); + + query.TvdbInfo.Season = episode.SeasonNumber; + query.TvdbInfo.Episode = episode.EpisodeNumber; + + requests.Add(GetRequest(query)); + } + } + + return requests; + } + + private bool TryAddSearchParameters(TorrentQuery query, SearchCriteriaBase searchCriteria) + { + if (searchCriteria.Series.TvdbId != 0) + { + query.TvdbInfo = query.TvdbInfo ?? new TvdbInfo(); + query.TvdbInfo.Id = searchCriteria.Series.TvdbId; + return true; + } + return false; + } + + private IEnumerable GetRequest(TorrentQuery query) + { + var builder = new HttpRequestBuilder(Settings.BaseUrl); + var request = builder.Build("/api/torrents"); + + request.Method = HttpMethod.POST; + const string appJson = "application/json"; + request.Headers.Accept = appJson; + request.Headers.ContentType = appJson; + + query.Username = Settings.Username; + query.Passkey = Settings.ApiKey; + + request.Body = query.ToJson(); + + yield return new IndexerRequest(request); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/HDBits/HdBitsSettings.cs b/src/NzbDrone.Core/Indexers/HDBits/HdBitsSettings.cs new file mode 100644 index 000000000..779ee574d --- /dev/null +++ b/src/NzbDrone.Core/Indexers/HDBits/HdBitsSettings.cs @@ -0,0 +1,70 @@ +using System; +using FluentValidation; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.Validation; +using NzbDrone.Core.Annotations; + +namespace NzbDrone.Core.Indexers.HDBits +{ + public class HdBitsSettingsValidator : AbstractValidator + { + public HdBitsSettingsValidator() + { + RuleFor(c => c.BaseUrl).ValidRootUrl(); + RuleFor(c => c.ApiKey).NotEmpty(); + } + } + + public class HdBitsSettings : IProviderConfig + { + private static readonly HdBitsSettingsValidator Validator = new HdBitsSettingsValidator(); + + public HdBitsSettings() + { + BaseUrl = "https://hdbits.org"; + } + + [FieldDefinition(0, Label = "Username")] + public string Username { get; set; } + + [FieldDefinition(1, Label = "API Key")] + public string ApiKey { get; set; } + + [FieldDefinition(2, Label = "API URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since your API key will be sent to that host.")] + public string BaseUrl { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } + + public enum HdBitsCategory + { + Movie = 1, + Tv = 2, + Documentary = 3, + Music = 4, + Sport = 5, + Audio = 6, + Xxx = 7, + MiscDemo = 8 + } + + public enum HdBitsCodec + { + H264 = 1, + Mpeg2 = 2, + Vc1 = 3, + Xvid = 4 + } + + public enum HdBitsMedium + { + Bluray = 1, + Encode = 3, + Capture = 4, + Remux = 5, + WebDl = 6 + } +} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index ce2f7577c..06f1432e1 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -479,6 +479,11 @@ + + + + +