From 1322633d0d71801ab4bd1579f94a2b83e3a8b965 Mon Sep 17 00:00:00 2001 From: Qstick Date: Fri, 22 Sep 2017 22:48:15 -0400 Subject: [PATCH] Add Support for Gazelle based indexers --- src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs | 81 ++++++++++ .../Indexers/Gazelle/GazelleApi.cs | 88 +++++++++++ .../Indexers/Gazelle/GazelleInfo.cs | 13 ++ .../Indexers/Gazelle/GazelleParser.cs | 112 ++++++++++++++ .../Gazelle/GazelleRequestGenerator.cs | 141 ++++++++++++++++++ .../Indexers/Gazelle/GazelleSettings.cs | 45 ++++++ src/NzbDrone.Core/NzbDrone.Core.csproj | 6 + 7 files changed, 486 insertions(+) create mode 100644 src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs create mode 100644 src/NzbDrone.Core/Indexers/Gazelle/GazelleApi.cs create mode 100644 src/NzbDrone.Core/Indexers/Gazelle/GazelleInfo.cs create mode 100644 src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs create mode 100644 src/NzbDrone.Core/Indexers/Gazelle/GazelleRequestGenerator.cs create mode 100644 src/NzbDrone.Core/Indexers/Gazelle/GazelleSettings.cs diff --git a/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs b/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs new file mode 100644 index 000000000..603d7ed85 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs @@ -0,0 +1,81 @@ +using NLog; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; +using NzbDrone.Core.ThingiProvider; +using System.Collections.Generic; + +namespace NzbDrone.Core.Indexers.Gazelle +{ + public class Gazelle : HttpIndexerBase + { + public override string Name => "Gazelle API"; + public override DownloadProtocol Protocol => DownloadProtocol.Torrent; + public override bool SupportsRss => true; + public override bool SupportsSearch => true; + public override int PageSize => 50; + + private readonly ICached> _authCookieCache; + private readonly IHttpClient _httpClient; + private readonly Logger _logger; + + public Gazelle(IHttpClient httpClient, ICacheManager cacheManager, IIndexerStatusService indexerStatusService, + IConfigService configService, IParsingService parsingService, Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) + { + _httpClient = httpClient; + _logger = logger; + _authCookieCache = cacheManager.GetCache>(GetType(), "authCookies"); + } + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new GazelleRequestGenerator() + { + Settings = Settings, + HttpClient = _httpClient, + Logger = _logger, + AuthCookieCache = _authCookieCache + }; + } + + public override IParseIndexerResponse GetParser() + { + return new GazelleParser(Settings); + } + + public override IEnumerable DefaultDefinitions + { + get + { + yield return GetDefinition("Apollo.Rip", GetSettings("https://apollo.rip")); + yield return GetDefinition("REDacted", GetSettings("https://redacted.ch")); + yield return GetDefinition("Not What CD", GetSettings("https://notwhat.cd")); + + } + } + + private IndexerDefinition GetDefinition(string name, GazelleSettings settings) + { + return new IndexerDefinition + { + EnableRss = false, + EnableSearch = false, + Name = name, + Implementation = GetType().Name, + Settings = settings, + Protocol = DownloadProtocol.Torrent, + SupportsRss = SupportsRss, + SupportsSearch = SupportsSearch + }; + } + + private GazelleSettings GetSettings(string url) + { + var settings = new GazelleSettings { BaseUrl = url }; + + return settings; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Gazelle/GazelleApi.cs b/src/NzbDrone.Core/Indexers/Gazelle/GazelleApi.cs new file mode 100644 index 000000000..77318f24f --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleApi.cs @@ -0,0 +1,88 @@ +using System; +using Newtonsoft.Json; +using System.Collections.Generic; + +namespace NzbDrone.Core.Indexers.Gazelle +{ + public class GazelleArtist + { + public string Name { get; set; } + public string Id { get; set; } + public string Aliasid { get; set; } + } + + public class GazelleTorrent + { + public int TorrentId { get; set; } + public int EditionId { get; set; } + public List Artists { get; set; } + public bool Remastered { get; set; } + public string RemasterYear { get; set; } + public string RemasterTitle { get; set; } + public string Media { get; set; } + public string Encoding { get; set; } + public string Format { get; set; } + public bool HasLog { get; set; } + public int LogScore { get; set; } + public bool HasQueue { get; set; } + public bool Scene { get; set; } + public bool VanityHouse { get; set; } + public int FileCount { get; set; } + public DateTime Time { get; set; } + public string Size { get; set; } + public string Snatches { get; set; } + public string Seeders { get; set; } + public string Leechers { get; set; } + public bool IsFreeLeech { get; set; } + public bool IsNeutralLeech { get; set; } + public bool IsPersonalFreeLeech { get; set; } + public bool CanUseToken { get; set; } + } + + public class GazelleRelease + { + public string GroupId { get; set; } + public string GroupName { get; set; } + public string Artist { get; set; } + public string GroupYear { get; set; } + public string Cover { get; set; } + public List Tags { get; set; } + public string ReleaseType { get; set; } + public int TotalLeechers { get; set; } + public int TotalSeeders { get; set; } + public int TotalSnatched { get; set; } + public long MaxSize { get; set; } + public string GroupTime { get; set; } + public List Torrents { get; set; } + } + + public class GazelleResponse + { + public string Status { get; set; } + public GazelleBrowseResponse Response { get; set; } + } + + public class GazelleBrowseResponse + { + public List Results { get; set; } + public string CurrentPage { get; set; } + public string Pages { get; set; } + } + + public class GazelleAuthResponse + { + public string Status { get; set; } + public GazelleIndexResponse Response { get; set; } + + } + + public class GazelleIndexResponse + { + public string Username { get; set; } + public string Id { get; set; } + public string Authkey { get; set; } + public string Passkey { get; set; } + + } + +} diff --git a/src/NzbDrone.Core/Indexers/Gazelle/GazelleInfo.cs b/src/NzbDrone.Core/Indexers/Gazelle/GazelleInfo.cs new file mode 100644 index 000000000..84915f605 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleInfo.cs @@ -0,0 +1,13 @@ +using NzbDrone.Core.Parser.Model; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; + +namespace NzbDrone.Core.Indexers.Gazelle +{ + public class GazelleInfo : TorrentInfo + { + public bool? Scene { get; set; } + } +} diff --git a/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs b/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs new file mode 100644 index 000000000..f60695486 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs @@ -0,0 +1,112 @@ +using System.Collections.Generic; +using System.Net; +using Newtonsoft.Json; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Parser.Model; +using System.Linq; +using NzbDrone.Common.Cache; +using NzbDrone.Common.Extensions; + +namespace NzbDrone.Core.Indexers.Gazelle +{ + public class GazelleParser : IParseIndexerResponse + { + private readonly GazelleSettings _settings; + public ICached> AuthCookieCache { get; set; } + + public GazelleParser(GazelleSettings settings) + { + _settings = settings; + } + + public IList ParseResponse(IndexerResponse indexerResponse) + { + var torrentInfos = new List(); + + if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + // Remove cookie cache + AuthCookieCache.Remove(_settings.BaseUrl.Trim().TrimEnd('/')); + + throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); + } + + if (!indexerResponse.HttpResponse.Headers.ContentType.Contains(HttpAccept.Json.Value)) + { + // Remove cookie cache + AuthCookieCache.Remove(_settings.BaseUrl.Trim().TrimEnd('/')); + + throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}"); + } + + var jsonResponse = JsonConvert.DeserializeObject(indexerResponse.Content); + if (jsonResponse.Status != "success" || + jsonResponse.Status.IsNullOrWhiteSpace() || + jsonResponse.Response == null) + { + return torrentInfos; + } + + + foreach (var result in jsonResponse.Response.Results) + { + if (result.Torrents != null) + { + foreach (var torrent in result.Torrents) + { + var id = torrent.TorrentId; + + torrentInfos.Add(new GazelleInfo() + { + Guid = string.Format("Gazelle-{0}", id), + Artist = result.Artist, + // Splice Title from info to avoid calling API again for every torrent. + Title = result.Artist + " - " + result.GroupName + " (" + result.GroupYear +") (" + torrent.Format + " " + torrent.Encoding + ")", + Album = result.GroupName, + Container = torrent.Encoding, + Codec = torrent.Format, + Size = long.Parse(torrent.Size), + DownloadUrl = GetDownloadUrl(id, _settings.AuthKey, _settings.PassKey), + InfoUrl = GetInfoUrl(result.GroupId, id), + Seeders = int.Parse(torrent.Seeders), + Peers = int.Parse(torrent.Leechers) + int.Parse(torrent.Seeders), + PublishDate = torrent.Time.ToUniversalTime(), + Scene = torrent.Scene, + }); + } + } + } + + var torr = torrentInfos; + // order by date + return + torrentInfos + .OrderByDescending(o => o.PublishDate) + .ToArray(); + + } + + private string GetDownloadUrl(int torrentId, string authKey, string passKey) + { + var url = new HttpUri(_settings.BaseUrl) + .CombinePath("/torrents.php") + .AddQueryParam("action", "download") + .AddQueryParam("id", torrentId) + .AddQueryParam("authkey", authKey) + .AddQueryParam("torrent_pass", passKey); + + return url.FullUri; + } + + private string GetInfoUrl(string groupId, int torrentId) + { + var url = new HttpUri(_settings.BaseUrl) + .CombinePath("/torrents.php") + .AddQueryParam("id", groupId) + .AddQueryParam("torrentid", torrentId); + + return url.FullUri; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Gazelle/GazelleRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Gazelle/GazelleRequestGenerator.cs new file mode 100644 index 000000000..c27403193 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleRequestGenerator.cs @@ -0,0 +1,141 @@ +using System; +using System.Collections.Generic; +using NzbDrone.Common.Http; +using NzbDrone.Core.IndexerSearch.Definitions; +using NzbDrone.Common.Cache; +using NLog; +using NzbDrone.Common.Serializer; + +namespace NzbDrone.Core.Indexers.Gazelle +{ + public class GazelleRequestGenerator : IIndexerRequestGenerator + { + + public GazelleSettings Settings { get; set; } + + public ICached> AuthCookieCache { get; set; } + public IHttpClient HttpClient { get; set; } + public Logger Logger { get; set; } + + public virtual IndexerPageableRequestChain GetRecentRequests() + { + var pageableRequests = new IndexerPageableRequestChain(); + + pageableRequests.Add(GetRequest(null)); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + pageableRequests.Add(GetRequest(string.Format("&artistname={0}&groupname={1}", searchCriteria.Artist.Name, searchCriteria.AlbumTitle))); + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + pageableRequests.Add(GetRequest(string.Format("&artistname={0}",searchCriteria.Artist.Name))); + return pageableRequests; + } + + private IEnumerable GetRequest(string searchParameters) + { + Authenticate(); + + var filter = ""; + if (searchParameters == null) + { + + } + + var request = + new IndexerRequest( + $"{Settings.BaseUrl.Trim().TrimEnd('/')}/ajax.php?action=browse&searchstr={searchParameters}{filter}", + HttpAccept.Json); + + var cookies = AuthCookieCache.Find(Settings.BaseUrl.Trim().TrimEnd('/')); + foreach (var cookie in cookies) + { + request.HttpRequest.Cookies[cookie.Key] = cookie.Value; + } + + yield return request; + } + + private GazelleAuthResponse GetIndex(Dictionary cookies) + { + var indexRequestBuilder = new HttpRequestBuilder($"{Settings.BaseUrl.Trim().TrimEnd('/')}") + { + LogResponseContent = true + }; + + indexRequestBuilder.SetCookies(cookies); + indexRequestBuilder.Method = HttpMethod.GET; + indexRequestBuilder.Resource("ajax.php?action=index"); + + var authIndexRequest = indexRequestBuilder + .Accept(HttpAccept.Json) + .Build(); + + var indexResponse = HttpClient.Execute(authIndexRequest); + + var result = Json.Deserialize(indexResponse.Content); + + return result; + } + + private void Authenticate() + { + + var requestBuilder = new HttpRequestBuilder($"{Settings.BaseUrl.Trim().TrimEnd('/')}") + { + LogResponseContent = true + }; + + requestBuilder.Method = HttpMethod.POST; + requestBuilder.Resource("login.php"); + requestBuilder.PostProcess += r => r.RequestTimeout = TimeSpan.FromSeconds(15); + + var authKey = Settings.BaseUrl.Trim().TrimEnd('/'); + var cookies = AuthCookieCache.Find(authKey); + + if (cookies == null) + { + AuthCookieCache.Remove(authKey); + var authLoginRequest = requestBuilder + .AddFormParameter("username", Settings.Username) + .AddFormParameter("password", Settings.Password) + .AddFormParameter("keeplogged", "1") + .SetHeader("Content-Type", "multipart/form-data") + .Accept(HttpAccept.Json) + .Build(); + + var response = HttpClient.Execute(authLoginRequest); + + cookies = response.GetCookies(); + AuthCookieCache.Set(authKey, cookies, new TimeSpan(7, 0, 0, 0, 0)); // re-auth every 7 days + requestBuilder.SetCookies(cookies); + } + else + { + requestBuilder.SetCookies(cookies); + } + + var index = GetIndex(cookies); + + if (index.Status != "success" || string.IsNullOrWhiteSpace(index.Status)) + { + Logger.Debug("Gazelle authentication failed."); + throw new Exception("Failed to authenticate with Gazelle."); + } + + Logger.Debug("Gazelle authentication succeeded."); + + Settings.AuthKey = index.Response.Authkey; + Settings.PassKey = index.Response.Passkey; + + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Gazelle/GazelleSettings.cs b/src/NzbDrone.Core/Indexers/Gazelle/GazelleSettings.cs new file mode 100644 index 000000000..6b428512b --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleSettings.cs @@ -0,0 +1,45 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.ThingiProvider; +using NzbDrone.Core.Validation; +using System.Text.RegularExpressions; + +namespace NzbDrone.Core.Indexers.Gazelle +{ + public class GazelleSettingsValidator : AbstractValidator + { + public GazelleSettingsValidator() + { + RuleFor(c => c.BaseUrl).ValidRootUrl(); + RuleFor(c => c.Username).NotEmpty(); + RuleFor(c => c.Password).NotEmpty(); + } + } + + public class GazelleSettings : IProviderConfig + { + private static readonly GazelleSettingsValidator Validator = new GazelleSettingsValidator(); + + public GazelleSettings() + { + + } + + public string AuthKey; + public string PassKey; + + [FieldDefinition(0, Label = "URL", Advanced = true, HelpText = "Do not change this unless you know what you're doing. Since your cookie will be sent to that host.")] + public string BaseUrl { get; set; } + + [FieldDefinition(1, Label = "Username", HelpText = "Username")] + public string Username { get; set; } + + [FieldDefinition(2, Label = "Password", Type = FieldType.Password, HelpText = "Password")] + public string Password { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 2ac10de77..68ad7bfdb 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -616,6 +616,12 @@ + + + + + +