From 6c2cd7fe167f528fd1b53105adb1a3ecfdd2cdf9 Mon Sep 17 00:00:00 2001 From: ta264 Date: Tue, 11 Aug 2020 21:09:18 +0100 Subject: [PATCH] New: Support for Redacted API keys Fixes #1127 --- .../RedactedTests/RedactedFixture.cs | 67 +++++++++++ .../Checks/RedactedGazelleCheck.cs | 37 ++++++ src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs | 1 - .../Indexers/Gazelle/GazelleParser.cs | 2 - .../Indexers/Redacted/Redacted.cs | 49 ++++++++ .../Indexers/Redacted/RedactedParser.cs | 106 ++++++++++++++++++ .../Redacted/RedactedRequestGenerator.cs | 87 ++++++++++++++ .../Indexers/Redacted/RedactedSettings.cs | 45 ++++++++ 8 files changed, 391 insertions(+), 3 deletions(-) create mode 100644 src/NzbDrone.Core.Test/IndexerTests/RedactedTests/RedactedFixture.cs create mode 100644 src/NzbDrone.Core/HealthCheck/Checks/RedactedGazelleCheck.cs create mode 100644 src/NzbDrone.Core/Indexers/Redacted/Redacted.cs create mode 100644 src/NzbDrone.Core/Indexers/Redacted/RedactedParser.cs create mode 100644 src/NzbDrone.Core/Indexers/Redacted/RedactedRequestGenerator.cs create mode 100644 src/NzbDrone.Core/Indexers/Redacted/RedactedSettings.cs diff --git a/src/NzbDrone.Core.Test/IndexerTests/RedactedTests/RedactedFixture.cs b/src/NzbDrone.Core.Test/IndexerTests/RedactedTests/RedactedFixture.cs new file mode 100644 index 000000000..fd77d17c1 --- /dev/null +++ b/src/NzbDrone.Core.Test/IndexerTests/RedactedTests/RedactedFixture.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using FluentAssertions; +using FluentValidation.Results; +using Moq; +using NUnit.Framework; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.Redacted; +using NzbDrone.Core.Test.Framework; + +namespace NzbDrone.Core.Test.IndexerTests.GazelleTests +{ + [TestFixture] + public class RedactedFixture : CoreTest + { + [SetUp] + public void Setup() + { + Subject.Definition = new IndexerDefinition() + { + Name = "Redacted", + Settings = new RedactedSettings + { + ApiKey = "key" + } + }; + } + + [Test] + public void should_parse_recent_feed_from_redacted() + { + var recentFeed = ReadAllText(@"Files/Indexers/Gazelle/Gazelle.json"); + var indexFeed = ReadAllText(@"Files/Indexers/Gazelle/GazelleIndex.json"); + + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET && + v.Url.FullUri.Contains("ajax.php?action=browse") && + v.Headers.Get("Authorization") == ((RedactedSettings)Subject.Definition.Settings).ApiKey))) + .Returns(r => new HttpResponse(r, new HttpHeader { ContentType = "application/json" }, recentFeed)); + + Mocker.GetMock() + .Setup(o => o.Execute(It.Is(v => v.Method == HttpMethod.GET && + v.Url.FullUri.Contains("ajax.php?action=index") && + v.Headers.Get("Authorization") == ((RedactedSettings)Subject.Definition.Settings).ApiKey))) + .Returns(r => new HttpResponse(r, new HttpHeader(), indexFeed)); + + ((RedactedRequestGenerator)Subject.GetRequestGenerator()).Authenticate(); + var releases = Subject.FetchRecent(); + + releases.Should().HaveCount(4); + + var releaseInfo = releases.First(); + + releaseInfo.Title.Should().Be("Shania Twain - Shania Twain (1993) [FLAC 24bit Lossless] [WEB]"); + releaseInfo.DownloadProtocol.Should().Be(DownloadProtocol.Torrent); + releaseInfo.DownloadUrl.Should() + .Be("https://redacted.ch/torrents.php?action=download&id=1541452&authkey=lidarr&torrent_pass=redacted"); + releaseInfo.InfoUrl.Should().Be("https://redacted.ch/torrents.php?id=106951&torrentid=1541452"); + releaseInfo.CommentUrl.Should().Be(null); + releaseInfo.Indexer.Should().Be(Subject.Definition.Name); + releaseInfo.PublishDate.Should().Be(DateTime.Parse("2017-12-11 00:17:53")); + releaseInfo.Size.Should().Be(653734702); + } + } +} diff --git a/src/NzbDrone.Core/HealthCheck/Checks/RedactedGazelleCheck.cs b/src/NzbDrone.Core/HealthCheck/Checks/RedactedGazelleCheck.cs new file mode 100644 index 000000000..7f27a2027 --- /dev/null +++ b/src/NzbDrone.Core/HealthCheck/Checks/RedactedGazelleCheck.cs @@ -0,0 +1,37 @@ +using NzbDrone.Core.Indexers; +using NzbDrone.Core.Indexers.Gazelle; +using NzbDrone.Core.ThingiProvider.Events; + +namespace NzbDrone.Core.HealthCheck.Checks +{ + [CheckOn(typeof(ProviderAddedEvent))] + [CheckOn(typeof(ProviderDeletedEvent))] + [CheckOn(typeof(ProviderUpdatedEvent))] + public class RedactedGazelleCheck : HealthCheckBase + { + private readonly IIndexerFactory _indexerFactory; + + public RedactedGazelleCheck(IIndexerFactory indexerFactory) + { + _indexerFactory = indexerFactory; + } + + public override HealthCheck Check() + { + var indexers = _indexerFactory.GetAvailableProviders(); + + foreach (var indexer in indexers) + { + var definition = (IndexerDefinition)indexer.Definition; + + if (definition.Settings is GazelleSettings s && + s.BaseUrl == "https://redacted.ch") + { + return new HealthCheck(GetType(), HealthCheckResult.Warning, "You have set up Redacted as a Gazelle indexer, please reconfigure using the Redacted indexer setting"); + } + } + + return new HealthCheck(GetType()); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs b/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs index 1bcd5a000..c30a7edfe 100644 --- a/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs +++ b/src/NzbDrone.Core/Indexers/Gazelle/Gazelle.cs @@ -51,7 +51,6 @@ namespace NzbDrone.Core.Indexers.Gazelle get { yield return GetDefinition("Orpheus Network", GetSettings("https://orpheus.network")); - yield return GetDefinition("REDacted", GetSettings("https://redacted.ch")); yield return GetDefinition("Not What CD", GetSettings("https://notwhat.cd")); } } diff --git a/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs b/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs index df3cdbf9d..34f1b076f 100644 --- a/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs +++ b/src/NzbDrone.Core/Indexers/Gazelle/GazelleParser.cs @@ -79,8 +79,6 @@ namespace NzbDrone.Core.Indexers.Gazelle } } - var torr = torrentInfos; - // order by date return torrentInfos diff --git a/src/NzbDrone.Core/Indexers/Redacted/Redacted.cs b/src/NzbDrone.Core/Indexers/Redacted/Redacted.cs new file mode 100644 index 000000000..45495af5a --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Redacted/Redacted.cs @@ -0,0 +1,49 @@ +using System.Collections.Generic; +using FluentValidation.Results; +using NLog; +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Parser; + +namespace NzbDrone.Core.Indexers.Redacted +{ + public class Redacted : HttpIndexerBase + { + public override string Name => "Redacted"; + public override DownloadProtocol Protocol => DownloadProtocol.Torrent; + public override bool SupportsRss => true; + public override bool SupportsSearch => true; + public override int PageSize => 50; + + public Redacted(IHttpClient httpClient, + IIndexerStatusService indexerStatusService, + IConfigService configService, + IParsingService parsingService, + Logger logger) + : base(httpClient, indexerStatusService, configService, parsingService, logger) + { + } + + public override IIndexerRequestGenerator GetRequestGenerator() + { + return new RedactedRequestGenerator() + { + Settings = Settings, + HttpClient = _httpClient, + Logger = _logger, + }; + } + + public override IParseIndexerResponse GetParser() + { + return new RedactedParser(Settings); + } + + protected override void Test(List failures) + { + ((RedactedRequestGenerator)GetRequestGenerator()).Authenticate(); + + base.Test(failures); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Redacted/RedactedParser.cs b/src/NzbDrone.Core/Indexers/Redacted/RedactedParser.cs new file mode 100644 index 000000000..002b55fea --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Redacted/RedactedParser.cs @@ -0,0 +1,106 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Core.Indexers.Exceptions; +using NzbDrone.Core.Indexers.Gazelle; +using NzbDrone.Core.Parser.Model; + +namespace NzbDrone.Core.Indexers.Redacted +{ + public class RedactedParser : IParseIndexerResponse + { + private readonly RedactedSettings _settings; + + public RedactedParser(RedactedSettings settings) + { + _settings = settings; + } + + public IList ParseResponse(IndexerResponse indexerResponse) + { + var torrentInfos = new List(); + + if (indexerResponse.HttpResponse.StatusCode != HttpStatusCode.OK) + { + throw new IndexerException(indexerResponse, $"Unexpected response status {indexerResponse.HttpResponse.StatusCode} code from API request"); + } + + if (!indexerResponse.HttpResponse.Headers.ContentType.Contains(HttpAccept.Json.Value)) + { + throw new IndexerException(indexerResponse, $"Unexpected response header {indexerResponse.HttpResponse.Headers.ContentType} from API request, expected {HttpAccept.Json.Value}"); + } + + var jsonResponse = new HttpResponse(indexerResponse.HttpResponse); + if (jsonResponse.Resource.Status != "success" || + jsonResponse.Resource.Status.IsNullOrWhiteSpace() || + jsonResponse.Resource.Response == null) + { + return torrentInfos; + } + + foreach (var result in jsonResponse.Resource.Response.Results) + { + if (result.Torrents != null) + { + foreach (var torrent in result.Torrents) + { + var id = torrent.TorrentId; + var artist = WebUtility.HtmlDecode(result.Artist); + var album = WebUtility.HtmlDecode(result.GroupName); + + torrentInfos.Add(new GazelleInfo() + { + Guid = string.Format("Redacted-{0}", id), + Artist = artist, + + // Splice Title from info to avoid calling API again for every torrent. + Title = WebUtility.HtmlDecode(result.Artist + " - " + result.GroupName + " (" + result.GroupYear + ") [" + torrent.Format + " " + torrent.Encoding + "]"), + Album = album, + Container = torrent.Encoding, + Codec = torrent.Format, + Size = long.Parse(torrent.Size), + DownloadUrl = GetDownloadUrl(id, _settings.PassKey), + InfoUrl = GetInfoUrl(result.GroupId, id), + Seeders = int.Parse(torrent.Seeders), + Peers = int.Parse(torrent.Leechers) + int.Parse(torrent.Seeders), + PublishDate = torrent.Time.ToUniversalTime(), + Scene = torrent.Scene, + }); + } + } + } + + // order by date + return + torrentInfos + .OrderByDescending(o => o.PublishDate) + .ToArray(); + } + + private string GetDownloadUrl(int torrentId, string passKey) + { + // AuthKey is required but not checked, just pass in a dummy variable + // to avoid having to track authkey, which is randomly cycled + var url = new HttpUri(_settings.BaseUrl) + .CombinePath("/torrents.php") + .AddQueryParam("action", "download") + .AddQueryParam("id", torrentId) + .AddQueryParam("authkey", "lidarr") + .AddQueryParam("torrent_pass", passKey); + + return url.FullUri; + } + + private string GetInfoUrl(string groupId, int torrentId) + { + var url = new HttpUri(_settings.BaseUrl) + .CombinePath("/torrents.php") + .AddQueryParam("id", groupId) + .AddQueryParam("torrentid", torrentId); + + return url.FullUri; + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Redacted/RedactedRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Redacted/RedactedRequestGenerator.cs new file mode 100644 index 000000000..a20ceaa69 --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Redacted/RedactedRequestGenerator.cs @@ -0,0 +1,87 @@ +using System; +using System.Collections.Generic; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Indexers.Gazelle; +using NzbDrone.Core.IndexerSearch.Definitions; + +namespace NzbDrone.Core.Indexers.Redacted +{ + public class RedactedRequestGenerator : IIndexerRequestGenerator + { + public RedactedSettings Settings { get; set; } + + public IHttpClient HttpClient { get; set; } + public Logger Logger { get; set; } + + public virtual IndexerPageableRequestChain GetRecentRequests() + { + var pageableRequests = new IndexerPageableRequestChain(); + + pageableRequests.Add(GetRequest(null)); + + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(AlbumSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + pageableRequests.Add(GetRequest(string.Format("&artistname={0}&groupname={1}", searchCriteria.ArtistQuery, searchCriteria.AlbumQuery))); + return pageableRequests; + } + + public IndexerPageableRequestChain GetSearchRequests(ArtistSearchCriteria searchCriteria) + { + var pageableRequests = new IndexerPageableRequestChain(); + pageableRequests.Add(GetRequest(string.Format("&artistname={0}", searchCriteria.ArtistQuery))); + return pageableRequests; + } + + public void Authenticate() + { + var index = GetIndex(); + + if (index == null || + index.Status.IsNullOrWhiteSpace() || + index.Status != "success" || + index.Response.Passkey.IsNullOrWhiteSpace()) + { + Logger.Debug("Redacted authentication failed."); + throw new Exception("Failed to authenticate with Redacted."); + } + + Logger.Debug("Redacted authentication succeeded."); + + Settings.PassKey = index.Response.Passkey; + } + + private IEnumerable GetRequest(string searchParameters) + { + var req = RequestBuilder() + .Resource($"ajax.php?action=browse&searchstr={searchParameters}") + .Build(); + + yield return new IndexerRequest(req); + } + + private GazelleAuthResponse GetIndex() + { + var request = RequestBuilder().Resource("ajax.php?action=index").Build(); + + var indexResponse = HttpClient.Execute(request); + + var result = Json.Deserialize(indexResponse.Content); + + return result; + } + + private HttpRequestBuilder RequestBuilder() + { + return new HttpRequestBuilder($"{Settings.BaseUrl.Trim().TrimEnd('/')}") + .Accept(HttpAccept.Json) + .SetHeader("Authorization", Settings.ApiKey); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Redacted/RedactedSettings.cs b/src/NzbDrone.Core/Indexers/Redacted/RedactedSettings.cs new file mode 100644 index 000000000..24a1f27ec --- /dev/null +++ b/src/NzbDrone.Core/Indexers/Redacted/RedactedSettings.cs @@ -0,0 +1,45 @@ +using FluentValidation; +using NzbDrone.Core.Annotations; +using NzbDrone.Core.Validation; + +namespace NzbDrone.Core.Indexers.Redacted +{ + public class RedactedSettingsValidator : AbstractValidator + { + public RedactedSettingsValidator() + { + RuleFor(c => c.ApiKey).NotEmpty(); + } + } + + public class RedactedSettings : ITorrentIndexerSettings + { + private static readonly RedactedSettingsValidator Validator = new RedactedSettingsValidator(); + + public RedactedSettings() + { + BaseUrl = "https://redacted.ch"; + MinimumSeeders = IndexerDefaults.MINIMUM_SEEDERS; + } + + public string BaseUrl { get; set; } + public string PassKey { get; set; } + + [FieldDefinition(1, Label = "ApiKey", HelpText = "Generate this in 'Access Settings' in your Redacted profile")] + public string ApiKey { get; set; } + + [FieldDefinition(3, Type = FieldType.Textbox, Label = "Minimum Seeders", HelpText = "Minimum number of seeders required.", Advanced = true)] + public int MinimumSeeders { get; set; } + + [FieldDefinition(4)] + public SeedCriteriaSettings SeedCriteria { get; } = new SeedCriteriaSettings(); + + [FieldDefinition(5, Type = FieldType.Number, Label = "Early Download Limit", Unit = "days", HelpText = "Time before release date Lidarr will download from this indexer, empty is no limit", Advanced = true)] + public int? EarlyReleaseLimit { get; set; } + + public NzbDroneValidationResult Validate() + { + return new NzbDroneValidationResult(Validator.Validate(this)); + } + } +}