From 11a6b593ddc5936ecf5142cd9220b6f726842a8e Mon Sep 17 00:00:00 2001 From: Bogdan Date: Thu, 21 Sep 2023 15:46:19 +0300 Subject: [PATCH] New: (Gazelle) Options to prevent downloads without FL tokens --- .../Migration/041_freeleech_token_options.cs | 60 ++++++++++ .../Indexers/Definitions/AlphaRatio.cs | 2 +- .../Definitions/Gazelle/GazelleParser.cs | 6 +- .../Definitions/Gazelle/GazelleSettings.cs | 16 ++- .../Indexers/Definitions/GreatPosterWall.cs | 4 +- .../Indexers/Definitions/Orpheus.cs | 73 +++++++++--- .../Indexers/Definitions/Redacted.cs | 111 ++++++++++++++---- .../Indexers/Definitions/SecretCinema.cs | 2 +- 8 files changed, 230 insertions(+), 44 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/041_freeleech_token_options.cs diff --git a/src/NzbDrone.Core/Datastore/Migration/041_freeleech_token_options.cs b/src/NzbDrone.Core/Datastore/Migration/041_freeleech_token_options.cs new file mode 100644 index 000000000..a1d0912fb --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/041_freeleech_token_options.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using System.Data; +using Dapper; +using FluentMigrator; +using Newtonsoft.Json.Linq; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(041)] + public class freeleech_token_options : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Execute.WithConnection(MigrateIndexersToTokenOptions); + } + + private void MigrateIndexersToTokenOptions(IDbConnection conn, IDbTransaction tran) + { + var updated = new List(); + + using (var cmd = conn.CreateCommand()) + { + cmd.Transaction = tran; + cmd.CommandText = "SELECT \"Id\", \"Settings\" FROM \"Indexers\" WHERE \"Implementation\" IN ('Orpheus', 'Redacted', 'GazelleBase')"; + + using (var reader = cmd.ExecuteReader()) + { + while (reader.Read()) + { + var id = reader.GetInt32(0); + var settings = Json.Deserialize(reader.GetString(1)); + + if (settings.ContainsKey("useFreeleechToken") && settings.Value("useFreeleechToken").Type == JTokenType.Boolean) + { + var optionValue = settings.Value("useFreeleechToken") switch + { + true => 1, // Preferred + _ => 0 // Never + }; + + settings.Remove("useFreeleechToken"); + settings.Add("useFreeleechToken", optionValue); + } + + updated.Add(new + { + Id = id, + Settings = settings.ToJson() + }); + } + } + } + + var updateSql = "UPDATE \"Indexers\" SET \"Settings\" = @Settings WHERE \"Id\" = @Id"; + conn.Execute(updateSql, updated, transaction: tran); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs b/src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs index d40710da9..ae712d2e7 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs @@ -143,7 +143,7 @@ public class AlphaRatioParser : GazelleParser .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (Settings.UseFreeleechToken && canUseToken) + if (Settings.UseFreeleechToken is (int)GazelleUseFreeleechTokens.Preferred or (int)GazelleUseFreeleechTokens.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs index f24954d18..4cb49a458 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs @@ -60,7 +60,7 @@ public class GazelleParser : IParseIndexerResponse foreach (var torrent in result.Torrents) { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (Settings.UseFreeleechToken && !torrent.CanUseToken) + if (Settings.UseFreeleechToken is (int)GazelleUseFreeleechTokens.Preferred or (int)GazelleUseFreeleechTokens.Required && !torrent.CanUseToken) { continue; } @@ -111,7 +111,7 @@ public class GazelleParser : IParseIndexerResponse else { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (Settings.UseFreeleechToken && !result.CanUseToken) + if (Settings.UseFreeleechToken is (int)GazelleUseFreeleechTokens.Preferred or (int)GazelleUseFreeleechTokens.Required && !result.CanUseToken) { continue; } @@ -165,7 +165,7 @@ public class GazelleParser : IParseIndexerResponse .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (Settings.UseFreeleechToken) + if (Settings.UseFreeleechToken is (int)GazelleUseFreeleechTokens.Preferred or (int)GazelleUseFreeleechTokens.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs index 64a394a77..e841b8772 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs @@ -16,11 +16,23 @@ public class GazelleSettings : UserPassTorrentBaseSettings public string AuthKey { get; set; } public string PassKey { get; set; } - [FieldDefinition(5, Type = FieldType.Checkbox, Label = "Use Freeleech Token", HelpText = "Use freeleech tokens when available")] - public bool UseFreeleechToken { get; set; } + [FieldDefinition(3, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(GazelleUseFreeleechTokens), HelpText = "When to use freeleech tokens")] + public int UseFreeleechToken { get; set; } public override NzbDroneValidationResult Validate() { return new NzbDroneValidationResult(Validator.Validate(this)); } } + +internal enum GazelleUseFreeleechTokens +{ + [FieldOption(Label = "Never", Hint = "Do not use tokens")] + Never = 0, + + [FieldOption(Label = "Preferred", Hint = "Use token if possible")] + Preferred = 1, + + [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")] + Required = 2, +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs b/src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs index 765de7aa7..5dd89fadb 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs @@ -169,7 +169,7 @@ public class GreatPosterWallParser : GazelleParser foreach (var torrent in result.Torrents) { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !torrent.CanUseToken) + if (_settings.UseFreeleechToken is (int)GazelleUseFreeleechTokens.Preferred or (int)GazelleUseFreeleechTokens.Required && !torrent.CanUseToken) { continue; } @@ -240,7 +240,7 @@ public class GreatPosterWallParser : GazelleParser .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (_settings.UseFreeleechToken && canUseToken) + if (_settings.UseFreeleechToken is (int)GazelleUseFreeleechTokens.Preferred or (int)GazelleUseFreeleechTokens.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs b/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs index 67f8ff55a..cccfc2a0f 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs @@ -10,6 +10,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Annotations; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers.Definitions.Gazelle; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.Indexers.Settings; @@ -95,40 +96,70 @@ namespace NzbDrone.Core.Indexers.Definitions .SetHeader("Authorization", $"token {Settings.Apikey}") .Build(); - var downloadBytes = Array.Empty(); + byte[] fileData; try { var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - downloadBytes = response.ResponseData; + fileData = response.ResponseData; - if (downloadBytes.Length >= 1 - && downloadBytes[0] != 'd' // simple test for torrent vs HTML content + if (Settings.UseFreeleechToken == (int)OrpheusUseFreeleechTokens.Preferred + && fileData.Length >= 1 + && fileData[0] != 'd' // simple test for torrent vs HTML content && link.Query.Contains("usetoken=1")) { - var html = Encoding.GetString(downloadBytes); + var html = Encoding.GetString(fileData); + if (html.Contains("You do not have any freeleech tokens left.") || html.Contains("You do not have enough freeleech tokens") || html.Contains("This torrent is too large.") || html.Contains("You cannot use tokens here")) { - // download again without usetoken + // Try to download again without usetoken request.Url = new HttpUri(link.ToString().Replace("&usetoken=1", "")); response = await _httpClient.ExecuteProxiedAsync(request, Definition); - downloadBytes = response.ResponseData; + fileData = response.ResponseData; } } + + _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri); + } + catch (HttpException ex) + { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri); + throw new ReleaseUnavailableException("Download failed", ex); + } + + if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) + { + _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); + } + else + { + _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri); + } + + throw new ReleaseDownloadException("Download failed", ex); + } + catch (WebException ex) + { + _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri); + + throw new ReleaseDownloadException("Download failed", ex); } catch (Exception) { _indexerStatusService.RecordFailure(Definition.Id); _logger.Error("Download failed"); + throw; } - ValidateDownloadData(downloadBytes); + ValidateDownloadData(fileData); - return downloadBytes; + return fileData; } } @@ -277,7 +308,7 @@ namespace NzbDrone.Core.Indexers.Definitions foreach (var torrent in result.Torrents) { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !torrent.CanUseToken) + if (_settings.UseFreeleechToken is (int)OrpheusUseFreeleechTokens.Preferred or (int)OrpheusUseFreeleechTokens.Required && !torrent.CanUseToken) { continue; } @@ -327,7 +358,7 @@ namespace NzbDrone.Core.Indexers.Definitions else { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !result.CanUseToken) + if (_settings.UseFreeleechToken is (int)OrpheusUseFreeleechTokens.Preferred or (int)OrpheusUseFreeleechTokens.Required && !result.CanUseToken) { continue; } @@ -413,7 +444,7 @@ namespace NzbDrone.Core.Indexers.Definitions .AddQueryParam("id", torrentId); // Orpheus fails to download if usetoken=0 so we need to only add if we will use one - if (_settings.UseFreeleechToken && canUseToken) + if (_settings.UseFreeleechToken is (int)OrpheusUseFreeleechTokens.Preferred or (int)OrpheusUseFreeleechTokens.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } @@ -447,18 +478,30 @@ namespace NzbDrone.Core.Indexers.Definitions public OrpheusSettings() { Apikey = ""; - UseFreeleechToken = false; + UseFreeleechToken = (int)OrpheusUseFreeleechTokens.Never; } [FieldDefinition(2, Label = "ApiKey", HelpText = "IndexerOrpheusSettingsApiKeyHelpText", Privacy = PrivacyLevel.ApiKey)] public string Apikey { get; set; } - [FieldDefinition(3, Label = "Use Freeleech Tokens", HelpText = "Use freeleech tokens when available", Type = FieldType.Checkbox)] - public bool UseFreeleechToken { get; set; } + [FieldDefinition(3, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(OrpheusUseFreeleechTokens), HelpText = "When to use freeleech tokens")] + public int UseFreeleechToken { get; set; } public override NzbDroneValidationResult Validate() { return new NzbDroneValidationResult(Validator.Validate(this)); } } + + internal enum OrpheusUseFreeleechTokens + { + [FieldOption(Label = "Never", Hint = "Do not use tokens")] + Never = 0, + + [FieldOption(Label = "Preferred", Hint = "Use token if possible")] + Preferred = 1, + + [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")] + Required = 2, + } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs b/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs index 2af30fb10..7d841ec87 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs @@ -10,6 +10,7 @@ using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Annotations; using NzbDrone.Core.Configuration; +using NzbDrone.Core.Exceptions; using NzbDrone.Core.Indexers.Definitions.Gazelle; using NzbDrone.Core.Indexers.Exceptions; using NzbDrone.Core.Indexers.Settings; @@ -50,20 +51,6 @@ namespace NzbDrone.Core.Indexers.Definitions return new RedactedParser(Settings, Capabilities.Categories); } - protected override Task GetDownloadRequest(Uri link) - { - var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri) - { - AllowAutoRedirect = FollowRedirect - }; - - var request = requestBuilder - .SetHeader("Authorization", Settings.Apikey) - .Build(); - - return Task.FromResult(request); - } - protected override IList CleanupReleases(IEnumerable releases, SearchCriteriaBase searchCriteria) { var cleanReleases = base.CleanupReleases(releases, searchCriteria); @@ -102,6 +89,78 @@ namespace NzbDrone.Core.Indexers.Definitions return caps; } + + public override async Task Download(Uri link) + { + var request = new HttpRequestBuilder(link.AbsoluteUri) + .SetHeader("Authorization", Settings.Apikey) + .Build(); + + byte[] fileData; + + try + { + var response = await _httpClient.ExecuteProxiedAsync(request, Definition); + fileData = response.ResponseData; + + if (Settings.UseFreeleechToken == (int)RedactedUseFreeleechTokens.Preferred + && fileData.Length >= 1 + && fileData[0] != 'd' // simple test for torrent vs HTML content + && link.Query.Contains("usetoken=1")) + { + var html = Encoding.GetString(fileData); + + if (html.Contains("You do not have any freeleech tokens left.") + || html.Contains("You do not have enough freeleech tokens") + || html.Contains("This torrent is too large.") + || html.Contains("You cannot use tokens here")) + { + // Try to download again without usetoken + request.Url = new HttpUri(link.ToString().Replace("&usetoken=1", "")); + + response = await _httpClient.ExecuteProxiedAsync(request, Definition); + fileData = response.ResponseData; + } + } + + _logger.Debug("Downloaded for release finished ({0} bytes from {1})", fileData.Length, link.AbsoluteUri); + } + catch (HttpException ex) + { + if (ex.Response.StatusCode == HttpStatusCode.NotFound) + { + _logger.Error(ex, "Downloading file for release failed since it no longer exists ({0})", link.AbsoluteUri); + throw new ReleaseUnavailableException("Download failed", ex); + } + + if (ex.Response.StatusCode == HttpStatusCode.TooManyRequests) + { + _logger.Error("API Grab Limit reached for {0}", link.AbsoluteUri); + } + else + { + _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri); + } + + throw new ReleaseDownloadException("Download failed", ex); + } + catch (WebException ex) + { + _logger.Error(ex, "Downloading for release failed ({0})", link.AbsoluteUri); + + throw new ReleaseDownloadException("Download failed", ex); + } + catch (Exception) + { + _indexerStatusService.RecordFailure(Definition.Id); + _logger.Error("Download failed"); + throw; + } + + ValidateDownloadData(fileData); + + return fileData; + } } public class RedactedRequestGenerator : IIndexerRequestGenerator @@ -248,7 +307,7 @@ namespace NzbDrone.Core.Indexers.Definitions foreach (var torrent in result.Torrents) { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !torrent.CanUseToken) + if (_settings.UseFreeleechToken is (int)RedactedUseFreeleechTokens.Preferred or (int)RedactedUseFreeleechTokens.Required && !torrent.CanUseToken) { continue; } @@ -304,7 +363,7 @@ namespace NzbDrone.Core.Indexers.Definitions else { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !result.CanUseToken) + if (_settings.UseFreeleechToken is (int)RedactedUseFreeleechTokens.Preferred or (int)RedactedUseFreeleechTokens.Required && !result.CanUseToken) { continue; } @@ -395,7 +454,7 @@ namespace NzbDrone.Core.Indexers.Definitions .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (_settings.UseFreeleechToken && canUseToken) + if (_settings.UseFreeleechToken is (int)RedactedUseFreeleechTokens.Preferred or (int)RedactedUseFreeleechTokens.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } @@ -429,14 +488,14 @@ namespace NzbDrone.Core.Indexers.Definitions public RedactedSettings() { Apikey = ""; - UseFreeleechToken = false; + UseFreeleechToken = (int)RedactedUseFreeleechTokens.Never; } [FieldDefinition(2, Label = "ApiKey", Privacy = PrivacyLevel.ApiKey, HelpText = "IndexerRedactedSettingsApiKeyHelpText")] public string Apikey { get; set; } - [FieldDefinition(3, Label = "Use Freeleech Tokens", Type = FieldType.Checkbox, HelpText = "Use freeleech tokens when available")] - public bool UseFreeleechToken { get; set; } + [FieldDefinition(3, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(RedactedUseFreeleechTokens), HelpText = "When to use freeleech tokens")] + public int UseFreeleechToken { get; set; } [FieldDefinition(4, Label = "Freeload Only", Type = FieldType.Checkbox, Advanced = true, HelpTextWarning = "Search freeload torrents only. End date: 31 January 2024, 23:59 UTC.")] public bool FreeloadOnly { get; set; } @@ -446,4 +505,16 @@ namespace NzbDrone.Core.Indexers.Definitions return new NzbDroneValidationResult(Validator.Validate(this)); } } + + internal enum RedactedUseFreeleechTokens + { + [FieldOption(Label = "Never", Hint = "Do not use tokens")] + Never = 0, + + [FieldOption(Label = "Preferred", Hint = "Use token if possible")] + Preferred = 1, + + [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")] + Required = 2, + } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs b/src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs index 1950974c0..50956d7f8 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs @@ -226,7 +226,7 @@ public class SecretCinemaParser : IParseIndexerResponse .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (_settings.UseFreeleechToken) + if (_settings.UseFreeleechToken is (int)GazelleUseFreeleechTokens.Preferred or (int)GazelleUseFreeleechTokens.Required) { url = url.AddQueryParam("useToken", "1"); }