From 710ab7ae090b2bb0a86dde08aa830595b2ea8bcc Mon Sep 17 00:00:00 2001 From: Bogdan Date: Thu, 21 Sep 2023 15:46:19 +0300 Subject: [PATCH] New: (Gazelle/OPS/RED) Prevent downloads without FL tokens --- .../041_gazelle_freeleech_token_options.cs | 60 +++++++++++++ .../Indexers/Definitions/AlphaRatio.cs | 2 +- .../Definitions/Gazelle/GazelleBase.cs | 18 ++-- .../Definitions/Gazelle/GazelleParser.cs | 6 +- .../Definitions/Gazelle/GazelleSettings.cs | 16 +++- .../Indexers/Definitions/GreatPosterWall.cs | 4 +- .../Indexers/Definitions/Orpheus.cs | 85 ++++++++++--------- .../Indexers/Definitions/Redacted.cs | 50 +++++++++-- .../Indexers/Definitions/SecretCinema.cs | 2 +- 9 files changed, 181 insertions(+), 62 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/041_gazelle_freeleech_token_options.cs diff --git a/src/NzbDrone.Core/Datastore/Migration/041_gazelle_freeleech_token_options.cs b/src/NzbDrone.Core/Datastore/Migration/041_gazelle_freeleech_token_options.cs new file mode 100644 index 000000000..56a11c732 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/041_gazelle_freeleech_token_options.cs @@ -0,0 +1,60 @@ +using System.Collections.Generic; +using System.Data; +using Dapper; +using FluentMigrator; +using Newtonsoft.Json.Linq; +using NzbDrone.Common.Serializer; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(041)] + public class gazelle_freeleech_token_options : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Execute.WithConnection(MigrateIndexersToTokenOptions); + } + + private void MigrateIndexersToTokenOptions(IDbConnection conn, IDbTransaction tran) + { + var updated = new List(); + + using (var cmd = conn.CreateCommand()) + { + cmd.Transaction = tran; + cmd.CommandText = "SELECT \"Id\", \"Settings\" FROM \"Indexers\" WHERE \"Implementation\" IN ('Orpheus', 'Redacted', 'AlphaRatio', 'BrokenStones', 'CGPeers', 'DICMusic', 'GreatPosterWall', 'SecretCinema')"; + + using (var reader = cmd.ExecuteReader()) + { + while (reader.Read()) + { + var id = reader.GetInt32(0); + var settings = Json.Deserialize(reader.GetString(1)); + + if (settings.ContainsKey("useFreeleechToken") && settings.Value("useFreeleechToken").Type == JTokenType.Boolean) + { + var optionValue = settings.Value("useFreeleechToken") switch + { + true => 2, // Required + _ => 0 // Never + }; + + settings.Remove("useFreeleechToken"); + settings.Add("useFreeleechToken", optionValue); + } + + updated.Add(new + { + Id = id, + Settings = settings.ToJson() + }); + } + } + } + + var updateSql = "UPDATE \"Indexers\" SET \"Settings\" = @Settings WHERE \"Id\" = @Id"; + conn.Execute(updateSql, updated, transaction: tran); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs b/src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs index d40710da9..c11d57b57 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/AlphaRatio.cs @@ -143,7 +143,7 @@ public class AlphaRatioParser : GazelleParser .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (Settings.UseFreeleechToken && canUseToken) + if (Settings.UseFreeleechToken is (int)GazelleFreeleechTokenAction.Preferred or (int)GazelleFreeleechTokenAction.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleBase.cs b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleBase.cs index 924a67cbe..584efd1e5 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleBase.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; using NLog; +using NzbDrone.Common.Extensions; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; using NzbDrone.Core.Messaging.Events; @@ -85,22 +86,23 @@ public abstract class GazelleBase : TorrentIndexerBase public override async Task Download(Uri link) { var downloadResponse = await base.Download(link); - var response = downloadResponse.Data; - if (response.Length >= 1 - && response[0] != 'd' // simple test for torrent vs HTML content + var fileData = downloadResponse.Data; + + if (Settings.UseFreeleechToken == (int)GazelleFreeleechTokenAction.Preferred + && fileData.Length >= 1 + && fileData[0] != 'd' // simple test for torrent vs HTML content && link.Query.Contains("usetoken=1")) { - var html = Encoding.GetString(response); + var html = Encoding.GetString(fileData); + if (html.Contains("You do not have any freeleech tokens left.") || html.Contains("You do not have enough freeleech tokens") || html.Contains("This torrent is too large.") || html.Contains("You cannot use tokens here")) { - // download again without usetoken=1 - var requestLinkNew = link.ToString().Replace("&usetoken=1", ""); - - downloadResponse = await base.Download(new Uri(requestLinkNew)); + // Try to download again without usetoken + downloadResponse = await base.Download(link.RemoveQueryParam("usetoken")); } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs index 10bb085f3..b5253ed6f 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleParser.cs @@ -62,7 +62,7 @@ public class GazelleParser : IParseIndexerResponse var isFreeLeech = torrent.IsFreeLeech || torrent.IsNeutralLeech || torrent.IsPersonalFreeLeech; // skip releases that cannot be used with freeleech tokens when the option is enabled - if (Settings.UseFreeleechToken && !torrent.CanUseToken && !isFreeLeech) + if (Settings.UseFreeleechToken == (int)GazelleFreeleechTokenAction.Required && !torrent.CanUseToken && !isFreeLeech) { continue; } @@ -115,7 +115,7 @@ public class GazelleParser : IParseIndexerResponse var isFreeLeech = result.IsFreeLeech || result.IsNeutralLeech || result.IsPersonalFreeLeech; // skip releases that cannot be used with freeleech tokens when the option is enabled - if (Settings.UseFreeleechToken && !result.CanUseToken && !isFreeLeech) + if (Settings.UseFreeleechToken == (int)GazelleFreeleechTokenAction.Required && !result.CanUseToken && !isFreeLeech) { continue; } @@ -169,7 +169,7 @@ public class GazelleParser : IParseIndexerResponse .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (Settings.UseFreeleechToken) + if (Settings.UseFreeleechToken is (int)GazelleFreeleechTokenAction.Preferred or (int)GazelleFreeleechTokenAction.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs index 64a394a77..628436549 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Gazelle/GazelleSettings.cs @@ -16,11 +16,23 @@ public class GazelleSettings : UserPassTorrentBaseSettings public string AuthKey { get; set; } public string PassKey { get; set; } - [FieldDefinition(5, Type = FieldType.Checkbox, Label = "Use Freeleech Token", HelpText = "Use freeleech tokens when available")] - public bool UseFreeleechToken { get; set; } + [FieldDefinition(5, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(GazelleFreeleechTokenAction), HelpText = "When to use freeleech tokens")] + public int UseFreeleechToken { get; set; } public override NzbDroneValidationResult Validate() { return new NzbDroneValidationResult(Validator.Validate(this)); } } + +public enum GazelleFreeleechTokenAction +{ + [FieldOption(Label = "Never", Hint = "Do not use tokens")] + Never = 0, + + [FieldOption(Label = "Preferred", Hint = "Use token if possible")] + Preferred = 1, + + [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")] + Required = 2, +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs b/src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs index b7dcfd4ff..4b8302e5a 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/GreatPosterWall.cs @@ -171,7 +171,7 @@ public class GreatPosterWallParser : GazelleParser var isFreeLeech = torrent.IsFreeleech || torrent.IsNeutralLeech || torrent.IsPersonalFreeleech; // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !torrent.CanUseToken && !isFreeLeech) + if (_settings.UseFreeleechToken == (int)GazelleFreeleechTokenAction.Required && !torrent.CanUseToken && !isFreeLeech) { continue; } @@ -242,7 +242,7 @@ public class GreatPosterWallParser : GazelleParser .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (_settings.UseFreeleechToken && canUseToken) + if (_settings.UseFreeleechToken is (int)GazelleFreeleechTokenAction.Preferred or (int)GazelleFreeleechTokenAction.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs b/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs index 4596b351d..14ee4a518 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Orpheus.cs @@ -50,6 +50,20 @@ namespace NzbDrone.Core.Indexers.Definitions return new OrpheusParser(Settings, Capabilities.Categories); } + protected override Task GetDownloadRequest(Uri link) + { + var requestBuilder = new HttpRequestBuilder(link.AbsoluteUri) + { + AllowAutoRedirect = FollowRedirect + }; + + var request = requestBuilder + .SetHeader("Authorization", $"token {Settings.Apikey}") + .Build(); + + return Task.FromResult(request); + } + protected override IList CleanupReleases(IEnumerable releases, SearchCriteriaBase searchCriteria) { var cleanReleases = base.CleanupReleases(releases, searchCriteria); @@ -91,47 +105,28 @@ namespace NzbDrone.Core.Indexers.Definitions public override async Task Download(Uri link) { - var request = new HttpRequestBuilder(link.AbsoluteUri) - .SetHeader("Authorization", $"token {Settings.Apikey}") - .Build(); + var downloadResponse = await base.Download(link); - byte[] downloadBytes; - long elapsedTime; + var fileData = downloadResponse.Data; - try + if (Settings.UseFreeleechToken == (int)OrpheusFreeleechTokenAction.Preferred + && fileData.Length >= 1 + && fileData[0] != 'd' // simple test for torrent vs HTML content + && link.Query.Contains("usetoken=1")) { - var response = await _httpClient.ExecuteProxiedAsync(request, Definition); - downloadBytes = response.ResponseData; - elapsedTime = response.ElapsedTime; + var html = Encoding.GetString(fileData); - if (downloadBytes.Length >= 1 - && downloadBytes[0] != 'd' // simple test for torrent vs HTML content - && link.Query.Contains("usetoken=1")) + if (html.Contains("You do not have any freeleech tokens left.") + || html.Contains("You do not have enough freeleech tokens") + || html.Contains("This torrent is too large.") + || html.Contains("You cannot use tokens here")) { - var html = Encoding.GetString(downloadBytes); - if (html.Contains("You do not have any freeleech tokens left.") - || html.Contains("You do not have enough freeleech tokens") - || html.Contains("This torrent is too large.") - || html.Contains("You cannot use tokens here")) - { - // download again without usetoken - request.Url = new HttpUri(link.ToString().Replace("&usetoken=1", "")); - - response = await _httpClient.ExecuteProxiedAsync(request, Definition); - downloadBytes = response.ResponseData; - } + // Try to download again without usetoken + downloadResponse = await base.Download(link.RemoveQueryParam("usetoken")); } } - catch (Exception) - { - _indexerStatusService.RecordFailure(Definition.Id); - _logger.Error("Download failed"); - throw; - } - - ValidateDownloadData(downloadBytes); - return new IndexerDownloadResponse(downloadBytes, elapsedTime); + return downloadResponse; } } @@ -280,7 +275,7 @@ namespace NzbDrone.Core.Indexers.Definitions foreach (var torrent in result.Torrents) { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !torrent.CanUseToken) + if (_settings.UseFreeleechToken == (int)OrpheusFreeleechTokenAction.Required && !torrent.CanUseToken) { continue; } @@ -331,7 +326,7 @@ namespace NzbDrone.Core.Indexers.Definitions else { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !result.CanUseToken) + if (_settings.UseFreeleechToken == (int)OrpheusFreeleechTokenAction.Required && !result.CanUseToken) { continue; } @@ -418,7 +413,7 @@ namespace NzbDrone.Core.Indexers.Definitions .AddQueryParam("id", torrentId); // Orpheus fails to download if usetoken=0 so we need to only add if we will use one - if (_settings.UseFreeleechToken && canUseToken) + if (_settings.UseFreeleechToken is (int)OrpheusFreeleechTokenAction.Preferred or (int)OrpheusFreeleechTokenAction.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } @@ -452,18 +447,30 @@ namespace NzbDrone.Core.Indexers.Definitions public OrpheusSettings() { Apikey = ""; - UseFreeleechToken = false; + UseFreeleechToken = (int)OrpheusFreeleechTokenAction.Never; } [FieldDefinition(2, Label = "ApiKey", HelpText = "IndexerOrpheusSettingsApiKeyHelpText", Privacy = PrivacyLevel.ApiKey)] public string Apikey { get; set; } - [FieldDefinition(3, Label = "Use Freeleech Tokens", HelpText = "Use freeleech tokens when available", Type = FieldType.Checkbox)] - public bool UseFreeleechToken { get; set; } + [FieldDefinition(3, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(OrpheusFreeleechTokenAction), HelpText = "When to use freeleech tokens")] + public int UseFreeleechToken { get; set; } public override NzbDroneValidationResult Validate() { return new NzbDroneValidationResult(Validator.Validate(this)); } } + + public enum OrpheusFreeleechTokenAction + { + [FieldOption(Label = "Never", Hint = "Do not use tokens")] + Never = 0, + + [FieldOption(Label = "Preferred", Hint = "Use token if possible")] + Preferred = 1, + + [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")] + Required = 2, + } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs b/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs index eef6adf3b..a624f9214 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Redacted.cs @@ -102,6 +102,32 @@ namespace NzbDrone.Core.Indexers.Definitions return caps; } + + public override async Task Download(Uri link) + { + var downloadResponse = await base.Download(link); + + var fileData = downloadResponse.Data; + + if (Settings.UseFreeleechToken == (int)RedactedFreeleechTokenAction.Preferred + && fileData.Length >= 1 + && fileData[0] != 'd' // simple test for torrent vs HTML content + && link.Query.Contains("usetoken=1")) + { + var html = Encoding.GetString(fileData); + + if (html.Contains("You do not have any freeleech tokens left.") + || html.Contains("You do not have enough freeleech tokens") + || html.Contains("This torrent is too large.") + || html.Contains("You cannot use tokens here")) + { + // Try to download again without usetoken + downloadResponse = await base.Download(link.RemoveQueryParam("usetoken")); + } + } + + return downloadResponse; + } } public class RedactedRequestGenerator : IIndexerRequestGenerator @@ -248,7 +274,7 @@ namespace NzbDrone.Core.Indexers.Definitions foreach (var torrent in result.Torrents) { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !torrent.CanUseToken) + if (_settings.UseFreeleechToken == (int)RedactedFreeleechTokenAction.Required && !torrent.CanUseToken) { continue; } @@ -305,7 +331,7 @@ namespace NzbDrone.Core.Indexers.Definitions else { // skip releases that cannot be used with freeleech tokens when the option is enabled - if (_settings.UseFreeleechToken && !result.CanUseToken) + if (_settings.UseFreeleechToken == (int)RedactedFreeleechTokenAction.Required && !result.CanUseToken) { continue; } @@ -397,7 +423,7 @@ namespace NzbDrone.Core.Indexers.Definitions .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (_settings.UseFreeleechToken && canUseToken) + if (_settings.UseFreeleechToken is (int)RedactedFreeleechTokenAction.Preferred or (int)RedactedFreeleechTokenAction.Required && canUseToken) { url = url.AddQueryParam("usetoken", "1"); } @@ -431,14 +457,14 @@ namespace NzbDrone.Core.Indexers.Definitions public RedactedSettings() { Apikey = ""; - UseFreeleechToken = false; + UseFreeleechToken = (int)RedactedFreeleechTokenAction.Never; } [FieldDefinition(2, Label = "ApiKey", Privacy = PrivacyLevel.ApiKey, HelpText = "IndexerRedactedSettingsApiKeyHelpText")] public string Apikey { get; set; } - [FieldDefinition(3, Label = "Use Freeleech Tokens", Type = FieldType.Checkbox, HelpText = "Use freeleech tokens when available")] - public bool UseFreeleechToken { get; set; } + [FieldDefinition(3, Type = FieldType.Select, Label = "Use Freeleech Tokens", SelectOptions = typeof(RedactedFreeleechTokenAction), HelpText = "When to use freeleech tokens")] + public int UseFreeleechToken { get; set; } [FieldDefinition(4, Label = "Freeload Only", Type = FieldType.Checkbox, Advanced = true, HelpTextWarning = "Search freeload torrents only. End date: 31 January 2024, 23:59 UTC.")] public bool FreeloadOnly { get; set; } @@ -448,4 +474,16 @@ namespace NzbDrone.Core.Indexers.Definitions return new NzbDroneValidationResult(Validator.Validate(this)); } } + + public enum RedactedFreeleechTokenAction + { + [FieldOption(Label = "Never", Hint = "Do not use tokens")] + Never = 0, + + [FieldOption(Label = "Preferred", Hint = "Use token if possible")] + Preferred = 1, + + [FieldOption(Label = "Required", Hint = "Abort download if unable to use token")] + Required = 2, + } } diff --git a/src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs b/src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs index 1950974c0..709868b14 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/SecretCinema.cs @@ -226,7 +226,7 @@ public class SecretCinemaParser : IParseIndexerResponse .AddQueryParam("action", "download") .AddQueryParam("id", torrentId); - if (_settings.UseFreeleechToken) + if (_settings.UseFreeleechToken is (int)GazelleFreeleechTokenAction.Preferred or (int)GazelleFreeleechTokenAction.Required) { url = url.AddQueryParam("useToken", "1"); }