From df8ef83e4097e4f4ceba1db05886ac5e69d19830 Mon Sep 17 00:00:00 2001 From: Qstick Date: Wed, 24 Feb 2021 23:28:00 -0500 Subject: [PATCH] New: Support Indexer Grab Redirects --- .gitignore | 1 + .../Indexer/Index/Table/IndexerIndexRow.js | 3 ++ .../Indexer/Index/Table/IndexerStatusCell.js | 9 ++++-- .../Indexers/EditIndexerModalContent.js | 15 ++++++++++ .../Datastore/Migration/003_indexer_props.cs | 30 +++++++++++++++++++ src/NzbDrone.Core/Datastore/TableMapping.cs | 1 + src/NzbDrone.Core/History/HistoryService.cs | 2 ++ .../Definitions/Cardigann/Cardigann.cs | 1 + .../Definitions/Cardigann/CardigannBase.cs | 4 +-- .../Indexers/Definitions/Newznab/Newznab.cs | 2 ++ .../TorrentPotato/TorrentPotato.cs | 3 +- .../Indexers/Definitions/Torznab/Torznab.cs | 1 + src/NzbDrone.Core/Indexers/DownloadService.cs | 16 ++++++---- .../Indexers/Events/IndexerDownloadEvent.cs | 6 +++- src/NzbDrone.Core/Indexers/HttpIndexerBase.cs | 1 + src/NzbDrone.Core/Indexers/IIndexer.cs | 1 + src/NzbDrone.Core/Indexers/IndexerBase.cs | 2 ++ .../Indexers/IndexerDefinition.cs | 2 ++ src/NzbDrone.Core/Indexers/IndexerFactory.cs | 1 + src/Prowlarr.Api.V1/Indexers/IndexerModule.cs | 18 ++++++++--- .../Indexers/IndexerResource.cs | 5 ++++ 21 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 src/NzbDrone.Core/Datastore/Migration/003_indexer_props.cs diff --git a/.gitignore b/.gitignore index 11a1b4059..45e6b1a39 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,7 @@ src/**/[Oo]bj/ *.sln.docstates .vs/ .vscode/ +.idea/ # Build results *_i.c diff --git a/frontend/src/Indexer/Index/Table/IndexerIndexRow.js b/frontend/src/Indexer/Index/Table/IndexerIndexRow.js index 40c54d3b2..afc3abf49 100644 --- a/frontend/src/Indexer/Index/Table/IndexerIndexRow.js +++ b/frontend/src/Indexer/Index/Table/IndexerIndexRow.js @@ -63,6 +63,7 @@ class IndexerIndexRow extends Component { name, baseUrl, enable, + redirect, tags, protocol, privacy, @@ -114,6 +115,7 @@ class IndexerIndexRow extends Component { key={column.name} className={styles[column.name]} enabled={enable} + redirect={redirect} status={status} longDateFormat={longDateFormat} timeFormat={timeFormat} @@ -258,6 +260,7 @@ IndexerIndexRow.propTypes = { priority: PropTypes.number.isRequired, name: PropTypes.string.isRequired, enable: PropTypes.bool.isRequired, + redirect: PropTypes.bool.isRequired, status: PropTypes.object, capabilities: PropTypes.object.isRequired, added: PropTypes.string.isRequired, diff --git a/frontend/src/Indexer/Index/Table/IndexerStatusCell.js b/frontend/src/Indexer/Index/Table/IndexerStatusCell.js index 2c3191aa2..89fa9b1fe 100644 --- a/frontend/src/Indexer/Index/Table/IndexerStatusCell.js +++ b/frontend/src/Indexer/Index/Table/IndexerStatusCell.js @@ -10,6 +10,7 @@ function IndexerStatusCell(props) { const { className, enabled, + redirect, status, longDateFormat, timeFormat, @@ -17,6 +18,9 @@ function IndexerStatusCell(props) { ...otherProps } = props; + const enableKind = redirect ? kinds.WARNING : kinds.SUCCESS; + const enableTitle = redirect ? 'Indexer is Enabled, Redirect is Enabled' : 'Indexer is Enabled'; + return ( } { @@ -46,6 +50,7 @@ function IndexerStatusCell(props) { IndexerStatusCell.propTypes = { className: PropTypes.string.isRequired, enabled: PropTypes.bool.isRequired, + redirect: PropTypes.bool.isRequired, status: PropTypes.object, longDateFormat: PropTypes.string.isRequired, timeFormat: PropTypes.string.isRequired, diff --git a/frontend/src/Settings/Indexers/Indexers/EditIndexerModalContent.js b/frontend/src/Settings/Indexers/Indexers/EditIndexerModalContent.js index 5b78a5be9..0732e0470 100644 --- a/frontend/src/Settings/Indexers/Indexers/EditIndexerModalContent.js +++ b/frontend/src/Settings/Indexers/Indexers/EditIndexerModalContent.js @@ -39,7 +39,9 @@ function EditIndexerModalContent(props) { implementationName, name, enable, + redirect, supportsRss, + supportsRedirect, fields, priority } = item; @@ -90,6 +92,19 @@ function EditIndexerModalContent(props) { /> + + {translate('Redirect')} + + + + { fields.map((field) => { return ( diff --git a/src/NzbDrone.Core/Datastore/Migration/003_indexer_props.cs b/src/NzbDrone.Core/Datastore/Migration/003_indexer_props.cs new file mode 100644 index 000000000..8f9cc9554 --- /dev/null +++ b/src/NzbDrone.Core/Datastore/Migration/003_indexer_props.cs @@ -0,0 +1,30 @@ +using FluentMigrator; +using NzbDrone.Core.Datastore.Migration.Framework; + +namespace NzbDrone.Core.Datastore.Migration +{ + [Migration(3)] + public class IndexerProps : NzbDroneMigrationBase + { + protected override void MainDbUpgrade() + { + Alter.Table("Indexers") + .AddColumn("Redirect").AsBoolean().NotNullable().WithDefaultValue(false); + + Create.TableForModel("DownloadClients") + .WithColumn("Enable").AsBoolean().NotNullable() + .WithColumn("Name").AsString().NotNullable() + .WithColumn("Implementation").AsString().NotNullable() + .WithColumn("Settings").AsString().NotNullable() + .WithColumn("ConfigContract").AsString().NotNullable() + .WithColumn("Priority").AsInt32().WithDefaultValue(1); + + Create.TableForModel("DownloadClientStatus") + .WithColumn("ProviderId").AsInt32().NotNullable().Unique() + .WithColumn("InitialFailure").AsDateTime().Nullable() + .WithColumn("MostRecentFailure").AsDateTime().Nullable() + .WithColumn("EscalationLevel").AsInt32().NotNullable() + .WithColumn("DisabledTill").AsDateTime().Nullable(); + } + } +} diff --git a/src/NzbDrone.Core/Datastore/TableMapping.cs b/src/NzbDrone.Core/Datastore/TableMapping.cs index 1bc3dd8a2..0a9eed911 100644 --- a/src/NzbDrone.Core/Datastore/TableMapping.cs +++ b/src/NzbDrone.Core/Datastore/TableMapping.cs @@ -45,6 +45,7 @@ namespace NzbDrone.Core.Datastore .Ignore(i => i.Privacy) .Ignore(i => i.SupportsRss) .Ignore(i => i.SupportsSearch) + .Ignore(i => i.SupportsRedirect) .Ignore(i => i.Capabilities) .Ignore(d => d.Tags); diff --git a/src/NzbDrone.Core/History/HistoryService.cs b/src/NzbDrone.Core/History/HistoryService.cs index 532436b0b..dcd4c807d 100644 --- a/src/NzbDrone.Core/History/HistoryService.cs +++ b/src/NzbDrone.Core/History/HistoryService.cs @@ -112,6 +112,8 @@ namespace NzbDrone.Core.History history.Data.Add("Successful", message.Successful.ToString()); history.Data.Add("Source", message.Source ?? string.Empty); + history.Data.Add("GrabMethod", message.Redirect ? "Proxy" : "Redirect"); + history.Data.Add("Title", message.Title); _historyRepository.Insert(history); } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs index 8cec3017b..6c1c03b66 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/Cardigann.cs @@ -78,6 +78,7 @@ namespace NzbDrone.Core.Indexers.Cardigann Privacy = definition.Type == "private" ? IndexerPrivacy.Private : IndexerPrivacy.Public, SupportsRss = SupportsRss, SupportsSearch = SupportsSearch, + SupportsRedirect = SupportsRedirect, Capabilities = new IndexerCapabilities(), ExtraFields = settings }; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs index 2d51907df..85a71db98 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Cardigann/CardigannBase.cs @@ -223,7 +223,7 @@ namespace NzbDrone.Core.Indexers.Cardigann if (setting.Type != "password") { - _logger.Debug($"{name} got value {value.ToJson()}"); + _logger.Trace($"{name} got value {value.ToJson()}"); } if (setting.Type == "text" || setting.Type == "password") @@ -236,7 +236,7 @@ namespace NzbDrone.Core.Indexers.Cardigann } else if (setting.Type == "select") { - _logger.Debug($"Setting options: {setting.Options.ToJson()}"); + _logger.Trace($"Setting options: {setting.Options.ToJson()}"); var sorted = setting.Options.OrderBy(x => x.Key).ToList(); var selected = sorted[(int)(long)value]; diff --git a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs index 38dd6944f..cecbd10f6 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Newznab/Newznab.cs @@ -18,6 +18,7 @@ namespace NzbDrone.Core.Indexers.Newznab public override string Name => "Newznab"; public override string BaseUrl => GetBaseUrlFromSettings(); public override bool FollowRedirect => true; + public override bool SupportsRedirect => true; public override DownloadProtocol Protocol => DownloadProtocol.Usenet; public override IndexerPrivacy Privacy => IndexerPrivacy.Private; @@ -113,6 +114,7 @@ namespace NzbDrone.Core.Indexers.Newznab Privacy = IndexerPrivacy.Private, SupportsRss = SupportsRss, SupportsSearch = SupportsSearch, + SupportsRedirect = SupportsRedirect, Capabilities = Capabilities }; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs b/src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs index c89892867..d819752db 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/TorrentPotato/TorrentPotato.cs @@ -29,7 +29,8 @@ namespace NzbDrone.Core.Indexers.TorrentPotato Settings = settings, Protocol = DownloadProtocol.Torrent, SupportsRss = SupportsRss, - SupportsSearch = SupportsSearch + SupportsSearch = SupportsSearch, + SupportsRedirect = SupportsRedirect }; } diff --git a/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs b/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs index 6aab1a176..343f03a0f 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Torznab/Torznab.cs @@ -64,6 +64,7 @@ namespace NzbDrone.Core.Indexers.Torznab Protocol = DownloadProtocol.Usenet, SupportsRss = SupportsRss, SupportsSearch = SupportsSearch, + SupportsRedirect = SupportsRedirect, Capabilities = new IndexerCapabilities() }; } diff --git a/src/NzbDrone.Core/Indexers/DownloadService.cs b/src/NzbDrone.Core/Indexers/DownloadService.cs index 781578ded..52b4ff4d5 100644 --- a/src/NzbDrone.Core/Indexers/DownloadService.cs +++ b/src/NzbDrone.Core/Indexers/DownloadService.cs @@ -12,7 +12,8 @@ namespace NzbDrone.Core.Indexers { public interface IDownloadService { - byte[] DownloadReport(string link, int indexerId, string source); + byte[] DownloadReport(string link, int indexerId, string source, string title); + void RecordRedirect(string link, int indexerId, string source, string title); } public class DownloadService : IDownloadService @@ -36,7 +37,7 @@ namespace NzbDrone.Core.Indexers _logger = logger; } - public byte[] DownloadReport(string link, int indexerId, string source) + public byte[] DownloadReport(string link, int indexerId, string source, string title) { var url = new HttpUri(link); @@ -59,7 +60,7 @@ namespace NzbDrone.Core.Indexers catch (ReleaseUnavailableException) { _logger.Trace("Release {0} no longer available on indexer.", link); - _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, title)); throw; } catch (ReleaseDownloadException ex) @@ -74,12 +75,17 @@ namespace NzbDrone.Core.Indexers _indexerStatusService.RecordFailure(indexerId); } - _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, title)); throw; } - _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source)); + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success, source, title)); return downloadedBytes; } + + public void RecordRedirect(string link, int indexerId, string source, string title) + { + _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, true, source, title, true)); + } } } diff --git a/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs b/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs index d10512bf6..3ac88ea28 100644 --- a/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs +++ b/src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs @@ -7,12 +7,16 @@ namespace NzbDrone.Core.Indexers.Events public int IndexerId { get; set; } public bool Successful { get; set; } public string Source { get; set; } + public string Title { get; set; } + public bool Redirect { get; set; } - public IndexerDownloadEvent(int indexerId, bool successful, string source) + public IndexerDownloadEvent(int indexerId, bool successful, string source, string title, bool redirect = false) { IndexerId = indexerId; Successful = successful; Source = source; + Title = title; + Redirect = redirect; } } } diff --git a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs index fade9fcbd..58c8d4b8e 100644 --- a/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/HttpIndexerBase.cs @@ -26,6 +26,7 @@ namespace NzbDrone.Core.Indexers public override bool SupportsRss => true; public override bool SupportsSearch => true; + public override bool SupportsRedirect => false; public override bool FollowRedirect => false; public override IndexerCapabilities Capabilities { get; protected set; } diff --git a/src/NzbDrone.Core/Indexers/IIndexer.cs b/src/NzbDrone.Core/Indexers/IIndexer.cs index e9f6a94de..cd46b636a 100644 --- a/src/NzbDrone.Core/Indexers/IIndexer.cs +++ b/src/NzbDrone.Core/Indexers/IIndexer.cs @@ -8,6 +8,7 @@ namespace NzbDrone.Core.Indexers { bool SupportsRss { get; } bool SupportsSearch { get; } + bool SupportsRedirect { get; } IndexerCapabilities Capabilities { get; } string BaseUrl { get; } diff --git a/src/NzbDrone.Core/Indexers/IndexerBase.cs b/src/NzbDrone.Core/Indexers/IndexerBase.cs index baf4abd08..f4c7a383c 100644 --- a/src/NzbDrone.Core/Indexers/IndexerBase.cs +++ b/src/NzbDrone.Core/Indexers/IndexerBase.cs @@ -25,9 +25,11 @@ namespace NzbDrone.Core.Indexers public abstract DownloadProtocol Protocol { get; } public abstract IndexerPrivacy Privacy { get; } public int Priority { get; set; } + public bool Redirect { get; set; } public abstract bool SupportsRss { get; } public abstract bool SupportsSearch { get; } + public abstract bool SupportsRedirect { get; } public abstract IndexerCapabilities Capabilities { get; protected set; } public IndexerBase(IIndexerStatusService indexerStatusService, IConfigService configService, Logger logger) diff --git a/src/NzbDrone.Core/Indexers/IndexerDefinition.cs b/src/NzbDrone.Core/Indexers/IndexerDefinition.cs index 31f42d5ee..f6808889d 100644 --- a/src/NzbDrone.Core/Indexers/IndexerDefinition.cs +++ b/src/NzbDrone.Core/Indexers/IndexerDefinition.cs @@ -12,8 +12,10 @@ namespace NzbDrone.Core.Indexers public IndexerPrivacy Privacy { get; set; } public bool SupportsRss { get; set; } public bool SupportsSearch { get; set; } + public bool SupportsRedirect { get; set; } public IndexerCapabilities Capabilities { get; set; } public int Priority { get; set; } = 25; + public bool Redirect { get; set; } public DateTime Added { get; set; } public IndexerStatus Status { get; set; } diff --git a/src/NzbDrone.Core/Indexers/IndexerFactory.cs b/src/NzbDrone.Core/Indexers/IndexerFactory.cs index 4af697fdb..e24331440 100644 --- a/src/NzbDrone.Core/Indexers/IndexerFactory.cs +++ b/src/NzbDrone.Core/Indexers/IndexerFactory.cs @@ -156,6 +156,7 @@ namespace NzbDrone.Core.Indexers definition.Protocol = provider.Protocol; definition.SupportsRss = provider.SupportsRss; definition.SupportsSearch = provider.SupportsSearch; + definition.SupportsRedirect = provider.SupportsRedirect; //We want to use the definition Caps and Privacy for Cardigann instead of the provider. if (definition.Implementation != typeof(Cardigann.Cardigann).Name) diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs b/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs index c5f64857d..af4a9cac0 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerModule.cs @@ -4,6 +4,7 @@ using System.Net; using System.Text; using Nancy; using Nancy.ModelBinding; +using Nancy.Responses; using NzbDrone.Common.Extensions; using NzbDrone.Core.Indexers; using NzbDrone.Core.IndexerSearch; @@ -107,17 +108,26 @@ namespace Prowlarr.Api.V1.Indexers throw new BadRequestException("Invalid Prowlarr link"); } + file = WebUtility.UrlDecode(file); + if (indexer == null) { throw new NotFoundException("Indexer Not Found"); } - var indexerInstance = _indexerFactory.GetInstance(indexer); - var source = UserAgentParser.ParseSource(Request.Headers.UserAgent); + var unprotectedlLink = _downloadMappingService.ConvertToNormalLink((string)link.Value); + + // If Indexer is set to download via Redirect then just redirect to the link + if (indexer.SupportsRedirect && indexer.Redirect) + { + _downloadService.RecordRedirect(unprotectedlLink, id, source, file); + return Response.AsRedirect(unprotectedlLink, RedirectResponse.RedirectType.Temporary); + } + var downloadBytes = Array.Empty(); - downloadBytes = _downloadService.DownloadReport(_downloadMappingService.ConvertToNormalLink(link), id, source); + downloadBytes = _downloadService.DownloadReport(unprotectedlLink, id, source, file); // handle magnet URLs if (downloadBytes.Length >= 7 @@ -130,7 +140,7 @@ namespace Prowlarr.Api.V1.Indexers && downloadBytes[6] == 0x3a) { var magnetUrl = Encoding.UTF8.GetString(downloadBytes); - return Response.AsRedirect(magnetUrl); + return Response.AsRedirect(magnetUrl, RedirectResponse.RedirectType.Temporary); } var contentType = indexer.Protocol == DownloadProtocol.Torrent ? "application/x-bittorrent" : "application/x-nzb"; diff --git a/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs b/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs index 9e88fe38a..ebd93c6ce 100644 --- a/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs +++ b/src/Prowlarr.Api.V1/Indexers/IndexerResource.cs @@ -14,8 +14,10 @@ namespace Prowlarr.Api.V1.Indexers { public string BaseUrl { get; set; } public bool Enable { get; set; } + public bool Redirect { get; set; } public bool SupportsRss { get; set; } public bool SupportsSearch { get; set; } + public bool SupportsRedirect { get; set; } public DownloadProtocol Protocol { get; set; } public IndexerPrivacy Privacy { get; set; } public IndexerCapabilityResource Capabilities { get; set; } @@ -61,8 +63,10 @@ namespace Prowlarr.Api.V1.Indexers resource.BaseUrl = definition.BaseUrl; resource.Enable = definition.Enable; + resource.Redirect = definition.Redirect; resource.SupportsRss = definition.SupportsRss; resource.SupportsSearch = definition.SupportsSearch; + resource.SupportsRedirect = definition.SupportsRedirect; resource.Capabilities = definition.Capabilities.ToResource(); resource.Protocol = definition.Protocol; resource.Privacy = definition.Privacy; @@ -100,6 +104,7 @@ namespace Prowlarr.Api.V1.Indexers } definition.Enable = resource.Enable; + definition.Redirect = resource.Redirect; definition.BaseUrl = resource.BaseUrl; definition.Priority = resource.Priority; definition.Privacy = resource.Privacy;