Proxy Nzb/Torrent Downloads thru Prowlarr

src/NzbDrone.Core/Authentication/ProtectionService.cs

@@ -0,0 +1,101 @@
+using System;
+using System.IO;
+using System.Security.Cryptography;
+using System.Text;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Configuration;
+
+namespace NzbDrone.Core.Authentication
+{
+    public interface IProtectionService
+    {
+        string Protect(string plainText);
+        string UnProtect(string plainText);
+    }
+
+    public class ProtectionService : IProtectionService
+    {
+        private readonly IConfigFileProvider _configService;
+
+        public ProtectionService(IConfigFileProvider configService)
+        {
+            _configService = configService;
+        }
+
+        public string Protect(string text)
+        {
+            var key = Encoding.UTF8.GetBytes(_configService.ApiKey);
+
+            using (var aesAlg = Aes.Create())
+            {
+                using (var encryptor = aesAlg.CreateEncryptor(key, aesAlg.IV))
+                {
+                    using (var msEncrypt = new MemoryStream())
+                    {
+                        using (var csEncrypt = new CryptoStream(msEncrypt, encryptor, CryptoStreamMode.Write))
+                        using (var swEncrypt = new StreamWriter(csEncrypt))
+                        {
+                            swEncrypt.Write(text);
+                        }
+
+                        var iv = aesAlg.IV;
+
+                        var decryptedContent = msEncrypt.ToArray();
+
+                        var result = new byte[iv.Length + decryptedContent.Length];
+
+                        Buffer.BlockCopy(iv, 0, result, 0, iv.Length);
+                        Buffer.BlockCopy(decryptedContent, 0, result, iv.Length, decryptedContent.Length);
+
+                        return Convert.ToBase64String(result);
+                    }
+                }
+            }
+        }
+
+        public string UnProtect(string value)
+        {
+            if (value.IsNullOrWhiteSpace())
+            {
+                return value;
+            }
+
+            try
+            {
+                value = value.Replace(" ", "+");
+                var fullCipher = Convert.FromBase64String(value);
+
+                var iv = new byte[16];
+                var cipher = new byte[fullCipher.Length - iv.Length];
+
+                Buffer.BlockCopy(fullCipher, 0, iv, 0, iv.Length);
+                Buffer.BlockCopy(fullCipher, iv.Length, cipher, 0, fullCipher.Length - iv.Length);
+                var key = Encoding.UTF8.GetBytes(_configService.ApiKey);
+
+                using (var aesAlg = Aes.Create())
+                {
+                    using (var decryptor = aesAlg.CreateDecryptor(key, iv))
+                    {
+                        string result;
+                        using (var msDecrypt = new MemoryStream(cipher))
+                        {
+                            using (var csDecrypt = new CryptoStream(msDecrypt, decryptor, CryptoStreamMode.Read))
+                            {
+                                using (var srDecrypt = new StreamReader(csDecrypt))
+                                {
+                                    result = srDecrypt.ReadToEnd();
+                                }
+                            }
+                        }
+
+                        return result;
+                    }
+                }
+            }
+            catch (Exception)
+            {
+                return string.Empty;
+            }
+        }
+    }
+}

src/NzbDrone.Core/History/HistoryService.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using NLog;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.ThingiProvider.Events;
 
@@ -24,7 +25,8 @@ namespace NzbDrone.Core.History
 
     public class HistoryService : IHistoryService,
                                   IHandle<ProviderDeletedEvent<IIndexer>>,
-                                  IHandle<IndexerQueryEvent>
+                                  IHandle<IndexerQueryEvent>,
+                                  IHandle<IndexerDownloadEvent>
     {
         private readonly IHistoryRepository _historyRepository;
         private readonly Logger _logger;
@@ -99,6 +101,20 @@ namespace NzbDrone.Core.History
             _historyRepository.Insert(history);
         }
 
+        public void Handle(IndexerDownloadEvent message)
+        {
+            var history = new History
+            {
+                Date = DateTime.UtcNow,
+                IndexerId = message.IndexerId,
+                EventType = HistoryEventType.ReleaseGrabbed
+            };
+
+            history.Data.Add("Successful", message.Successful.ToString());
+
+            _historyRepository.Insert(history);
+        }
+
         public void Handle(ProviderDeletedEvent<IIndexer> message)
         {
             _historyRepository.DeleteForIndexers(new List<int> { message.ProviderId });

src/NzbDrone.Core/IndexerSearch/NzbSearchService.cs

@@ -6,6 +6,7 @@ using NLog;
 using NzbDrone.Common.Instrumentation.Extensions;
 using NzbDrone.Common.TPL;
 using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;

src/NzbDrone.Core/Indexers/DownloadMappingService.cs

@@ -0,0 +1,52 @@
+using System;
+using System.Net;
+using System.Text;
+using Microsoft.AspNetCore.WebUtilities;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Core.Authentication;
+using NzbDrone.Core.Configuration;
+
+namespace NzbDrone.Core.Indexers
+{
+    public interface IDownloadMappingService
+    {
+        Uri ConvertToProxyLink(Uri link, string serverUrl, int indexerId, string file = "t");
+        string ConvertToNormalLink(string link);
+    }
+
+    public class DownloadMappingService : IDownloadMappingService
+    {
+        private readonly IProtectionService _protectionService;
+        private readonly IConfigFileProvider _configFileProvider;
+
+        public DownloadMappingService(IProtectionService protectionService, IConfigFileProvider configFileProvider)
+        {
+            _protectionService = protectionService;
+            _configFileProvider = configFileProvider;
+        }
+
+        public Uri ConvertToProxyLink(Uri link, string serverUrl, int indexerId, string file = "t")
+        {
+            var urlBase = _configFileProvider.UrlBase;
+
+            if (urlBase.IsNotNullOrWhiteSpace() && !urlBase.StartsWith("/"))
+            {
+                urlBase = "/" + urlBase;
+            }
+
+            var encryptedLink = _protectionService.Protect(link.ToString());
+            var encodedLink = WebEncoders.Base64UrlEncode(Encoding.UTF8.GetBytes(encryptedLink));
+            var urlEncodedFile = WebUtility.UrlEncode(file);
+            var proxyLink = $"{serverUrl}{urlBase}/api/v1/indexer/{indexerId}/download?apikey={_configFileProvider.ApiKey}&link={encodedLink}&file={urlEncodedFile}";
+            return new Uri(proxyLink);
+        }
+
+        public string ConvertToNormalLink(string link)
+        {
+            var encodedLink = Encoding.UTF8.GetString(WebEncoders.Base64UrlDecode(link));
+            var decryptedLink = _protectionService.UnProtect(encodedLink);
+
+            return decryptedLink;
+        }
+    }
+}

src/NzbDrone.Core/Indexers/DownloadService.cs

@@ -0,0 +1,86 @@
+using System;
+using NLog;
+using NzbDrone.Common.EnsureThat;
+using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
+using NzbDrone.Common.TPL;
+using NzbDrone.Core.Exceptions;
+using NzbDrone.Core.Indexers;
+using NzbDrone.Core.Indexers.Events;
+using NzbDrone.Core.Messaging.Events;
+
+namespace NzbDrone.Core.Indexers
+{
+    public interface IDownloadService
+    {
+        byte[] DownloadReport(string link, int indexerId);
+    }
+
+    public class DownloadService : IDownloadService
+    {
+        private readonly IIndexerFactory _indexerFactory;
+        private readonly IIndexerStatusService _indexerStatusService;
+        private readonly IRateLimitService _rateLimitService;
+        private readonly IEventAggregator _eventAggregator;
+        private readonly Logger _logger;
+
+        public DownloadService(IIndexerFactory indexerFactory,
+                               IIndexerStatusService indexerStatusService,
+                               IRateLimitService rateLimitService,
+                               IEventAggregator eventAggregator,
+                               Logger logger)
+        {
+            _indexerFactory = indexerFactory;
+            _indexerStatusService = indexerStatusService;
+            _rateLimitService = rateLimitService;
+            _eventAggregator = eventAggregator;
+            _logger = logger;
+        }
+
+        public byte[] DownloadReport(string link, int indexerId)
+        {
+            var url = new HttpUri(link);
+
+            // Limit grabs to 2 per second.
+            if (link.IsNotNullOrWhiteSpace() && !link.StartsWith("magnet:"))
+            {
+                _rateLimitService.WaitAndPulse(url.Host, TimeSpan.FromSeconds(2));
+            }
+
+            var indexer = _indexerFactory.GetInstance(_indexerFactory.Get(indexerId));
+            bool success;
+            var downloadedBytes = Array.Empty<byte>();
+
+            try
+            {
+                downloadedBytes = indexer.Download(url);
+                _indexerStatusService.RecordSuccess(indexerId);
+                success = true;
+            }
+            catch (ReleaseUnavailableException)
+            {
+                _logger.Trace("Release {0} no longer available on indexer.", link);
+                _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, false));
+                throw;
+            }
+            catch (ReleaseDownloadException ex)
+            {
+                var http429 = ex.InnerException as TooManyRequestsException;
+                if (http429 != null)
+                {
+                    _indexerStatusService.RecordFailure(indexerId, http429.RetryAfter);
+                }
+                else
+                {
+                    _indexerStatusService.RecordFailure(indexerId);
+                }
+
+                _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, false));
+                throw;
+            }
+
+            _eventAggregator.PublishEvent(new IndexerDownloadEvent(indexerId, success));
+            return downloadedBytes;
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Events/IndexerDownloadEvent.cs

@@ -0,0 +1,21 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NzbDrone.Common.Messaging;
+
+namespace NzbDrone.Core.Indexers.Events
+{
+    public class IndexerDownloadEvent : IEvent
+    {
+        public int IndexerId { get; set; }
+        public bool Successful { get; set; }
+
+        public IndexerDownloadEvent(int indexerId, bool successful)
+        {
+            IndexerId = indexerId;
+            Successful = successful;
+        }
+    }
+}

src/NzbDrone.Core/Indexers/Events/IndexerQueryEvent.cs

@@ -1,7 +1,7 @@
 using NzbDrone.Common.Messaging;
 using NzbDrone.Core.IndexerSearch.Definitions;
 
-namespace NzbDrone.Core.Indexers
+namespace NzbDrone.Core.Indexers.Events
 {
     public class IndexerQueryEvent : IEvent
     {

src/NzbDrone.Core/Indexers/HttpIndexerBase.cs

@@ -9,8 +9,10 @@ using NzbDrone.Common.Extensions;
 using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.Http.CloudFlare;
+using NzbDrone.Core.Indexers.Events;
 using NzbDrone.Core.Indexers.Exceptions;
 using NzbDrone.Core.IndexerSearch.Definitions;
+using NzbDrone.Core.Messaging.Events;
 using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 
@@ -89,6 +91,32 @@ namespace NzbDrone.Core.Indexers
             return FetchReleases(g => SetCookieFunctions(g).GetSearchRequests(searchCriteria));
         }
 
+        public override byte[] Download(HttpUri link)
+        {
+            Cookies = GetCookies();
+
+            var requestBuilder = new HttpRequestBuilder(link.FullUri);
+
+            if (Cookies != null)
+            {
+                requestBuilder.SetCookies(Cookies);
+            }
+
+            var downloadBytes = Array.Empty<byte>();
+
+            try
+            {
+                downloadBytes = _httpClient.Execute(requestBuilder.Build()).ResponseData;
+            }
+            catch (Exception)
+            {
+                _indexerStatusService.RecordFailure(Definition.Id);
+                _logger.Error("Download failed");
+            }
+
+            return downloadBytes;
+        }
+
         protected IIndexerRequestGenerator SetCookieFunctions(IIndexerRequestGenerator generator)
         {
             //A func ensures cookies are always updated to the latest. This way, the first page could update the cookies and then can be reused by the second page.

src/NzbDrone.Core/Indexers/IIndexer.cs

@@ -1,6 +1,5 @@
-using System.Collections.Generic;
+using NzbDrone.Common.Http;
 using NzbDrone.Core.IndexerSearch.Definitions;
-using NzbDrone.Core.Parser.Model;
 using NzbDrone.Core.ThingiProvider;
 
 namespace NzbDrone.Core.Indexers
@@ -20,6 +19,8 @@ namespace NzbDrone.Core.Indexers
         IndexerPageableQueryResult Fetch(BookSearchCriteria searchCriteria);
         IndexerPageableQueryResult Fetch(BasicSearchCriteria searchCriteria);
 
+        byte[] Download(HttpUri link);
+
         IndexerCapabilities GetCapabilities();
     }
 }

src/NzbDrone.Core/Indexers/IndexerBase.cs

@@ -4,6 +4,7 @@ using System.Linq;
 using FluentValidation.Results;
 using NLog;
 using NzbDrone.Common.Extensions;
+using NzbDrone.Common.Http;
 using NzbDrone.Core.Configuration;
 using NzbDrone.Core.IndexerSearch.Definitions;
 using NzbDrone.Core.Parser.Model;
@@ -71,6 +72,7 @@ namespace NzbDrone.Core.Indexers
         public abstract IndexerPageableQueryResult Fetch(TvSearchCriteria searchCriteria);
         public abstract IndexerPageableQueryResult Fetch(BookSearchCriteria searchCriteria);
         public abstract IndexerPageableQueryResult Fetch(BasicSearchCriteria searchCriteria);
+        public abstract byte[] Download(HttpUri searchCriteria);
 
         public abstract IndexerCapabilities GetCapabilities();
 

src/NzbDrone.Core/Prowlarr.Core.csproj

@@ -5,6 +5,7 @@
   <ItemGroup>
     <PackageReference Include="Dapper" Version="2.0.78" />
     <PackageReference Include="MailKit" Version="2.10.1" />
+    <PackageReference Include="Microsoft.AspNetCore.WebUtilities" Version="2.2.0" />
     <PackageReference Include="System.Memory" Version="4.5.4" />
     <PackageReference Include="System.ServiceModel.Syndication" Version="5.0.0" />
     <PackageReference Include="FluentMigrator.Runner" Version="4.0.0-alpha.268" />

src/Prowlarr.Api.V1/Indexers/IndexerModule.cs

@@ -1,10 +1,14 @@
+using System;
 using System.Collections.Generic;
+using System.Text;
 using Nancy;
 using Nancy.ModelBinding;
 using NzbDrone.Common.Extensions;
 using NzbDrone.Core.Indexers;
 using NzbDrone.Core.IndexerSearch;
 using NzbDrone.Core.Parser;
+using NzbDrone.Http.Extensions;
+using Prowlarr.Http.Extensions;
 using Prowlarr.Http.REST;
 
 namespace Prowlarr.Api.V1.Indexers
@@ -15,18 +19,26 @@ namespace Prowlarr.Api.V1.Indexers
 
         private IIndexerFactory _indexerFactory { get; set; }
         private ISearchForNzb _nzbSearchService { get; set; }
+        private IDownloadMappingService _downloadMappingService { get; set; }
+        private IDownloadService _downloadService { get; set; }
 
-        public IndexerModule(IndexerFactory indexerFactory, ISearchForNzb nzbSearchService)
+        public IndexerModule(IndexerFactory indexerFactory, ISearchForNzb nzbSearchService, IDownloadMappingService downloadMappingService, IDownloadService downloadService)
             : base(indexerFactory, "indexer", ResourceMapper)
         {
             _indexerFactory = indexerFactory;
             _nzbSearchService = nzbSearchService;
+            _downloadMappingService = downloadMappingService;
+            _downloadService = downloadService;
 
             Get("{id}/newznab", x =>
             {
                 var request = this.Bind<NewznabRequest>();
                 return GetNewznabResponse(request);
             });
+            Get("{id}/download", x =>
+            {
+                return GetDownload(x.id);
+            });
         }
 
         protected override void Validate(IndexerDefinition definition, bool includeWarnings)
@@ -57,6 +69,7 @@ namespace Prowlarr.Api.V1.Indexers
             }
 
             var indexerInstance = _indexerFactory.GetInstance(indexer);
+            var serverUrl = Request.GetServerUrl();
 
             switch (requestType)
             {
@@ -69,12 +82,61 @@ namespace Prowlarr.Api.V1.Indexers
                 case "music":
                 case "book":
                 case "movie":
-                    Response searchResponse = _nzbSearchService.Search(request, new List<int> { indexer.Id }, false).ToXml(indexerInstance.Protocol);
+                    var results = _nzbSearchService.Search(request, new List<int> { indexer.Id }, false);
+
+                    foreach (var result in results.Releases)
+                    {
+                        result.DownloadUrl = _downloadMappingService.ConvertToProxyLink(new Uri(result.DownloadUrl), serverUrl, indexer.Id, result.Title).ToString();
+                    }
+
+                    Response searchResponse = results.ToXml(indexerInstance.Protocol);
                     searchResponse.ContentType = "application/rss+xml";
                     return searchResponse;
                 default:
                     throw new BadRequestException("Function Not Available");
             }
         }
+
+        private object GetDownload(int id)
+        {
+            var indexer = _indexerFactory.Get(id);
+            var link = Request.Query.Link;
+            var file = Request.Query.File;
+
+            if (!link.HasValue || !file.HasValue)
+            {
+                throw new BadRequestException("Invalid Prowlarr link");
+            }
+
+            if (indexer == null)
+            {
+                throw new NotFoundException("Indexer Not Found");
+            }
+
+            var indexerInstance = _indexerFactory.GetInstance(indexer);
+
+            var downloadBytes = Array.Empty<byte>();
+            downloadBytes = _downloadService.DownloadReport(_downloadMappingService.ConvertToNormalLink(link), id);
+
+            // handle magnet URLs
+            if (downloadBytes.Length >= 7
+                && downloadBytes[0] == 0x6d
+                && downloadBytes[1] == 0x61
+                && downloadBytes[2] == 0x67
+                && downloadBytes[3] == 0x6e
+                && downloadBytes[4] == 0x65
+                && downloadBytes[5] == 0x74
+                && downloadBytes[6] == 0x3a)
+            {
+                var magnetUrl = Encoding.UTF8.GetString(downloadBytes);
+                return Response.AsRedirect(magnetUrl);
+            }
+
+            var contentType = indexer.Protocol == DownloadProtocol.Torrent ? "application/x-bittorrent" : "application/x-nzb";
+            var extension = indexer.Protocol == DownloadProtocol.Torrent ? "torrent" : "nzb";
+            var filename = $"{file}.{extension}";
+
+            return Response.FromByteArray(downloadBytes, contentType).AsAttachment(filename, contentType);
+        }
     }
 }

src/Prowlarr.Http/Extensions/RequestExtensions.cs

@@ -124,5 +124,34 @@ namespace Prowlarr.Http.Extensions
 
             return remoteAddress;
         }
+
+        public static string GetServerUrl(this Request request)
+        {
+            var scheme = request.Url.Scheme;
+            var port = request.Url.Port;
+
+            // Check for protocol headers added by reverse proxys
+            // X-Forwarded-Proto: A de facto standard for identifying the originating protocol of an HTTP request
+            var xForwardedProto = request.Headers.Where(x => x.Key == "X-Forwarded-Proto").Select(x => x.Value).FirstOrDefault();
+
+            if (xForwardedProto != null)
+            {
+                scheme = xForwardedProto.First();
+            }
+
+            // Front-End-Https: Non-standard header field used by Microsoft applications and load-balancers
+            else if (request.Headers.Where(x => x.Key == "Front-End-Https" && x.Value.FirstOrDefault() == "on").Any())
+            {
+                scheme = "https";
+            }
+
+            //default to 443 if the Host header doesn't contain the port (needed for reverse proxy setups)
+            if (scheme == "https" && !request.Url.HostName.Contains(":"))
+            {
+                port = 443;
+            }
+
+            return $"{scheme}://{request.Url.HostName}:{port}";
+        }
     }
 }

src/Prowlarr.Http/Extensions/ResponseExtensions.cs

@@ -0,0 +1,29 @@
+using System.IO;
+using Nancy;
+
+namespace NzbDrone.Http.Extensions
+{
+    public static class ResponseExtensions
+    {
+        public static Response FromByteArray(this IResponseFormatter formatter, byte[] body, string contentType = null)
+        {
+            return new ByteArrayResponse(body, contentType);
+        }
+    }
+
+    public class ByteArrayResponse : Response
+    {
+        public ByteArrayResponse(byte[] body, string contentType = null)
+        {
+            this.ContentType = contentType ?? "application/octet-stream";
+
+            this.Contents = stream =>
+            {
+                using (var writer = new BinaryWriter(stream))
+                {
+                    writer.Write(body);
+                }
+            };
+        }
+    }
+}