diff --git a/src/NzbDrone.Common/Http/Dispatchers/CurlHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/CurlHttpDispatcher.cs index 109e673d6..9c6118270 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/CurlHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/CurlHttpDispatcher.cs @@ -20,7 +20,8 @@ namespace NzbDrone.Common.Http.Dispatchers { private static readonly Regex ExpiryDate = new Regex(@"(expires=)([^;]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled); - private static readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(CurlHttpDispatcher)); + private readonly IHttpProxySettingsProvider _proxySettingsProvider; + private readonly Logger _logger; private const string _caBundleFileName = "curl-ca-bundle.crt"; private static readonly string _caBundleFilePath; @@ -36,8 +37,14 @@ namespace NzbDrone.Common.Http.Dispatchers _caBundleFilePath = _caBundleFileName; } } + + public CurlHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, Logger logger) + { + _proxySettingsProvider = proxySettingsProvider; + _logger = logger; + } - public static bool CheckAvailability() + public bool CheckAvailability() { try { @@ -76,32 +83,10 @@ namespace NzbDrone.Common.Http.Dispatchers return s * n; }; - if(request.Proxy != null && !request.Proxy.ShouldProxyBeBypassed(new Uri(request.Url.FullUri))) - - { - switch (request.Proxy.Type) - { - case ProxyType.Http: - curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Http); - curlEasy.SetOpt(CurlOption.ProxyAuth, CurlHttpAuth.Basic); - curlEasy.SetOpt(CurlOption.ProxyUserPwd, request.Proxy.Username + ":" + request.Proxy.Password.ToString()); - break; - case ProxyType.Socks4: - curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Socks4); - curlEasy.SetOpt(CurlOption.ProxyUsername, request.Proxy.Username); - curlEasy.SetOpt(CurlOption.ProxyPassword, request.Proxy.Password); - break; - case ProxyType.Socks5: - curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Socks5); - curlEasy.SetOpt(CurlOption.ProxyUsername, request.Proxy.Username); - curlEasy.SetOpt(CurlOption.ProxyPassword, request.Proxy.Password); - break; - } - curlEasy.SetOpt(CurlOption.Proxy, request.Proxy.Host + ":" + request.Proxy.Port.ToString()); - } - + AddProxy(curlEasy, request); + curlEasy.Url = request.Url.FullUri; - + switch (request.Method) { case HttpMethod.GET: @@ -174,6 +159,34 @@ namespace NzbDrone.Common.Http.Dispatchers } } + private void AddProxy(CurlEasy curlEasy, HttpRequest request) + { + var proxySettings = _proxySettingsProvider.GetProxySettings(request); + if (proxySettings != null) + + { + switch (proxySettings.Type) + { + case ProxyType.Http: + curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Http); + curlEasy.SetOpt(CurlOption.ProxyAuth, CurlHttpAuth.Basic); + curlEasy.SetOpt(CurlOption.ProxyUserPwd, proxySettings.Username + ":" + proxySettings.Password.ToString()); + break; + case ProxyType.Socks4: + curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Socks4); + curlEasy.SetOpt(CurlOption.ProxyUsername, proxySettings.Username); + curlEasy.SetOpt(CurlOption.ProxyPassword, proxySettings.Password); + break; + case ProxyType.Socks5: + curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Socks5); + curlEasy.SetOpt(CurlOption.ProxyUsername, proxySettings.Username); + curlEasy.SetOpt(CurlOption.ProxyPassword, proxySettings.Password); + break; + } + curlEasy.SetOpt(CurlOption.Proxy, proxySettings.Host + ":" + proxySettings.Port.ToString()); + } + } + private CurlSlist SerializeHeaders(HttpRequest request) { if (!request.Headers.ContainsKey("Accept-Encoding")) diff --git a/src/NzbDrone.Common/Http/Dispatchers/FallbackHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/FallbackHttpDispatcher.cs index 0fc64af8e..109d4aec2 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/FallbackHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/FallbackHttpDispatcher.cs @@ -8,17 +8,18 @@ namespace NzbDrone.Common.Http.Dispatchers { public class FallbackHttpDispatcher : IHttpDispatcher { - private readonly Logger _logger; - private readonly ICached _curlTLSFallbackCache; private readonly ManagedHttpDispatcher _managedDispatcher; private readonly CurlHttpDispatcher _curlDispatcher; + private readonly Logger _logger; + + private readonly ICached _curlTLSFallbackCache; - public FallbackHttpDispatcher(ICached curlTLSFallbackCache, Logger logger) + public FallbackHttpDispatcher(ManagedHttpDispatcher managedDispatcher, CurlHttpDispatcher curlDispatcher, ICacheManager cacheManager, Logger logger) { + _managedDispatcher = managedDispatcher; + _curlDispatcher = curlDispatcher; + _curlTLSFallbackCache = cacheManager.GetCache(GetType(), "curlTLSFallback"); _logger = logger; - _curlTLSFallbackCache = curlTLSFallbackCache; - _managedDispatcher = new ManagedHttpDispatcher(); - _curlDispatcher = new CurlHttpDispatcher(); } public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) @@ -46,7 +47,7 @@ namespace NzbDrone.Common.Http.Dispatchers } } - if (CurlHttpDispatcher.CheckAvailability()) + if (_curlDispatcher.CheckAvailability()) { return _curlDispatcher.GetResponse(request, cookies); } diff --git a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs index a195e9185..5d2c1a0f7 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs @@ -4,12 +4,22 @@ using NzbDrone.Common.Extensions; using com.LandonKey.SocksWebProxy.Proxy; using com.LandonKey.SocksWebProxy; using System.Net.Sockets; -using System.Linq; namespace NzbDrone.Common.Http.Dispatchers { public class ManagedHttpDispatcher : IHttpDispatcher { + private readonly IHttpProxySettingsProvider _proxySettingsProvider; + + private readonly ICached _webProxyCache; + + public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, ICacheManager cacheManager) + { + _proxySettingsProvider = proxySettingsProvider; + + _webProxyCache = cacheManager.GetCache(GetType(), "webProxy"); + } + public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url); @@ -30,42 +40,7 @@ namespace NzbDrone.Common.Http.Dispatchers webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds); } - if (request.Proxy != null && !request.Proxy.ShouldProxyBeBypassed(new Uri(request.Url.FullUri))) - { - var addresses = Dns.GetHostAddresses(request.Proxy.Host); - - if(addresses.Length > 1) - { - var ipv4Only = addresses.Where(a => a.AddressFamily == AddressFamily.InterNetwork); - if (ipv4Only.Any()) - { - addresses = ipv4Only.ToArray(); - } - } - - var socksUsername = request.Proxy.Username == null ? string.Empty : request.Proxy.Username; - var socksPassword = request.Proxy.Password == null ? string.Empty : request.Proxy.Password; - - switch (request.Proxy.Type) - { - case ProxyType.Http: - if(request.Proxy.Username.IsNotNullOrWhiteSpace() && request.Proxy.Password.IsNotNullOrWhiteSpace()) - { - webRequest.Proxy = new WebProxy(request.Proxy.Host + ":" + request.Proxy.Port, request.Proxy.BypassLocalAddress, request.Proxy.SubnetFilterAsArray, new NetworkCredential(request.Proxy.Username, request.Proxy.Password)); - } - else - { - webRequest.Proxy = new WebProxy(request.Proxy.Host + ":" + request.Proxy.Port, request.Proxy.BypassLocalAddress, request.Proxy.SubnetFilterAsArray); - } - break; - case ProxyType.Socks4: - webRequest.Proxy = new SocksWebProxy(new ProxyConfig(IPAddress.Parse("127.0.0.1"), GetNextFreePort(), addresses[0], request.Proxy.Port, ProxyConfig.SocksVersion.Four, socksUsername, socksPassword), false); - break; - case ProxyType.Socks5: - webRequest.Proxy = new SocksWebProxy(new ProxyConfig(IPAddress.Parse("127.0.0.1"), GetNextFreePort(), addresses[0], request.Proxy.Port, ProxyConfig.SocksVersion.Five, socksUsername, socksPassword), false); - break; - } - } + AddProxy(webRequest, request); if (request.Headers != null) { @@ -110,6 +85,50 @@ namespace NzbDrone.Common.Http.Dispatchers return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode); } + protected virtual void AddProxy(HttpWebRequest webRequest, HttpRequest request) + { + var proxySettings = _proxySettingsProvider.GetProxySettings(request); + if (proxySettings != null) + { + webRequest.Proxy = _webProxyCache.Get(proxySettings.Key, () => CreateWebProxy(proxySettings), TimeSpan.FromMinutes(5)); + } + + _webProxyCache.ClearExpired(); + } + + private IWebProxy CreateWebProxy(HttpRequestProxySettings proxySettings) + { + var addresses = Dns.GetHostAddresses(proxySettings.Host); + + if(addresses.Length > 1) + { + var ipv4Only = addresses.Where(a => a.AddressFamily == AddressFamily.InterNetwork); + if (ipv4Only.Any()) + { + addresses = ipv4Only.ToArray(); + } + } + + switch (proxySettings.Type) + { + case ProxyType.Http: + if (proxySettings.Username.IsNotNullOrWhiteSpace() && proxySettings.Password.IsNotNullOrWhiteSpace()) + { + return new WebProxy(proxySettings.Host + ":" + proxySettings.Port, proxySettings.BypassLocalAddress, proxySettings.SubnetFilterAsArray, new NetworkCredential(proxySettings.Username, proxySettings.Password)); + } + else + { + return new WebProxy(proxySettings.Host + ":" + proxySettings.Port, proxySettings.BypassLocalAddress, proxySettings.SubnetFilterAsArray); + } + case ProxyType.Socks4: + return new SocksWebProxy(new ProxyConfig(IPAddress.Loopback, GetNextFreePort(), addresses[0], proxySettings.Port, ProxyConfig.SocksVersion.Four, proxySettings.Username, proxySettings.Password), false); + case ProxyType.Socks5: + return new SocksWebProxy(new ProxyConfig(IPAddress.Loopback, GetNextFreePort(), addresses[0], proxySettings.Port, ProxyConfig.SocksVersion.Five, proxySettings.Username, proxySettings.Password), false); + } + + return null; + } + protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers) { foreach (var header in headers) diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index baf78a2f3..87b04da62 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -35,21 +35,15 @@ namespace NzbDrone.Common.Http public HttpClient(IEnumerable requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, IHttpDispatcher httpDispatcher, Logger logger) { - _logger = logger; - _rateLimitService = rateLimitService; _requestInterceptors = requestInterceptors.ToList(); - ServicePointManager.DefaultConnectionLimit = 12; + _rateLimitService = rateLimitService; _httpDispatcher = httpDispatcher; + _logger = logger; + ServicePointManager.DefaultConnectionLimit = 12; _cookieContainerCache = cacheManager.GetCache(typeof(HttpClient)); } - public HttpClient(IEnumerable requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, Logger logger) - : this(requestInterceptors, cacheManager, rateLimitService, null, logger) - { - _httpDispatcher = new FallbackHttpDispatcher(cacheManager.GetCache(typeof(HttpClient), "curlTLSFallback"), _logger); - } - public HttpResponse Execute(HttpRequest request) { foreach (var interceptor in _requestInterceptors) diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs index 966a3b534..94b5880d8 100644 --- a/src/NzbDrone.Common/Http/HttpRequest.cs +++ b/src/NzbDrone.Common/Http/HttpRequest.cs @@ -41,7 +41,6 @@ namespace NzbDrone.Common.Http public bool StoreResponseCookie { get; set; } public TimeSpan RequestTimeout { get; set; } public TimeSpan RateLimit { get; set; } - public HttpRequestProxySettings Proxy {get; set;} public override string ToString() { diff --git a/src/NzbDrone.Common/Http/HttpRequestProxySettings.cs b/src/NzbDrone.Common/Http/HttpRequestProxySettings.cs index 41c15ccfb..cbbff60bb 100644 --- a/src/NzbDrone.Common/Http/HttpRequestProxySettings.cs +++ b/src/NzbDrone.Common/Http/HttpRequestProxySettings.cs @@ -1,18 +1,19 @@ using System; using System.Net; +using NzbDrone.Common.Extensions; namespace NzbDrone.Common.Http { public class HttpRequestProxySettings { - public HttpRequestProxySettings(ProxyType type, string host, int port, string filterSubnet, bool bypassLocalAddress, string username = null, string password = null) + public HttpRequestProxySettings(ProxyType type, string host, int port, string bypassFilter, bool bypassLocalAddress, string username = null, string password = null) { Type = type; - Host = host; + Host = host.IsNullOrWhiteSpace() ? "127.0.0.1" : host; Port = port; - Username = username; - Password = password; - SubnetFilter = filterSubnet; + Username = username ?? string.Empty; + Password = password ?? string.Empty; + BypassFilter = bypassFilter ?? string.Empty; BypassLocalAddress = bypassLocalAddress; } @@ -21,27 +22,34 @@ namespace NzbDrone.Common.Http public int Port { get; private set; } public string Username { get; private set; } public string Password { get; private set; } - public string SubnetFilter { get; private set; } + public string BypassFilter { get; private set; } public bool BypassLocalAddress { get; private set; } public string[] SubnetFilterAsArray { get { - if (!string.IsNullOrWhiteSpace(SubnetFilter)) + if (!string.IsNullOrWhiteSpace(BypassFilter)) { - return SubnetFilter.Split(';'); + return BypassFilter.Split(';'); } return new string[] { }; } } - public bool ShouldProxyBeBypassed(Uri url) + public string Key { - //We are utilising the WebProxy implementation here to save us having to reimplement it. This way we use Microsofts implementation - WebProxy proxy = new WebProxy(Host + ":" + Port, BypassLocalAddress, SubnetFilterAsArray); - - return proxy.IsBypassed(url); + get + { + return string.Join("_", + Type, + Host, + Port, + Username, + Password, + BypassFilter, + BypassLocalAddress); + } } } } diff --git a/src/NzbDrone.Common/Http/IHttpProxySettingsProvider.cs b/src/NzbDrone.Common/Http/IHttpProxySettingsProvider.cs new file mode 100644 index 000000000..73fdd5fda --- /dev/null +++ b/src/NzbDrone.Common/Http/IHttpProxySettingsProvider.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace NzbDrone.Common.Http +{ + public interface IHttpProxySettingsProvider + { + HttpRequestProxySettings GetProxySettings(HttpRequest request); + } +} diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index 8d040a199..c335256f5 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -171,6 +171,7 @@ + diff --git a/src/NzbDrone.Core.Test/Framework/CoreTest.cs b/src/NzbDrone.Core.Test/Framework/CoreTest.cs index 89a3a02c1..932356a55 100644 --- a/src/NzbDrone.Core.Test/Framework/CoreTest.cs +++ b/src/NzbDrone.Core.Test/Framework/CoreTest.cs @@ -3,6 +3,7 @@ using NUnit.Framework; using NzbDrone.Common.Cache; using NzbDrone.Common.Cloud; using NzbDrone.Common.Http; +using NzbDrone.Common.Http.Dispatchers; using NzbDrone.Common.TPL; using NzbDrone.Test.Common; @@ -13,7 +14,7 @@ namespace NzbDrone.Core.Test.Framework protected void UseRealHttp() { Mocker.SetConstant(new HttpProvider(TestLogger)); - Mocker.SetConstant(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), TestLogger)); + Mocker.SetConstant(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); Mocker.SetConstant(new SonarrCloudRequestBuilder()); } } diff --git a/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs b/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs new file mode 100644 index 000000000..511b32948 --- /dev/null +++ b/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs @@ -0,0 +1,52 @@ +using NzbDrone.Common.Http; +using NzbDrone.Core.Configuration; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Net; +using NzbDrone.Common.Extensions; + +namespace NzbDrone.Core.Http +{ + public class HttpProxySettingsProvider : IHttpProxySettingsProvider + { + private readonly IConfigService _configService; + + public HttpProxySettingsProvider(IConfigService configService) + { + _configService = configService; + } + + public HttpRequestProxySettings GetProxySettings(HttpRequest request) + { + if (!_configService.ProxyEnabled) + { + return null; + } + + var proxySettings = new HttpRequestProxySettings(_configService.ProxyType, + _configService.ProxyHostname, + _configService.ProxyPort, + _configService.ProxySubnetFilter, + _configService.ProxyBypassLocalAddresses, + _configService.ProxyUsername, + _configService.ProxyPassword); + + if (ShouldProxyBeBypassed(proxySettings, request.Url)) + { + return null; + } + + return proxySettings; + } + + public bool ShouldProxyBeBypassed(HttpRequestProxySettings proxySettings, HttpUri url) + { + //We are utilising the WebProxy implementation here to save us having to reimplement it. This way we use Microsofts implementation + var proxy = new WebProxy(proxySettings.Host + ":" + proxySettings.Port, proxySettings.BypassLocalAddress, proxySettings.SubnetFilterAsArray); + + return proxy.IsBypassed((Uri)url); + } + } +} diff --git a/src/NzbDrone.Core/Http/ProxyHttpInterceptor.cs b/src/NzbDrone.Core/Http/ProxyHttpInterceptor.cs deleted file mode 100644 index 1207e1281..000000000 --- a/src/NzbDrone.Core/Http/ProxyHttpInterceptor.cs +++ /dev/null @@ -1,40 +0,0 @@ -using NzbDrone.Common.Http; -using NzbDrone.Core.Configuration; -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; - -namespace NzbDrone.Core.Http -{ - public class ProxyHttpInterceptor : IHttpRequestInterceptor - { - private readonly IConfigService _configService; - - public ProxyHttpInterceptor(IConfigService configService) - { - this._configService = configService; - } - - public HttpResponse PostResponse(HttpResponse response) - { - return response; - } - - public HttpRequest PreRequest(HttpRequest request) - { - if(_configService.ProxyEnabled) - { - request.Proxy = new HttpRequestProxySettings(_configService.ProxyType, - _configService.ProxyHostname, - _configService.ProxyPort, - _configService.ProxySubnetFilter, - _configService.ProxyBypassLocalAddresses, - _configService.ProxyUsername, - _configService.ProxyPassword); - } - - return request; - } - } -} diff --git a/src/NzbDrone.Core/NzbDrone.Core.csproj b/src/NzbDrone.Core/NzbDrone.Core.csproj index 920b002fd..3cddc0350 100644 --- a/src/NzbDrone.Core/NzbDrone.Core.csproj +++ b/src/NzbDrone.Core/NzbDrone.Core.csproj @@ -515,7 +515,7 @@ - + diff --git a/src/NzbDrone.Host/MainAppContainerBuilder.cs b/src/NzbDrone.Host/MainAppContainerBuilder.cs index 7c2cdc800..33c05878e 100644 --- a/src/NzbDrone.Host/MainAppContainerBuilder.cs +++ b/src/NzbDrone.Host/MainAppContainerBuilder.cs @@ -4,6 +4,7 @@ using Nancy.Bootstrapper; using NzbDrone.Api; using NzbDrone.Common.Composition; using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Common.Http.Dispatchers; using NzbDrone.Core.Datastore; using NzbDrone.Core.Organizer; using NzbDrone.SignalR; @@ -42,6 +43,7 @@ namespace NzbDrone.Host AutoRegisterImplementations(); Container.Register(); + Container.Register(); } } } \ No newline at end of file diff --git a/src/NzbDrone.Update/UpdateContainerBuilder.cs b/src/NzbDrone.Update/UpdateContainerBuilder.cs index 4dc5e9662..e3f449586 100644 --- a/src/NzbDrone.Update/UpdateContainerBuilder.cs +++ b/src/NzbDrone.Update/UpdateContainerBuilder.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using NzbDrone.Common.Composition; using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Common.Http.Dispatchers; namespace NzbDrone.Update { @@ -10,7 +11,7 @@ namespace NzbDrone.Update private UpdateContainerBuilder(IStartupContext startupContext, string[] assemblies) : base(startupContext, assemblies) { - + Container.Register(); } public static IContainer Build(IStartupContext startupContext)