Create HttpProxySettingsProvider and fixed related issues.

src/NzbDrone.Common/Http/Dispatchers/CurlHttpDispatcher.cs

@@ -20,7 +20,8 @@ namespace NzbDrone.Common.Http.Dispatchers
     {
         private static readonly Regex ExpiryDate = new Regex(@"(expires=)([^;]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
 
-        private static readonly Logger _logger = NzbDroneLogger.GetLogger(typeof(CurlHttpDispatcher));
+        private readonly IHttpProxySettingsProvider _proxySettingsProvider;
+        private readonly Logger _logger;
 
         private const string _caBundleFileName = "curl-ca-bundle.crt";
         private static readonly string _caBundleFilePath;
@@ -37,7 +38,13 @@ namespace NzbDrone.Common.Http.Dispatchers
             }
         }
         
-        public static bool CheckAvailability()
+        public CurlHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, Logger logger)
+        {
+            _proxySettingsProvider = proxySettingsProvider;
+            _logger = logger;
+        }
+
+        public bool CheckAvailability()
         {
             try
             {
@@ -76,29 +83,7 @@ namespace NzbDrone.Common.Http.Dispatchers
                         return s * n;
                     };
 
-                    if(request.Proxy != null && !request.Proxy.ShouldProxyBeBypassed(new Uri(request.Url.FullUri)))
-                    
-                    {
-                        switch (request.Proxy.Type)
-                        {
-                            case ProxyType.Http:
-                                curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Http);
-                                curlEasy.SetOpt(CurlOption.ProxyAuth, CurlHttpAuth.Basic);
-                                curlEasy.SetOpt(CurlOption.ProxyUserPwd, request.Proxy.Username + ":" + request.Proxy.Password.ToString());
-                                break;
-                            case ProxyType.Socks4:
-                                curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Socks4);
-                                curlEasy.SetOpt(CurlOption.ProxyUsername, request.Proxy.Username);
-                                curlEasy.SetOpt(CurlOption.ProxyPassword, request.Proxy.Password);
-                                break;
-                            case ProxyType.Socks5:
-                                curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Socks5);
-                                curlEasy.SetOpt(CurlOption.ProxyUsername, request.Proxy.Username);
-                                curlEasy.SetOpt(CurlOption.ProxyPassword, request.Proxy.Password);
-                                break;
-                        }
-                        curlEasy.SetOpt(CurlOption.Proxy, request.Proxy.Host + ":" + request.Proxy.Port.ToString());
-                    }
+                    AddProxy(curlEasy, request);
 
                     curlEasy.Url = request.Url.FullUri;
 
@@ -174,6 +159,34 @@ namespace NzbDrone.Common.Http.Dispatchers
             }
         }
 
+        private void AddProxy(CurlEasy curlEasy, HttpRequest request)
+        {
+            var proxySettings = _proxySettingsProvider.GetProxySettings(request);
+            if (proxySettings != null)
+
+            {
+                switch (proxySettings.Type)
+                {
+                    case ProxyType.Http:
+                        curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Http);
+                        curlEasy.SetOpt(CurlOption.ProxyAuth, CurlHttpAuth.Basic);
+                        curlEasy.SetOpt(CurlOption.ProxyUserPwd, proxySettings.Username + ":" + proxySettings.Password.ToString());
+                        break;
+                    case ProxyType.Socks4:
+                        curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Socks4);
+                        curlEasy.SetOpt(CurlOption.ProxyUsername, proxySettings.Username);
+                        curlEasy.SetOpt(CurlOption.ProxyPassword, proxySettings.Password);
+                        break;
+                    case ProxyType.Socks5:
+                        curlEasy.SetOpt(CurlOption.ProxyType, CurlProxyType.Socks5);
+                        curlEasy.SetOpt(CurlOption.ProxyUsername, proxySettings.Username);
+                        curlEasy.SetOpt(CurlOption.ProxyPassword, proxySettings.Password);
+                        break;
+                }
+                curlEasy.SetOpt(CurlOption.Proxy, proxySettings.Host + ":" + proxySettings.Port.ToString());
+            }
+        }
+
         private CurlSlist SerializeHeaders(HttpRequest request)
         {
             if (!request.Headers.ContainsKey("Accept-Encoding"))

src/NzbDrone.Common/Http/Dispatchers/FallbackHttpDispatcher.cs

@@ -8,17 +8,18 @@ namespace NzbDrone.Common.Http.Dispatchers
 {
     public class FallbackHttpDispatcher : IHttpDispatcher
     {
-        private readonly Logger _logger;
-        private readonly ICached<bool> _curlTLSFallbackCache;
         private readonly ManagedHttpDispatcher _managedDispatcher;
         private readonly CurlHttpDispatcher _curlDispatcher;
+        private readonly Logger _logger;
+
+        private readonly ICached<bool> _curlTLSFallbackCache;
 
-        public FallbackHttpDispatcher(ICached<bool> curlTLSFallbackCache, Logger logger)
+        public FallbackHttpDispatcher(ManagedHttpDispatcher managedDispatcher, CurlHttpDispatcher curlDispatcher, ICacheManager cacheManager, Logger logger)
         {
+            _managedDispatcher = managedDispatcher;
+            _curlDispatcher = curlDispatcher;
+            _curlTLSFallbackCache = cacheManager.GetCache<bool>(GetType(), "curlTLSFallback");
             _logger = logger;
-            _curlTLSFallbackCache = curlTLSFallbackCache;
-            _managedDispatcher = new ManagedHttpDispatcher();
-            _curlDispatcher = new CurlHttpDispatcher();
         }
 
         public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
@@ -46,7 +47,7 @@ namespace NzbDrone.Common.Http.Dispatchers
                     }
                 }
 
-                if (CurlHttpDispatcher.CheckAvailability())
+                if (_curlDispatcher.CheckAvailability())
                 {
                     return _curlDispatcher.GetResponse(request, cookies);
                 }

src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs

@@ -4,12 +4,22 @@ using NzbDrone.Common.Extensions;
 using com.LandonKey.SocksWebProxy.Proxy;
 using com.LandonKey.SocksWebProxy;
 using System.Net.Sockets;
-using System.Linq;
 
 namespace NzbDrone.Common.Http.Dispatchers
 {
     public class ManagedHttpDispatcher : IHttpDispatcher
     {
+        private readonly IHttpProxySettingsProvider _proxySettingsProvider;
+
+        private readonly ICached<IWebProxy> _webProxyCache;
+
+        public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, ICacheManager cacheManager)
+        {
+            _proxySettingsProvider = proxySettingsProvider;
+
+            _webProxyCache = cacheManager.GetCache<IWebProxy>(GetType(), "webProxy");
+        }
+
         public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
         {
             var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url);
@@ -30,42 +40,7 @@ namespace NzbDrone.Common.Http.Dispatchers
                 webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds);
             }
 
-            if (request.Proxy != null && !request.Proxy.ShouldProxyBeBypassed(new Uri(request.Url.FullUri)))
-            {
-                var addresses = Dns.GetHostAddresses(request.Proxy.Host);
-
-                if(addresses.Length > 1)
-                {
-                    var ipv4Only = addresses.Where(a => a.AddressFamily == AddressFamily.InterNetwork);
-                    if (ipv4Only.Any())
-                    {
-                        addresses = ipv4Only.ToArray();
-                    }
-                }
-
-                var socksUsername = request.Proxy.Username == null ? string.Empty : request.Proxy.Username;
-                var socksPassword = request.Proxy.Password == null ? string.Empty : request.Proxy.Password;
-
-                switch (request.Proxy.Type)
-                {
-                    case ProxyType.Http:
-                        if(request.Proxy.Username.IsNotNullOrWhiteSpace() && request.Proxy.Password.IsNotNullOrWhiteSpace())
-                        {
-                            webRequest.Proxy = new WebProxy(request.Proxy.Host + ":" + request.Proxy.Port, request.Proxy.BypassLocalAddress, request.Proxy.SubnetFilterAsArray, new NetworkCredential(request.Proxy.Username, request.Proxy.Password));
-                        }
-                        else
-                        {
-                            webRequest.Proxy = new WebProxy(request.Proxy.Host + ":" + request.Proxy.Port, request.Proxy.BypassLocalAddress, request.Proxy.SubnetFilterAsArray);
-                        }
-                        break;
-                    case ProxyType.Socks4:
-                        webRequest.Proxy = new SocksWebProxy(new ProxyConfig(IPAddress.Parse("127.0.0.1"), GetNextFreePort(), addresses[0], request.Proxy.Port, ProxyConfig.SocksVersion.Four, socksUsername, socksPassword), false);
-                        break;
-                    case ProxyType.Socks5:
-                        webRequest.Proxy = new SocksWebProxy(new ProxyConfig(IPAddress.Parse("127.0.0.1"), GetNextFreePort(), addresses[0], request.Proxy.Port, ProxyConfig.SocksVersion.Five, socksUsername, socksPassword), false);
-                        break;
-                }                
-            }
+            AddProxy(webRequest, request);
 
             if (request.Headers != null)
             {
@@ -110,6 +85,50 @@ namespace NzbDrone.Common.Http.Dispatchers
             return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode);
         }
 
+        protected virtual void AddProxy(HttpWebRequest webRequest, HttpRequest request)
+        {
+            var proxySettings = _proxySettingsProvider.GetProxySettings(request);
+            if (proxySettings != null)
+            {
+                webRequest.Proxy = _webProxyCache.Get(proxySettings.Key, () => CreateWebProxy(proxySettings), TimeSpan.FromMinutes(5));
+            }
+
+            _webProxyCache.ClearExpired();
+        }
+
+        private IWebProxy CreateWebProxy(HttpRequestProxySettings proxySettings)
+        {
+            var addresses = Dns.GetHostAddresses(proxySettings.Host);
+
+            if(addresses.Length > 1)
+            {
+                var ipv4Only = addresses.Where(a => a.AddressFamily == AddressFamily.InterNetwork);
+                if (ipv4Only.Any())
+                {
+                    addresses = ipv4Only.ToArray();
+                }
+            }
+
+            switch (proxySettings.Type)
+            {
+                case ProxyType.Http:
+                    if (proxySettings.Username.IsNotNullOrWhiteSpace() && proxySettings.Password.IsNotNullOrWhiteSpace())
+                    {
+                        return new WebProxy(proxySettings.Host + ":" + proxySettings.Port, proxySettings.BypassLocalAddress, proxySettings.SubnetFilterAsArray, new NetworkCredential(proxySettings.Username, proxySettings.Password));
+                    }
+                    else
+                    {
+                        return new WebProxy(proxySettings.Host + ":" + proxySettings.Port, proxySettings.BypassLocalAddress, proxySettings.SubnetFilterAsArray);
+                    }
+                case ProxyType.Socks4:
+                    return new SocksWebProxy(new ProxyConfig(IPAddress.Loopback, GetNextFreePort(), addresses[0], proxySettings.Port, ProxyConfig.SocksVersion.Four, proxySettings.Username, proxySettings.Password), false);
+                case ProxyType.Socks5:
+                    return new SocksWebProxy(new ProxyConfig(IPAddress.Loopback, GetNextFreePort(), addresses[0], proxySettings.Port, ProxyConfig.SocksVersion.Five, proxySettings.Username, proxySettings.Password), false);
+            }
+
+            return null;
+        }
+
         protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers)
         {
             foreach (var header in headers)

src/NzbDrone.Common/Http/HttpClient.cs

@@ -35,21 +35,15 @@ namespace NzbDrone.Common.Http
 
         public HttpClient(IEnumerable<IHttpRequestInterceptor> requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, IHttpDispatcher httpDispatcher, Logger logger)
         {
-            _logger = logger;
-            _rateLimitService = rateLimitService;
             _requestInterceptors = requestInterceptors.ToList();
-            ServicePointManager.DefaultConnectionLimit = 12;
+            _rateLimitService = rateLimitService;
             _httpDispatcher = httpDispatcher;
+            _logger = logger;
 
+            ServicePointManager.DefaultConnectionLimit = 12;
             _cookieContainerCache = cacheManager.GetCache<CookieContainer>(typeof(HttpClient));
         }
 
-        public HttpClient(IEnumerable<IHttpRequestInterceptor> requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, Logger logger)
-            : this(requestInterceptors, cacheManager, rateLimitService, null, logger)
-        {
-            _httpDispatcher = new FallbackHttpDispatcher(cacheManager.GetCache<bool>(typeof(HttpClient), "curlTLSFallback"), _logger);
-        }
-
         public HttpResponse Execute(HttpRequest request)
         {
             foreach (var interceptor in _requestInterceptors)

src/NzbDrone.Common/Http/HttpRequest.cs

@@ -41,7 +41,6 @@ namespace NzbDrone.Common.Http
         public bool StoreResponseCookie { get; set; }
         public TimeSpan RequestTimeout { get; set; }
         public TimeSpan RateLimit { get; set; }
-        public HttpRequestProxySettings Proxy {get; set;}
 
         public override string ToString()
         {

src/NzbDrone.Common/Http/HttpRequestProxySettings.cs

@@ -1,18 +1,19 @@
 using System;
 using System.Net;
+using NzbDrone.Common.Extensions;
 
 namespace NzbDrone.Common.Http
 {
     public class HttpRequestProxySettings
     {
-        public HttpRequestProxySettings(ProxyType type, string host, int port, string filterSubnet, bool bypassLocalAddress, string username = null, string password = null)
+        public HttpRequestProxySettings(ProxyType type, string host, int port, string bypassFilter, bool bypassLocalAddress, string username = null, string password = null)
         {
             Type = type;
-            Host = host;
+            Host = host.IsNullOrWhiteSpace() ? "127.0.0.1" : host;
             Port = port;
-            Username = username;
-            Password = password;
-            SubnetFilter = filterSubnet;
+            Username = username ?? string.Empty;
+            Password = password ?? string.Empty;
+            BypassFilter = bypassFilter ?? string.Empty;
             BypassLocalAddress = bypassLocalAddress;
         }
 
@@ -21,27 +22,34 @@ namespace NzbDrone.Common.Http
         public int Port { get; private set; }
         public string Username { get; private set; }
         public string Password { get; private set; }
-        public string SubnetFilter { get; private set; }
+        public string BypassFilter { get; private set; }
         public bool BypassLocalAddress { get; private set; }
 
         public string[] SubnetFilterAsArray
         {
             get
             {
-                if (!string.IsNullOrWhiteSpace(SubnetFilter))
+                if (!string.IsNullOrWhiteSpace(BypassFilter))
                 {
-                    return SubnetFilter.Split(';');
+                    return BypassFilter.Split(';');
                 }
                 return new string[] { };
             }
         }
 
-        public bool ShouldProxyBeBypassed(Uri url)
+        public string Key
         {
-            //We are utilising the WebProxy implementation here to save us having to reimplement it. This way we use Microsofts implementation
-            WebProxy proxy = new WebProxy(Host + ":" + Port, BypassLocalAddress, SubnetFilterAsArray);
-
-            return proxy.IsBypassed(url);
+            get
+            {
+                return string.Join("_",
+                    Type,
+                    Host,
+                    Port,
+                    Username,
+                    Password,
+                    BypassFilter,
+                    BypassLocalAddress);
+            }
         }
     }
 }

src/NzbDrone.Common/Http/IHttpProxySettingsProvider.cs

@@ -0,0 +1,11 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace NzbDrone.Common.Http
+{
+    public interface IHttpProxySettingsProvider
+    {
+        HttpRequestProxySettings GetProxySettings(HttpRequest request);
+    }
+}

src/NzbDrone.Common/NzbDrone.Common.csproj

@@ -171,6 +171,7 @@
     <Compile Include="Http\HttpRequestProxySettings.cs" />
     <Compile Include="Http\HttpResponse.cs" />
     <Compile Include="Http\HttpUri.cs" />
+    <Compile Include="Http\IHttpProxySettingsProvider.cs" />
     <Compile Include="Http\IHttpRequestInterceptor.cs" />
     <Compile Include="Http\JsonRpcRequestBuilder.cs" />
     <Compile Include="Http\JsonRpcResponse.cs" />

src/NzbDrone.Core.Test/Framework/CoreTest.cs

@@ -3,6 +3,7 @@ using NUnit.Framework;
 using NzbDrone.Common.Cache;
 using NzbDrone.Common.Cloud;
 using NzbDrone.Common.Http;
+using NzbDrone.Common.Http.Dispatchers;
 using NzbDrone.Common.TPL;
 using NzbDrone.Test.Common;
 
@@ -13,7 +14,7 @@ namespace NzbDrone.Core.Test.Framework
         protected void UseRealHttp()
         {
             Mocker.SetConstant<IHttpProvider>(new HttpProvider(TestLogger));
-            Mocker.SetConstant<IHttpClient>(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), TestLogger));
+            Mocker.SetConstant<IHttpClient>(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), Mocker.Resolve<FallbackHttpDispatcher>(), TestLogger));
             Mocker.SetConstant<ISonarrCloudRequestBuilder>(new SonarrCloudRequestBuilder());
         }
     }

src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs

@@ -0,0 +1,52 @@
+using NzbDrone.Common.Http;
+using NzbDrone.Core.Configuration;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Net;
+using NzbDrone.Common.Extensions;
+
+namespace NzbDrone.Core.Http
+{
+    public class HttpProxySettingsProvider : IHttpProxySettingsProvider
+    {
+        private readonly IConfigService _configService;
+
+        public HttpProxySettingsProvider(IConfigService configService)
+        {
+            _configService = configService;
+        }
+
+        public HttpRequestProxySettings GetProxySettings(HttpRequest request)
+        {
+            if (!_configService.ProxyEnabled)
+            {
+                return null;
+            }
+            
+            var proxySettings = new HttpRequestProxySettings(_configService.ProxyType,
+                                _configService.ProxyHostname,
+                                _configService.ProxyPort,
+                                _configService.ProxySubnetFilter,
+                                _configService.ProxyBypassLocalAddresses,
+                                _configService.ProxyUsername,
+                                _configService.ProxyPassword);
+
+            if (ShouldProxyBeBypassed(proxySettings, request.Url))
+            {
+                return null;
+            }
+
+            return proxySettings;
+        }
+
+        public bool ShouldProxyBeBypassed(HttpRequestProxySettings proxySettings, HttpUri url)
+        {
+            //We are utilising the WebProxy implementation here to save us having to reimplement it. This way we use Microsofts implementation
+            var proxy = new WebProxy(proxySettings.Host + ":" + proxySettings.Port, proxySettings.BypassLocalAddress, proxySettings.SubnetFilterAsArray);
+
+            return proxy.IsBypassed((Uri)url);
+        }
+    }
+}

src/NzbDrone.Core/Http/ProxyHttpInterceptor.cs

@@ -1,40 +0,0 @@
-using NzbDrone.Common.Http;
-using NzbDrone.Core.Configuration;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-
-namespace NzbDrone.Core.Http
-{
-    public class ProxyHttpInterceptor : IHttpRequestInterceptor
-    {
-        private readonly IConfigService _configService;
-
-        public ProxyHttpInterceptor(IConfigService configService)
-        {
-            this._configService = configService;
-        }
-
-        public HttpResponse PostResponse(HttpResponse response)
-        {
-            return response;
-        }
-
-        public HttpRequest PreRequest(HttpRequest request)
-        {
-            if(_configService.ProxyEnabled)
-            {
-                request.Proxy = new HttpRequestProxySettings(_configService.ProxyType,
-                    _configService.ProxyHostname,
-                    _configService.ProxyPort,
-                    _configService.ProxySubnetFilter,
-                    _configService.ProxyBypassLocalAddresses,
-                    _configService.ProxyUsername,
-                    _configService.ProxyPassword);
-            }
-
-            return request;
-        }
-    }
-}

src/NzbDrone.Core/NzbDrone.Core.csproj

@@ -515,7 +515,7 @@
     <Compile Include="Housekeeping\HousekeepingCommand.cs" />
     <Compile Include="Housekeeping\HousekeepingService.cs" />
     <Compile Include="Housekeeping\IHousekeepingTask.cs" />
-    <Compile Include="Http\ProxyHttpInterceptor.cs" />
+    <Compile Include="Http\HttpProxySettingsProvider.cs" />
     <Compile Include="Http\TorcacheHttpInterceptor.cs" />
     <Compile Include="Indexers\BitMeTv\BitMeTv.cs" />
     <Compile Include="Indexers\BitMeTv\BitMeTvSettings.cs" />

src/NzbDrone.Host/MainAppContainerBuilder.cs

@@ -4,6 +4,7 @@ using Nancy.Bootstrapper;
 using NzbDrone.Api;
 using NzbDrone.Common.Composition;
 using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.Http.Dispatchers;
 using NzbDrone.Core.Datastore;
 using NzbDrone.Core.Organizer;
 using NzbDrone.SignalR;
@@ -42,6 +43,7 @@ namespace NzbDrone.Host
             AutoRegisterImplementations<NzbDronePersistentConnection>();
 
             Container.Register<INancyBootstrapper, NancyBootstrapper>();
+            Container.Register<IHttpDispatcher, FallbackHttpDispatcher>();
         }
     }
 }

src/NzbDrone.Update/UpdateContainerBuilder.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using NzbDrone.Common.Composition;
 using NzbDrone.Common.EnvironmentInfo;
+using NzbDrone.Common.Http.Dispatchers;
 
 namespace NzbDrone.Update
 {
@@ -10,7 +11,7 @@ namespace NzbDrone.Update
         private UpdateContainerBuilder(IStartupContext startupContext, string[] assemblies)
             : base(startupContext, assemblies)
         {
-
+            Container.Register<IHttpDispatcher, FallbackHttpDispatcher>();
         }
 
         public static IContainer Build(IStartupContext startupContext)