diff --git a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs index 668957cf5..9b3e84046 100644 --- a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs +++ b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs @@ -99,9 +99,9 @@ namespace NzbDrone.Common.Test.Http Mocker.SetConstant(Mocker.Resolve()); // Used for manual testing of socks proxies. - //Mocker.GetMock() - // .Setup(v => v.GetProxySettings(It.IsAny())) - // .Returns(new HttpProxySettings(ProxyType.Socks5, "127.0.0.1", 5476, "", false)); + // Mocker.GetMock() + // .Setup(v => v.GetProxySettings(It.IsAny())) + // .Returns(new HttpProxySettings(ProxyType.Socks5, "127.0.0.1", 5476, "", false)); // Roundrobin over the two servers, to reduce the chance of hitting the ratelimiter. _httpBinHost2 = _httpBinHosts[_httpBinRandom++ % _httpBinHosts.Length]; @@ -283,16 +283,31 @@ namespace NzbDrone.Common.Test.Http { var file = GetTempFilePath(); - var url = "https://sonarr.tv/img/slider/seriesdetails.png"; + var url = "https://lidarr.audio/img/slider/artistdetails.png"; Subject.DownloadFile(url, file); - File.Exists(file).Should().BeTrue(); - File.Exists(file + ".part").Should().BeFalse(); - var fileInfo = new FileInfo(file); + fileInfo.Exists.Should().BeTrue(); + fileInfo.Length.Should().Be(146122); + } - fileInfo.Length.Should().Be(307054); + [Test] + public void should_download_file_with_redirect() + { + var file = GetTempFilePath(); + + var request = new HttpRequestBuilder($"https://{_httpBinHost}/redirect-to") + .AddQueryParam("url", $"https://lidarr.audio/img/slider/artistdetails.png") + .Build(); + + Subject.DownloadFile(request.Url.FullUri, file); + + ExceptionVerification.ExpectedErrors(0); + + var fileInfo = new FileInfo(file); + fileInfo.Exists.Should().BeTrue(); + fileInfo.Length.Should().Be(146122); } [Test] @@ -303,9 +318,8 @@ namespace NzbDrone.Common.Test.Http Assert.Throws(() => Subject.DownloadFile("https://download.readarr.com/wrongpath", file)); File.Exists(file).Should().BeFalse(); - File.Exists(file + ".part").Should().BeFalse(); - ExceptionVerification.ExpectedWarns(0); + ExceptionVerification.ExpectedWarns(1); } [Test] @@ -333,7 +347,7 @@ namespace NzbDrone.Common.Test.Http var oldRequest = new HttpRequest($"https://{_httpBinHost2}/get"); oldRequest.Cookies["my"] = "cookie"; - var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.GetMock().Object, Mocker.Resolve()); + var oldClient = new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve()); oldClient.Should().NotBeSameAs(Subject); diff --git a/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs index 8e665ceed..b4cae7ff8 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs @@ -5,5 +5,6 @@ namespace NzbDrone.Common.Http.Dispatchers public interface IHttpDispatcher { HttpResponse GetResponse(HttpRequest request, CookieContainer cookies); + void DownloadFile(string url, string fileName); } } diff --git a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs index be1b53a65..95e070deb 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs @@ -1,5 +1,7 @@ using System; +using System.Diagnostics; using System.IO; +using System.IO.Compression; using System.Net; using System.Reflection; using System.Text; @@ -74,7 +76,7 @@ namespace NzbDrone.Common.Http.Dispatchers webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds); } - AddProxy(webRequest, request); + webRequest.Proxy = GetProxy(request.Url); if (request.Headers != null) { @@ -163,13 +165,54 @@ namespace NzbDrone.Common.Http.Dispatchers return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode); } - protected virtual void AddProxy(HttpWebRequest webRequest, HttpRequest request) + public void DownloadFile(string url, string fileName) { - var proxySettings = _proxySettingsProvider.GetProxySettings(request); + try + { + var fileInfo = new FileInfo(fileName); + if (fileInfo.Directory != null && !fileInfo.Directory.Exists) + { + fileInfo.Directory.Create(); + } + + _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); + + var stopWatch = Stopwatch.StartNew(); + var uri = new HttpUri(url); + + using (var webClient = new GZipWebClient()) + { + webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent()); + webClient.Proxy = GetProxy(uri); + webClient.DownloadFile(uri.FullUri, fileName); + stopWatch.Stop(); + _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); + } + } + catch (WebException e) + { + _logger.Warn("Failed to get response from: {0} {1}", url, e.Message); + throw; + } + catch (Exception e) + { + _logger.Warn(e, "Failed to get response from: " + url); + throw; + } + } + + protected virtual IWebProxy GetProxy(HttpUri uri) + { + IWebProxy proxy = null; + + var proxySettings = _proxySettingsProvider.GetProxySettings(uri); + if (proxySettings != null) { - webRequest.Proxy = _createManagedWebProxy.GetWebProxy(proxySettings); + proxy = _createManagedWebProxy.GetWebProxy(proxySettings); } + + return proxy; } protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers) diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index 856643b3e..1c415249e 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.Diagnostics; -using System.IO; using System.Linq; using System.Net; using NLog; @@ -33,19 +32,16 @@ namespace NzbDrone.Common.Http private readonly ICached _cookieContainerCache; private readonly List _requestInterceptors; private readonly IHttpDispatcher _httpDispatcher; - private readonly IUserAgentBuilder _userAgentBuilder; public HttpClient(IEnumerable requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, IHttpDispatcher httpDispatcher, - IUserAgentBuilder userAgentBuilder, Logger logger) { _requestInterceptors = requestInterceptors.ToList(); _rateLimitService = rateLimitService; _httpDispatcher = httpDispatcher; - _userAgentBuilder = userAgentBuilder; _logger = logger; ServicePointManager.DefaultConnectionLimit = 12; @@ -231,48 +227,7 @@ namespace NzbDrone.Common.Http public void DownloadFile(string url, string fileName, string userAgent = null) { - var fileNamePart = fileName + ".part"; - - try - { - var fileInfo = new FileInfo(fileName); - if (fileInfo.Directory != null && !fileInfo.Directory.Exists) - { - fileInfo.Directory.Create(); - } - - _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); - - var stopWatch = Stopwatch.StartNew(); - using (var fileStream = new FileStream(fileNamePart, FileMode.Create, FileAccess.ReadWrite)) - { - var request = new HttpRequest(url); - - if (userAgent.IsNotNullOrWhiteSpace()) - { - request.Headers.Set("User-Agent", userAgent); - } - - request.ResponseStream = fileStream; - var response = Get(request); - } - - stopWatch.Stop(); - if (File.Exists(fileName)) - { - File.Delete(fileName); - } - - File.Move(fileNamePart, fileName); - _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); - } - finally - { - if (File.Exists(fileNamePart)) - { - File.Delete(fileNamePart); - } - } + _httpDispatcher.DownloadFile(url, fileName); } public HttpResponse Get(HttpRequest request) diff --git a/src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs b/src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs index a288a46c4..ed223fe63 100644 --- a/src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs +++ b/src/NzbDrone.Common/Http/Proxy/IHttpProxySettingsProvider.cs @@ -2,6 +2,7 @@ namespace NzbDrone.Common.Http.Proxy { public interface IHttpProxySettingsProvider { - HttpProxySettings GetProxySettings(HttpRequest request); + HttpProxySettings GetProxySettings(HttpUri uri); + HttpProxySettings GetProxySettings(); } } diff --git a/src/NzbDrone.Core.Test/Framework/CoreTest.cs b/src/NzbDrone.Core.Test/Framework/CoreTest.cs index b3967c9b8..789e40e22 100644 --- a/src/NzbDrone.Core.Test/Framework/CoreTest.cs +++ b/src/NzbDrone.Core.Test/Framework/CoreTest.cs @@ -26,7 +26,7 @@ namespace NzbDrone.Core.Test.Framework Mocker.SetConstant(new HttpProxySettingsProvider(Mocker.Resolve())); Mocker.SetConstant(new ManagedWebProxyFactory(Mocker.Resolve())); Mocker.SetConstant(new ManagedHttpDispatcher(Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); - Mocker.SetConstant(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); + Mocker.SetConstant(new HttpClient(new IHttpRequestInterceptor[0], Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); Mocker.SetConstant(new ReadarrCloudRequestBuilder()); Mocker.SetConstant(Mocker.Resolve()); diff --git a/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs b/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs index 26c7d9341..a24370c98 100644 --- a/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs +++ b/src/NzbDrone.Core/Http/HttpProxySettingsProvider.cs @@ -15,27 +15,36 @@ namespace NzbDrone.Core.Http _configService = configService; } - public HttpProxySettings GetProxySettings(HttpRequest request) + public HttpProxySettings GetProxySettings(HttpUri uri) + { + var proxySettings = GetProxySettings(); + if (proxySettings == null) + { + return null; + } + + if (ShouldProxyBeBypassed(proxySettings, uri)) + { + return null; + } + + return proxySettings; + } + + public HttpProxySettings GetProxySettings() { if (!_configService.ProxyEnabled) { return null; } - var proxySettings = new HttpProxySettings(_configService.ProxyType, + return new HttpProxySettings(_configService.ProxyType, _configService.ProxyHostname, _configService.ProxyPort, _configService.ProxyBypassFilter, _configService.ProxyBypassLocalAddresses, _configService.ProxyUsername, _configService.ProxyPassword); - - if (ShouldProxyBeBypassed(proxySettings, request.Url)) - { - return null; - } - - return proxySettings; } public bool ShouldProxyBeBypassed(HttpProxySettings proxySettings, HttpUri url)