From 527122452cac373a83c11797b5c15af6e6fafa3b Mon Sep 17 00:00:00 2001 From: Taloth Saldono Date: Thu, 18 Jun 2015 09:08:29 +0200 Subject: [PATCH] New: Fallback to libcurl/libssl on mono for https connections. --- .gitmodules | 4 + build.ps1 | 3 + src/ExternalModules/CurlSharp | 1 + .../Http/HttpClientFixture.cs | 12 + src/NzbDrone.Common/CurlSharp.dll.config | 7 + src/NzbDrone.Common/Http/CurlHttpClient.cs | 233 ++++++++++++++++++ src/NzbDrone.Common/Http/HttpClient.cs | 91 +++++-- src/NzbDrone.Common/NzbDrone.Common.csproj | 8 + src/NzbDrone.sln | 9 + 9 files changed, 348 insertions(+), 20 deletions(-) create mode 100644 .gitmodules create mode 160000 src/ExternalModules/CurlSharp create mode 100644 src/NzbDrone.Common/CurlSharp.dll.config create mode 100644 src/NzbDrone.Common/Http/CurlHttpClient.cs diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 000000000..bc941f3dd --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "src/ExternalModules/CurlSharp"] + path = src/ExternalModules/CurlSharp + url = https://github.com/Sonarr/CurlSharp.git + branch = master diff --git a/build.ps1 b/build.ps1 index 0ca821f4a..e699a4646 100644 --- a/build.ps1 +++ b/build.ps1 @@ -116,6 +116,9 @@ Function PackageMono() Write-Host "Adding NzbDrone.Core.dll.config (for dllmap)" Copy-Item "$sourceFolder\NzbDrone.Core\NzbDrone.Core.dll.config" $outputFolderMono + + Write-Host "Adding CurlSharp.dll.config (for dllmap)" + Copy-Item "$sourceFolder\NzbDrone.Common\CurlSharp.dll.config" $outputFolderMono Write-Host Renaming NzbDrone.Console.exe to NzbDrone.exe Get-ChildItem $outputFolderMono -File -Filter "NzbDrone.exe*" -Recurse | foreach ($_) {remove-item $_.fullname} diff --git a/src/ExternalModules/CurlSharp b/src/ExternalModules/CurlSharp new file mode 160000 index 000000000..cfdbbbd9c --- /dev/null +++ b/src/ExternalModules/CurlSharp @@ -0,0 +1 @@ +Subproject commit cfdbbbd9c6b9612c2756245049a8234ce87dc576 diff --git a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs index f6eadb4e5..1a0f0b87e 100644 --- a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs +++ b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs @@ -44,6 +44,17 @@ namespace NzbDrone.Common.Test.Http response.Resource.Url.Should().Be(request.Url.ToString()); } + [Test] + public void should_execute_simple_post() + { + var request = new HttpRequest("http://eu.httpbin.org/post"); + request.Body = "{ my: 1 }"; + + var response = Subject.Post(request); + + response.Resource.Data.Should().Be(request.Body); + } + [TestCase("gzip")] public void should_execute_get_using_gzip(string compression) { @@ -224,5 +235,6 @@ namespace NzbDrone.Common.Test.Http public Dictionary Headers { get; set; } public string Origin { get; set; } public string Url { get; set; } + public string Data { get; set; } } } \ No newline at end of file diff --git a/src/NzbDrone.Common/CurlSharp.dll.config b/src/NzbDrone.Common/CurlSharp.dll.config new file mode 100644 index 000000000..2b2e7ac1f --- /dev/null +++ b/src/NzbDrone.Common/CurlSharp.dll.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/NzbDrone.Common/Http/CurlHttpClient.cs b/src/NzbDrone.Common/Http/CurlHttpClient.cs new file mode 100644 index 000000000..f5a158684 --- /dev/null +++ b/src/NzbDrone.Common/Http/CurlHttpClient.cs @@ -0,0 +1,233 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Runtime.InteropServices; +using System.Text; +using CurlSharp; +using NLog; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Instrumentation; + +namespace NzbDrone.Common.Http +{ + public class CurlHttpClient + { + private static Logger Logger = NzbDroneLogger.GetLogger(typeof(CurlHttpClient)); + + public CurlHttpClient() + { + if (!CheckAvailability()) + { + throw new ApplicationException("Curl failed to initialize."); + } + } + + public static bool CheckAvailability() + { + try + { + return CurlGlobalHandle.Instance.Initialize(); + } + catch (Exception ex) + { + Logger.TraceException("Initializing curl failed", ex); + return false; + } + } + + public HttpResponse GetResponse(HttpRequest httpRequest, HttpWebRequest webRequest) + { + Stream responseStream = new MemoryStream(); + Stream headerStream = new MemoryStream(); + + var curlEasy = new CurlEasy(); + curlEasy.AutoReferer = false; + curlEasy.WriteFunction = (b, s, n, o) => + { + responseStream.Write(b, 0, s * n); + return s * n; + }; + curlEasy.HeaderFunction = (b, s, n, o) => + { + headerStream.Write(b, 0, s * n); + return s * n; + }; + + curlEasy.UserAgent = webRequest.UserAgent; + curlEasy.FollowLocation = webRequest.AllowAutoRedirect; + curlEasy.HttpGet = webRequest.Method == "GET"; + curlEasy.Post = webRequest.Method == "POST"; + curlEasy.Put = webRequest.Method == "PUT"; + curlEasy.Url = webRequest.RequestUri.ToString(); + + if (webRequest.CookieContainer != null) + { + curlEasy.Cookie = webRequest.CookieContainer.GetCookieHeader(webRequest.RequestUri); + } + + if (!httpRequest.Body.IsNullOrWhiteSpace()) + { + // TODO: This might not go well with encoding. + curlEasy.PostFields = httpRequest.Body; + curlEasy.PostFieldSize = httpRequest.Body.Length; + } + + curlEasy.HttpHeader = SerializeHeaders(webRequest); + + var result = curlEasy.Perform(); + + if (result != CurlCode.Ok) + { + throw new WebException(string.Format("Curl Error {0} for Url {1}", result, curlEasy.Url)); + } + + var webHeaderCollection = ProcessHeaderStream(webRequest, headerStream); + var responseData = ProcessResponseStream(webRequest, responseStream, webHeaderCollection); + + var httpHeader = new HttpHeader(webHeaderCollection); + + return new HttpResponse(httpRequest, httpHeader, responseData, (HttpStatusCode)curlEasy.ResponseCode); + } + + private CurlSlist SerializeHeaders(HttpWebRequest webRequest) + { + if (webRequest.SendChunked) + { + throw new NotSupportedException("Chunked transfer is not supported"); + } + + if (webRequest.ContentLength > 0) + { + webRequest.Headers.Add("Content-Length", webRequest.ContentLength.ToString()); + } + + if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.GZip)) + { + if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate)) + { + webRequest.Headers.Add("Accept-Encoding", "gzip, deflate"); + } + else + { + webRequest.Headers.Add("Accept-Encoding", "gzip"); + } + } + else + { + if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate)) + { + webRequest.Headers.Add("Accept-Encoding", "deflate"); + } + } + + + var curlHeaders = new CurlSlist(); + for (int i = 0; i < webRequest.Headers.Count; i++) + { + curlHeaders.Append(webRequest.Headers.GetKey(i) + ": " + webRequest.Headers.Get(i)); + } + + curlHeaders.Append("Content-Type: " + webRequest.ContentType ?? string.Empty); + + return curlHeaders; + } + + private WebHeaderCollection ProcessHeaderStream(HttpWebRequest webRequest, Stream headerStream) + { + headerStream.Position = 0; + var headerData = headerStream.ToBytes(); + var headerString = Encoding.ASCII.GetString(headerData); + + var webHeaderCollection = new WebHeaderCollection(); + + foreach (var header in headerString.Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).Skip(1)) + { + webHeaderCollection.Add(header); + } + + var setCookie = webHeaderCollection.Get("Set-Cookie"); + if (setCookie != null && setCookie.Length > 0 && webRequest.CookieContainer != null) + { + webRequest.CookieContainer.SetCookies(webRequest.RequestUri, setCookie); + } + + return webHeaderCollection; + } + + private byte[] ProcessResponseStream(HttpWebRequest webRequest, Stream responseStream, WebHeaderCollection webHeaderCollection) + { + responseStream.Position = 0; + + if (responseStream.Length != 0 && webRequest.AutomaticDecompression != DecompressionMethods.None) + { + var encoding = webHeaderCollection["Content-Encoding"]; + if (encoding != null) + { + if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.GZip) && encoding.IndexOf("gzip") != -1) + { + responseStream = new GZipStream(responseStream, CompressionMode.Decompress); + + webHeaderCollection.Remove("Content-Encoding"); + } + else if (webRequest.AutomaticDecompression.HasFlag(DecompressionMethods.Deflate) && encoding.IndexOf("deflate") != -1) + { + responseStream = new DeflateStream(responseStream, CompressionMode.Decompress); + + webHeaderCollection.Remove("Content-Encoding"); + } + } + } + + return responseStream.ToBytes(); + + } + } + + internal sealed class CurlGlobalHandle : SafeHandle + { + public static readonly CurlGlobalHandle Instance = new CurlGlobalHandle(); + + private bool _initialized; + private bool _available; + + protected override void Dispose(bool disposing) + { + base.Dispose(disposing); + } + + private CurlGlobalHandle() + : base(IntPtr.Zero, true) + { + + } + + public bool Initialize() + { + if (_initialized) + return _available; + + _initialized = true; + _available = Curl.GlobalInit(CurlInitFlag.All) == CurlCode.Ok; + + return _available; + } + + protected override bool ReleaseHandle() + { + if (_initialized && _available) + { + Curl.GlobalCleanup(); + _available = false; + } + return true; + } + + public override bool IsInvalid + { + get { return !_initialized || !_available; } + } + } +} diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index e68895813..adeed3533 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -26,6 +26,7 @@ namespace NzbDrone.Common.Http private readonly Logger _logger; private readonly IRateLimitService _rateLimitService; private readonly ICached _cookieContainerCache; + private readonly ICached _curlTLSFallbackCache; public HttpClient(ICacheManager cacheManager, IRateLimitService rateLimitService, Logger logger) { @@ -34,6 +35,7 @@ namespace NzbDrone.Common.Http ServicePointManager.DefaultConnectionLimit = 12; _cookieContainerCache = cacheManager.GetCache(typeof(HttpClient)); + _curlTLSFallbackCache = cacheManager.GetCache(typeof(HttpClient), "curlTLSFallback"); } public HttpResponse Execute(HttpRequest request) @@ -94,6 +96,74 @@ namespace NzbDrone.Common.Http webRequest.CookieContainer.Add(cookieContainer.GetCookies(request.Url)); } + var response = ExecuteRequest(request, webRequest); + + stopWatch.Stop(); + + _logger.Trace("{0} ({1:n0} ms)", response, stopWatch.ElapsedMilliseconds); + + if (request.AllowAutoRedirect && !RuntimeInfoBase.IsProduction && + (response.StatusCode == HttpStatusCode.Moved || + response.StatusCode == HttpStatusCode.MovedPermanently || + response.StatusCode == HttpStatusCode.Found)) + { + _logger.Error("Server requested a redirect to [" + response.Headers["Location"] + "]. Update the request URL to avoid this redirect."); + } + + if (!request.SuppressHttpError && response.HasHttpError) + { + _logger.Warn("HTTP Error - {0}", response); + throw new HttpException(request, response); + } + + return response; + } + + private HttpResponse ExecuteRequest(HttpRequest request, HttpWebRequest webRequest) + { + if (OsInfo.IsMonoRuntime && webRequest.RequestUri.Scheme == "https") + { + if (!_curlTLSFallbackCache.Find(webRequest.RequestUri.Host)) + { + try + { + return ExecuteWebRequest(request, webRequest); + } + catch (Exception ex) + { + if (ex.ToString().Contains("The authentication or decryption has failed.")) + { + _logger.Debug("https request failed in tls error for {0}, trying curl fallback.", webRequest.RequestUri.Host); + + _curlTLSFallbackCache.Set(webRequest.RequestUri.Host, true); + } + else + { + throw; + } + } + } + + if (CurlHttpClient.CheckAvailability()) + { + return ExecuteCurlRequest(request, webRequest); + } + + _logger.Trace("Curl not available, using default WebClient."); + } + + return ExecuteWebRequest(request, webRequest); + } + + private HttpResponse ExecuteCurlRequest(HttpRequest request, HttpWebRequest webRequest) + { + var curlClient = new CurlHttpClient(); + + return curlClient.GetResponse(request, webRequest); + } + + private HttpResponse ExecuteWebRequest(HttpRequest request, HttpWebRequest webRequest) + { if (!request.Body.IsNullOrWhiteSpace()) { var bytes = request.Headers.GetEncodingFromContentType().GetBytes(request.Body.ToCharArray()); @@ -131,26 +201,7 @@ namespace NzbDrone.Common.Http } } - stopWatch.Stop(); - - var response = new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode); - _logger.Trace("{0} ({1:n0} ms)", response, stopWatch.ElapsedMilliseconds); - - if (request.AllowAutoRedirect && !RuntimeInfoBase.IsProduction && - (response.StatusCode == HttpStatusCode.Moved || - response.StatusCode == HttpStatusCode.MovedPermanently || - response.StatusCode == HttpStatusCode.Found)) - { - _logger.Error("Server requested a redirect to [" + response.Headers["Location"] + "]. Update the request URL to avoid this redirect."); - } - - if (!request.SuppressHttpError && response.HasHttpError) - { - _logger.Warn("HTTP Error - {0}", response); - throw new HttpException(request, response); - } - - return response; + return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), data, httpWebResponse.StatusCode); } public void DownloadFile(string url, string fileName) diff --git a/src/NzbDrone.Common/NzbDrone.Common.csproj b/src/NzbDrone.Common/NzbDrone.Common.csproj index e430f4f34..76f210985 100644 --- a/src/NzbDrone.Common/NzbDrone.Common.csproj +++ b/src/NzbDrone.Common/NzbDrone.Common.csproj @@ -140,6 +140,7 @@ + Component @@ -196,6 +197,9 @@ + + Always + Designer @@ -208,6 +212,10 @@ + + {74420a79-cc16-442c-8b1e-7c1b913844f0} + CurlSharp + {9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB} LogentriesNLog diff --git a/src/NzbDrone.sln b/src/NzbDrone.sln index d80b15706..e6afdcb7a 100644 --- a/src/NzbDrone.sln +++ b/src/NzbDrone.sln @@ -88,6 +88,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LogentriesNLog", "Logentrie {90D6E9FC-7B88-4E1B-B018-8FA742274558} = {90D6E9FC-7B88-4E1B-B018-8FA742274558} EndProjectSection EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CurlSharp", "ExternalModules\CurlSharp\CurlSharp\CurlSharp.csproj", "{74420A79-CC16-442C-8B1E-7C1B913844F0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|x86 = Debug|x86 @@ -268,6 +270,12 @@ Global {9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB}.Mono|x86.Build.0 = Release|x86 {9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB}.Release|x86.ActiveCfg = Release|x86 {9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB}.Release|x86.Build.0 = Release|x86 + {74420A79-CC16-442C-8B1E-7C1B913844F0}.Debug|x86.ActiveCfg = Debug|Any CPU + {74420A79-CC16-442C-8B1E-7C1B913844F0}.Debug|x86.Build.0 = Debug|Any CPU + {74420A79-CC16-442C-8B1E-7C1B913844F0}.Mono|x86.ActiveCfg = Release|Any CPU + {74420A79-CC16-442C-8B1E-7C1B913844F0}.Mono|x86.Build.0 = Release|Any CPU + {74420A79-CC16-442C-8B1E-7C1B913844F0}.Release|x86.ActiveCfg = Release|Any CPU + {74420A79-CC16-442C-8B1E-7C1B913844F0}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -300,6 +308,7 @@ Global {411A9E0E-FDC6-4E25-828A-0C2CD1CD96F8} = {F6E3A728-AE77-4D02-BAC8-82FBC1402DDA} {90D6E9FC-7B88-4E1B-B018-8FA742274558} = {F6E3A728-AE77-4D02-BAC8-82FBC1402DDA} {9DC31DE3-79FF-47A8-96B4-6BA18F6BB1CB} = {F6E3A728-AE77-4D02-BAC8-82FBC1402DDA} + {74420A79-CC16-442C-8B1E-7C1B913844F0} = {F6E3A728-AE77-4D02-BAC8-82FBC1402DDA} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.0\lib\NET35;packages\Unity.2.1.505.2\lib\NET35