From b7b5a6e7e1150e2a34aac37deff84f02840111cf Mon Sep 17 00:00:00 2001 From: Qstick Date: Mon, 4 Jul 2022 18:37:31 -0500 Subject: [PATCH] Modern HTTP Client (#685) --- .../Http/HttpClientFixture.cs | 135 ++++++- .../Extensions/StringExtensions.cs | 21 ++ .../Http/BasicNetworkCredential.cs | 12 + src/NzbDrone.Common/Http/CookieUtil.cs | 3 +- .../ICertificateValidationService.cs | 11 + .../Http/Dispatchers/IHttpDispatcher.cs | 1 - .../Http/Dispatchers/ManagedHttpDispatcher.cs | 342 +++++++++++------- src/NzbDrone.Common/Http/GZipWebClient.cs | 15 - src/NzbDrone.Common/Http/HttpClient.cs | 154 ++++++-- src/NzbDrone.Common/Http/HttpHeader.cs | 21 ++ src/NzbDrone.Common/Http/HttpRequest.cs | 14 +- .../Http/HttpRequestBuilder.cs | 13 +- src/NzbDrone.Common/Http/HttpResponse.cs | 9 +- .../Http/XmlRpcRequestBuilder.cs | 103 ++++++ src/NzbDrone.Core.Test/Framework/CoreTest.cs | 4 +- .../Download/Clients/Aria2/Aria2Containers.cs | 230 +++++++----- .../Download/Clients/Aria2/Aria2Proxy.cs | 160 ++++---- .../Clients/rTorrent/RTorrentFault.cs | 28 ++ .../Clients/rTorrent/RTorrentProxy.cs | 251 +++++-------- .../Clients/rTorrent/RTorrentTorrent.cs | 31 +- .../Download/Extensions/XmlExtensions.cs | 54 +++ .../Definitions/Headphones/Headphones.cs | 2 +- .../Headphones/HeadphonesRequestGenerator.cs | 2 +- .../Notifications/Email/Email.cs | 7 +- .../PushBullet/PushBulletProxy.cs | 4 +- .../Notifications/Twitter/TwitterProxy.cs | 110 ++++++ .../Notifications/Twitter/TwitterService.cs | 46 +-- .../Notifications/Webhook/WebhookProxy.cs | 2 +- .../X509CertificateValidationService.cs | 38 +- src/NzbDrone.Core/TinyTwitter.cs | 235 ------------ 30 files changed, 1222 insertions(+), 836 deletions(-) create mode 100644 src/NzbDrone.Common/Http/BasicNetworkCredential.cs create mode 100644 src/NzbDrone.Common/Http/Dispatchers/ICertificateValidationService.cs delete mode 100644 src/NzbDrone.Common/Http/GZipWebClient.cs create mode 100644 src/NzbDrone.Common/Http/XmlRpcRequestBuilder.cs create mode 100644 src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentFault.cs create mode 100644 src/NzbDrone.Core/Download/Extensions/XmlExtensions.cs create mode 100644 src/NzbDrone.Core/Notifications/Twitter/TwitterProxy.cs delete mode 100644 src/NzbDrone.Core/TinyTwitter.cs diff --git a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs index 23f08adbe..8979c2a19 100644 --- a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs +++ b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs @@ -4,6 +4,7 @@ using System.Globalization; using System.IO; using System.Linq; using System.Net; +using System.Net.Http; using System.Threading; using FluentAssertions; using Moq; @@ -15,8 +16,11 @@ using NzbDrone.Common.Http; using NzbDrone.Common.Http.Dispatchers; using NzbDrone.Common.Http.Proxy; using NzbDrone.Common.TPL; +using NzbDrone.Core.Configuration; +using NzbDrone.Core.Security; using NzbDrone.Test.Common; using NzbDrone.Test.Common.Categories; +using HttpClient = NzbDrone.Common.Http.HttpClient; namespace NzbDrone.Common.Test.Http { @@ -31,6 +35,8 @@ namespace NzbDrone.Common.Test.Http private string _httpBinHost; private string _httpBinHost2; + private System.Net.Http.HttpClient _httpClient = new (); + [OneTimeSetUp] public void FixtureSetUp() { @@ -38,7 +44,7 @@ namespace NzbDrone.Common.Test.Http var mainHost = "httpbin.servarr.com"; // Use mirrors for tests that use two hosts - var candidates = new[] { "eu.httpbin.org", /* "httpbin.org", */ "www.httpbin.org" }; + var candidates = new[] { "httpbin1.servarr.com" }; // httpbin.org is broken right now, occassionally redirecting to https if it's unavailable. _httpBinHost = mainHost; @@ -46,29 +52,20 @@ namespace NzbDrone.Common.Test.Http TestLogger.Info($"{candidates.Length} TestSites available."); - _httpBinSleep = _httpBinHosts.Length < 2 ? 100 : 10; + _httpBinSleep = 10; } private bool IsTestSiteAvailable(string site) { try { - var req = WebRequest.Create($"https://{site}/get") as HttpWebRequest; - var res = req.GetResponse() as HttpWebResponse; + var res = _httpClient.GetAsync($"https://{site}/get").GetAwaiter().GetResult(); if (res.StatusCode != HttpStatusCode.OK) { return false; } - try - { - req = WebRequest.Create($"https://{site}/status/429") as HttpWebRequest; - res = req.GetResponse() as HttpWebResponse; - } - catch (WebException ex) - { - res = ex.Response as HttpWebResponse; - } + res = _httpClient.GetAsync($"https://{site}/status/429").GetAwaiter().GetResult(); if (res == null || res.StatusCode != (HttpStatusCode)429) { @@ -95,10 +92,13 @@ namespace NzbDrone.Common.Test.Http Mocker.GetMock().Setup(c => c.Name).Returns("TestOS"); Mocker.GetMock().Setup(c => c.Version).Returns("9.0.0"); + Mocker.GetMock().SetupGet(x => x.CertificateValidation).Returns(CertificateValidationType.Enabled); + Mocker.SetConstant(Mocker.Resolve()); Mocker.SetConstant(Mocker.Resolve()); Mocker.SetConstant(Mocker.Resolve()); + Mocker.SetConstant(new X509CertificateValidationService(Mocker.GetMock().Object, TestLogger)); Mocker.SetConstant(Mocker.Resolve()); Mocker.SetConstant>(Array.Empty()); Mocker.SetConstant(Mocker.Resolve()); @@ -138,6 +138,28 @@ namespace NzbDrone.Common.Test.Http response.Content.Should().NotBeNullOrWhiteSpace(); } + [TestCase(CertificateValidationType.Enabled)] + [TestCase(CertificateValidationType.DisabledForLocalAddresses)] + public void bad_ssl_should_fail_when_remote_validation_enabled(CertificateValidationType validationType) + { + Mocker.GetMock().SetupGet(x => x.CertificateValidation).Returns(validationType); + var request = new HttpRequest($"https://expired.badssl.com"); + + Assert.Throws(() => Subject.Execute(request)); + ExceptionVerification.ExpectedErrors(1); + } + + [Test] + public void bad_ssl_should_pass_if_remote_validation_disabled() + { + Mocker.GetMock().SetupGet(x => x.CertificateValidation).Returns(CertificateValidationType.Disabled); + + var request = new HttpRequest($"https://expired.badssl.com"); + + Subject.Execute(request); + ExceptionVerification.ExpectedErrors(0); + } + [Test] public void should_execute_typed_get() { @@ -162,15 +184,44 @@ namespace NzbDrone.Common.Test.Http response.Resource.Data.Should().Be(message); } - [TestCase("gzip")] - public void should_execute_get_using_gzip(string compression) + [Test] + public void should_execute_post_with_content_type() { - var request = new HttpRequest($"https://{_httpBinHost}/{compression}"); + var message = "{ my: 1 }"; + + var request = new HttpRequest($"https://{_httpBinHost}/post"); + request.SetContent(message); + request.Headers.ContentType = "application/json"; + + var response = Subject.Post(request); + + response.Resource.Data.Should().Be(message); + } + + [Test] + public void should_execute_get_using_gzip() + { + var request = new HttpRequest($"https://{_httpBinHost}/gzip"); var response = Subject.Get(request); - response.Resource.Headers["Accept-Encoding"].ToString().Should().Be(compression); + response.Resource.Headers["Accept-Encoding"].ToString().Should().Contain("gzip"); + response.Resource.Gzipped.Should().BeTrue(); + response.Resource.Brotli.Should().BeFalse(); + } + + [Test] + public void should_execute_get_using_brotli() + { + var request = new HttpRequest($"https://{_httpBinHost}/brotli"); + + var response = Subject.Get(request); + + response.Resource.Headers["Accept-Encoding"].ToString().Should().Contain("br"); + + response.Resource.Gzipped.Should().BeFalse(); + response.Resource.Brotli.Should().BeTrue(); } [TestCase(HttpStatusCode.Unauthorized)] @@ -190,6 +241,28 @@ namespace NzbDrone.Common.Test.Http ExceptionVerification.IgnoreWarns(); } + [Test] + public void should_not_throw_on_suppressed_status_codes() + { + var request = new HttpRequest($"https://{_httpBinHost}/status/{HttpStatusCode.NotFound}"); + request.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.NotFound }; + + Assert.Throws(() => Subject.Get(request)); + + ExceptionVerification.IgnoreWarns(); + } + + [Test] + public void should_not_log_unsuccessful_status_codes() + { + var request = new HttpRequest($"https://{_httpBinHost}/status/{HttpStatusCode.NotFound}"); + request.LogHttpError = false; + + Assert.Throws(() => Subject.Get(request)); + + ExceptionVerification.ExpectedWarns(0); + } + [Test] public void should_not_follow_redirects_when_not_in_production() { @@ -315,13 +388,38 @@ namespace NzbDrone.Common.Test.Http { var file = GetTempFilePath(); - Assert.Throws(() => Subject.DownloadFile("https://download.sonarr.tv/wrongpath", file)); + Assert.Throws(() => Subject.DownloadFile("https://download.sonarr.tv/wrongpath", file)); File.Exists(file).Should().BeFalse(); ExceptionVerification.ExpectedWarns(1); } + [Test] + public void should_not_write_redirect_content_to_stream() + { + var file = GetTempFilePath(); + + using (var fileStream = new FileStream(file, FileMode.Create)) + { + var request = new HttpRequest($"http://{_httpBinHost}/redirect/1"); + request.AllowAutoRedirect = false; + request.ResponseStream = fileStream; + + var response = Subject.Get(request); + + response.StatusCode.Should().Be(HttpStatusCode.Moved); + } + + ExceptionVerification.ExpectedErrors(1); + + File.Exists(file).Should().BeTrue(); + + var fileInfo = new FileInfo(file); + + fileInfo.Length.Should().Be(0); + } + [Test] public void should_send_cookie() { @@ -753,6 +851,7 @@ namespace NzbDrone.Common.Test.Http public string Url { get; set; } public string Data { get; set; } public bool Gzipped { get; set; } + public bool Brotli { get; set; } } public class HttpCookieResource diff --git a/src/NzbDrone.Common/Extensions/StringExtensions.cs b/src/NzbDrone.Common/Extensions/StringExtensions.cs index 744309caf..28725f269 100644 --- a/src/NzbDrone.Common/Extensions/StringExtensions.cs +++ b/src/NzbDrone.Common/Extensions/StringExtensions.cs @@ -210,5 +210,26 @@ namespace NzbDrone.Common.Extensions return result.TrimStart(' ', '.').TrimEnd(' '); } + + public static string EncodeRFC3986(this string value) + { + // From Twitterizer http://www.twitterizer.net/ + if (string.IsNullOrEmpty(value)) + { + return string.Empty; + } + + var encoded = Uri.EscapeDataString(value); + + return Regex + .Replace(encoded, "(%[0-9a-f][0-9a-f])", c => c.Value.ToUpper()) + .Replace("(", "%28") + .Replace(")", "%29") + .Replace("$", "%24") + .Replace("!", "%21") + .Replace("*", "%2A") + .Replace("'", "%27") + .Replace("%7E", "~"); + } } } diff --git a/src/NzbDrone.Common/Http/BasicNetworkCredential.cs b/src/NzbDrone.Common/Http/BasicNetworkCredential.cs new file mode 100644 index 000000000..26710f766 --- /dev/null +++ b/src/NzbDrone.Common/Http/BasicNetworkCredential.cs @@ -0,0 +1,12 @@ +using System.Net; + +namespace NzbDrone.Common.Http +{ + public class BasicNetworkCredential : NetworkCredential + { + public BasicNetworkCredential(string user, string pass) + : base(user, pass) + { + } + } +} diff --git a/src/NzbDrone.Common/Http/CookieUtil.cs b/src/NzbDrone.Common/Http/CookieUtil.cs index 5b57c06a1..38c3c756b 100644 --- a/src/NzbDrone.Common/Http/CookieUtil.cs +++ b/src/NzbDrone.Common/Http/CookieUtil.cs @@ -10,6 +10,7 @@ namespace NzbDrone.Common.Http // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie // NOTE: we are not checking non-ascii characters and we should private static readonly Regex _CookieRegex = new Regex(@"([^\(\)<>@,;:\\""/\[\]\?=\{\}\s]+)=([^,;\\""\s]+)"); + private static readonly string[] FilterProps = { "COMMENT", "COMMENTURL", "DISCORD", "DOMAIN", "EXPIRES", "MAX-AGE", "PATH", "PORT", "SECURE", "VERSION", "HTTPONLY", "SAMESITE" }; private static readonly char[] InvalidKeyChars = { '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t', '\n' }; private static readonly char[] InvalidValueChars = { '"', ',', ';', '\\', ' ', '\t', '\n' }; @@ -24,7 +25,7 @@ namespace NzbDrone.Common.Http var matches = _CookieRegex.Match(cookieHeader); while (matches.Success) { - if (matches.Groups.Count > 2) + if (matches.Groups.Count > 2 && !FilterProps.Contains(matches.Groups[1].Value.ToUpperInvariant())) { cookieDictionary[matches.Groups[1].Value] = matches.Groups[2].Value; } diff --git a/src/NzbDrone.Common/Http/Dispatchers/ICertificateValidationService.cs b/src/NzbDrone.Common/Http/Dispatchers/ICertificateValidationService.cs new file mode 100644 index 000000000..187c1fd43 --- /dev/null +++ b/src/NzbDrone.Common/Http/Dispatchers/ICertificateValidationService.cs @@ -0,0 +1,11 @@ +using System.Net.Http; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; + +namespace NzbDrone.Common.Http.Dispatchers +{ + public interface ICertificateValidationService + { + bool ShouldByPassValidationError(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors); + } +} diff --git a/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs index 9afd1f5ad..a5565f26b 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs @@ -6,6 +6,5 @@ namespace NzbDrone.Common.Http.Dispatchers public interface IHttpDispatcher { Task GetResponseAsync(HttpRequest request, CookieContainer cookies); - Task DownloadFileAsync(string url, string fileName); } } diff --git a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs index 1be10eb56..20b407a69 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs @@ -1,13 +1,16 @@ using System; using System.Diagnostics; using System.IO; -using System.IO.Compression; +using System.Linq; using System.Net; -using System.Reflection; +using System.Net.Http; +using System.Net.Security; +using System.Net.Sockets; +using System.Text; +using System.Threading; using System.Threading.Tasks; using NLog; -using NLog.Fluent; -using NzbDrone.Common.EnvironmentInfo; +using NzbDrone.Common.Cache; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http.Proxy; @@ -15,221 +18,223 @@ namespace NzbDrone.Common.Http.Dispatchers { public class ManagedHttpDispatcher : IHttpDispatcher { + private const string NO_PROXY_KEY = "no-proxy"; + + private const int connection_establish_timeout = 2000; + private static bool useIPv6 = Socket.OSSupportsIPv6; + private static bool hasResolvedIPv6Availability; + private readonly IHttpProxySettingsProvider _proxySettingsProvider; private readonly ICreateManagedWebProxy _createManagedWebProxy; + private readonly ICertificateValidationService _certificateValidationService; private readonly IUserAgentBuilder _userAgentBuilder; - private readonly IPlatformInfo _platformInfo; - private readonly Logger _logger; + private readonly ICached _httpClientCache; + private readonly ICached _credentialCache; - public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, ICreateManagedWebProxy createManagedWebProxy, IUserAgentBuilder userAgentBuilder, IPlatformInfo platformInfo, Logger logger) + public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, + ICreateManagedWebProxy createManagedWebProxy, + ICertificateValidationService certificateValidationService, + IUserAgentBuilder userAgentBuilder, + ICacheManager cacheManager) { _proxySettingsProvider = proxySettingsProvider; _createManagedWebProxy = createManagedWebProxy; + _certificateValidationService = certificateValidationService; _userAgentBuilder = userAgentBuilder; - _platformInfo = platformInfo; - _logger = logger; + + _httpClientCache = cacheManager.GetCache(typeof(ManagedHttpDispatcher)); + _credentialCache = cacheManager.GetCache(typeof(ManagedHttpDispatcher), "credentialcache"); } public async Task GetResponseAsync(HttpRequest request, CookieContainer cookies) { - var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url); - - // Deflate is not a standard and could break depending on implementation. - // we should just stick with the more compatible Gzip - //http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net - webRequest.AutomaticDecompression = DecompressionMethods.GZip; + var requestMessage = new HttpRequestMessage(request.Method, (Uri)request.Url); + requestMessage.Headers.UserAgent.ParseAdd(_userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent)); + requestMessage.Headers.ConnectionClose = !request.ConnectionKeepAlive; - webRequest.Method = request.Method.ToString(); - webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent); - webRequest.KeepAlive = request.ConnectionKeepAlive; - webRequest.AllowAutoRedirect = false; - webRequest.CookieContainer = cookies; + var cookieHeader = cookies.GetCookieHeader((Uri)request.Url); + if (cookieHeader.IsNotNullOrWhiteSpace()) + { + requestMessage.Headers.Add("Cookie", cookieHeader); + } + using var cts = new CancellationTokenSource(); if (request.RequestTimeout != TimeSpan.Zero) { - webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds); + cts.CancelAfter(request.RequestTimeout); + } + else + { + // The default for System.Net.Http.HttpClient + cts.CancelAfter(TimeSpan.FromSeconds(100)); } - webRequest.Proxy = request.Proxy ?? GetProxy(request.Url); + if (request.Credentials != null) + { + if (request.Credentials is BasicNetworkCredential bc) + { + // Manually set header to avoid initial challenge response + var authInfo = bc.UserName + ":" + bc.Password; + authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(authInfo)); + requestMessage.Headers.Add("Authorization", "Basic " + authInfo); + } + else if (request.Credentials is NetworkCredential nc) + { + var creds = GetCredentialCache(); + foreach (var authtype in new[] { "Basic", "Digest" }) + { + creds.Remove((Uri)request.Url, authtype); + creds.Add((Uri)request.Url, authtype, nc); + } + } + } + + if (request.ContentData != null) + { + requestMessage.Content = new ByteArrayContent(request.ContentData); + } if (request.Headers != null) { - AddRequestHeaders(webRequest, request.Headers); + AddRequestHeaders(requestMessage, request.Headers); } - HttpWebResponse httpWebResponse; + var httpClient = GetClient(request.Url); var sw = new Stopwatch(); sw.Start(); - try - { - if (request.ContentData != null) - { - webRequest.ContentLength = request.ContentData.Length; - using (var writeStream = webRequest.GetRequestStream()) - { - writeStream.Write(request.ContentData, 0, request.ContentData.Length); - } - } - - httpWebResponse = (HttpWebResponse)await webRequest.GetResponseAsync(); - } - catch (WebException e) + using var responseMessage = await httpClient.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token); { - httpWebResponse = (HttpWebResponse)e.Response; + byte[] data = null; - if (httpWebResponse == null) + try { - // The default messages for WebException on mono are pretty horrible. - if (e.Status == WebExceptionStatus.NameResolutionFailure) + if (request.ResponseStream != null && responseMessage.StatusCode == HttpStatusCode.OK) { - throw new WebException($"DNS Name Resolution Failure: '{webRequest.RequestUri.Host}'", e.Status); - } - else if (e.ToString().Contains("TLS Support not")) - { - throw new TlsFailureException(webRequest, e); - } - else if (e.ToString().Contains("The authentication or decryption has failed.")) - { - throw new TlsFailureException(webRequest, e); - } - else if (OsInfo.IsNotWindows) - { - throw new WebException($"{e.Message}: '{webRequest.RequestUri}'", e, e.Status, e.Response); + responseMessage.Content.CopyTo(request.ResponseStream, null, cts.Token); } else { - throw; + data = responseMessage.Content.ReadAsByteArrayAsync(cts.Token).GetAwaiter().GetResult(); } } - } + catch (Exception ex) + { + throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, null); + } - byte[] data = null; + var headers = responseMessage.Headers.ToNameValueCollection(); - using (var responseStream = httpWebResponse.GetResponseStream()) - { - if (responseStream != null && responseStream != Stream.Null) + headers.Add(responseMessage.Content.Headers.ToNameValueCollection()); + + CookieContainer responseCookies = new CookieContainer(); + + if (responseMessage.Headers.TryGetValues("Set-Cookie", out var cookieHeaders)) { - try - { - data = await responseStream.ToBytes(); - } - catch (Exception ex) + foreach (var responseCookieHeader in cookieHeaders) { - throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, httpWebResponse); + try + { + cookies.SetCookies(responseMessage.RequestMessage.RequestUri, responseCookieHeader); + } + catch + { + // Ignore invalid cookies + } } } - } - sw.Stop(); + var cookieCollection = cookies.GetCookies(responseMessage.RequestMessage.RequestUri); + + sw.Stop(); - return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), httpWebResponse.Cookies, data, sw.ElapsedMilliseconds, httpWebResponse.StatusCode); + return new HttpResponse(request, new HttpHeader(headers), cookieCollection, data, sw.ElapsedMilliseconds, responseMessage.StatusCode); + } } - public async Task DownloadFileAsync(string url, string fileName) + protected virtual System.Net.Http.HttpClient GetClient(HttpUri uri) { - try - { - var fileInfo = new FileInfo(fileName); - if (fileInfo.Directory != null && !fileInfo.Directory.Exists) - { - fileInfo.Directory.Create(); - } + var proxySettings = _proxySettingsProvider.GetProxySettings(uri); - _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); + var key = proxySettings?.Key ?? NO_PROXY_KEY; - var stopWatch = Stopwatch.StartNew(); - var uri = new HttpUri(url); + return _httpClientCache.Get(key, () => CreateHttpClient(proxySettings)); + } - using (var webClient = new GZipWebClient()) - { - webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent()); - webClient.Proxy = GetProxy(uri); - await webClient.DownloadFileTaskAsync(url, fileName); - stopWatch.Stop(); - _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); - } - } - catch (WebException e) + protected virtual System.Net.Http.HttpClient CreateHttpClient(HttpProxySettings proxySettings) + { + var handler = new SocketsHttpHandler() { - _logger.Warn("Failed to get response from: {0} {1}", url, e.Message); - - if (File.Exists(fileName)) + AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Brotli, + UseCookies = false, // sic - we don't want to use a shared cookie container + AllowAutoRedirect = false, + Credentials = GetCredentialCache(), + PreAuthenticate = true, + MaxConnectionsPerServer = 12, + ConnectCallback = onConnect, + SslOptions = new SslClientAuthenticationOptions { - File.Delete(fileName); + RemoteCertificateValidationCallback = _certificateValidationService.ShouldByPassValidationError } + }; - throw; - } - catch (Exception e) + if (proxySettings != null) { - _logger.Warn(e, "Failed to get response from: " + url); - - if (File.Exists(fileName)) - { - File.Delete(fileName); - } - - throw; + handler.Proxy = _createManagedWebProxy.GetWebProxy(proxySettings); } - } - - protected virtual IWebProxy GetProxy(HttpUri uri) - { - IWebProxy proxy = null; - - var proxySettings = _proxySettingsProvider.GetProxySettings(uri); - if (proxySettings != null) + var client = new System.Net.Http.HttpClient(handler) { - proxy = _createManagedWebProxy.GetWebProxy(proxySettings); - } + Timeout = Timeout.InfiniteTimeSpan + }; - return proxy; + return client; } - protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers) + protected virtual void AddRequestHeaders(HttpRequestMessage webRequest, HttpHeader headers) { foreach (var header in headers) { switch (header.Key) { case "Accept": - webRequest.Accept = header.Value; + webRequest.Headers.Accept.ParseAdd(header.Value); break; case "Connection": - webRequest.Connection = header.Value; + webRequest.Headers.Connection.Clear(); + webRequest.Headers.Connection.Add(header.Value); break; case "Content-Length": - webRequest.ContentLength = Convert.ToInt64(header.Value); + AddContentHeader(webRequest, "Content-Length", header.Value); break; case "Content-Type": - webRequest.ContentType = header.Value; + AddContentHeader(webRequest, "Content-Type", header.Value); break; case "Date": - webRequest.Date = HttpHeader.ParseDateTime(header.Value); + webRequest.Headers.Remove("Date"); + webRequest.Headers.Date = HttpHeader.ParseDateTime(header.Value); break; case "Expect": - webRequest.Expect = header.Value; + webRequest.Headers.Expect.ParseAdd(header.Value); break; case "Host": - webRequest.Host = header.Value; + webRequest.Headers.Host = header.Value; break; case "If-Modified-Since": - webRequest.IfModifiedSince = HttpHeader.ParseDateTime(header.Value); + webRequest.Headers.IfModifiedSince = HttpHeader.ParseDateTime(header.Value); break; case "Range": throw new NotImplementedException(); case "Referer": - webRequest.Referer = header.Value; + webRequest.Headers.Add("Referer", header.Value); break; case "Transfer-Encoding": - webRequest.TransferEncoding = header.Value; + webRequest.Headers.TransferEncoding.ParseAdd(header.Value); break; case "User-Agent": - webRequest.UserAgent = header.Value; + webRequest.Headers.UserAgent.ParseAdd(header.Value); break; case "Proxy-Connection": throw new NotImplementedException(); @@ -239,5 +244,84 @@ namespace NzbDrone.Common.Http.Dispatchers } } } + + private void AddContentHeader(HttpRequestMessage request, string header, string value) + { + var headers = request.Content?.Headers; + if (headers == null) + { + return; + } + + headers.Remove(header); + headers.Add(header, value); + } + + private CredentialCache GetCredentialCache() + { + return _credentialCache.Get("credentialCache", () => new CredentialCache()); + } + + private static async ValueTask onConnect(SocketsHttpConnectionContext context, CancellationToken cancellationToken) + { + // Until .NET supports an implementation of Happy Eyeballs (https://tools.ietf.org/html/rfc8305#section-2), let's make IPv4 fallback work in a simple way. + // This issue is being tracked at https://github.com/dotnet/runtime/issues/26177 and expected to be fixed in .NET 6. + if (useIPv6) + { + try + { + var localToken = cancellationToken; + + if (!hasResolvedIPv6Availability) + { + // to make things move fast, use a very low timeout for the initial ipv6 attempt. + var quickFailCts = new CancellationTokenSource(connection_establish_timeout); + var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, quickFailCts.Token); + + localToken = linkedTokenSource.Token; + } + + return await attemptConnection(AddressFamily.InterNetworkV6, context, localToken); + } + catch + { + // very naively fallback to ipv4 permanently for this execution based on the response of the first connection attempt. + // note that this may cause users to eventually get switched to ipv4 (on a random failure when they are switching networks, for instance) + // but in the interest of keeping this implementation simple, this is acceptable. + useIPv6 = false; + } + finally + { + hasResolvedIPv6Availability = true; + } + } + + // fallback to IPv4. + return await attemptConnection(AddressFamily.InterNetwork, context, cancellationToken); + } + + private static async ValueTask attemptConnection(AddressFamily addressFamily, SocketsHttpConnectionContext context, CancellationToken cancellationToken) + { + // The following socket constructor will create a dual-mode socket on systems where IPV6 is available. + var socket = new Socket(addressFamily, SocketType.Stream, ProtocolType.Tcp) + { + // Turn off Nagle's algorithm since it degrades performance in most HttpClient scenarios. + NoDelay = true + }; + + try + { + await socket.ConnectAsync(context.DnsEndPoint, cancellationToken).ConfigureAwait(false); + + // The stream should take the ownership of the underlying socket, + // closing it when it's disposed. + return new NetworkStream(socket, ownsSocket: true); + } + catch + { + socket.Dispose(); + throw; + } + } } } diff --git a/src/NzbDrone.Common/Http/GZipWebClient.cs b/src/NzbDrone.Common/Http/GZipWebClient.cs deleted file mode 100644 index 191bfb10b..000000000 --- a/src/NzbDrone.Common/Http/GZipWebClient.cs +++ /dev/null @@ -1,15 +0,0 @@ -using System; -using System.Net; - -namespace NzbDrone.Common.Http -{ - public class GZipWebClient : WebClient - { - protected override WebRequest GetWebRequest(Uri address) - { - var request = (HttpWebRequest)base.GetWebRequest(address); - request.AutomaticDecompression = DecompressionMethods.GZip; - return request; - } - } -} diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index 7be59b663..6fb27fbb3 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.IO; using System.Linq; using System.Net; using System.Net.Http; @@ -86,13 +87,21 @@ namespace NzbDrone.Common.Http } // 302 or 303 should default to GET on redirect even if POST on original - if (response.StatusCode == HttpStatusCode.Redirect || response.StatusCode == HttpStatusCode.RedirectMethod) + if (RequestRequiresForceGet(response.StatusCode, response.Request.Method)) { request.Method = HttpMethod.Get; request.ContentData = null; } - response = await ExecuteRequestAsync(request, cookieContainer); + // Save to add to final response + var responseCookies = response.Cookies; + + // Update cookiecontainer for next request with any cookies recieved on last request + var responseContainer = HandleRedirectCookies(request, response); + + response = await ExecuteRequestAsync(request, responseContainer); + + response.Cookies.Add(responseCookies); } while (response.HasHttpRedirect); } @@ -102,9 +111,12 @@ namespace NzbDrone.Common.Http _logger.Error("Server requested a redirect to [{0}] while in developer mode. Update the request URL to avoid this redirect.", response.Headers["Location"]); } - if (!request.SuppressHttpError && response.HasHttpError) + if (!request.SuppressHttpError && response.HasHttpError && (request.SuppressHttpErrorStatusCodes == null || !request.SuppressHttpErrorStatusCodes.Contains(response.StatusCode))) { - _logger.Warn("HTTP Error - {0}", response); + if (request.LogHttpError) + { + _logger.Warn("HTTP Error - {0}", response); + } if ((int)response.StatusCode == 429) { @@ -124,6 +136,21 @@ namespace NzbDrone.Common.Http return ExecuteAsync(request).GetAwaiter().GetResult(); } + private static bool RequestRequiresForceGet(HttpStatusCode statusCode, HttpMethod requestMethod) + { + switch (statusCode) + { + case HttpStatusCode.Moved: + case HttpStatusCode.Found: + case HttpStatusCode.MultipleChoices: + return requestMethod == HttpMethod.Post; + case HttpStatusCode.SeeOther: + return requestMethod != HttpMethod.Get && requestMethod != HttpMethod.Head; + default: + return false; + } + } + private async Task ExecuteRequestAsync(HttpRequest request, CookieContainer cookieContainer) { foreach (var interceptor in _requestInterceptors) @@ -140,8 +167,6 @@ namespace NzbDrone.Common.Http var stopWatch = Stopwatch.StartNew(); - PrepareRequestCookies(request, cookieContainer); - var response = await _httpDispatcher.GetResponseAsync(request, cookieContainer); HandleResponseCookies(response, cookieContainer); @@ -208,52 +233,125 @@ namespace NzbDrone.Common.Http } } - private void PrepareRequestCookies(HttpRequest request, CookieContainer cookieContainer) + private CookieContainer HandleRedirectCookies(HttpRequest request, HttpResponse response) { - // Don't collect persistnet cookies for intermediate/redirected urls. - /*lock (_cookieContainerCache) + var sourceContainer = new CookieContainer(); + var responseCookies = response.GetCookies(); + if (responseCookies.Count != 0) { - var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); - var persistentCookies = presistentContainer.GetCookies((Uri)request.Url); - var existingCookies = cookieContainer.GetCookies((Uri)request.Url); + foreach (var pair in responseCookies) + { + Cookie cookie; + if (pair.Value == null) + { + cookie = new Cookie(pair.Key, "", "/") + { + Expires = DateTime.Now.AddDays(-1) + }; + } + else + { + cookie = new Cookie(pair.Key, pair.Value, "/") + { + // Use Now rather than UtcNow to work around Mono cookie expiry bug. + // See https://gist.github.com/ta264/7822b1424f72e5b4c961 + Expires = DateTime.Now.AddHours(1) + }; + } - cookieContainer.Add(persistentCookies); - cookieContainer.Add(existingCookies); - }*/ + sourceContainer.Add((Uri)request.Url, cookie); + } + } + + return sourceContainer; } - private void HandleResponseCookies(HttpResponse response, CookieContainer cookieContainer) + private void HandleResponseCookies(HttpResponse response, CookieContainer container) { + foreach (Cookie cookie in container.GetAllCookies()) + { + cookie.Expired = true; + } + var cookieHeaders = response.GetCookieHeaders(); if (cookieHeaders.Empty()) { return; } + AddCookiesToContainer(response.Request.Url, cookieHeaders, container); + if (response.Request.StoreResponseCookie) { lock (_cookieContainerCache) { var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); - foreach (var cookieHeader in cookieHeaders) - { - try - { - persistentCookieContainer.SetCookies((Uri)response.Request.Url, cookieHeader); - } - catch (Exception ex) - { - _logger.Debug(ex, "Invalid cookie in {0}", response.Request.Url); - } - } + AddCookiesToContainer(response.Request.Url, cookieHeaders, persistentCookieContainer); + } + } + } + + private void AddCookiesToContainer(HttpUri url, string[] cookieHeaders, CookieContainer container) + { + foreach (var cookieHeader in cookieHeaders) + { + try + { + container.SetCookies((Uri)url, cookieHeader); + } + catch (Exception ex) + { + _logger.Debug(ex, "Invalid cookie in {0}", url); } } } public async Task DownloadFileAsync(string url, string fileName) { - await _httpDispatcher.DownloadFileAsync(url, fileName); + var fileNamePart = fileName + ".part"; + + try + { + var fileInfo = new FileInfo(fileName); + if (fileInfo.Directory != null && !fileInfo.Directory.Exists) + { + fileInfo.Directory.Create(); + } + + _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); + + var stopWatch = Stopwatch.StartNew(); + using (var fileStream = new FileStream(fileNamePart, FileMode.Create, FileAccess.ReadWrite)) + { + var request = new HttpRequest(url); + request.AllowAutoRedirect = true; + request.ResponseStream = fileStream; + var response = await GetAsync(request); + + if (response.Headers.ContentType != null && response.Headers.ContentType.Contains("text/html")) + { + throw new HttpException(request, response, "Site responded with html content."); + } + } + + stopWatch.Stop(); + + if (File.Exists(fileName)) + { + File.Delete(fileName); + } + + File.Move(fileNamePart, fileName); + _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); + } + finally + { + if (File.Exists(fileNamePart)) + { + File.Delete(fileNamePart); + } + } } public void DownloadFile(string url, string fileName) diff --git a/src/NzbDrone.Common/Http/HttpHeader.cs b/src/NzbDrone.Common/Http/HttpHeader.cs index b78e27460..a5047f4b6 100644 --- a/src/NzbDrone.Common/Http/HttpHeader.cs +++ b/src/NzbDrone.Common/Http/HttpHeader.cs @@ -4,11 +4,27 @@ using System.Collections.Generic; using System.Collections.Specialized; using System.Globalization; using System.Linq; +using System.Net; +using System.Net.Http.Headers; using System.Text; using NzbDrone.Common.Extensions; namespace NzbDrone.Common.Http { + public static class WebHeaderCollectionExtensions + { + public static NameValueCollection ToNameValueCollection(this HttpHeaders headers) + { + var result = new NameValueCollection(); + foreach (var header in headers) + { + result.Add(header.Key, header.Value.ConcatToString(";")); + } + + return result; + } + } + public class HttpHeader : NameValueCollection, IEnumerable>, IEnumerable { public HttpHeader(NameValueCollection headers) @@ -16,6 +32,11 @@ namespace NzbDrone.Common.Http { } + public HttpHeader(HttpHeaders headers) + : base(headers.ToNameValueCollection()) + { + } + public HttpHeader() { } diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs index 4a0b7a85a..5092207bc 100644 --- a/src/NzbDrone.Common/Http/HttpRequest.cs +++ b/src/NzbDrone.Common/Http/HttpRequest.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Net; using System.Net.Http; using System.Text; @@ -12,11 +13,13 @@ namespace NzbDrone.Common.Http { public HttpRequest(string url, HttpAccept httpAccept = null) { + Method = HttpMethod.Get; Url = new HttpUri(url); Headers = new HttpHeader(); Method = HttpMethod.Get; ConnectionKeepAlive = true; AllowAutoRedirect = true; + LogHttpError = true; Cookies = new Dictionary(); if (!RuntimeInfo.IsProduction) @@ -37,16 +40,20 @@ namespace NzbDrone.Common.Http public IWebProxy Proxy { get; set; } public byte[] ContentData { get; set; } public string ContentSummary { get; set; } + public ICredentials Credentials { get; set; } public bool SuppressHttpError { get; set; } + public IEnumerable SuppressHttpErrorStatusCodes { get; set; } public bool UseSimplifiedUserAgent { get; set; } public bool AllowAutoRedirect { get; set; } public bool ConnectionKeepAlive { get; set; } public bool LogResponseContent { get; set; } + public bool LogHttpError { get; set; } public Dictionary Cookies { get; private set; } public bool StoreRequestCookie { get; set; } public bool StoreResponseCookie { get; set; } public TimeSpan RequestTimeout { get; set; } public TimeSpan RateLimit { get; set; } + public Stream ResponseStream { get; set; } public override string ToString() { @@ -103,12 +110,5 @@ namespace NzbDrone.Common.Http return encoding.GetString(ContentData); } } - - public void AddBasicAuthentication(string username, string password) - { - var authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes($"{username}:{password}")); - - Headers.Set("Authorization", "Basic " + authInfo); - } } } diff --git a/src/NzbDrone.Common/Http/HttpRequestBuilder.cs b/src/NzbDrone.Common/Http/HttpRequestBuilder.cs index 86b40b319..7d75291b3 100644 --- a/src/NzbDrone.Common/Http/HttpRequestBuilder.cs +++ b/src/NzbDrone.Common/Http/HttpRequestBuilder.cs @@ -21,12 +21,13 @@ namespace NzbDrone.Common.Http public Dictionary Segments { get; private set; } public HttpHeader Headers { get; private set; } public bool SuppressHttpError { get; set; } + public bool LogHttpError { get; set; } public bool UseSimplifiedUserAgent { get; set; } public bool AllowAutoRedirect { get; set; } public bool ConnectionKeepAlive { get; set; } public TimeSpan RateLimit { get; set; } public bool LogResponseContent { get; set; } - public NetworkCredential NetworkCredential { get; set; } + public ICredentials NetworkCredential { get; set; } public Dictionary Cookies { get; private set; } public bool StoreRequestCookie { get; set; } public bool StoreResponseCookie { get; set; } @@ -46,6 +47,7 @@ namespace NzbDrone.Common.Http Headers = new HttpHeader(); Cookies = new Dictionary(); FormData = new List(); + LogHttpError = true; } public HttpRequestBuilder(bool useHttps, string host, int port, string urlBase = null) @@ -106,6 +108,7 @@ namespace NzbDrone.Common.Http request.Method = Method; request.Encoding = Encoding; request.SuppressHttpError = SuppressHttpError; + request.LogHttpError = LogHttpError; request.UseSimplifiedUserAgent = UseSimplifiedUserAgent; request.AllowAutoRedirect = AllowAutoRedirect; request.StoreRequestCookie = StoreRequestCookie; @@ -113,13 +116,7 @@ namespace NzbDrone.Common.Http request.ConnectionKeepAlive = ConnectionKeepAlive; request.RateLimit = RateLimit; request.LogResponseContent = LogResponseContent; - - if (NetworkCredential != null) - { - var authInfo = NetworkCredential.UserName + ":" + NetworkCredential.Password; - authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(authInfo)); - request.Headers.Set("Authorization", "Basic " + authInfo); - } + request.Credentials = NetworkCredential; foreach (var header in Headers) { diff --git a/src/NzbDrone.Common/Http/HttpResponse.cs b/src/NzbDrone.Common/Http/HttpResponse.cs index 819b3b805..c92444d76 100644 --- a/src/NzbDrone.Common/Http/HttpResponse.cs +++ b/src/NzbDrone.Common/Http/HttpResponse.cs @@ -64,10 +64,11 @@ namespace NzbDrone.Common.Http public bool HasHttpError => (int)StatusCode >= 400; public bool HasHttpRedirect => StatusCode == HttpStatusCode.Moved || - StatusCode == HttpStatusCode.MovedPermanently || - StatusCode == HttpStatusCode.RedirectMethod || - StatusCode == HttpStatusCode.TemporaryRedirect || StatusCode == HttpStatusCode.Found || + StatusCode == HttpStatusCode.SeeOther || + StatusCode == HttpStatusCode.TemporaryRedirect || + StatusCode == HttpStatusCode.MultipleChoices || + StatusCode == HttpStatusCode.PermanentRedirect || Headers.ContainsKey("Refresh"); public string RedirectUrl @@ -117,7 +118,7 @@ namespace NzbDrone.Common.Http public override string ToString() { - var result = string.Format("Res: [{0}] {1}: {2}.{3}", Request.Method, Request.Url, (int)StatusCode, StatusCode); + var result = string.Format("Res: [{0}] {1}: {2}.{3} ({4} bytes)", Request.Method, Request.Url, (int)StatusCode, StatusCode, ResponseData?.Length ?? 0); if (HasHttpError && Headers.ContentType.IsNotNullOrWhiteSpace() && !Headers.ContentType.Equals("text/html", StringComparison.InvariantCultureIgnoreCase)) { diff --git a/src/NzbDrone.Common/Http/XmlRpcRequestBuilder.cs b/src/NzbDrone.Common/Http/XmlRpcRequestBuilder.cs new file mode 100644 index 000000000..e03161702 --- /dev/null +++ b/src/NzbDrone.Common/Http/XmlRpcRequestBuilder.cs @@ -0,0 +1,103 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Net.Http; +using System.Xml.Linq; +using NLog; +using NzbDrone.Common.Instrumentation; + +namespace NzbDrone.Common.Http +{ + public class XmlRpcRequestBuilder : HttpRequestBuilder + { + public static string XmlRpcContentType = "text/xml"; + + private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(XmlRpcRequestBuilder)); + + public string XmlMethod { get; private set; } + public List XmlParameters { get; private set; } + + public XmlRpcRequestBuilder(string baseUrl) + : base(baseUrl) + { + Method = HttpMethod.Post; + XmlParameters = new List(); + } + + public XmlRpcRequestBuilder(bool useHttps, string host, int port, string urlBase = null) + : this(BuildBaseUrl(useHttps, host, port, urlBase)) + { + } + + public override HttpRequestBuilder Clone() + { + var clone = base.Clone() as XmlRpcRequestBuilder; + clone.XmlParameters = new List(XmlParameters); + return clone; + } + + public XmlRpcRequestBuilder Call(string method, params object[] parameters) + { + var clone = Clone() as XmlRpcRequestBuilder; + clone.XmlMethod = method; + clone.XmlParameters = parameters.ToList(); + return clone; + } + + protected override void Apply(HttpRequest request) + { + base.Apply(request); + + request.Headers.ContentType = XmlRpcContentType; + + var methodCallElements = new List { new XElement("methodName", XmlMethod) }; + + if (XmlParameters.Any()) + { + var argElements = XmlParameters.Select(x => new XElement("param", ConvertParameter(x))).ToList(); + var paramsElement = new XElement("params", argElements); + methodCallElements.Add(paramsElement); + } + + var message = new XDocument( + new XDeclaration("1.0", "utf-8", "yes"), + new XElement("methodCall", methodCallElements)); + + var body = message.ToString(); + + Logger.Debug($"Executing remote method: {XmlMethod}"); + + Logger.Trace($"methodCall {XmlMethod} body:\n{body}"); + + request.SetContent(body); + } + + private static XElement ConvertParameter(object value) + { + XElement data; + + if (value is string s) + { + data = new XElement("string", s); + } + else if (value is List l) + { + data = new XElement("array", new XElement("data", l.Select(x => new XElement("value", new XElement("string", x))))); + } + else if (value is int i) + { + data = new XElement("int", i); + } + else if (value is byte[] bytes) + { + data = new XElement("base64", Convert.ToBase64String(bytes)); + } + else + { + throw new InvalidOperationException($"Unhandled argument type {value.GetType().Name}"); + } + + return new XElement("value", data); + } + } +} diff --git a/src/NzbDrone.Core.Test/Framework/CoreTest.cs b/src/NzbDrone.Core.Test/Framework/CoreTest.cs index 9da38329f..ef5202734 100644 --- a/src/NzbDrone.Core.Test/Framework/CoreTest.cs +++ b/src/NzbDrone.Core.Test/Framework/CoreTest.cs @@ -11,6 +11,7 @@ using NzbDrone.Common.TPL; using NzbDrone.Core.Configuration; using NzbDrone.Core.Http; using NzbDrone.Core.Parser; +using NzbDrone.Core.Security; using NzbDrone.Test.Common; namespace NzbDrone.Core.Test.Framework @@ -25,7 +26,8 @@ namespace NzbDrone.Core.Test.Framework Mocker.SetConstant(new HttpProxySettingsProvider(Mocker.Resolve())); Mocker.SetConstant(new ManagedWebProxyFactory(Mocker.Resolve())); - Mocker.SetConstant(new ManagedHttpDispatcher(Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); + Mocker.SetConstant(new X509CertificateValidationService(Mocker.Resolve(), TestLogger)); + Mocker.SetConstant(new ManagedHttpDispatcher(Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve())); Mocker.SetConstant(new HttpClient(Array.Empty(), Mocker.Resolve(), Mocker.Resolve(), Mocker.Resolve(), TestLogger)); Mocker.SetConstant(new ProwlarrCloudRequestBuilder()); } diff --git a/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs index d4ab5a49c..24122b65c 100644 --- a/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs +++ b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Containers.cs @@ -1,111 +1,161 @@ -using CookComputing.XmlRpc; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; +using System.Xml.XPath; +using NzbDrone.Core.Download.Extensions; namespace NzbDrone.Core.Download.Clients.Aria2 { - public class Aria2Version + public class Aria2Fault { - [XmlRpcMember("version")] - public string Version; - - [XmlRpcMember("enabledFeatures")] - public string[] EnabledFeatures; + public Aria2Fault(XElement element) + { + foreach (var e in element.XPathSelectElements("./value/struct/member")) + { + var name = e.ElementAsString("name"); + if (name == "faultCode") + { + FaultCode = e.Element("value").ElementAsInt("int"); + } + else if (name == "faultString") + { + FaultString = e.Element("value").GetStringValue(); + } + } + } + + public int FaultCode { get; set; } + public string FaultString { get; set; } } - public class Aria2Uri + public class Aria2Version { - [XmlRpcMember("status")] - public string Status; - - [XmlRpcMember("uri")] - public string Uri; + public Aria2Version(XElement element) + { + foreach (var e in element.XPathSelectElements("./struct/member")) + { + if (e.ElementAsString("name") == "version") + { + Version = e.Element("value").GetStringValue(); + } + } + } + + public string Version { get; set; } } public class Aria2File { - [XmlRpcMember("index")] - public string Index; - - [XmlRpcMember("length")] - public string Length; + public Aria2File(XElement element) + { + foreach (var e in element.XPathSelectElements("./struct/member")) + { + var name = e.ElementAsString("name"); + + if (name == "path") + { + Path = e.Element("value").GetStringValue(); + } + } + } + + public string Path { get; set; } + } - [XmlRpcMember("completedLength")] - public string CompletedLength; + public class Aria2Dict + { + public Aria2Dict(XElement element) + { + Dict = new Dictionary(); - [XmlRpcMember("path")] - public string Path; + foreach (var e in element.XPathSelectElements("./struct/member")) + { + Dict.Add(e.ElementAsString("name"), e.Element("value").GetStringValue()); + } + } - [XmlRpcMember("selected")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string Selected; + public Dictionary Dict { get; set; } + } - [XmlRpcMember("uris")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public Aria2Uri[] Uris; + public class Aria2Bittorrent + { + public Aria2Bittorrent(XElement element) + { + foreach (var e in element.Descendants("member")) + { + if (e.ElementAsString("name") == "name") + { + Name = e.Element("value").GetStringValue(); + } + } + } + + public string Name; } public class Aria2Status { - [XmlRpcMember("bittorrent")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public XmlRpcStruct Bittorrent; - - [XmlRpcMember("bitfield")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string Bitfield; - - [XmlRpcMember("infoHash")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string InfoHash; - - [XmlRpcMember("completedLength")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string CompletedLength; - - [XmlRpcMember("connections")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string Connections; - - [XmlRpcMember("dir")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string Dir; - - [XmlRpcMember("downloadSpeed")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string DownloadSpeed; - - [XmlRpcMember("files")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public Aria2File[] Files; - - [XmlRpcMember("gid")] - public string Gid; - - [XmlRpcMember("numPieces")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string NumPieces; - - [XmlRpcMember("pieceLength")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string PieceLength; - - [XmlRpcMember("status")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string Status; - - [XmlRpcMember("totalLength")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string TotalLength; - - [XmlRpcMember("uploadLength")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string UploadLength; - - [XmlRpcMember("uploadSpeed")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string UploadSpeed; - - [XmlRpcMember("errorMessage")] - [XmlRpcMissingMapping(MappingAction.Ignore)] - public string ErrorMessage; + public Aria2Status(XElement element) + { + foreach (var e in element.XPathSelectElements("./struct/member")) + { + var name = e.ElementAsString("name"); + + if (name == "bittorrent") + { + Bittorrent = new Aria2Bittorrent(e.Element("value")); + } + else if (name == "infoHash") + { + InfoHash = e.Element("value").GetStringValue(); + } + else if (name == "completedLength") + { + CompletedLength = e.Element("value").GetStringValue(); + } + else if (name == "downloadSpeed") + { + DownloadSpeed = e.Element("value").GetStringValue(); + } + else if (name == "files") + { + Files = e.XPathSelectElement("./value/array/data") + .Elements() + .Select(x => new Aria2File(x)) + .ToArray(); + } + else if (name == "gid") + { + Gid = e.Element("value").GetStringValue(); + } + else if (name == "status") + { + Status = e.Element("value").GetStringValue(); + } + else if (name == "totalLength") + { + TotalLength = e.Element("value").GetStringValue(); + } + else if (name == "uploadLength") + { + UploadLength = e.Element("value").GetStringValue(); + } + else if (name == "errorMessage") + { + ErrorMessage = e.Element("value").GetStringValue(); + } + } + } + + public Aria2Bittorrent Bittorrent { get; set; } + public string InfoHash { get; set; } + public string CompletedLength { get; set; } + public string DownloadSpeed { get; set; } + public Aria2File[] Files { get; set; } + public string Gid { get; set; } + public string Status { get; set; } + public string TotalLength { get; set; } + public string UploadLength { get; set; } + public string ErrorMessage { get; set; } } } diff --git a/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs index 855d300ba..9cc2d9793 100644 --- a/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs +++ b/src/NzbDrone.Core/Download/Clients/Aria2/Aria2Proxy.cs @@ -1,9 +1,9 @@ -using System; -using System.Collections; using System.Collections.Generic; -using System.Net; -using CookComputing.XmlRpc; -using NLog; +using System.Linq; +using System.Xml.Linq; +using System.Xml.XPath; +using NzbDrone.Common.Http; +using NzbDrone.Core.Download.Extensions; namespace NzbDrone.Core.Download.Clients.Aria2 { @@ -12,133 +12,133 @@ namespace NzbDrone.Core.Download.Clients.Aria2 string GetVersion(Aria2Settings settings); string AddUri(Aria2Settings settings, string magnet); string AddTorrent(Aria2Settings settings, byte[] torrent); + Dictionary GetGlobals(Aria2Settings settings); + List GetTorrents(Aria2Settings settings); Aria2Status GetFromGID(Aria2Settings settings, string gid); } - public interface IAria2 : IXmlRpcProxy + public class Aria2Proxy : IAria2Proxy { - [XmlRpcMethod("aria2.getVersion")] - Aria2Version GetVersion(string token); - - [XmlRpcMethod("aria2.addUri")] - string AddUri(string token, string[] uri); - - [XmlRpcMethod("aria2.addTorrent")] - string AddTorrent(string token, byte[] torrent); + private readonly IHttpClient _httpClient; - [XmlRpcMethod("aria2.forceRemove")] - string Remove(string token, string gid); + public Aria2Proxy(IHttpClient httpClient) + { + _httpClient = httpClient; + } - [XmlRpcMethod("aria2.tellStatus")] - Aria2Status GetFromGid(string token, string gid); + public string GetVersion(Aria2Settings settings) + { + var response = ExecuteRequest(settings, "aria2.getVersion", GetToken(settings)); - [XmlRpcMethod("aria2.getGlobalOption")] - XmlRpcStruct GetGlobalOption(string token); + var element = response.XPathSelectElement("./methodResponse/params/param/value"); - [XmlRpcMethod("aria2.tellActive")] - Aria2Status[] GetActives(string token); + var version = new Aria2Version(element); - [XmlRpcMethod("aria2.tellWaiting")] - Aria2Status[] GetWaitings(string token, int offset, int num); + return version.Version; + } - [XmlRpcMethod("aria2.tellStopped")] - Aria2Status[] GetStoppeds(string token, int offset, int num); - } + public Aria2Status GetFromGID(Aria2Settings settings, string gid) + { + var response = ExecuteRequest(settings, "aria2.tellStatus", GetToken(settings), gid); - public class Aria2Proxy : IAria2Proxy - { - private readonly Logger _logger; + var element = response.XPathSelectElement("./methodResponse/params/param/value"); - public Aria2Proxy(Logger logger) - { - _logger = logger; + return new Aria2Status(element); } - private string GetToken(Aria2Settings settings) + private List GetTorrentsMethod(Aria2Settings settings, string method, params object[] args) { - return $"token:{settings?.SecretToken}"; - } + var allArgs = new List { GetToken(settings) }; + if (args.Any()) + { + allArgs.AddRange(args); + } - private string GetURL(Aria2Settings settings) - { - return $"http{(settings.UseSsl ? "s" : "")}://{settings.Host}:{settings.Port}{settings.RpcPath}"; + var response = ExecuteRequest(settings, method, allArgs.ToArray()); + + var element = response.XPathSelectElement("./methodResponse/params/param/value/array/data"); + + var torrents = element?.Elements() + .Select(x => new Aria2Status(x)) + .ToList() + ?? new List(); + return torrents; } - public string GetVersion(Aria2Settings settings) + public List GetTorrents(Aria2Settings settings) { - _logger.Debug("> aria2.getVersion"); + var active = GetTorrentsMethod(settings, "aria2.tellActive"); - var client = BuildClient(settings); - var version = ExecuteRequest(() => client.GetVersion(GetToken(settings))); + var waiting = GetTorrentsMethod(settings, "aria2.tellWaiting", 0, 10 * 1024); - _logger.Debug("< aria2.getVersion"); + var stopped = GetTorrentsMethod(settings, "aria2.tellStopped", 0, 10 * 1024); - return version.Version; + var items = new List(); + + items.AddRange(active); + items.AddRange(waiting); + items.AddRange(stopped); + + return items; } - public Aria2Status GetFromGID(Aria2Settings settings, string gid) + public Dictionary GetGlobals(Aria2Settings settings) { - _logger.Debug("> aria2.tellStatus"); + var response = ExecuteRequest(settings, "aria2.getGlobalOption", GetToken(settings)); - var client = BuildClient(settings); - var found = ExecuteRequest(() => client.GetFromGid(GetToken(settings), gid)); + var element = response.XPathSelectElement("./methodResponse/params/param/value"); - _logger.Debug("< aria2.tellStatus"); + var result = new Aria2Dict(element); - return found; + return result.Dict; } public string AddUri(Aria2Settings settings, string magnet) { - _logger.Debug("> aria2.addUri"); - - var client = BuildClient(settings); - var gid = ExecuteRequest(() => client.AddUri(GetToken(settings), new[] { magnet })); + var response = ExecuteRequest(settings, "aria2.addUri", GetToken(settings), new List { magnet }); - _logger.Debug("< aria2.addUri"); + var gid = response.GetStringResponse(); return gid; } public string AddTorrent(Aria2Settings settings, byte[] torrent) { - _logger.Debug("> aria2.addTorrent"); - - var client = BuildClient(settings); - var gid = ExecuteRequest(() => client.AddTorrent(GetToken(settings), torrent)); + var response = ExecuteRequest(settings, "aria2.addTorrent", GetToken(settings), torrent); - _logger.Debug("< aria2.addTorrent"); + var gid = response.GetStringResponse(); return gid; } - private IAria2 BuildClient(Aria2Settings settings) + private string GetToken(Aria2Settings settings) { - var client = XmlRpcProxyGen.Create(); - client.Url = GetURL(settings); - - return client; + return $"token:{settings?.SecretToken}"; } - private T ExecuteRequest(Func task) + private XDocument ExecuteRequest(Aria2Settings settings, string methodName, params object[] args) { - try - { - return task(); - } - catch (XmlRpcServerException ex) + var requestBuilder = new XmlRpcRequestBuilder(settings.UseSsl, settings.Host, settings.Port, settings.RpcPath) { - throw new DownloadClientException("Unable to connect to aria2, please check your settings", ex); - } - catch (WebException ex) + LogResponseContent = true, + }; + + var request = requestBuilder.Call(methodName, args).Build(); + + var response = _httpClient.Execute(request); + + var doc = XDocument.Parse(response.Content); + + var faultElement = doc.XPathSelectElement("./methodResponse/fault"); + + if (faultElement != null) { - if (ex.Status == WebExceptionStatus.TrustFailure) - { - throw new DownloadClientUnavailableException("Unable to connect to aria2, certificate validation failed.", ex); - } + var fault = new Aria2Fault(faultElement); - throw new DownloadClientUnavailableException("Unable to connect to aria2, please check your settings", ex); + throw new DownloadClientException($"Aria2 returned error code {fault.FaultCode}: {fault.FaultString}"); } + + return doc; } } } diff --git a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentFault.cs b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentFault.cs new file mode 100644 index 000000000..ba1a23962 --- /dev/null +++ b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentFault.cs @@ -0,0 +1,28 @@ +using System.Xml.Linq; +using System.Xml.XPath; +using NzbDrone.Core.Download.Extensions; + +namespace NzbDrone.Core.Download.Clients.RTorrent +{ + public class RTorrentFault + { + public RTorrentFault(XElement element) + { + foreach (var e in element.XPathSelectElements("./value/struct/member")) + { + var name = e.ElementAsString("name"); + if (name == "faultCode") + { + FaultCode = e.Element("value").GetIntValue(); + } + else if (name == "faultString") + { + FaultString = e.Element("value").GetStringValue(); + } + } + } + + public int FaultCode { get; set; } + public string FaultString { get; set; } + } +} diff --git a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs index c7f72952f..fb32c0598 100644 --- a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs +++ b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentProxy.cs @@ -1,10 +1,12 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Net; -using CookComputing.XmlRpc; -using NLog; +using System.Xml.Linq; +using System.Xml.XPath; using NzbDrone.Common.Extensions; -using NzbDrone.Common.Serializer; +using NzbDrone.Common.Http; +using NzbDrone.Core.Download.Extensions; namespace NzbDrone.Core.Download.Clients.RTorrent { @@ -18,122 +20,70 @@ namespace NzbDrone.Core.Download.Clients.RTorrent void RemoveTorrent(string hash, RTorrentSettings settings); void SetTorrentLabel(string hash, string label, RTorrentSettings settings); bool HasHashTorrent(string hash, RTorrentSettings settings); - } - - public interface IRTorrent : IXmlRpcProxy - { - [XmlRpcMethod("d.multicall2")] - object[] TorrentMulticall(params string[] parameters); - - [XmlRpcMethod("load.normal")] - int LoadNormal(string target, string data, params string[] commands); - - [XmlRpcMethod("load.start")] - int LoadStart(string target, string data, params string[] commands); - - [XmlRpcMethod("load.raw")] - int LoadRaw(string target, byte[] data, params string[] commands); - - [XmlRpcMethod("load.raw_start")] - int LoadRawStart(string target, byte[] data, params string[] commands); - - [XmlRpcMethod("d.erase")] - int Remove(string hash); - - [XmlRpcMethod("d.name")] - string GetName(string hash); - - [XmlRpcMethod("d.custom1.set")] - string SetLabel(string hash, string label); - - [XmlRpcMethod("system.client_version")] - string GetVersion(); + void PushTorrentUniqueView(string hash, string view, RTorrentSettings settings); } public class RTorrentProxy : IRTorrentProxy { - private readonly Logger _logger; + private readonly IHttpClient _httpClient; - public RTorrentProxy(Logger logger) + public RTorrentProxy(IHttpClient httpClient) { - _logger = logger; + _httpClient = httpClient; } public string GetVersion(RTorrentSettings settings) { - _logger.Debug("Executing remote method: system.client_version"); - - var client = BuildClient(settings); - var version = ExecuteRequest(() => client.GetVersion()); + var document = ExecuteRequest(settings, "system.client_version"); - return version; + return document.Descendants("string").FirstOrDefault()?.Value ?? "0.0.0"; } public List GetTorrents(RTorrentSettings settings) { - _logger.Debug("Executing remote method: d.multicall2"); - - var client = BuildClient(settings); - var ret = ExecuteRequest(() => client.TorrentMulticall( - "", - "", - "d.name=", // string - "d.hash=", // string - "d.base_path=", // string - "d.custom1=", // string (label) - "d.size_bytes=", // long - "d.left_bytes=", // long - "d.down.rate=", // long (in bytes / s) - "d.ratio=", // long - "d.is_open=", // long - "d.is_active=", // long - "d.complete=")); //long - - _logger.Trace(ret.ToJson()); - - var items = new List(); - - foreach (object[] torrent in ret) - { - var labelDecoded = System.Web.HttpUtility.UrlDecode((string)torrent[3]); - - var item = new RTorrentTorrent(); - item.Name = (string)torrent[0]; - item.Hash = (string)torrent[1]; - item.Path = (string)torrent[2]; - item.Category = labelDecoded; - item.TotalSize = (long)torrent[4]; - item.RemainingSize = (long)torrent[5]; - item.DownRate = (long)torrent[6]; - item.Ratio = (long)torrent[7]; - item.IsOpen = Convert.ToBoolean((long)torrent[8]); - item.IsActive = Convert.ToBoolean((long)torrent[9]); - item.IsFinished = Convert.ToBoolean((long)torrent[10]); - - items.Add(item); - } - - return items; + var document = ExecuteRequest(settings, + "d.multicall2", + "", + "", + "d.name=", // string + "d.hash=", // string + "d.base_path=", // string + "d.custom1=", // string (label) + "d.size_bytes=", // long + "d.left_bytes=", // long + "d.down.rate=", // long (in bytes / s) + "d.ratio=", // long + "d.is_open=", // long + "d.is_active=", // long + "d.complete=", //long + "d.timestamp.finished="); // long (unix timestamp) + + var torrents = document.XPathSelectElement("./methodResponse/params/param/value/array/data") + ?.Elements() + .Select(x => new RTorrentTorrent(x)) + .ToList() + ?? new List(); + + return torrents; } public void AddTorrentFromUrl(string torrentUrl, string label, RTorrentPriority priority, string directory, RTorrentSettings settings) { - var client = BuildClient(settings); - var response = ExecuteRequest(() => + var args = new List { "", torrentUrl }; + args.AddRange(GetCommands(label, priority, directory)); + + XDocument response; + + if (settings.AddStopped) { - if (settings.AddStopped) - { - _logger.Debug("Executing remote method: load.normal"); - return client.LoadNormal("", torrentUrl, GetCommands(label, priority, directory)); - } - else - { - _logger.Debug("Executing remote method: load.start"); - return client.LoadStart("", torrentUrl, GetCommands(label, priority, directory)); - } - }); + response = ExecuteRequest(settings, "load.normal", args.ToArray()); + } + else + { + response = ExecuteRequest(settings, "load.start", args.ToArray()); + } - if (response != 0) + if (response.GetIntResponse() != 0) { throw new DownloadClientException("Could not add torrent: {0}.", torrentUrl); } @@ -141,22 +91,21 @@ namespace NzbDrone.Core.Download.Clients.RTorrent public void AddTorrentFromFile(string fileName, byte[] fileContent, string label, RTorrentPriority priority, string directory, RTorrentSettings settings) { - var client = BuildClient(settings); - var response = ExecuteRequest(() => + var args = new List { "", fileContent }; + args.AddRange(GetCommands(label, priority, directory)); + + XDocument response; + + if (settings.AddStopped) { - if (settings.AddStopped) - { - _logger.Debug("Executing remote method: load.raw"); - return client.LoadRaw("", fileContent, GetCommands(label, priority, directory)); - } - else - { - _logger.Debug("Executing remote method: load.raw_start"); - return client.LoadRawStart("", fileContent, GetCommands(label, priority, directory)); - } - }); + response = ExecuteRequest(settings, "load.raw", args.ToArray()); + } + else + { + response = ExecuteRequest(settings, "load.raw_start", args.ToArray()); + } - if (response != 0) + if (response.GetIntResponse() != 0) { throw new DownloadClientException("Could not add torrent: {0}.", fileName); } @@ -164,25 +113,29 @@ namespace NzbDrone.Core.Download.Clients.RTorrent public void SetTorrentLabel(string hash, string label, RTorrentSettings settings) { - _logger.Debug("Executing remote method: d.custom1.set"); - - var client = BuildClient(settings); - var response = ExecuteRequest(() => client.SetLabel(hash, label)); + var response = ExecuteRequest(settings, "d.custom1.set", hash, label); - if (response != label) + if (response.GetStringResponse() != label) { throw new DownloadClientException("Could not set label to {1} for torrent: {0}.", hash, label); } } - public void RemoveTorrent(string hash, RTorrentSettings settings) + public void PushTorrentUniqueView(string hash, string view, RTorrentSettings settings) { - _logger.Debug("Executing remote method: d.erase"); + var response = ExecuteRequest(settings, "d.views.push_back_unique", hash, view); - var client = BuildClient(settings); - var response = ExecuteRequest(() => client.Remove(hash)); + if (response.GetIntResponse() != 0) + { + throw new DownloadClientException("Could not push unique view {0} for torrent: {1}.", view, hash); + } + } - if (response != 0) + public void RemoveTorrent(string hash, RTorrentSettings settings) + { + var response = ExecuteRequest(settings, "d.erase", hash); + + if (response.GetIntResponse() != 0) { throw new DownloadClientException("Could not remove torrent: {0}.", hash); } @@ -190,13 +143,10 @@ namespace NzbDrone.Core.Download.Clients.RTorrent public bool HasHashTorrent(string hash, RTorrentSettings settings) { - _logger.Debug("Executing remote method: d.name"); - - var client = BuildClient(settings); - try { - var name = ExecuteRequest(() => client.GetName(hash)); + var response = ExecuteRequest(settings, "d.name", hash); + var name = response.GetStringResponse(); if (name.IsNullOrWhiteSpace()) { @@ -235,45 +185,34 @@ namespace NzbDrone.Core.Download.Clients.RTorrent return result.ToArray(); } - private IRTorrent BuildClient(RTorrentSettings settings) + private XDocument ExecuteRequest(RTorrentSettings settings, string methodName, params object[] args) { - var client = XmlRpcProxyGen.Create(); - - client.Url = string.Format(@"{0}://{1}:{2}/{3}", - settings.UseSsl ? "https" : "http", - settings.Host, - settings.Port, - settings.UrlBase); - - client.EnableCompression = true; + var requestBuilder = new XmlRpcRequestBuilder(settings.UseSsl, settings.Host, settings.Port, settings.UrlBase) + { + LogResponseContent = true, + }; if (!settings.Username.IsNullOrWhiteSpace()) { - client.Credentials = new NetworkCredential(settings.Username, settings.Password); + requestBuilder.NetworkCredential = new NetworkCredential(settings.Username, settings.Password); } - return client; - } + var request = requestBuilder.Call(methodName, args).Build(); - private T ExecuteRequest(Func task) - { - try - { - return task(); - } - catch (XmlRpcServerException ex) - { - throw new DownloadClientException("Unable to connect to rTorrent, please check your settings", ex); - } - catch (WebException ex) + var response = _httpClient.Execute(request); + + var doc = XDocument.Parse(response.Content); + + var faultElement = doc.XPathSelectElement("./methodResponse/fault"); + + if (faultElement != null) { - if (ex.Status == WebExceptionStatus.TrustFailure) - { - throw new DownloadClientUnavailableException("Unable to connect to rTorrent, certificate validation failed.", ex); - } + var fault = new RTorrentFault(faultElement); - throw new DownloadClientUnavailableException("Unable to connect to rTorrent, please check your settings", ex); + throw new DownloadClientException($"rTorrent returned error code {fault.FaultCode}: {fault.FaultString}"); } + + return doc; } } } diff --git a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentTorrent.cs b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentTorrent.cs index d00df188f..0dc1165d0 100644 --- a/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentTorrent.cs +++ b/src/NzbDrone.Core/Download/Clients/rTorrent/RTorrentTorrent.cs @@ -1,7 +1,35 @@ -namespace NzbDrone.Core.Download.Clients.RTorrent +using System; +using System.Linq; +using System.Web; +using System.Xml.Linq; +using NzbDrone.Core.Download.Extensions; + +namespace NzbDrone.Core.Download.Clients.RTorrent { public class RTorrentTorrent { + public RTorrentTorrent() + { + } + + public RTorrentTorrent(XElement element) + { + var data = element.Descendants("value").ToList(); + + Name = data[0].GetStringValue(); + Hash = data[1].GetStringValue(); + Path = data[2].GetStringValue(); + Category = HttpUtility.UrlDecode(data[3].GetStringValue()); + TotalSize = data[4].GetLongValue(); + RemainingSize = data[5].GetLongValue(); + DownRate = data[6].GetLongValue(); + Ratio = data[7].GetLongValue(); + IsOpen = Convert.ToBoolean(data[8].GetLongValue()); + IsActive = Convert.ToBoolean(data[9].GetLongValue()); + IsFinished = Convert.ToBoolean(data[10].GetLongValue()); + FinishedTime = data[11].GetLongValue(); + } + public string Name { get; set; } public string Hash { get; set; } public string Path { get; set; } @@ -10,6 +38,7 @@ public long RemainingSize { get; set; } public long DownRate { get; set; } public long Ratio { get; set; } + public long FinishedTime { get; set; } public bool IsFinished { get; set; } public bool IsOpen { get; set; } public bool IsActive { get; set; } diff --git a/src/NzbDrone.Core/Download/Extensions/XmlExtensions.cs b/src/NzbDrone.Core/Download/Extensions/XmlExtensions.cs new file mode 100644 index 000000000..bce66c8bc --- /dev/null +++ b/src/NzbDrone.Core/Download/Extensions/XmlExtensions.cs @@ -0,0 +1,54 @@ +using System.Xml.Linq; +using System.Xml.XPath; + +namespace NzbDrone.Core.Download.Extensions +{ + internal static class XmlExtensions + { + public static string GetStringValue(this XElement element) + { + return element.ElementAsString("string"); + } + + public static long GetLongValue(this XElement element) + { + return element.ElementAsLong("i8"); + } + + public static int GetIntValue(this XElement element) + { + return element.ElementAsInt("i4"); + } + + public static string ElementAsString(this XElement element, XName name, bool trim = false) + { + var el = element.Element(name); + + return string.IsNullOrWhiteSpace(el?.Value) + ? null + : (trim ? el.Value.Trim() : el.Value); + } + + public static long ElementAsLong(this XElement element, XName name) + { + var el = element.Element(name); + return long.TryParse(el?.Value, out long value) ? value : default; + } + + public static int ElementAsInt(this XElement element, XName name) + { + var el = element.Element(name); + return int.TryParse(el?.Value, out int value) ? value : default(int); + } + + public static int GetIntResponse(this XDocument document) + { + return document.XPathSelectElement("./methodResponse/params/param/value").GetIntValue(); + } + + public static string GetStringResponse(this XDocument document) + { + return document.XPathSelectElement("./methodResponse/params/param/value").GetStringValue(); + } + } +} diff --git a/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs b/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs index 264aed562..65a0734af 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Headphones/Headphones.cs @@ -59,7 +59,7 @@ namespace NzbDrone.Core.Indexers.Headphones var request = requestBuilder.Build(); - request.AddBasicAuthentication(Settings.Username, Settings.Password); + request.Credentials = new BasicNetworkCredential(Settings.Username, Settings.Password); try { diff --git a/src/NzbDrone.Core/Indexers/Definitions/Headphones/HeadphonesRequestGenerator.cs b/src/NzbDrone.Core/Indexers/Definitions/Headphones/HeadphonesRequestGenerator.cs index 22eca7982..c2169d9cb 100644 --- a/src/NzbDrone.Core/Indexers/Definitions/Headphones/HeadphonesRequestGenerator.cs +++ b/src/NzbDrone.Core/Indexers/Definitions/Headphones/HeadphonesRequestGenerator.cs @@ -131,7 +131,7 @@ namespace NzbDrone.Core.Indexers.Headphones } var request = new IndexerRequest(string.Format("{0}&{1}", baseUrl, parameters.GetQueryString()), HttpAccept.Rss); - request.HttpRequest.AddBasicAuthentication(Settings.Username, Settings.Password); + request.HttpRequest.Credentials = new BasicNetworkCredential(Settings.Username, Settings.Password); yield return request; } diff --git a/src/NzbDrone.Core/Notifications/Email/Email.cs b/src/NzbDrone.Core/Notifications/Email/Email.cs index f40c20da6..6844f132f 100644 --- a/src/NzbDrone.Core/Notifications/Email/Email.cs +++ b/src/NzbDrone.Core/Notifications/Email/Email.cs @@ -7,17 +7,20 @@ using MailKit.Security; using MimeKit; using NLog; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http.Dispatchers; namespace NzbDrone.Core.Notifications.Email { public class Email : NotificationBase { + private readonly ICertificateValidationService _certificateValidationService; private readonly Logger _logger; public override string Name => "Email"; - public Email(Logger logger) + public Email(ICertificateValidationService certificateValidationService, Logger logger) { + _certificateValidationService = certificateValidationService; _logger = logger; } @@ -113,6 +116,8 @@ namespace NzbDrone.Core.Notifications.Email } } + client.ServerCertificateValidationCallback = _certificateValidationService.ShouldByPassValidationError; + _logger.Debug("Connecting to mail server"); client.Connect(settings.Server, settings.Port, serverOption); diff --git a/src/NzbDrone.Core/Notifications/PushBullet/PushBulletProxy.cs b/src/NzbDrone.Core/Notifications/PushBullet/PushBulletProxy.cs index f39382ae8..da36b9970 100644 --- a/src/NzbDrone.Core/Notifications/PushBullet/PushBulletProxy.cs +++ b/src/NzbDrone.Core/Notifications/PushBullet/PushBulletProxy.cs @@ -102,7 +102,7 @@ namespace NzbDrone.Core.Notifications.PushBullet var request = requestBuilder.Build(); request.Method = HttpMethod.Get; - request.AddBasicAuthentication(settings.ApiKey, string.Empty); + request.Credentials = new BasicNetworkCredential(settings.ApiKey, string.Empty); var response = _httpClient.Execute(request); @@ -198,7 +198,7 @@ namespace NzbDrone.Core.Notifications.PushBullet var request = requestBuilder.Build(); - request.AddBasicAuthentication(settings.ApiKey, string.Empty); + request.Credentials = new BasicNetworkCredential(settings.ApiKey, string.Empty); _httpClient.Execute(request); } diff --git a/src/NzbDrone.Core/Notifications/Twitter/TwitterProxy.cs b/src/NzbDrone.Core/Notifications/Twitter/TwitterProxy.cs new file mode 100644 index 000000000..3330e4a07 --- /dev/null +++ b/src/NzbDrone.Core/Notifications/Twitter/TwitterProxy.cs @@ -0,0 +1,110 @@ +using System.Collections.Generic; +using System.Collections.Specialized; +using System.Linq; +using System.Net.Http; +using System.Text; +using System.Web; +using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http; +using NzbDrone.Common.OAuth; + +namespace NzbDrone.Core.Notifications.Twitter +{ + public interface ITwitterProxy + { + NameValueCollection GetOAuthToken(string consumerKey, string consumerSecret, string oauthToken, string oauthVerifier); + string GetOAuthRedirect(string consumerKey, string consumerSecret, string callbackUrl); + void UpdateStatus(string message, TwitterSettings settings); + void DirectMessage(string message, TwitterSettings settings); + } + + public class TwitterProxy : ITwitterProxy + { + private readonly IHttpClient _httpClient; + + public TwitterProxy(IHttpClient httpClient) + { + _httpClient = httpClient; + } + + public string GetOAuthRedirect(string consumerKey, string consumerSecret, string callbackUrl) + { + // Creating a new instance with a helper method + var oAuthRequest = OAuthRequest.ForRequestToken(consumerKey, consumerSecret, callbackUrl); + oAuthRequest.RequestUrl = "https://api.twitter.com/oauth/request_token"; + var qscoll = HttpUtility.ParseQueryString(ExecuteRequest(GetRequest(oAuthRequest, new Dictionary())).Content); + + return string.Format("https://api.twitter.com/oauth/authorize?oauth_token={0}", qscoll["oauth_token"]); + } + + public NameValueCollection GetOAuthToken(string consumerKey, string consumerSecret, string oauthToken, string oauthVerifier) + { + // Creating a new instance with a helper method + var oAuthRequest = OAuthRequest.ForAccessToken(consumerKey, consumerSecret, oauthToken, "", oauthVerifier); + oAuthRequest.RequestUrl = "https://api.twitter.com/oauth/access_token"; + + return HttpUtility.ParseQueryString(ExecuteRequest(GetRequest(oAuthRequest, new Dictionary())).Content); + } + + public void UpdateStatus(string message, TwitterSettings settings) + { + var oAuthRequest = OAuthRequest.ForProtectedResource("POST", settings.ConsumerKey, settings.ConsumerSecret, settings.AccessToken, settings.AccessTokenSecret); + + oAuthRequest.RequestUrl = "https://api.twitter.com/1.1/statuses/update.json"; + + var customParams = new Dictionary + { + { "status", message.EncodeRFC3986() } + }; + + var request = GetRequest(oAuthRequest, customParams); + + request.Headers.ContentType = "application/x-www-form-urlencoded"; + request.SetContent(Encoding.ASCII.GetBytes(GetCustomParametersString(customParams))); + + ExecuteRequest(request); + } + + public void DirectMessage(string message, TwitterSettings settings) + { + var oAuthRequest = OAuthRequest.ForProtectedResource("POST", settings.ConsumerKey, settings.ConsumerSecret, settings.AccessToken, settings.AccessTokenSecret); + + oAuthRequest.RequestUrl = "https://api.twitter.com/1.1/direct_messages/new.json"; + + var customParams = new Dictionary + { + { "text", message.EncodeRFC3986() }, + { "screenname", settings.Mention.EncodeRFC3986() } + }; + + var request = GetRequest(oAuthRequest, customParams); + + request.Headers.ContentType = "application/x-www-form-urlencoded"; + request.SetContent(Encoding.ASCII.GetBytes(GetCustomParametersString(customParams))); + + ExecuteRequest(request); + } + + private string GetCustomParametersString(Dictionary customParams) + { + return customParams.Select(x => string.Format("{0}={1}", x.Key, x.Value)).Join("&"); + } + + private HttpRequest GetRequest(OAuthRequest oAuthRequest, Dictionary customParams) + { + var auth = oAuthRequest.GetAuthorizationHeader(customParams); + var request = new HttpRequest(oAuthRequest.RequestUrl); + + request.Headers.Add("Authorization", auth); + + request.Method = oAuthRequest.Method == "POST" ? HttpMethod.Post : HttpMethod.Get; + + return request; + } + + private HttpResponse ExecuteRequest(HttpRequest request) + { + return _httpClient.Execute(request); + } + } +} diff --git a/src/NzbDrone.Core/Notifications/Twitter/TwitterService.cs b/src/NzbDrone.Core/Notifications/Twitter/TwitterService.cs index 85a9646e7..bce42d483 100644 --- a/src/NzbDrone.Core/Notifications/Twitter/TwitterService.cs +++ b/src/NzbDrone.Core/Notifications/Twitter/TwitterService.cs @@ -1,13 +1,9 @@ using System; -using System.Collections.Specialized; using System.IO; using System.Net; -using System.Web; using FluentValidation.Results; using NLog; using NzbDrone.Common.Extensions; -using NzbDrone.Common.Http; -using NzbDrone.Common.OAuth; namespace NzbDrone.Core.Notifications.Twitter { @@ -21,31 +17,18 @@ namespace NzbDrone.Core.Notifications.Twitter public class TwitterService : ITwitterService { - private readonly IHttpClient _httpClient; + private readonly ITwitterProxy _twitterProxy; private readonly Logger _logger; - public TwitterService(IHttpClient httpClient, Logger logger) + public TwitterService(ITwitterProxy twitterProxy, Logger logger) { - _httpClient = httpClient; + _twitterProxy = twitterProxy; _logger = logger; } - private NameValueCollection OAuthQuery(OAuthRequest oAuthRequest) - { - var auth = oAuthRequest.GetAuthorizationHeader(); - var request = new Common.Http.HttpRequest(oAuthRequest.RequestUrl); - request.Headers.Add("Authorization", auth); - var response = _httpClient.Get(request); - - return HttpUtility.ParseQueryString(response.Content); - } - public OAuthToken GetOAuthToken(string consumerKey, string consumerSecret, string oauthToken, string oauthVerifier) { - // Creating a new instance with a helper method - var oAuthRequest = OAuthRequest.ForAccessToken(consumerKey, consumerSecret, oauthToken, "", oauthVerifier); - oAuthRequest.RequestUrl = "https://api.twitter.com/oauth/access_token"; - var qscoll = OAuthQuery(oAuthRequest); + var qscoll = _twitterProxy.GetOAuthToken(consumerKey, consumerSecret, oauthToken, oauthVerifier); return new OAuthToken { @@ -56,31 +39,16 @@ namespace NzbDrone.Core.Notifications.Twitter public string GetOAuthRedirect(string consumerKey, string consumerSecret, string callbackUrl) { - // Creating a new instance with a helper method - var oAuthRequest = OAuthRequest.ForRequestToken(consumerKey, consumerSecret, callbackUrl); - oAuthRequest.RequestUrl = "https://api.twitter.com/oauth/request_token"; - var qscoll = OAuthQuery(oAuthRequest); - - return string.Format("https://api.twitter.com/oauth/authorize?oauth_token={0}", qscoll["oauth_token"]); + return _twitterProxy.GetOAuthRedirect(consumerKey, consumerSecret, callbackUrl); } public void SendNotification(string message, TwitterSettings settings) { try { - var oAuth = new TinyTwitter.OAuthInfo - { - ConsumerKey = settings.ConsumerKey, - ConsumerSecret = settings.ConsumerSecret, - AccessToken = settings.AccessToken, - AccessSecret = settings.AccessTokenSecret - }; - - var twitter = new TinyTwitter.TinyTwitter(oAuth); - if (settings.DirectMessage) { - twitter.DirectMessage(message, settings.Mention); + _twitterProxy.DirectMessage(message, settings); } else { @@ -89,7 +57,7 @@ namespace NzbDrone.Core.Notifications.Twitter message += string.Format(" @{0}", settings.Mention); } - twitter.UpdateStatus(message); + _twitterProxy.UpdateStatus(message, settings); } } catch (WebException ex) diff --git a/src/NzbDrone.Core/Notifications/Webhook/WebhookProxy.cs b/src/NzbDrone.Core/Notifications/Webhook/WebhookProxy.cs index d196b5579..23a7fbdc8 100755 --- a/src/NzbDrone.Core/Notifications/Webhook/WebhookProxy.cs +++ b/src/NzbDrone.Core/Notifications/Webhook/WebhookProxy.cs @@ -40,7 +40,7 @@ namespace NzbDrone.Core.Notifications.Webhook if (settings.Username.IsNotNullOrWhiteSpace() || settings.Password.IsNotNullOrWhiteSpace()) { - request.AddBasicAuthentication(settings.Username, settings.Password); + request.Credentials = new BasicNetworkCredential(settings.Username, settings.Password); } _httpClient.Execute(request); diff --git a/src/NzbDrone.Core/Security/X509CertificateValidationService.cs b/src/NzbDrone.Core/Security/X509CertificateValidationService.cs index 6d505ca0a..7efc877ad 100644 --- a/src/NzbDrone.Core/Security/X509CertificateValidationService.cs +++ b/src/NzbDrone.Core/Security/X509CertificateValidationService.cs @@ -4,13 +4,12 @@ using System.Net.Security; using System.Security.Cryptography.X509Certificates; using NLog; using NzbDrone.Common.Extensions; +using NzbDrone.Common.Http.Dispatchers; using NzbDrone.Core.Configuration; -using NzbDrone.Core.Lifecycle; -using NzbDrone.Core.Messaging.Events; namespace NzbDrone.Core.Security { - public class X509CertificateValidationService : IHandle + public class X509CertificateValidationService : ICertificateValidationService { private readonly IConfigService _configService; private readonly Logger _logger; @@ -21,19 +20,29 @@ namespace NzbDrone.Core.Security _logger = logger; } - private bool ShouldByPassValidationError(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors) + public bool ShouldByPassValidationError(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors) { - var request = sender as HttpWebRequest; + var targetHostName = string.Empty; - if (request == null) + if (sender is not SslStream && sender is not string) { return true; } - var cert2 = certificate as X509Certificate2; - if (cert2 != null && request != null && cert2.SignatureAlgorithm.FriendlyName == "md5RSA") + if (sender is SslStream request) { - _logger.Error("https://{0} uses the obsolete md5 hash in it's https certificate, if that is your certificate, please (re)create certificate with better algorithm as soon as possible.", request.RequestUri.Authority); + targetHostName = request.TargetHostName; + } + + // Mailkit passes host in sender as string + if (sender is string stringHost) + { + targetHostName = stringHost; + } + + if (certificate is X509Certificate2 cert2 && cert2.SignatureAlgorithm.FriendlyName == "md5RSA") + { + _logger.Error("https://{0} uses the obsolete md5 hash in it's https certificate, if that is your certificate, please (re)create certificate with better algorithm as soon as possible.", targetHostName); } if (sslPolicyErrors == SslPolicyErrors.None) @@ -41,12 +50,12 @@ namespace NzbDrone.Core.Security return true; } - if (request.RequestUri.Host == "localhost" || request.RequestUri.Host == "127.0.0.1") + if (targetHostName == "localhost" || targetHostName == "127.0.0.1") { return true; } - var ipAddresses = GetIPAddresses(request.RequestUri.Host); + var ipAddresses = GetIPAddresses(targetHostName); var certificateValidation = _configService.CertificateValidation; if (certificateValidation == CertificateValidationType.Disabled) @@ -60,7 +69,7 @@ namespace NzbDrone.Core.Security return true; } - _logger.Error("Certificate validation for {0} failed. {1}", request.Address, sslPolicyErrors); + _logger.Error("Certificate validation for {0} failed. {1}", targetHostName, sslPolicyErrors); return false; } @@ -74,10 +83,5 @@ namespace NzbDrone.Core.Security return Dns.GetHostEntry(host).AddressList; } - - public void Handle(ApplicationStartedEvent message) - { - ServicePointManager.ServerCertificateValidationCallback = ShouldByPassValidationError; - } } } diff --git a/src/NzbDrone.Core/TinyTwitter.cs b/src/NzbDrone.Core/TinyTwitter.cs deleted file mode 100644 index 9f772095d..000000000 --- a/src/NzbDrone.Core/TinyTwitter.cs +++ /dev/null @@ -1,235 +0,0 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Net; -using System.Security.Cryptography; -using System.Text; -using System.Text.RegularExpressions; - -namespace TinyTwitter -{ - public class OAuthInfo - { - public string ConsumerKey { get; set; } - public string ConsumerSecret { get; set; } - public string AccessToken { get; set; } - public string AccessSecret { get; set; } - } - - public class Tweet - { - public long Id { get; set; } - public DateTime CreatedAt { get; set; } - public string UserName { get; set; } - public string ScreenName { get; set; } - public string Text { get; set; } - } - - public class TinyTwitter - { - private readonly OAuthInfo _oauth; - - public TinyTwitter(OAuthInfo oauth) - { - _oauth = oauth; - } - - public void UpdateStatus(string message) - { - new RequestBuilder(_oauth, "POST", "https://api.twitter.com/1.1/statuses/update.json") - .AddParameter("status", message) - .Execute(); - } - - /** - * - * As of June 26th 2015 Direct Messaging is not part of TinyTwitter. - * I have added it to Sonarr's copy to make our implementation easier - * and added this banner so it's not blindly updated. - * - **/ - public void DirectMessage(string message, string screenName) - { - new RequestBuilder(_oauth, "POST", "https://api.twitter.com/1.1/direct_messages/new.json") - .AddParameter("text", message) - .AddParameter("screen_name", screenName) - .Execute(); - } - - public class RequestBuilder - { - private const string VERSION = "1.0"; - private const string SIGNATURE_METHOD = "HMAC-SHA1"; - - private readonly OAuthInfo _oauth; - private readonly string _method; - private readonly IDictionary _customParameters; - private readonly string _url; - - public RequestBuilder(OAuthInfo oauth, string method, string url) - { - _oauth = oauth; - _method = method; - _url = url; - _customParameters = new Dictionary(); - } - - public RequestBuilder AddParameter(string name, string value) - { - _customParameters.Add(name, value.EncodeRFC3986()); - return this; - } - - public string Execute() - { - var timespan = GetTimestamp(); - var nonce = CreateNonce(); - - var parameters = new Dictionary(_customParameters); - AddOAuthParameters(parameters, timespan, nonce); - - var signature = GenerateSignature(parameters); - var headerValue = GenerateAuthorizationHeaderValue(parameters, signature); - - var request = (HttpWebRequest)WebRequest.Create(GetRequestUrl()); - request.Method = _method; - request.ContentType = "application/x-www-form-urlencoded"; - - request.Headers.Add("Authorization", headerValue); - - WriteRequestBody(request); - - // It looks like a bug in HttpWebRequest. It throws random TimeoutExceptions - // after some requests. Abort the request seems to work. More info: - // http://stackoverflow.com/questions/2252762/getrequeststream-throws-timeout-exception-randomly - var response = request.GetResponse(); - - string content; - - using (var stream = response.GetResponseStream()) - { - using (var reader = new StreamReader(stream)) - { - content = reader.ReadToEnd(); - } - } - - request.Abort(); - - return content; - } - - private void WriteRequestBody(HttpWebRequest request) - { - if (_method == "GET") - { - return; - } - - var requestBody = Encoding.ASCII.GetBytes(GetCustomParametersString()); - using (var stream = request.GetRequestStream()) - { - stream.Write(requestBody, 0, requestBody.Length); - } - } - - private string GetRequestUrl() - { - if (_method != "GET" || _customParameters.Count == 0) - { - return _url; - } - - return string.Format("{0}?{1}", _url, GetCustomParametersString()); - } - - private string GetCustomParametersString() - { - return _customParameters.Select(x => string.Format("{0}={1}", x.Key, x.Value)).Join("&"); - } - - private string GenerateAuthorizationHeaderValue(IEnumerable> parameters, string signature) - { - return new StringBuilder("OAuth ") - .Append(parameters.Concat(new KeyValuePair("oauth_signature", signature)) - .Where(x => x.Key.StartsWith("oauth_")) - .Select(x => string.Format("{0}=\"{1}\"", x.Key, x.Value.EncodeRFC3986())) - .Join(",")) - .ToString(); - } - - private string GenerateSignature(IEnumerable> parameters) - { - var dataToSign = new StringBuilder() - .Append(_method).Append('&') - .Append(_url.EncodeRFC3986()).Append('&') - .Append(parameters - .OrderBy(x => x.Key) - .Select(x => string.Format("{0}={1}", x.Key, x.Value)) - .Join("&") - .EncodeRFC3986()); - - var signatureKey = string.Format("{0}&{1}", _oauth.ConsumerSecret.EncodeRFC3986(), _oauth.AccessSecret.EncodeRFC3986()); - var sha1 = new HMACSHA1(Encoding.ASCII.GetBytes(signatureKey)); - - var signatureBytes = sha1.ComputeHash(Encoding.ASCII.GetBytes(dataToSign.ToString())); - return Convert.ToBase64String(signatureBytes); - } - - private void AddOAuthParameters(IDictionary parameters, string timestamp, string nonce) - { - parameters.Add("oauth_version", VERSION); - parameters.Add("oauth_consumer_key", _oauth.ConsumerKey); - parameters.Add("oauth_nonce", nonce); - parameters.Add("oauth_signature_method", SIGNATURE_METHOD); - parameters.Add("oauth_timestamp", timestamp); - parameters.Add("oauth_token", _oauth.AccessToken); - } - - private static string GetTimestamp() - { - return ((int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds).ToString(); - } - - private static string CreateNonce() - { - return new Random().Next(0x0000000, 0x7fffffff).ToString("X8"); - } - } - } - - public static class TinyTwitterHelperExtensions - { - public static string Join(this IEnumerable items, string separator) - { - return string.Join(separator, items.ToArray()); - } - - public static IEnumerable Concat(this IEnumerable items, T value) - { - return items.Concat(new[] { value }); - } - - public static string EncodeRFC3986(this string value) - { - // From Twitterizer http://www.twitterizer.net/ - if (string.IsNullOrEmpty(value)) - { - return string.Empty; - } - - var encoded = Uri.EscapeDataString(value); - - return Regex - .Replace(encoded, "(%[0-9a-f][0-9a-f])", c => c.Value.ToUpper()) - .Replace("(", "%28") - .Replace(")", "%29") - .Replace("$", "%24") - .Replace("!", "%21") - .Replace("*", "%2A") - .Replace("'", "%27") - .Replace("%7E", "~"); - } - } -}