Modern HTTP Client (#685)

pull/1056/head
Qstick 2 years ago committed by GitHub
parent 593a649045
commit b7b5a6e7e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -4,6 +4,7 @@ using System.Globalization;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
using System.Threading;
using FluentAssertions;
using Moq;
@ -15,8 +16,11 @@ using NzbDrone.Common.Http;
using NzbDrone.Common.Http.Dispatchers;
using NzbDrone.Common.Http.Proxy;
using NzbDrone.Common.TPL;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Security;
using NzbDrone.Test.Common;
using NzbDrone.Test.Common.Categories;
using HttpClient = NzbDrone.Common.Http.HttpClient;
namespace NzbDrone.Common.Test.Http
{
@ -31,6 +35,8 @@ namespace NzbDrone.Common.Test.Http
private string _httpBinHost;
private string _httpBinHost2;
private System.Net.Http.HttpClient _httpClient = new ();
[OneTimeSetUp]
public void FixtureSetUp()
{
@ -38,7 +44,7 @@ namespace NzbDrone.Common.Test.Http
var mainHost = "httpbin.servarr.com";
// Use mirrors for tests that use two hosts
var candidates = new[] { "eu.httpbin.org", /* "httpbin.org", */ "www.httpbin.org" };
var candidates = new[] { "httpbin1.servarr.com" };
// httpbin.org is broken right now, occassionally redirecting to https if it's unavailable.
_httpBinHost = mainHost;
@ -46,29 +52,20 @@ namespace NzbDrone.Common.Test.Http
TestLogger.Info($"{candidates.Length} TestSites available.");
_httpBinSleep = _httpBinHosts.Length < 2 ? 100 : 10;
_httpBinSleep = 10;
}
private bool IsTestSiteAvailable(string site)
{
try
{
var req = WebRequest.Create($"https://{site}/get") as HttpWebRequest;
var res = req.GetResponse() as HttpWebResponse;
var res = _httpClient.GetAsync($"https://{site}/get").GetAwaiter().GetResult();
if (res.StatusCode != HttpStatusCode.OK)
{
return false;
}
try
{
req = WebRequest.Create($"https://{site}/status/429") as HttpWebRequest;
res = req.GetResponse() as HttpWebResponse;
}
catch (WebException ex)
{
res = ex.Response as HttpWebResponse;
}
res = _httpClient.GetAsync($"https://{site}/status/429").GetAwaiter().GetResult();
if (res == null || res.StatusCode != (HttpStatusCode)429)
{
@ -95,10 +92,13 @@ namespace NzbDrone.Common.Test.Http
Mocker.GetMock<IOsInfo>().Setup(c => c.Name).Returns("TestOS");
Mocker.GetMock<IOsInfo>().Setup(c => c.Version).Returns("9.0.0");
Mocker.GetMock<IConfigService>().SetupGet(x => x.CertificateValidation).Returns(CertificateValidationType.Enabled);
Mocker.SetConstant<IUserAgentBuilder>(Mocker.Resolve<UserAgentBuilder>());
Mocker.SetConstant<ICacheManager>(Mocker.Resolve<CacheManager>());
Mocker.SetConstant<ICreateManagedWebProxy>(Mocker.Resolve<ManagedWebProxyFactory>());
Mocker.SetConstant<ICertificateValidationService>(new X509CertificateValidationService(Mocker.GetMock<IConfigService>().Object, TestLogger));
Mocker.SetConstant<IRateLimitService>(Mocker.Resolve<RateLimitService>());
Mocker.SetConstant<IEnumerable<IHttpRequestInterceptor>>(Array.Empty<IHttpRequestInterceptor>());
Mocker.SetConstant<IHttpDispatcher>(Mocker.Resolve<TDispatcher>());
@ -138,6 +138,28 @@ namespace NzbDrone.Common.Test.Http
response.Content.Should().NotBeNullOrWhiteSpace();
}
[TestCase(CertificateValidationType.Enabled)]
[TestCase(CertificateValidationType.DisabledForLocalAddresses)]
public void bad_ssl_should_fail_when_remote_validation_enabled(CertificateValidationType validationType)
{
Mocker.GetMock<IConfigService>().SetupGet(x => x.CertificateValidation).Returns(validationType);
var request = new HttpRequest($"https://expired.badssl.com");
Assert.Throws<HttpRequestException>(() => Subject.Execute(request));
ExceptionVerification.ExpectedErrors(1);
}
[Test]
public void bad_ssl_should_pass_if_remote_validation_disabled()
{
Mocker.GetMock<IConfigService>().SetupGet(x => x.CertificateValidation).Returns(CertificateValidationType.Disabled);
var request = new HttpRequest($"https://expired.badssl.com");
Subject.Execute(request);
ExceptionVerification.ExpectedErrors(0);
}
[Test]
public void should_execute_typed_get()
{
@ -162,15 +184,44 @@ namespace NzbDrone.Common.Test.Http
response.Resource.Data.Should().Be(message);
}
[TestCase("gzip")]
public void should_execute_get_using_gzip(string compression)
[Test]
public void should_execute_post_with_content_type()
{
var request = new HttpRequest($"https://{_httpBinHost}/{compression}");
var message = "{ my: 1 }";
var request = new HttpRequest($"https://{_httpBinHost}/post");
request.SetContent(message);
request.Headers.ContentType = "application/json";
var response = Subject.Post<HttpBinResource>(request);
response.Resource.Data.Should().Be(message);
}
[Test]
public void should_execute_get_using_gzip()
{
var request = new HttpRequest($"https://{_httpBinHost}/gzip");
var response = Subject.Get<HttpBinResource>(request);
response.Resource.Headers["Accept-Encoding"].ToString().Should().Be(compression);
response.Resource.Headers["Accept-Encoding"].ToString().Should().Contain("gzip");
response.Resource.Gzipped.Should().BeTrue();
response.Resource.Brotli.Should().BeFalse();
}
[Test]
public void should_execute_get_using_brotli()
{
var request = new HttpRequest($"https://{_httpBinHost}/brotli");
var response = Subject.Get<HttpBinResource>(request);
response.Resource.Headers["Accept-Encoding"].ToString().Should().Contain("br");
response.Resource.Gzipped.Should().BeFalse();
response.Resource.Brotli.Should().BeTrue();
}
[TestCase(HttpStatusCode.Unauthorized)]
@ -190,6 +241,28 @@ namespace NzbDrone.Common.Test.Http
ExceptionVerification.IgnoreWarns();
}
[Test]
public void should_not_throw_on_suppressed_status_codes()
{
var request = new HttpRequest($"https://{_httpBinHost}/status/{HttpStatusCode.NotFound}");
request.SuppressHttpErrorStatusCodes = new[] { HttpStatusCode.NotFound };
Assert.Throws<HttpException>(() => Subject.Get<HttpBinResource>(request));
ExceptionVerification.IgnoreWarns();
}
[Test]
public void should_not_log_unsuccessful_status_codes()
{
var request = new HttpRequest($"https://{_httpBinHost}/status/{HttpStatusCode.NotFound}");
request.LogHttpError = false;
Assert.Throws<HttpException>(() => Subject.Get<HttpBinResource>(request));
ExceptionVerification.ExpectedWarns(0);
}
[Test]
public void should_not_follow_redirects_when_not_in_production()
{
@ -315,13 +388,38 @@ namespace NzbDrone.Common.Test.Http
{
var file = GetTempFilePath();
Assert.Throws<WebException>(() => Subject.DownloadFile("https://download.sonarr.tv/wrongpath", file));
Assert.Throws<HttpException>(() => Subject.DownloadFile("https://download.sonarr.tv/wrongpath", file));
File.Exists(file).Should().BeFalse();
ExceptionVerification.ExpectedWarns(1);
}
[Test]
public void should_not_write_redirect_content_to_stream()
{
var file = GetTempFilePath();
using (var fileStream = new FileStream(file, FileMode.Create))
{
var request = new HttpRequest($"http://{_httpBinHost}/redirect/1");
request.AllowAutoRedirect = false;
request.ResponseStream = fileStream;
var response = Subject.Get(request);
response.StatusCode.Should().Be(HttpStatusCode.Moved);
}
ExceptionVerification.ExpectedErrors(1);
File.Exists(file).Should().BeTrue();
var fileInfo = new FileInfo(file);
fileInfo.Length.Should().Be(0);
}
[Test]
public void should_send_cookie()
{
@ -753,6 +851,7 @@ namespace NzbDrone.Common.Test.Http
public string Url { get; set; }
public string Data { get; set; }
public bool Gzipped { get; set; }
public bool Brotli { get; set; }
}
public class HttpCookieResource

@ -210,5 +210,26 @@ namespace NzbDrone.Common.Extensions
return result.TrimStart(' ', '.').TrimEnd(' ');
}
public static string EncodeRFC3986(this string value)
{
// From Twitterizer http://www.twitterizer.net/
if (string.IsNullOrEmpty(value))
{
return string.Empty;
}
var encoded = Uri.EscapeDataString(value);
return Regex
.Replace(encoded, "(%[0-9a-f][0-9a-f])", c => c.Value.ToUpper())
.Replace("(", "%28")
.Replace(")", "%29")
.Replace("$", "%24")
.Replace("!", "%21")
.Replace("*", "%2A")
.Replace("'", "%27")
.Replace("%7E", "~");
}
}
}

@ -0,0 +1,12 @@
using System.Net;
namespace NzbDrone.Common.Http
{
public class BasicNetworkCredential : NetworkCredential
{
public BasicNetworkCredential(string user, string pass)
: base(user, pass)
{
}
}
}

@ -10,6 +10,7 @@ namespace NzbDrone.Common.Http
// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Set-Cookie
// NOTE: we are not checking non-ascii characters and we should
private static readonly Regex _CookieRegex = new Regex(@"([^\(\)<>@,;:\\""/\[\]\?=\{\}\s]+)=([^,;\\""\s]+)");
private static readonly string[] FilterProps = { "COMMENT", "COMMENTURL", "DISCORD", "DOMAIN", "EXPIRES", "MAX-AGE", "PATH", "PORT", "SECURE", "VERSION", "HTTPONLY", "SAMESITE" };
private static readonly char[] InvalidKeyChars = { '(', ')', '<', '>', '@', ',', ';', ':', '\\', '"', '/', '[', ']', '?', '=', '{', '}', ' ', '\t', '\n' };
private static readonly char[] InvalidValueChars = { '"', ',', ';', '\\', ' ', '\t', '\n' };
@ -24,7 +25,7 @@ namespace NzbDrone.Common.Http
var matches = _CookieRegex.Match(cookieHeader);
while (matches.Success)
{
if (matches.Groups.Count > 2)
if (matches.Groups.Count > 2 && !FilterProps.Contains(matches.Groups[1].Value.ToUpperInvariant()))
{
cookieDictionary[matches.Groups[1].Value] = matches.Groups[2].Value;
}

@ -0,0 +1,11 @@
using System.Net.Http;
using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
namespace NzbDrone.Common.Http.Dispatchers
{
public interface ICertificateValidationService
{
bool ShouldByPassValidationError(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors);
}
}

@ -6,6 +6,5 @@ namespace NzbDrone.Common.Http.Dispatchers
public interface IHttpDispatcher
{
Task<HttpResponse> GetResponseAsync(HttpRequest request, CookieContainer cookies);
Task DownloadFileAsync(string url, string fileName);
}
}

@ -1,13 +1,16 @@
using System;
using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Net;
using System.Reflection;
using System.Net.Http;
using System.Net.Security;
using System.Net.Sockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using NLog;
using NLog.Fluent;
using NzbDrone.Common.EnvironmentInfo;
using NzbDrone.Common.Cache;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http.Proxy;
@ -15,221 +18,223 @@ namespace NzbDrone.Common.Http.Dispatchers
{
public class ManagedHttpDispatcher : IHttpDispatcher
{
private const string NO_PROXY_KEY = "no-proxy";
private const int connection_establish_timeout = 2000;
private static bool useIPv6 = Socket.OSSupportsIPv6;
private static bool hasResolvedIPv6Availability;
private readonly IHttpProxySettingsProvider _proxySettingsProvider;
private readonly ICreateManagedWebProxy _createManagedWebProxy;
private readonly ICertificateValidationService _certificateValidationService;
private readonly IUserAgentBuilder _userAgentBuilder;
private readonly IPlatformInfo _platformInfo;
private readonly Logger _logger;
private readonly ICached<System.Net.Http.HttpClient> _httpClientCache;
private readonly ICached<CredentialCache> _credentialCache;
public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider, ICreateManagedWebProxy createManagedWebProxy, IUserAgentBuilder userAgentBuilder, IPlatformInfo platformInfo, Logger logger)
public ManagedHttpDispatcher(IHttpProxySettingsProvider proxySettingsProvider,
ICreateManagedWebProxy createManagedWebProxy,
ICertificateValidationService certificateValidationService,
IUserAgentBuilder userAgentBuilder,
ICacheManager cacheManager)
{
_proxySettingsProvider = proxySettingsProvider;
_createManagedWebProxy = createManagedWebProxy;
_certificateValidationService = certificateValidationService;
_userAgentBuilder = userAgentBuilder;
_platformInfo = platformInfo;
_logger = logger;
_httpClientCache = cacheManager.GetCache<System.Net.Http.HttpClient>(typeof(ManagedHttpDispatcher));
_credentialCache = cacheManager.GetCache<CredentialCache>(typeof(ManagedHttpDispatcher), "credentialcache");
}
public async Task<HttpResponse> GetResponseAsync(HttpRequest request, CookieContainer cookies)
{
var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url);
// Deflate is not a standard and could break depending on implementation.
// we should just stick with the more compatible Gzip
//http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net
webRequest.AutomaticDecompression = DecompressionMethods.GZip;
var requestMessage = new HttpRequestMessage(request.Method, (Uri)request.Url);
requestMessage.Headers.UserAgent.ParseAdd(_userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent));
requestMessage.Headers.ConnectionClose = !request.ConnectionKeepAlive;
webRequest.Method = request.Method.ToString();
webRequest.UserAgent = _userAgentBuilder.GetUserAgent(request.UseSimplifiedUserAgent);
webRequest.KeepAlive = request.ConnectionKeepAlive;
webRequest.AllowAutoRedirect = false;
webRequest.CookieContainer = cookies;
var cookieHeader = cookies.GetCookieHeader((Uri)request.Url);
if (cookieHeader.IsNotNullOrWhiteSpace())
{
requestMessage.Headers.Add("Cookie", cookieHeader);
}
using var cts = new CancellationTokenSource();
if (request.RequestTimeout != TimeSpan.Zero)
{
webRequest.Timeout = (int)Math.Ceiling(request.RequestTimeout.TotalMilliseconds);
cts.CancelAfter(request.RequestTimeout);
}
else
{
// The default for System.Net.Http.HttpClient
cts.CancelAfter(TimeSpan.FromSeconds(100));
}
webRequest.Proxy = request.Proxy ?? GetProxy(request.Url);
if (request.Credentials != null)
{
if (request.Credentials is BasicNetworkCredential bc)
{
// Manually set header to avoid initial challenge response
var authInfo = bc.UserName + ":" + bc.Password;
authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(authInfo));
requestMessage.Headers.Add("Authorization", "Basic " + authInfo);
}
else if (request.Credentials is NetworkCredential nc)
{
var creds = GetCredentialCache();
foreach (var authtype in new[] { "Basic", "Digest" })
{
creds.Remove((Uri)request.Url, authtype);
creds.Add((Uri)request.Url, authtype, nc);
}
}
}
if (request.ContentData != null)
{
requestMessage.Content = new ByteArrayContent(request.ContentData);
}
if (request.Headers != null)
{
AddRequestHeaders(webRequest, request.Headers);
AddRequestHeaders(requestMessage, request.Headers);
}
HttpWebResponse httpWebResponse;
var httpClient = GetClient(request.Url);
var sw = new Stopwatch();
sw.Start();
try
{
if (request.ContentData != null)
{
webRequest.ContentLength = request.ContentData.Length;
using (var writeStream = webRequest.GetRequestStream())
{
writeStream.Write(request.ContentData, 0, request.ContentData.Length);
}
}
httpWebResponse = (HttpWebResponse)await webRequest.GetResponseAsync();
}
catch (WebException e)
using var responseMessage = await httpClient.SendAsync(requestMessage, HttpCompletionOption.ResponseHeadersRead, cts.Token);
{
httpWebResponse = (HttpWebResponse)e.Response;
byte[] data = null;
if (httpWebResponse == null)
try
{
// The default messages for WebException on mono are pretty horrible.
if (e.Status == WebExceptionStatus.NameResolutionFailure)
if (request.ResponseStream != null && responseMessage.StatusCode == HttpStatusCode.OK)
{
throw new WebException($"DNS Name Resolution Failure: '{webRequest.RequestUri.Host}'", e.Status);
}
else if (e.ToString().Contains("TLS Support not"))
{
throw new TlsFailureException(webRequest, e);
}
else if (e.ToString().Contains("The authentication or decryption has failed."))
{
throw new TlsFailureException(webRequest, e);
}
else if (OsInfo.IsNotWindows)
{
throw new WebException($"{e.Message}: '{webRequest.RequestUri}'", e, e.Status, e.Response);
responseMessage.Content.CopyTo(request.ResponseStream, null, cts.Token);
}
else
{
throw;
data = responseMessage.Content.ReadAsByteArrayAsync(cts.Token).GetAwaiter().GetResult();
}
}
}
catch (Exception ex)
{
throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, null);
}
byte[] data = null;
var headers = responseMessage.Headers.ToNameValueCollection();
using (var responseStream = httpWebResponse.GetResponseStream())
{
if (responseStream != null && responseStream != Stream.Null)
headers.Add(responseMessage.Content.Headers.ToNameValueCollection());
CookieContainer responseCookies = new CookieContainer();
if (responseMessage.Headers.TryGetValues("Set-Cookie", out var cookieHeaders))
{
try
{
data = await responseStream.ToBytes();
}
catch (Exception ex)
foreach (var responseCookieHeader in cookieHeaders)
{
throw new WebException("Failed to read complete http response", ex, WebExceptionStatus.ReceiveFailure, httpWebResponse);
try
{
cookies.SetCookies(responseMessage.RequestMessage.RequestUri, responseCookieHeader);
}
catch
{
// Ignore invalid cookies
}
}
}
}
sw.Stop();
var cookieCollection = cookies.GetCookies(responseMessage.RequestMessage.RequestUri);
sw.Stop();
return new HttpResponse(request, new HttpHeader(httpWebResponse.Headers), httpWebResponse.Cookies, data, sw.ElapsedMilliseconds, httpWebResponse.StatusCode);
return new HttpResponse(request, new HttpHeader(headers), cookieCollection, data, sw.ElapsedMilliseconds, responseMessage.StatusCode);
}
}
public async Task DownloadFileAsync(string url, string fileName)
protected virtual System.Net.Http.HttpClient GetClient(HttpUri uri)
{
try
{
var fileInfo = new FileInfo(fileName);
if (fileInfo.Directory != null && !fileInfo.Directory.Exists)
{
fileInfo.Directory.Create();
}
var proxySettings = _proxySettingsProvider.GetProxySettings(uri);
_logger.Debug("Downloading [{0}] to [{1}]", url, fileName);
var key = proxySettings?.Key ?? NO_PROXY_KEY;
var stopWatch = Stopwatch.StartNew();
var uri = new HttpUri(url);
return _httpClientCache.Get(key, () => CreateHttpClient(proxySettings));
}
using (var webClient = new GZipWebClient())
{
webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent());
webClient.Proxy = GetProxy(uri);
await webClient.DownloadFileTaskAsync(url, fileName);
stopWatch.Stop();
_logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
}
}
catch (WebException e)
protected virtual System.Net.Http.HttpClient CreateHttpClient(HttpProxySettings proxySettings)
{
var handler = new SocketsHttpHandler()
{
_logger.Warn("Failed to get response from: {0} {1}", url, e.Message);
if (File.Exists(fileName))
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Brotli,
UseCookies = false, // sic - we don't want to use a shared cookie container
AllowAutoRedirect = false,
Credentials = GetCredentialCache(),
PreAuthenticate = true,
MaxConnectionsPerServer = 12,
ConnectCallback = onConnect,
SslOptions = new SslClientAuthenticationOptions
{
File.Delete(fileName);
RemoteCertificateValidationCallback = _certificateValidationService.ShouldByPassValidationError
}
};
throw;
}
catch (Exception e)
if (proxySettings != null)
{
_logger.Warn(e, "Failed to get response from: " + url);
if (File.Exists(fileName))
{
File.Delete(fileName);
}
throw;
handler.Proxy = _createManagedWebProxy.GetWebProxy(proxySettings);
}
}
protected virtual IWebProxy GetProxy(HttpUri uri)
{
IWebProxy proxy = null;
var proxySettings = _proxySettingsProvider.GetProxySettings(uri);
if (proxySettings != null)
var client = new System.Net.Http.HttpClient(handler)
{
proxy = _createManagedWebProxy.GetWebProxy(proxySettings);
}
Timeout = Timeout.InfiniteTimeSpan
};
return proxy;
return client;
}
protected virtual void AddRequestHeaders(HttpWebRequest webRequest, HttpHeader headers)
protected virtual void AddRequestHeaders(HttpRequestMessage webRequest, HttpHeader headers)
{
foreach (var header in headers)
{
switch (header.Key)
{
case "Accept":
webRequest.Accept = header.Value;
webRequest.Headers.Accept.ParseAdd(header.Value);
break;
case "Connection":
webRequest.Connection = header.Value;
webRequest.Headers.Connection.Clear();
webRequest.Headers.Connection.Add(header.Value);
break;
case "Content-Length":
webRequest.ContentLength = Convert.ToInt64(header.Value);
AddContentHeader(webRequest, "Content-Length", header.Value);
break;
case "Content-Type":
webRequest.ContentType = header.Value;
AddContentHeader(webRequest, "Content-Type", header.Value);
break;
case "Date":
webRequest.Date = HttpHeader.ParseDateTime(header.Value);
webRequest.Headers.Remove("Date");
webRequest.Headers.Date = HttpHeader.ParseDateTime(header.Value);
break;
case "Expect":
webRequest.Expect = header.Value;
webRequest.Headers.Expect.ParseAdd(header.Value);
break;
case "Host":
webRequest.Host = header.Value;
webRequest.Headers.Host = header.Value;
break;
case "If-Modified-Since":
webRequest.IfModifiedSince = HttpHeader.ParseDateTime(header.Value);
webRequest.Headers.IfModifiedSince = HttpHeader.ParseDateTime(header.Value);
break;
case "Range":
throw new NotImplementedException();
case "Referer":
webRequest.Referer = header.Value;
webRequest.Headers.Add("Referer", header.Value);
break;
case "Transfer-Encoding":
webRequest.TransferEncoding = header.Value;
webRequest.Headers.TransferEncoding.ParseAdd(header.Value);
break;
case "User-Agent":
webRequest.UserAgent = header.Value;
webRequest.Headers.UserAgent.ParseAdd(header.Value);
break;
case "Proxy-Connection":
throw new NotImplementedException();
@ -239,5 +244,84 @@ namespace NzbDrone.Common.Http.Dispatchers
}
}
}
private void AddContentHeader(HttpRequestMessage request, string header, string value)
{
var headers = request.Content?.Headers;
if (headers == null)
{
return;
}
headers.Remove(header);
headers.Add(header, value);
}
private CredentialCache GetCredentialCache()
{
return _credentialCache.Get("credentialCache", () => new CredentialCache());
}
private static async ValueTask<Stream> onConnect(SocketsHttpConnectionContext context, CancellationToken cancellationToken)
{
// Until .NET supports an implementation of Happy Eyeballs (https://tools.ietf.org/html/rfc8305#section-2), let's make IPv4 fallback work in a simple way.
// This issue is being tracked at https://github.com/dotnet/runtime/issues/26177 and expected to be fixed in .NET 6.
if (useIPv6)
{
try
{
var localToken = cancellationToken;
if (!hasResolvedIPv6Availability)
{
// to make things move fast, use a very low timeout for the initial ipv6 attempt.
var quickFailCts = new CancellationTokenSource(connection_establish_timeout);
var linkedTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, quickFailCts.Token);
localToken = linkedTokenSource.Token;
}
return await attemptConnection(AddressFamily.InterNetworkV6, context, localToken);
}
catch
{
// very naively fallback to ipv4 permanently for this execution based on the response of the first connection attempt.
// note that this may cause users to eventually get switched to ipv4 (on a random failure when they are switching networks, for instance)
// but in the interest of keeping this implementation simple, this is acceptable.
useIPv6 = false;
}
finally
{
hasResolvedIPv6Availability = true;
}
}
// fallback to IPv4.
return await attemptConnection(AddressFamily.InterNetwork, context, cancellationToken);
}
private static async ValueTask<Stream> attemptConnection(AddressFamily addressFamily, SocketsHttpConnectionContext context, CancellationToken cancellationToken)
{
// The following socket constructor will create a dual-mode socket on systems where IPV6 is available.
var socket = new Socket(addressFamily, SocketType.Stream, ProtocolType.Tcp)
{
// Turn off Nagle's algorithm since it degrades performance in most HttpClient scenarios.
NoDelay = true
};
try
{
await socket.ConnectAsync(context.DnsEndPoint, cancellationToken).ConfigureAwait(false);
// The stream should take the ownership of the underlying socket,
// closing it when it's disposed.
return new NetworkStream(socket, ownsSocket: true);
}
catch
{
socket.Dispose();
throw;
}
}
}
}

@ -1,15 +0,0 @@
using System;
using System.Net;
namespace NzbDrone.Common.Http
{
public class GZipWebClient : WebClient
{
protected override WebRequest GetWebRequest(Uri address)
{
var request = (HttpWebRequest)base.GetWebRequest(address);
request.AutomaticDecompression = DecompressionMethods.GZip;
return request;
}
}
}

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Http;
@ -86,13 +87,21 @@ namespace NzbDrone.Common.Http
}
// 302 or 303 should default to GET on redirect even if POST on original
if (response.StatusCode == HttpStatusCode.Redirect || response.StatusCode == HttpStatusCode.RedirectMethod)
if (RequestRequiresForceGet(response.StatusCode, response.Request.Method))
{
request.Method = HttpMethod.Get;
request.ContentData = null;
}
response = await ExecuteRequestAsync(request, cookieContainer);
// Save to add to final response
var responseCookies = response.Cookies;
// Update cookiecontainer for next request with any cookies recieved on last request
var responseContainer = HandleRedirectCookies(request, response);
response = await ExecuteRequestAsync(request, responseContainer);
response.Cookies.Add(responseCookies);
}
while (response.HasHttpRedirect);
}
@ -102,9 +111,12 @@ namespace NzbDrone.Common.Http
_logger.Error("Server requested a redirect to [{0}] while in developer mode. Update the request URL to avoid this redirect.", response.Headers["Location"]);
}
if (!request.SuppressHttpError && response.HasHttpError)
if (!request.SuppressHttpError && response.HasHttpError && (request.SuppressHttpErrorStatusCodes == null || !request.SuppressHttpErrorStatusCodes.Contains(response.StatusCode)))
{
_logger.Warn("HTTP Error - {0}", response);
if (request.LogHttpError)
{
_logger.Warn("HTTP Error - {0}", response);
}
if ((int)response.StatusCode == 429)
{
@ -124,6 +136,21 @@ namespace NzbDrone.Common.Http
return ExecuteAsync(request).GetAwaiter().GetResult();
}
private static bool RequestRequiresForceGet(HttpStatusCode statusCode, HttpMethod requestMethod)
{
switch (statusCode)
{
case HttpStatusCode.Moved:
case HttpStatusCode.Found:
case HttpStatusCode.MultipleChoices:
return requestMethod == HttpMethod.Post;
case HttpStatusCode.SeeOther:
return requestMethod != HttpMethod.Get && requestMethod != HttpMethod.Head;
default:
return false;
}
}
private async Task<HttpResponse> ExecuteRequestAsync(HttpRequest request, CookieContainer cookieContainer)
{
foreach (var interceptor in _requestInterceptors)
@ -140,8 +167,6 @@ namespace NzbDrone.Common.Http
var stopWatch = Stopwatch.StartNew();
PrepareRequestCookies(request, cookieContainer);
var response = await _httpDispatcher.GetResponseAsync(request, cookieContainer);
HandleResponseCookies(response, cookieContainer);
@ -208,52 +233,125 @@ namespace NzbDrone.Common.Http
}
}
private void PrepareRequestCookies(HttpRequest request, CookieContainer cookieContainer)
private CookieContainer HandleRedirectCookies(HttpRequest request, HttpResponse response)
{
// Don't collect persistnet cookies for intermediate/redirected urls.
/*lock (_cookieContainerCache)
var sourceContainer = new CookieContainer();
var responseCookies = response.GetCookies();
if (responseCookies.Count != 0)
{
var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer());
var persistentCookies = presistentContainer.GetCookies((Uri)request.Url);
var existingCookies = cookieContainer.GetCookies((Uri)request.Url);
foreach (var pair in responseCookies)
{
Cookie cookie;
if (pair.Value == null)
{
cookie = new Cookie(pair.Key, "", "/")
{
Expires = DateTime.Now.AddDays(-1)
};
}
else
{
cookie = new Cookie(pair.Key, pair.Value, "/")
{
// Use Now rather than UtcNow to work around Mono cookie expiry bug.
// See https://gist.github.com/ta264/7822b1424f72e5b4c961
Expires = DateTime.Now.AddHours(1)
};
}
cookieContainer.Add(persistentCookies);
cookieContainer.Add(existingCookies);
}*/
sourceContainer.Add((Uri)request.Url, cookie);
}
}
return sourceContainer;
}
private void HandleResponseCookies(HttpResponse response, CookieContainer cookieContainer)
private void HandleResponseCookies(HttpResponse response, CookieContainer container)
{
foreach (Cookie cookie in container.GetAllCookies())
{
cookie.Expired = true;
}
var cookieHeaders = response.GetCookieHeaders();
if (cookieHeaders.Empty())
{
return;
}
AddCookiesToContainer(response.Request.Url, cookieHeaders, container);
if (response.Request.StoreResponseCookie)
{
lock (_cookieContainerCache)
{
var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer());
foreach (var cookieHeader in cookieHeaders)
{
try
{
persistentCookieContainer.SetCookies((Uri)response.Request.Url, cookieHeader);
}
catch (Exception ex)
{
_logger.Debug(ex, "Invalid cookie in {0}", response.Request.Url);
}
}
AddCookiesToContainer(response.Request.Url, cookieHeaders, persistentCookieContainer);
}
}
}
private void AddCookiesToContainer(HttpUri url, string[] cookieHeaders, CookieContainer container)
{
foreach (var cookieHeader in cookieHeaders)
{
try
{
container.SetCookies((Uri)url, cookieHeader);
}
catch (Exception ex)
{
_logger.Debug(ex, "Invalid cookie in {0}", url);
}
}
}
public async Task DownloadFileAsync(string url, string fileName)
{
await _httpDispatcher.DownloadFileAsync(url, fileName);
var fileNamePart = fileName + ".part";
try
{
var fileInfo = new FileInfo(fileName);
if (fileInfo.Directory != null && !fileInfo.Directory.Exists)
{
fileInfo.Directory.Create();
}
_logger.Debug("Downloading [{0}] to [{1}]", url, fileName);
var stopWatch = Stopwatch.StartNew();
using (var fileStream = new FileStream(fileNamePart, FileMode.Create, FileAccess.ReadWrite))
{
var request = new HttpRequest(url);
request.AllowAutoRedirect = true;
request.ResponseStream = fileStream;
var response = await GetAsync(request);
if (response.Headers.ContentType != null && response.Headers.ContentType.Contains("text/html"))
{
throw new HttpException(request, response, "Site responded with html content.");
}
}
stopWatch.Stop();
if (File.Exists(fileName))
{
File.Delete(fileName);
}
File.Move(fileNamePart, fileName);
_logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
}
finally
{
if (File.Exists(fileNamePart))
{
File.Delete(fileNamePart);
}
}
}
public void DownloadFile(string url, string fileName)

@ -4,11 +4,27 @@ using System.Collections.Generic;
using System.Collections.Specialized;
using System.Globalization;
using System.Linq;
using System.Net;
using System.Net.Http.Headers;
using System.Text;
using NzbDrone.Common.Extensions;
namespace NzbDrone.Common.Http
{
public static class WebHeaderCollectionExtensions
{
public static NameValueCollection ToNameValueCollection(this HttpHeaders headers)
{
var result = new NameValueCollection();
foreach (var header in headers)
{
result.Add(header.Key, header.Value.ConcatToString(";"));
}
return result;
}
}
public class HttpHeader : NameValueCollection, IEnumerable<KeyValuePair<string, string>>, IEnumerable
{
public HttpHeader(NameValueCollection headers)
@ -16,6 +32,11 @@ namespace NzbDrone.Common.Http
{
}
public HttpHeader(HttpHeaders headers)
: base(headers.ToNameValueCollection())
{
}
public HttpHeader()
{
}

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Text;
@ -12,11 +13,13 @@ namespace NzbDrone.Common.Http
{
public HttpRequest(string url, HttpAccept httpAccept = null)
{
Method = HttpMethod.Get;
Url = new HttpUri(url);
Headers = new HttpHeader();
Method = HttpMethod.Get;
ConnectionKeepAlive = true;
AllowAutoRedirect = true;
LogHttpError = true;
Cookies = new Dictionary<string, string>();
if (!RuntimeInfo.IsProduction)
@ -37,16 +40,20 @@ namespace NzbDrone.Common.Http
public IWebProxy Proxy { get; set; }
public byte[] ContentData { get; set; }
public string ContentSummary { get; set; }
public ICredentials Credentials { get; set; }
public bool SuppressHttpError { get; set; }
public IEnumerable<HttpStatusCode> SuppressHttpErrorStatusCodes { get; set; }
public bool UseSimplifiedUserAgent { get; set; }
public bool AllowAutoRedirect { get; set; }
public bool ConnectionKeepAlive { get; set; }
public bool LogResponseContent { get; set; }
public bool LogHttpError { get; set; }
public Dictionary<string, string> Cookies { get; private set; }
public bool StoreRequestCookie { get; set; }
public bool StoreResponseCookie { get; set; }
public TimeSpan RequestTimeout { get; set; }
public TimeSpan RateLimit { get; set; }
public Stream ResponseStream { get; set; }
public override string ToString()
{
@ -103,12 +110,5 @@ namespace NzbDrone.Common.Http
return encoding.GetString(ContentData);
}
}
public void AddBasicAuthentication(string username, string password)
{
var authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes($"{username}:{password}"));
Headers.Set("Authorization", "Basic " + authInfo);
}
}
}

@ -21,12 +21,13 @@ namespace NzbDrone.Common.Http
public Dictionary<string, string> Segments { get; private set; }
public HttpHeader Headers { get; private set; }
public bool SuppressHttpError { get; set; }
public bool LogHttpError { get; set; }
public bool UseSimplifiedUserAgent { get; set; }
public bool AllowAutoRedirect { get; set; }
public bool ConnectionKeepAlive { get; set; }
public TimeSpan RateLimit { get; set; }
public bool LogResponseContent { get; set; }
public NetworkCredential NetworkCredential { get; set; }
public ICredentials NetworkCredential { get; set; }
public Dictionary<string, string> Cookies { get; private set; }
public bool StoreRequestCookie { get; set; }
public bool StoreResponseCookie { get; set; }
@ -46,6 +47,7 @@ namespace NzbDrone.Common.Http
Headers = new HttpHeader();
Cookies = new Dictionary<string, string>();
FormData = new List<HttpFormData>();
LogHttpError = true;
}
public HttpRequestBuilder(bool useHttps, string host, int port, string urlBase = null)
@ -106,6 +108,7 @@ namespace NzbDrone.Common.Http
request.Method = Method;
request.Encoding = Encoding;
request.SuppressHttpError = SuppressHttpError;
request.LogHttpError = LogHttpError;
request.UseSimplifiedUserAgent = UseSimplifiedUserAgent;
request.AllowAutoRedirect = AllowAutoRedirect;
request.StoreRequestCookie = StoreRequestCookie;
@ -113,13 +116,7 @@ namespace NzbDrone.Common.Http
request.ConnectionKeepAlive = ConnectionKeepAlive;
request.RateLimit = RateLimit;
request.LogResponseContent = LogResponseContent;
if (NetworkCredential != null)
{
var authInfo = NetworkCredential.UserName + ":" + NetworkCredential.Password;
authInfo = Convert.ToBase64String(Encoding.GetEncoding("ISO-8859-1").GetBytes(authInfo));
request.Headers.Set("Authorization", "Basic " + authInfo);
}
request.Credentials = NetworkCredential;
foreach (var header in Headers)
{

@ -64,10 +64,11 @@ namespace NzbDrone.Common.Http
public bool HasHttpError => (int)StatusCode >= 400;
public bool HasHttpRedirect => StatusCode == HttpStatusCode.Moved ||
StatusCode == HttpStatusCode.MovedPermanently ||
StatusCode == HttpStatusCode.RedirectMethod ||
StatusCode == HttpStatusCode.TemporaryRedirect ||
StatusCode == HttpStatusCode.Found ||
StatusCode == HttpStatusCode.SeeOther ||
StatusCode == HttpStatusCode.TemporaryRedirect ||
StatusCode == HttpStatusCode.MultipleChoices ||
StatusCode == HttpStatusCode.PermanentRedirect ||
Headers.ContainsKey("Refresh");
public string RedirectUrl
@ -117,7 +118,7 @@ namespace NzbDrone.Common.Http
public override string ToString()
{
var result = string.Format("Res: [{0}] {1}: {2}.{3}", Request.Method, Request.Url, (int)StatusCode, StatusCode);
var result = string.Format("Res: [{0}] {1}: {2}.{3} ({4} bytes)", Request.Method, Request.Url, (int)StatusCode, StatusCode, ResponseData?.Length ?? 0);
if (HasHttpError && Headers.ContentType.IsNotNullOrWhiteSpace() && !Headers.ContentType.Equals("text/html", StringComparison.InvariantCultureIgnoreCase))
{

@ -0,0 +1,103 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Xml.Linq;
using NLog;
using NzbDrone.Common.Instrumentation;
namespace NzbDrone.Common.Http
{
public class XmlRpcRequestBuilder : HttpRequestBuilder
{
public static string XmlRpcContentType = "text/xml";
private static readonly Logger Logger = NzbDroneLogger.GetLogger(typeof(XmlRpcRequestBuilder));
public string XmlMethod { get; private set; }
public List<object> XmlParameters { get; private set; }
public XmlRpcRequestBuilder(string baseUrl)
: base(baseUrl)
{
Method = HttpMethod.Post;
XmlParameters = new List<object>();
}
public XmlRpcRequestBuilder(bool useHttps, string host, int port, string urlBase = null)
: this(BuildBaseUrl(useHttps, host, port, urlBase))
{
}
public override HttpRequestBuilder Clone()
{
var clone = base.Clone() as XmlRpcRequestBuilder;
clone.XmlParameters = new List<object>(XmlParameters);
return clone;
}
public XmlRpcRequestBuilder Call(string method, params object[] parameters)
{
var clone = Clone() as XmlRpcRequestBuilder;
clone.XmlMethod = method;
clone.XmlParameters = parameters.ToList();
return clone;
}
protected override void Apply(HttpRequest request)
{
base.Apply(request);
request.Headers.ContentType = XmlRpcContentType;
var methodCallElements = new List<XElement> { new XElement("methodName", XmlMethod) };
if (XmlParameters.Any())
{
var argElements = XmlParameters.Select(x => new XElement("param", ConvertParameter(x))).ToList();
var paramsElement = new XElement("params", argElements);
methodCallElements.Add(paramsElement);
}
var message = new XDocument(
new XDeclaration("1.0", "utf-8", "yes"),
new XElement("methodCall", methodCallElements));
var body = message.ToString();
Logger.Debug($"Executing remote method: {XmlMethod}");
Logger.Trace($"methodCall {XmlMethod} body:\n{body}");
request.SetContent(body);
}
private static XElement ConvertParameter(object value)
{
XElement data;
if (value is string s)
{
data = new XElement("string", s);
}
else if (value is List<string> l)
{
data = new XElement("array", new XElement("data", l.Select(x => new XElement("value", new XElement("string", x)))));
}
else if (value is int i)
{
data = new XElement("int", i);
}
else if (value is byte[] bytes)
{
data = new XElement("base64", Convert.ToBase64String(bytes));
}
else
{
throw new InvalidOperationException($"Unhandled argument type {value.GetType().Name}");
}
return new XElement("value", data);
}
}
}

@ -11,6 +11,7 @@ using NzbDrone.Common.TPL;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Http;
using NzbDrone.Core.Parser;
using NzbDrone.Core.Security;
using NzbDrone.Test.Common;
namespace NzbDrone.Core.Test.Framework
@ -25,7 +26,8 @@ namespace NzbDrone.Core.Test.Framework
Mocker.SetConstant<IHttpProxySettingsProvider>(new HttpProxySettingsProvider(Mocker.Resolve<ConfigService>()));
Mocker.SetConstant<ICreateManagedWebProxy>(new ManagedWebProxyFactory(Mocker.Resolve<CacheManager>()));
Mocker.SetConstant<IHttpDispatcher>(new ManagedHttpDispatcher(Mocker.Resolve<IHttpProxySettingsProvider>(), Mocker.Resolve<ICreateManagedWebProxy>(), Mocker.Resolve<UserAgentBuilder>(), Mocker.Resolve<IPlatformInfo>(), TestLogger));
Mocker.SetConstant<ICertificateValidationService>(new X509CertificateValidationService(Mocker.Resolve<ConfigService>(), TestLogger));
Mocker.SetConstant<IHttpDispatcher>(new ManagedHttpDispatcher(Mocker.Resolve<IHttpProxySettingsProvider>(), Mocker.Resolve<ICreateManagedWebProxy>(), Mocker.Resolve<ICertificateValidationService>(), Mocker.Resolve<UserAgentBuilder>(), Mocker.Resolve<CacheManager>()));
Mocker.SetConstant<IHttpClient>(new HttpClient(Array.Empty<IHttpRequestInterceptor>(), Mocker.Resolve<CacheManager>(), Mocker.Resolve<RateLimitService>(), Mocker.Resolve<IHttpDispatcher>(), TestLogger));
Mocker.SetConstant<IProwlarrCloudRequestBuilder>(new ProwlarrCloudRequestBuilder());
}

@ -1,111 +1,161 @@
using CookComputing.XmlRpc;
using System.Collections.Generic;
using System.Linq;
using System.Xml.Linq;
using System.Xml.XPath;
using NzbDrone.Core.Download.Extensions;
namespace NzbDrone.Core.Download.Clients.Aria2
{
public class Aria2Version
public class Aria2Fault
{
[XmlRpcMember("version")]
public string Version;
[XmlRpcMember("enabledFeatures")]
public string[] EnabledFeatures;
public Aria2Fault(XElement element)
{
foreach (var e in element.XPathSelectElements("./value/struct/member"))
{
var name = e.ElementAsString("name");
if (name == "faultCode")
{
FaultCode = e.Element("value").ElementAsInt("int");
}
else if (name == "faultString")
{
FaultString = e.Element("value").GetStringValue();
}
}
}
public int FaultCode { get; set; }
public string FaultString { get; set; }
}
public class Aria2Uri
public class Aria2Version
{
[XmlRpcMember("status")]
public string Status;
[XmlRpcMember("uri")]
public string Uri;
public Aria2Version(XElement element)
{
foreach (var e in element.XPathSelectElements("./struct/member"))
{
if (e.ElementAsString("name") == "version")
{
Version = e.Element("value").GetStringValue();
}
}
}
public string Version { get; set; }
}
public class Aria2File
{
[XmlRpcMember("index")]
public string Index;
[XmlRpcMember("length")]
public string Length;
public Aria2File(XElement element)
{
foreach (var e in element.XPathSelectElements("./struct/member"))
{
var name = e.ElementAsString("name");
if (name == "path")
{
Path = e.Element("value").GetStringValue();
}
}
}
public string Path { get; set; }
}
[XmlRpcMember("completedLength")]
public string CompletedLength;
public class Aria2Dict
{
public Aria2Dict(XElement element)
{
Dict = new Dictionary<string, string>();
[XmlRpcMember("path")]
public string Path;
foreach (var e in element.XPathSelectElements("./struct/member"))
{
Dict.Add(e.ElementAsString("name"), e.Element("value").GetStringValue());
}
}
[XmlRpcMember("selected")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string Selected;
public Dictionary<string, string> Dict { get; set; }
}
[XmlRpcMember("uris")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public Aria2Uri[] Uris;
public class Aria2Bittorrent
{
public Aria2Bittorrent(XElement element)
{
foreach (var e in element.Descendants("member"))
{
if (e.ElementAsString("name") == "name")
{
Name = e.Element("value").GetStringValue();
}
}
}
public string Name;
}
public class Aria2Status
{
[XmlRpcMember("bittorrent")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public XmlRpcStruct Bittorrent;
[XmlRpcMember("bitfield")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string Bitfield;
[XmlRpcMember("infoHash")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string InfoHash;
[XmlRpcMember("completedLength")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string CompletedLength;
[XmlRpcMember("connections")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string Connections;
[XmlRpcMember("dir")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string Dir;
[XmlRpcMember("downloadSpeed")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string DownloadSpeed;
[XmlRpcMember("files")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public Aria2File[] Files;
[XmlRpcMember("gid")]
public string Gid;
[XmlRpcMember("numPieces")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string NumPieces;
[XmlRpcMember("pieceLength")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string PieceLength;
[XmlRpcMember("status")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string Status;
[XmlRpcMember("totalLength")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string TotalLength;
[XmlRpcMember("uploadLength")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string UploadLength;
[XmlRpcMember("uploadSpeed")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string UploadSpeed;
[XmlRpcMember("errorMessage")]
[XmlRpcMissingMapping(MappingAction.Ignore)]
public string ErrorMessage;
public Aria2Status(XElement element)
{
foreach (var e in element.XPathSelectElements("./struct/member"))
{
var name = e.ElementAsString("name");
if (name == "bittorrent")
{
Bittorrent = new Aria2Bittorrent(e.Element("value"));
}
else if (name == "infoHash")
{
InfoHash = e.Element("value").GetStringValue();
}
else if (name == "completedLength")
{
CompletedLength = e.Element("value").GetStringValue();
}
else if (name == "downloadSpeed")
{
DownloadSpeed = e.Element("value").GetStringValue();
}
else if (name == "files")
{
Files = e.XPathSelectElement("./value/array/data")
.Elements()
.Select(x => new Aria2File(x))
.ToArray();
}
else if (name == "gid")
{
Gid = e.Element("value").GetStringValue();
}
else if (name == "status")
{
Status = e.Element("value").GetStringValue();
}
else if (name == "totalLength")
{
TotalLength = e.Element("value").GetStringValue();
}
else if (name == "uploadLength")
{
UploadLength = e.Element("value").GetStringValue();
}
else if (name == "errorMessage")
{
ErrorMessage = e.Element("value").GetStringValue();
}
}
}
public Aria2Bittorrent Bittorrent { get; set; }
public string InfoHash { get; set; }
public string CompletedLength { get; set; }
public string DownloadSpeed { get; set; }
public Aria2File[] Files { get; set; }
public string Gid { get; set; }
public string Status { get; set; }
public string TotalLength { get; set; }
public string UploadLength { get; set; }
public string ErrorMessage { get; set; }
}
}

@ -1,9 +1,9 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.Net;
using CookComputing.XmlRpc;
using NLog;
using System.Linq;
using System.Xml.Linq;
using System.Xml.XPath;
using NzbDrone.Common.Http;
using NzbDrone.Core.Download.Extensions;
namespace NzbDrone.Core.Download.Clients.Aria2
{
@ -12,133 +12,133 @@ namespace NzbDrone.Core.Download.Clients.Aria2
string GetVersion(Aria2Settings settings);
string AddUri(Aria2Settings settings, string magnet);
string AddTorrent(Aria2Settings settings, byte[] torrent);
Dictionary<string, string> GetGlobals(Aria2Settings settings);
List<Aria2Status> GetTorrents(Aria2Settings settings);
Aria2Status GetFromGID(Aria2Settings settings, string gid);
}
public interface IAria2 : IXmlRpcProxy
public class Aria2Proxy : IAria2Proxy
{
[XmlRpcMethod("aria2.getVersion")]
Aria2Version GetVersion(string token);
[XmlRpcMethod("aria2.addUri")]
string AddUri(string token, string[] uri);
[XmlRpcMethod("aria2.addTorrent")]
string AddTorrent(string token, byte[] torrent);
private readonly IHttpClient _httpClient;
[XmlRpcMethod("aria2.forceRemove")]
string Remove(string token, string gid);
public Aria2Proxy(IHttpClient httpClient)
{
_httpClient = httpClient;
}
[XmlRpcMethod("aria2.tellStatus")]
Aria2Status GetFromGid(string token, string gid);
public string GetVersion(Aria2Settings settings)
{
var response = ExecuteRequest(settings, "aria2.getVersion", GetToken(settings));
[XmlRpcMethod("aria2.getGlobalOption")]
XmlRpcStruct GetGlobalOption(string token);
var element = response.XPathSelectElement("./methodResponse/params/param/value");
[XmlRpcMethod("aria2.tellActive")]
Aria2Status[] GetActives(string token);
var version = new Aria2Version(element);
[XmlRpcMethod("aria2.tellWaiting")]
Aria2Status[] GetWaitings(string token, int offset, int num);
return version.Version;
}
[XmlRpcMethod("aria2.tellStopped")]
Aria2Status[] GetStoppeds(string token, int offset, int num);
}
public Aria2Status GetFromGID(Aria2Settings settings, string gid)
{
var response = ExecuteRequest(settings, "aria2.tellStatus", GetToken(settings), gid);
public class Aria2Proxy : IAria2Proxy
{
private readonly Logger _logger;
var element = response.XPathSelectElement("./methodResponse/params/param/value");
public Aria2Proxy(Logger logger)
{
_logger = logger;
return new Aria2Status(element);
}
private string GetToken(Aria2Settings settings)
private List<Aria2Status> GetTorrentsMethod(Aria2Settings settings, string method, params object[] args)
{
return $"token:{settings?.SecretToken}";
}
var allArgs = new List<object> { GetToken(settings) };
if (args.Any())
{
allArgs.AddRange(args);
}
private string GetURL(Aria2Settings settings)
{
return $"http{(settings.UseSsl ? "s" : "")}://{settings.Host}:{settings.Port}{settings.RpcPath}";
var response = ExecuteRequest(settings, method, allArgs.ToArray());
var element = response.XPathSelectElement("./methodResponse/params/param/value/array/data");
var torrents = element?.Elements()
.Select(x => new Aria2Status(x))
.ToList()
?? new List<Aria2Status>();
return torrents;
}
public string GetVersion(Aria2Settings settings)
public List<Aria2Status> GetTorrents(Aria2Settings settings)
{
_logger.Debug("> aria2.getVersion");
var active = GetTorrentsMethod(settings, "aria2.tellActive");
var client = BuildClient(settings);
var version = ExecuteRequest(() => client.GetVersion(GetToken(settings)));
var waiting = GetTorrentsMethod(settings, "aria2.tellWaiting", 0, 10 * 1024);
_logger.Debug("< aria2.getVersion");
var stopped = GetTorrentsMethod(settings, "aria2.tellStopped", 0, 10 * 1024);
return version.Version;
var items = new List<Aria2Status>();
items.AddRange(active);
items.AddRange(waiting);
items.AddRange(stopped);
return items;
}
public Aria2Status GetFromGID(Aria2Settings settings, string gid)
public Dictionary<string, string> GetGlobals(Aria2Settings settings)
{
_logger.Debug("> aria2.tellStatus");
var response = ExecuteRequest(settings, "aria2.getGlobalOption", GetToken(settings));
var client = BuildClient(settings);
var found = ExecuteRequest(() => client.GetFromGid(GetToken(settings), gid));
var element = response.XPathSelectElement("./methodResponse/params/param/value");
_logger.Debug("< aria2.tellStatus");
var result = new Aria2Dict(element);
return found;
return result.Dict;
}
public string AddUri(Aria2Settings settings, string magnet)
{
_logger.Debug("> aria2.addUri");
var client = BuildClient(settings);
var gid = ExecuteRequest(() => client.AddUri(GetToken(settings), new[] { magnet }));
var response = ExecuteRequest(settings, "aria2.addUri", GetToken(settings), new List<string> { magnet });
_logger.Debug("< aria2.addUri");
var gid = response.GetStringResponse();
return gid;
}
public string AddTorrent(Aria2Settings settings, byte[] torrent)
{
_logger.Debug("> aria2.addTorrent");
var client = BuildClient(settings);
var gid = ExecuteRequest(() => client.AddTorrent(GetToken(settings), torrent));
var response = ExecuteRequest(settings, "aria2.addTorrent", GetToken(settings), torrent);
_logger.Debug("< aria2.addTorrent");
var gid = response.GetStringResponse();
return gid;
}
private IAria2 BuildClient(Aria2Settings settings)
private string GetToken(Aria2Settings settings)
{
var client = XmlRpcProxyGen.Create<IAria2>();
client.Url = GetURL(settings);
return client;
return $"token:{settings?.SecretToken}";
}
private T ExecuteRequest<T>(Func<T> task)
private XDocument ExecuteRequest(Aria2Settings settings, string methodName, params object[] args)
{
try
{
return task();
}
catch (XmlRpcServerException ex)
var requestBuilder = new XmlRpcRequestBuilder(settings.UseSsl, settings.Host, settings.Port, settings.RpcPath)
{
throw new DownloadClientException("Unable to connect to aria2, please check your settings", ex);
}
catch (WebException ex)
LogResponseContent = true,
};
var request = requestBuilder.Call(methodName, args).Build();
var response = _httpClient.Execute(request);
var doc = XDocument.Parse(response.Content);
var faultElement = doc.XPathSelectElement("./methodResponse/fault");
if (faultElement != null)
{
if (ex.Status == WebExceptionStatus.TrustFailure)
{
throw new DownloadClientUnavailableException("Unable to connect to aria2, certificate validation failed.", ex);
}
var fault = new Aria2Fault(faultElement);
throw new DownloadClientUnavailableException("Unable to connect to aria2, please check your settings", ex);
throw new DownloadClientException($"Aria2 returned error code {fault.FaultCode}: {fault.FaultString}");
}
return doc;
}
}
}

@ -0,0 +1,28 @@
using System.Xml.Linq;
using System.Xml.XPath;
using NzbDrone.Core.Download.Extensions;
namespace NzbDrone.Core.Download.Clients.RTorrent
{
public class RTorrentFault
{
public RTorrentFault(XElement element)
{
foreach (var e in element.XPathSelectElements("./value/struct/member"))
{
var name = e.ElementAsString("name");
if (name == "faultCode")
{
FaultCode = e.Element("value").GetIntValue();
}
else if (name == "faultString")
{
FaultString = e.Element("value").GetStringValue();
}
}
}
public int FaultCode { get; set; }
public string FaultString { get; set; }
}
}

@ -1,10 +1,12 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using CookComputing.XmlRpc;
using NLog;
using System.Xml.Linq;
using System.Xml.XPath;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Serializer;
using NzbDrone.Common.Http;
using NzbDrone.Core.Download.Extensions;
namespace NzbDrone.Core.Download.Clients.RTorrent
{
@ -18,122 +20,70 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
void RemoveTorrent(string hash, RTorrentSettings settings);
void SetTorrentLabel(string hash, string label, RTorrentSettings settings);
bool HasHashTorrent(string hash, RTorrentSettings settings);
}
public interface IRTorrent : IXmlRpcProxy
{
[XmlRpcMethod("d.multicall2")]
object[] TorrentMulticall(params string[] parameters);
[XmlRpcMethod("load.normal")]
int LoadNormal(string target, string data, params string[] commands);
[XmlRpcMethod("load.start")]
int LoadStart(string target, string data, params string[] commands);
[XmlRpcMethod("load.raw")]
int LoadRaw(string target, byte[] data, params string[] commands);
[XmlRpcMethod("load.raw_start")]
int LoadRawStart(string target, byte[] data, params string[] commands);
[XmlRpcMethod("d.erase")]
int Remove(string hash);
[XmlRpcMethod("d.name")]
string GetName(string hash);
[XmlRpcMethod("d.custom1.set")]
string SetLabel(string hash, string label);
[XmlRpcMethod("system.client_version")]
string GetVersion();
void PushTorrentUniqueView(string hash, string view, RTorrentSettings settings);
}
public class RTorrentProxy : IRTorrentProxy
{
private readonly Logger _logger;
private readonly IHttpClient _httpClient;
public RTorrentProxy(Logger logger)
public RTorrentProxy(IHttpClient httpClient)
{
_logger = logger;
_httpClient = httpClient;
}
public string GetVersion(RTorrentSettings settings)
{
_logger.Debug("Executing remote method: system.client_version");
var client = BuildClient(settings);
var version = ExecuteRequest(() => client.GetVersion());
var document = ExecuteRequest(settings, "system.client_version");
return version;
return document.Descendants("string").FirstOrDefault()?.Value ?? "0.0.0";
}
public List<RTorrentTorrent> GetTorrents(RTorrentSettings settings)
{
_logger.Debug("Executing remote method: d.multicall2");
var client = BuildClient(settings);
var ret = ExecuteRequest(() => client.TorrentMulticall(
"",
"",
"d.name=", // string
"d.hash=", // string
"d.base_path=", // string
"d.custom1=", // string (label)
"d.size_bytes=", // long
"d.left_bytes=", // long
"d.down.rate=", // long (in bytes / s)
"d.ratio=", // long
"d.is_open=", // long
"d.is_active=", // long
"d.complete=")); //long
_logger.Trace(ret.ToJson());
var items = new List<RTorrentTorrent>();
foreach (object[] torrent in ret)
{
var labelDecoded = System.Web.HttpUtility.UrlDecode((string)torrent[3]);
var item = new RTorrentTorrent();
item.Name = (string)torrent[0];
item.Hash = (string)torrent[1];
item.Path = (string)torrent[2];
item.Category = labelDecoded;
item.TotalSize = (long)torrent[4];
item.RemainingSize = (long)torrent[5];
item.DownRate = (long)torrent[6];
item.Ratio = (long)torrent[7];
item.IsOpen = Convert.ToBoolean((long)torrent[8]);
item.IsActive = Convert.ToBoolean((long)torrent[9]);
item.IsFinished = Convert.ToBoolean((long)torrent[10]);
items.Add(item);
}
return items;
var document = ExecuteRequest(settings,
"d.multicall2",
"",
"",
"d.name=", // string
"d.hash=", // string
"d.base_path=", // string
"d.custom1=", // string (label)
"d.size_bytes=", // long
"d.left_bytes=", // long
"d.down.rate=", // long (in bytes / s)
"d.ratio=", // long
"d.is_open=", // long
"d.is_active=", // long
"d.complete=", //long
"d.timestamp.finished="); // long (unix timestamp)
var torrents = document.XPathSelectElement("./methodResponse/params/param/value/array/data")
?.Elements()
.Select(x => new RTorrentTorrent(x))
.ToList()
?? new List<RTorrentTorrent>();
return torrents;
}
public void AddTorrentFromUrl(string torrentUrl, string label, RTorrentPriority priority, string directory, RTorrentSettings settings)
{
var client = BuildClient(settings);
var response = ExecuteRequest(() =>
var args = new List<object> { "", torrentUrl };
args.AddRange(GetCommands(label, priority, directory));
XDocument response;
if (settings.AddStopped)
{
if (settings.AddStopped)
{
_logger.Debug("Executing remote method: load.normal");
return client.LoadNormal("", torrentUrl, GetCommands(label, priority, directory));
}
else
{
_logger.Debug("Executing remote method: load.start");
return client.LoadStart("", torrentUrl, GetCommands(label, priority, directory));
}
});
response = ExecuteRequest(settings, "load.normal", args.ToArray());
}
else
{
response = ExecuteRequest(settings, "load.start", args.ToArray());
}
if (response != 0)
if (response.GetIntResponse() != 0)
{
throw new DownloadClientException("Could not add torrent: {0}.", torrentUrl);
}
@ -141,22 +91,21 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
public void AddTorrentFromFile(string fileName, byte[] fileContent, string label, RTorrentPriority priority, string directory, RTorrentSettings settings)
{
var client = BuildClient(settings);
var response = ExecuteRequest(() =>
var args = new List<object> { "", fileContent };
args.AddRange(GetCommands(label, priority, directory));
XDocument response;
if (settings.AddStopped)
{
if (settings.AddStopped)
{
_logger.Debug("Executing remote method: load.raw");
return client.LoadRaw("", fileContent, GetCommands(label, priority, directory));
}
else
{
_logger.Debug("Executing remote method: load.raw_start");
return client.LoadRawStart("", fileContent, GetCommands(label, priority, directory));
}
});
response = ExecuteRequest(settings, "load.raw", args.ToArray());
}
else
{
response = ExecuteRequest(settings, "load.raw_start", args.ToArray());
}
if (response != 0)
if (response.GetIntResponse() != 0)
{
throw new DownloadClientException("Could not add torrent: {0}.", fileName);
}
@ -164,25 +113,29 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
public void SetTorrentLabel(string hash, string label, RTorrentSettings settings)
{
_logger.Debug("Executing remote method: d.custom1.set");
var client = BuildClient(settings);
var response = ExecuteRequest(() => client.SetLabel(hash, label));
var response = ExecuteRequest(settings, "d.custom1.set", hash, label);
if (response != label)
if (response.GetStringResponse() != label)
{
throw new DownloadClientException("Could not set label to {1} for torrent: {0}.", hash, label);
}
}
public void RemoveTorrent(string hash, RTorrentSettings settings)
public void PushTorrentUniqueView(string hash, string view, RTorrentSettings settings)
{
_logger.Debug("Executing remote method: d.erase");
var response = ExecuteRequest(settings, "d.views.push_back_unique", hash, view);
var client = BuildClient(settings);
var response = ExecuteRequest(() => client.Remove(hash));
if (response.GetIntResponse() != 0)
{
throw new DownloadClientException("Could not push unique view {0} for torrent: {1}.", view, hash);
}
}
if (response != 0)
public void RemoveTorrent(string hash, RTorrentSettings settings)
{
var response = ExecuteRequest(settings, "d.erase", hash);
if (response.GetIntResponse() != 0)
{
throw new DownloadClientException("Could not remove torrent: {0}.", hash);
}
@ -190,13 +143,10 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
public bool HasHashTorrent(string hash, RTorrentSettings settings)
{
_logger.Debug("Executing remote method: d.name");
var client = BuildClient(settings);
try
{
var name = ExecuteRequest(() => client.GetName(hash));
var response = ExecuteRequest(settings, "d.name", hash);
var name = response.GetStringResponse();
if (name.IsNullOrWhiteSpace())
{
@ -235,45 +185,34 @@ namespace NzbDrone.Core.Download.Clients.RTorrent
return result.ToArray();
}
private IRTorrent BuildClient(RTorrentSettings settings)
private XDocument ExecuteRequest(RTorrentSettings settings, string methodName, params object[] args)
{
var client = XmlRpcProxyGen.Create<IRTorrent>();
client.Url = string.Format(@"{0}://{1}:{2}/{3}",
settings.UseSsl ? "https" : "http",
settings.Host,
settings.Port,
settings.UrlBase);
client.EnableCompression = true;
var requestBuilder = new XmlRpcRequestBuilder(settings.UseSsl, settings.Host, settings.Port, settings.UrlBase)
{
LogResponseContent = true,
};
if (!settings.Username.IsNullOrWhiteSpace())
{
client.Credentials = new NetworkCredential(settings.Username, settings.Password);
requestBuilder.NetworkCredential = new NetworkCredential(settings.Username, settings.Password);
}
return client;
}
var request = requestBuilder.Call(methodName, args).Build();
private T ExecuteRequest<T>(Func<T> task)
{
try
{
return task();
}
catch (XmlRpcServerException ex)
{
throw new DownloadClientException("Unable to connect to rTorrent, please check your settings", ex);
}
catch (WebException ex)
var response = _httpClient.Execute(request);
var doc = XDocument.Parse(response.Content);
var faultElement = doc.XPathSelectElement("./methodResponse/fault");
if (faultElement != null)
{
if (ex.Status == WebExceptionStatus.TrustFailure)
{
throw new DownloadClientUnavailableException("Unable to connect to rTorrent, certificate validation failed.", ex);
}
var fault = new RTorrentFault(faultElement);
throw new DownloadClientUnavailableException("Unable to connect to rTorrent, please check your settings", ex);
throw new DownloadClientException($"rTorrent returned error code {fault.FaultCode}: {fault.FaultString}");
}
return doc;
}
}
}

@ -1,7 +1,35 @@
namespace NzbDrone.Core.Download.Clients.RTorrent
using System;
using System.Linq;
using System.Web;
using System.Xml.Linq;
using NzbDrone.Core.Download.Extensions;
namespace NzbDrone.Core.Download.Clients.RTorrent
{
public class RTorrentTorrent
{
public RTorrentTorrent()
{
}
public RTorrentTorrent(XElement element)
{
var data = element.Descendants("value").ToList();
Name = data[0].GetStringValue();
Hash = data[1].GetStringValue();
Path = data[2].GetStringValue();
Category = HttpUtility.UrlDecode(data[3].GetStringValue());
TotalSize = data[4].GetLongValue();
RemainingSize = data[5].GetLongValue();
DownRate = data[6].GetLongValue();
Ratio = data[7].GetLongValue();
IsOpen = Convert.ToBoolean(data[8].GetLongValue());
IsActive = Convert.ToBoolean(data[9].GetLongValue());
IsFinished = Convert.ToBoolean(data[10].GetLongValue());
FinishedTime = data[11].GetLongValue();
}
public string Name { get; set; }
public string Hash { get; set; }
public string Path { get; set; }
@ -10,6 +38,7 @@
public long RemainingSize { get; set; }
public long DownRate { get; set; }
public long Ratio { get; set; }
public long FinishedTime { get; set; }
public bool IsFinished { get; set; }
public bool IsOpen { get; set; }
public bool IsActive { get; set; }

@ -0,0 +1,54 @@
using System.Xml.Linq;
using System.Xml.XPath;
namespace NzbDrone.Core.Download.Extensions
{
internal static class XmlExtensions
{
public static string GetStringValue(this XElement element)
{
return element.ElementAsString("string");
}
public static long GetLongValue(this XElement element)
{
return element.ElementAsLong("i8");
}
public static int GetIntValue(this XElement element)
{
return element.ElementAsInt("i4");
}
public static string ElementAsString(this XElement element, XName name, bool trim = false)
{
var el = element.Element(name);
return string.IsNullOrWhiteSpace(el?.Value)
? null
: (trim ? el.Value.Trim() : el.Value);
}
public static long ElementAsLong(this XElement element, XName name)
{
var el = element.Element(name);
return long.TryParse(el?.Value, out long value) ? value : default;
}
public static int ElementAsInt(this XElement element, XName name)
{
var el = element.Element(name);
return int.TryParse(el?.Value, out int value) ? value : default(int);
}
public static int GetIntResponse(this XDocument document)
{
return document.XPathSelectElement("./methodResponse/params/param/value").GetIntValue();
}
public static string GetStringResponse(this XDocument document)
{
return document.XPathSelectElement("./methodResponse/params/param/value").GetStringValue();
}
}
}

@ -59,7 +59,7 @@ namespace NzbDrone.Core.Indexers.Headphones
var request = requestBuilder.Build();
request.AddBasicAuthentication(Settings.Username, Settings.Password);
request.Credentials = new BasicNetworkCredential(Settings.Username, Settings.Password);
try
{

@ -131,7 +131,7 @@ namespace NzbDrone.Core.Indexers.Headphones
}
var request = new IndexerRequest(string.Format("{0}&{1}", baseUrl, parameters.GetQueryString()), HttpAccept.Rss);
request.HttpRequest.AddBasicAuthentication(Settings.Username, Settings.Password);
request.HttpRequest.Credentials = new BasicNetworkCredential(Settings.Username, Settings.Password);
yield return request;
}

@ -7,17 +7,20 @@ using MailKit.Security;
using MimeKit;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http.Dispatchers;
namespace NzbDrone.Core.Notifications.Email
{
public class Email : NotificationBase<EmailSettings>
{
private readonly ICertificateValidationService _certificateValidationService;
private readonly Logger _logger;
public override string Name => "Email";
public Email(Logger logger)
public Email(ICertificateValidationService certificateValidationService, Logger logger)
{
_certificateValidationService = certificateValidationService;
_logger = logger;
}
@ -113,6 +116,8 @@ namespace NzbDrone.Core.Notifications.Email
}
}
client.ServerCertificateValidationCallback = _certificateValidationService.ShouldByPassValidationError;
_logger.Debug("Connecting to mail server");
client.Connect(settings.Server, settings.Port, serverOption);

@ -102,7 +102,7 @@ namespace NzbDrone.Core.Notifications.PushBullet
var request = requestBuilder.Build();
request.Method = HttpMethod.Get;
request.AddBasicAuthentication(settings.ApiKey, string.Empty);
request.Credentials = new BasicNetworkCredential(settings.ApiKey, string.Empty);
var response = _httpClient.Execute(request);
@ -198,7 +198,7 @@ namespace NzbDrone.Core.Notifications.PushBullet
var request = requestBuilder.Build();
request.AddBasicAuthentication(settings.ApiKey, string.Empty);
request.Credentials = new BasicNetworkCredential(settings.ApiKey, string.Empty);
_httpClient.Execute(request);
}

@ -0,0 +1,110 @@
using System.Collections.Generic;
using System.Collections.Specialized;
using System.Linq;
using System.Net.Http;
using System.Text;
using System.Web;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.OAuth;
namespace NzbDrone.Core.Notifications.Twitter
{
public interface ITwitterProxy
{
NameValueCollection GetOAuthToken(string consumerKey, string consumerSecret, string oauthToken, string oauthVerifier);
string GetOAuthRedirect(string consumerKey, string consumerSecret, string callbackUrl);
void UpdateStatus(string message, TwitterSettings settings);
void DirectMessage(string message, TwitterSettings settings);
}
public class TwitterProxy : ITwitterProxy
{
private readonly IHttpClient _httpClient;
public TwitterProxy(IHttpClient httpClient)
{
_httpClient = httpClient;
}
public string GetOAuthRedirect(string consumerKey, string consumerSecret, string callbackUrl)
{
// Creating a new instance with a helper method
var oAuthRequest = OAuthRequest.ForRequestToken(consumerKey, consumerSecret, callbackUrl);
oAuthRequest.RequestUrl = "https://api.twitter.com/oauth/request_token";
var qscoll = HttpUtility.ParseQueryString(ExecuteRequest(GetRequest(oAuthRequest, new Dictionary<string, string>())).Content);
return string.Format("https://api.twitter.com/oauth/authorize?oauth_token={0}", qscoll["oauth_token"]);
}
public NameValueCollection GetOAuthToken(string consumerKey, string consumerSecret, string oauthToken, string oauthVerifier)
{
// Creating a new instance with a helper method
var oAuthRequest = OAuthRequest.ForAccessToken(consumerKey, consumerSecret, oauthToken, "", oauthVerifier);
oAuthRequest.RequestUrl = "https://api.twitter.com/oauth/access_token";
return HttpUtility.ParseQueryString(ExecuteRequest(GetRequest(oAuthRequest, new Dictionary<string, string>())).Content);
}
public void UpdateStatus(string message, TwitterSettings settings)
{
var oAuthRequest = OAuthRequest.ForProtectedResource("POST", settings.ConsumerKey, settings.ConsumerSecret, settings.AccessToken, settings.AccessTokenSecret);
oAuthRequest.RequestUrl = "https://api.twitter.com/1.1/statuses/update.json";
var customParams = new Dictionary<string, string>
{
{ "status", message.EncodeRFC3986() }
};
var request = GetRequest(oAuthRequest, customParams);
request.Headers.ContentType = "application/x-www-form-urlencoded";
request.SetContent(Encoding.ASCII.GetBytes(GetCustomParametersString(customParams)));
ExecuteRequest(request);
}
public void DirectMessage(string message, TwitterSettings settings)
{
var oAuthRequest = OAuthRequest.ForProtectedResource("POST", settings.ConsumerKey, settings.ConsumerSecret, settings.AccessToken, settings.AccessTokenSecret);
oAuthRequest.RequestUrl = "https://api.twitter.com/1.1/direct_messages/new.json";
var customParams = new Dictionary<string, string>
{
{ "text", message.EncodeRFC3986() },
{ "screenname", settings.Mention.EncodeRFC3986() }
};
var request = GetRequest(oAuthRequest, customParams);
request.Headers.ContentType = "application/x-www-form-urlencoded";
request.SetContent(Encoding.ASCII.GetBytes(GetCustomParametersString(customParams)));
ExecuteRequest(request);
}
private string GetCustomParametersString(Dictionary<string, string> customParams)
{
return customParams.Select(x => string.Format("{0}={1}", x.Key, x.Value)).Join("&");
}
private HttpRequest GetRequest(OAuthRequest oAuthRequest, Dictionary<string, string> customParams)
{
var auth = oAuthRequest.GetAuthorizationHeader(customParams);
var request = new HttpRequest(oAuthRequest.RequestUrl);
request.Headers.Add("Authorization", auth);
request.Method = oAuthRequest.Method == "POST" ? HttpMethod.Post : HttpMethod.Get;
return request;
}
private HttpResponse ExecuteRequest(HttpRequest request)
{
return _httpClient.Execute(request);
}
}
}

@ -1,13 +1,9 @@
using System;
using System.Collections.Specialized;
using System.IO;
using System.Net;
using System.Web;
using FluentValidation.Results;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http;
using NzbDrone.Common.OAuth;
namespace NzbDrone.Core.Notifications.Twitter
{
@ -21,31 +17,18 @@ namespace NzbDrone.Core.Notifications.Twitter
public class TwitterService : ITwitterService
{
private readonly IHttpClient _httpClient;
private readonly ITwitterProxy _twitterProxy;
private readonly Logger _logger;
public TwitterService(IHttpClient httpClient, Logger logger)
public TwitterService(ITwitterProxy twitterProxy, Logger logger)
{
_httpClient = httpClient;
_twitterProxy = twitterProxy;
_logger = logger;
}
private NameValueCollection OAuthQuery(OAuthRequest oAuthRequest)
{
var auth = oAuthRequest.GetAuthorizationHeader();
var request = new Common.Http.HttpRequest(oAuthRequest.RequestUrl);
request.Headers.Add("Authorization", auth);
var response = _httpClient.Get(request);
return HttpUtility.ParseQueryString(response.Content);
}
public OAuthToken GetOAuthToken(string consumerKey, string consumerSecret, string oauthToken, string oauthVerifier)
{
// Creating a new instance with a helper method
var oAuthRequest = OAuthRequest.ForAccessToken(consumerKey, consumerSecret, oauthToken, "", oauthVerifier);
oAuthRequest.RequestUrl = "https://api.twitter.com/oauth/access_token";
var qscoll = OAuthQuery(oAuthRequest);
var qscoll = _twitterProxy.GetOAuthToken(consumerKey, consumerSecret, oauthToken, oauthVerifier);
return new OAuthToken
{
@ -56,31 +39,16 @@ namespace NzbDrone.Core.Notifications.Twitter
public string GetOAuthRedirect(string consumerKey, string consumerSecret, string callbackUrl)
{
// Creating a new instance with a helper method
var oAuthRequest = OAuthRequest.ForRequestToken(consumerKey, consumerSecret, callbackUrl);
oAuthRequest.RequestUrl = "https://api.twitter.com/oauth/request_token";
var qscoll = OAuthQuery(oAuthRequest);
return string.Format("https://api.twitter.com/oauth/authorize?oauth_token={0}", qscoll["oauth_token"]);
return _twitterProxy.GetOAuthRedirect(consumerKey, consumerSecret, callbackUrl);
}
public void SendNotification(string message, TwitterSettings settings)
{
try
{
var oAuth = new TinyTwitter.OAuthInfo
{
ConsumerKey = settings.ConsumerKey,
ConsumerSecret = settings.ConsumerSecret,
AccessToken = settings.AccessToken,
AccessSecret = settings.AccessTokenSecret
};
var twitter = new TinyTwitter.TinyTwitter(oAuth);
if (settings.DirectMessage)
{
twitter.DirectMessage(message, settings.Mention);
_twitterProxy.DirectMessage(message, settings);
}
else
{
@ -89,7 +57,7 @@ namespace NzbDrone.Core.Notifications.Twitter
message += string.Format(" @{0}", settings.Mention);
}
twitter.UpdateStatus(message);
_twitterProxy.UpdateStatus(message, settings);
}
}
catch (WebException ex)

@ -40,7 +40,7 @@ namespace NzbDrone.Core.Notifications.Webhook
if (settings.Username.IsNotNullOrWhiteSpace() || settings.Password.IsNotNullOrWhiteSpace())
{
request.AddBasicAuthentication(settings.Username, settings.Password);
request.Credentials = new BasicNetworkCredential(settings.Username, settings.Password);
}
_httpClient.Execute(request);

@ -4,13 +4,12 @@ using System.Net.Security;
using System.Security.Cryptography.X509Certificates;
using NLog;
using NzbDrone.Common.Extensions;
using NzbDrone.Common.Http.Dispatchers;
using NzbDrone.Core.Configuration;
using NzbDrone.Core.Lifecycle;
using NzbDrone.Core.Messaging.Events;
namespace NzbDrone.Core.Security
{
public class X509CertificateValidationService : IHandle<ApplicationStartedEvent>
public class X509CertificateValidationService : ICertificateValidationService
{
private readonly IConfigService _configService;
private readonly Logger _logger;
@ -21,19 +20,29 @@ namespace NzbDrone.Core.Security
_logger = logger;
}
private bool ShouldByPassValidationError(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
public bool ShouldByPassValidationError(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors sslPolicyErrors)
{
var request = sender as HttpWebRequest;
var targetHostName = string.Empty;
if (request == null)
if (sender is not SslStream && sender is not string)
{
return true;
}
var cert2 = certificate as X509Certificate2;
if (cert2 != null && request != null && cert2.SignatureAlgorithm.FriendlyName == "md5RSA")
if (sender is SslStream request)
{
_logger.Error("https://{0} uses the obsolete md5 hash in it's https certificate, if that is your certificate, please (re)create certificate with better algorithm as soon as possible.", request.RequestUri.Authority);
targetHostName = request.TargetHostName;
}
// Mailkit passes host in sender as string
if (sender is string stringHost)
{
targetHostName = stringHost;
}
if (certificate is X509Certificate2 cert2 && cert2.SignatureAlgorithm.FriendlyName == "md5RSA")
{
_logger.Error("https://{0} uses the obsolete md5 hash in it's https certificate, if that is your certificate, please (re)create certificate with better algorithm as soon as possible.", targetHostName);
}
if (sslPolicyErrors == SslPolicyErrors.None)
@ -41,12 +50,12 @@ namespace NzbDrone.Core.Security
return true;
}
if (request.RequestUri.Host == "localhost" || request.RequestUri.Host == "127.0.0.1")
if (targetHostName == "localhost" || targetHostName == "127.0.0.1")
{
return true;
}
var ipAddresses = GetIPAddresses(request.RequestUri.Host);
var ipAddresses = GetIPAddresses(targetHostName);
var certificateValidation = _configService.CertificateValidation;
if (certificateValidation == CertificateValidationType.Disabled)
@ -60,7 +69,7 @@ namespace NzbDrone.Core.Security
return true;
}
_logger.Error("Certificate validation for {0} failed. {1}", request.Address, sslPolicyErrors);
_logger.Error("Certificate validation for {0} failed. {1}", targetHostName, sslPolicyErrors);
return false;
}
@ -74,10 +83,5 @@ namespace NzbDrone.Core.Security
return Dns.GetHostEntry(host).AddressList;
}
public void Handle(ApplicationStartedEvent message)
{
ServicePointManager.ServerCertificateValidationCallback = ShouldByPassValidationError;
}
}
}

@ -1,235 +0,0 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
namespace TinyTwitter
{
public class OAuthInfo
{
public string ConsumerKey { get; set; }
public string ConsumerSecret { get; set; }
public string AccessToken { get; set; }
public string AccessSecret { get; set; }
}
public class Tweet
{
public long Id { get; set; }
public DateTime CreatedAt { get; set; }
public string UserName { get; set; }
public string ScreenName { get; set; }
public string Text { get; set; }
}
public class TinyTwitter
{
private readonly OAuthInfo _oauth;
public TinyTwitter(OAuthInfo oauth)
{
_oauth = oauth;
}
public void UpdateStatus(string message)
{
new RequestBuilder(_oauth, "POST", "https://api.twitter.com/1.1/statuses/update.json")
.AddParameter("status", message)
.Execute();
}
/**
*
* As of June 26th 2015 Direct Messaging is not part of TinyTwitter.
* I have added it to Sonarr's copy to make our implementation easier
* and added this banner so it's not blindly updated.
*
**/
public void DirectMessage(string message, string screenName)
{
new RequestBuilder(_oauth, "POST", "https://api.twitter.com/1.1/direct_messages/new.json")
.AddParameter("text", message)
.AddParameter("screen_name", screenName)
.Execute();
}
public class RequestBuilder
{
private const string VERSION = "1.0";
private const string SIGNATURE_METHOD = "HMAC-SHA1";
private readonly OAuthInfo _oauth;
private readonly string _method;
private readonly IDictionary<string, string> _customParameters;
private readonly string _url;
public RequestBuilder(OAuthInfo oauth, string method, string url)
{
_oauth = oauth;
_method = method;
_url = url;
_customParameters = new Dictionary<string, string>();
}
public RequestBuilder AddParameter(string name, string value)
{
_customParameters.Add(name, value.EncodeRFC3986());
return this;
}
public string Execute()
{
var timespan = GetTimestamp();
var nonce = CreateNonce();
var parameters = new Dictionary<string, string>(_customParameters);
AddOAuthParameters(parameters, timespan, nonce);
var signature = GenerateSignature(parameters);
var headerValue = GenerateAuthorizationHeaderValue(parameters, signature);
var request = (HttpWebRequest)WebRequest.Create(GetRequestUrl());
request.Method = _method;
request.ContentType = "application/x-www-form-urlencoded";
request.Headers.Add("Authorization", headerValue);
WriteRequestBody(request);
// It looks like a bug in HttpWebRequest. It throws random TimeoutExceptions
// after some requests. Abort the request seems to work. More info:
// http://stackoverflow.com/questions/2252762/getrequeststream-throws-timeout-exception-randomly
var response = request.GetResponse();
string content;
using (var stream = response.GetResponseStream())
{
using (var reader = new StreamReader(stream))
{
content = reader.ReadToEnd();
}
}
request.Abort();
return content;
}
private void WriteRequestBody(HttpWebRequest request)
{
if (_method == "GET")
{
return;
}
var requestBody = Encoding.ASCII.GetBytes(GetCustomParametersString());
using (var stream = request.GetRequestStream())
{
stream.Write(requestBody, 0, requestBody.Length);
}
}
private string GetRequestUrl()
{
if (_method != "GET" || _customParameters.Count == 0)
{
return _url;
}
return string.Format("{0}?{1}", _url, GetCustomParametersString());
}
private string GetCustomParametersString()
{
return _customParameters.Select(x => string.Format("{0}={1}", x.Key, x.Value)).Join("&");
}
private string GenerateAuthorizationHeaderValue(IEnumerable<KeyValuePair<string, string>> parameters, string signature)
{
return new StringBuilder("OAuth ")
.Append(parameters.Concat(new KeyValuePair<string, string>("oauth_signature", signature))
.Where(x => x.Key.StartsWith("oauth_"))
.Select(x => string.Format("{0}=\"{1}\"", x.Key, x.Value.EncodeRFC3986()))
.Join(","))
.ToString();
}
private string GenerateSignature(IEnumerable<KeyValuePair<string, string>> parameters)
{
var dataToSign = new StringBuilder()
.Append(_method).Append('&')
.Append(_url.EncodeRFC3986()).Append('&')
.Append(parameters
.OrderBy(x => x.Key)
.Select(x => string.Format("{0}={1}", x.Key, x.Value))
.Join("&")
.EncodeRFC3986());
var signatureKey = string.Format("{0}&{1}", _oauth.ConsumerSecret.EncodeRFC3986(), _oauth.AccessSecret.EncodeRFC3986());
var sha1 = new HMACSHA1(Encoding.ASCII.GetBytes(signatureKey));
var signatureBytes = sha1.ComputeHash(Encoding.ASCII.GetBytes(dataToSign.ToString()));
return Convert.ToBase64String(signatureBytes);
}
private void AddOAuthParameters(IDictionary<string, string> parameters, string timestamp, string nonce)
{
parameters.Add("oauth_version", VERSION);
parameters.Add("oauth_consumer_key", _oauth.ConsumerKey);
parameters.Add("oauth_nonce", nonce);
parameters.Add("oauth_signature_method", SIGNATURE_METHOD);
parameters.Add("oauth_timestamp", timestamp);
parameters.Add("oauth_token", _oauth.AccessToken);
}
private static string GetTimestamp()
{
return ((int)(DateTime.UtcNow - new DateTime(1970, 1, 1)).TotalSeconds).ToString();
}
private static string CreateNonce()
{
return new Random().Next(0x0000000, 0x7fffffff).ToString("X8");
}
}
}
public static class TinyTwitterHelperExtensions
{
public static string Join<T>(this IEnumerable<T> items, string separator)
{
return string.Join(separator, items.ToArray());
}
public static IEnumerable<T> Concat<T>(this IEnumerable<T> items, T value)
{
return items.Concat(new[] { value });
}
public static string EncodeRFC3986(this string value)
{
// From Twitterizer http://www.twitterizer.net/
if (string.IsNullOrEmpty(value))
{
return string.Empty;
}
var encoded = Uri.EscapeDataString(value);
return Regex
.Replace(encoded, "(%[0-9a-f][0-9a-f])", c => c.Value.ToUpper())
.Replace("(", "%28")
.Replace(")", "%29")
.Replace("$", "%24")
.Replace("!", "%21")
.Replace("*", "%2A")
.Replace("'", "%27")
.Replace("%7E", "~");
}
}
}
Loading…
Cancel
Save