@ -1,214 +1,231 @@
using System ; using System ;
using System.Diagnostics ; using System.Diagnostics ;
using System.IO ; using System.IO ;
using System.IO.Compression ;
using System.Net ; using System.Net ;
using System.Reflection ; using System.Net.Http ;
using System.Net.Security ;
using System.Net.Sockets ;
using System.Text ;
using System.Threading ;
using System.Threading.Tasks ;
using NLog ; using NLog ;
using NLog.Fluent ; using NzbDrone.Common.Cache ;
using NzbDrone.Common.EnvironmentInfo ;
using NzbDrone.Common.Extensions ; using NzbDrone.Common.Extensions ;
using NzbDrone.Common.Http.Proxy ; using NzbDrone.Common.Http.Proxy ;
using NzbDrone.Common.Instrumentation.Extensions ;
namespace NzbDrone.Common.Http.Dispatchers namespace NzbDrone.Common.Http.Dispatchers
{ {
public class ManagedHttpDispatcher : IHttpDispatcher
public class ManagedHttpDispatcher : IHttpDispatcher
{
{
private const string NO_PROXY_KEY = "no-proxy" ;
private const int connection_establish_timeout = 2000 ;
private static bool useIPv6 = Socket . OSSupportsIPv6 ;
private static bool hasResolvedIPv6Availability ;
private readonly IHttpProxySettingsProvider _proxySettingsProvider ;
private readonly IHttpProxySettingsProvider _proxySettingsProvider ;
private readonly ICreateManagedWebProxy _createManagedWebProxy ;
private readonly ICreateManagedWebProxy _createManagedWebProxy ;
private readonly ICertificateValidationService _certificateValidationService ;
private readonly IUserAgentBuilder _userAgentBuilder ;
private readonly IUserAgentBuilder _userAgentBuilder ;
private readonly IPlatformInfo _platformInfo ;
private readonly I Cached< System . Net . Http . HttpClient > _httpClientCache ;
private readonly Logger _logger ;
private readonly Logger _logger ;
private readonly ICached < CredentialCache > _credentialCache ;
public ManagedHttpDispatcher ( IHttpProxySettingsProvider proxySettingsProvider , ICreateManagedWebProxy createManagedWebProxy , IUserAgentBuilder userAgentBuilder , IPlatformInfo platformInfo , Logger logger )
public ManagedHttpDispatcher ( IHttpProxySettingsProvider proxySettingsProvider ,
ICreateManagedWebProxy createManagedWebProxy ,
ICertificateValidationService certificateValidationService ,
IUserAgentBuilder userAgentBuilder ,
ICacheManager cacheManager ,
Logger logger )
{
{
_proxySettingsProvider = proxySettingsProvider ;
_proxySettingsProvider = proxySettingsProvider ;
_createManagedWebProxy = createManagedWebProxy ;
_createManagedWebProxy = createManagedWebProxy ;
_certificateValidationService = certificateValidationService ;
_userAgentBuilder = userAgentBuilder ;
_userAgentBuilder = userAgentBuilder ;
_platformInfo = platformInfo ;
_logger = logger ;
_logger = logger ;
_httpClientCache = cacheManager . GetCache < System . Net . Http . HttpClient > ( typeof ( ManagedHttpDispatcher ) ) ;
_credentialCache = cacheManager . GetCache < CredentialCache > ( typeof ( ManagedHttpDispatcher ) , "credentialcache" ) ;
}
}
public HttpResponse GetResponse ( HttpRequest request , CookieContainer cookies )
public HttpResponse GetResponse ( HttpRequest request , CookieContainer cookies )
{
{
var webRequest = ( HttpWebRequest ) WebRequest . Create ( ( Uri ) request . Url ) ;
var requestMessage = new HttpRequestMessage ( request . Method , ( Uri ) request . Url ) ;
requestMessage . Headers . UserAgent . ParseAdd ( _userAgentBuilder . GetUserAgent ( request . UseSimplifiedUserAgent ) ) ;
// Deflate is not a standard and could break depending on implementation.
requestMessage . Headers . ConnectionClose = ! request . ConnectionKeepAlive ;
// we should just stick with the more compatible Gzip
//http://stackoverflow.com/questions/8490718/how-to-decompress-stream-deflated-with-java-util-zip-deflater-in-net
webRequest . AutomaticDecompression = DecompressionMethods . GZip ;
webRequest . Method = request . Method . ToString ( ) ;
var cookieHeader = cookies . GetCookieHeader ( ( Uri ) request . Url ) ;
webRequest . UserAgent = _userAgentBuilder . GetUserAgent ( request . UseSimplifiedUserAgent ) ;
if ( cookieHeader . IsNotNullOrWhiteSpace ( ) )
webRequest . KeepAlive = request . ConnectionKeepAlive ;
{
webRequest . AllowAutoRedirect = false ;
requestMessage . Headers . Add ( "Cookie" , cookieHeader ) ;
webRequest . CookieContainer = cookies ;
}
using var cts = new CancellationTokenSource ( ) ;
if ( request . RequestTimeout ! = TimeSpan . Zero )
if ( request . RequestTimeout ! = TimeSpan . Zero )
{
{
webRequest. Timeout = ( int ) Math . Ceiling ( request . RequestTimeout . TotalMilliseconds ) ;
cts. CancelAfter ( request . RequestTimeout ) ;
}
}
else
webRequest . Proxy = GetProxy ( request . Url ) ;
if ( request . Headers ! = null )
{
{
AddRequestHeaders ( webRequest , request . Headers ) ;
// The default for System.Net.Http.HttpClient
cts . CancelAfter ( TimeSpan . FromSeconds ( 100 ) ) ;
}
}
HttpWebResponse httpWebResponse ;
if ( request . Credentials ! = null )
try
{
{
if ( request . ContentData ! = null )
if ( request . Credentials is BasicNetworkCredential bc )
{
// Manually set header to avoid initial challenge response
var authInfo = bc . UserName + ":" + bc . Password ;
authInfo = Convert . ToBase64String ( Encoding . GetEncoding ( "ISO-8859-1" ) . GetBytes ( authInfo ) ) ;
requestMessage . Headers . Add ( "Authorization" , "Basic " + authInfo ) ;
}
else if ( request . Credentials is NetworkCredential nc )
{
{
webRequest . ContentLength = request . ContentData . Length ;
var creds = GetCredentialCache ( ) ;
using ( var writeStream = webRequest . GetRequestStream ( ) )
foreach ( var authtype in new [ ] { "Basic" , "Digest" } )
{
{
writeStream . Write ( request . ContentData , 0 , request . ContentData . Length ) ;
creds . Remove ( ( Uri ) request . Url , authtype ) ;
creds . Add ( ( Uri ) request . Url , authtype , nc ) ;
}
}
}
}
}
httpWebResponse = ( HttpWebResponse ) webRequest . GetResponse ( ) ;
if ( request . ContentData ! = null )
{
requestMessage . Content = new ByteArrayContent ( request . ContentData ) ;
}
}
catch ( WebException e )
if ( request . Headers ! = null )
{
{
httpWebResponse = ( HttpWebResponse ) e . Response ;
AddRequestHeaders ( requestMessage , request . Headers ) ;
}
if ( httpWebResponse = = null )
var httpClient = GetClient ( request . Url ) ;
{
// The default messages for WebException on mono are pretty horrible.
HttpResponseMessage responseMessage ;
if ( e . Status = = WebExceptionStatus . NameResolutionFailure )
{
try
throw new WebException ( $"DNS Name Resolution Failure: '{webRequest.RequestUri.Host}'" , e . Status ) ;
{
}
responseMessage = httpClient . Send ( requestMessage , cts . Token ) ;
else if ( e . ToString ( ) . Contains ( "TLS Support not" ) )
}
{
catch ( HttpRequestException e )
throw new TlsFailureException ( webRequest , e ) ;
{
}
_logger . Error ( e , "HttpClient error" ) ;
else if ( e . ToString ( ) . Contains ( "The authentication or decryption has failed." ) )
throw ;
{
throw new TlsFailureException ( webRequest , e ) ;
}
else if ( OsInfo . IsNotWindows )
{
throw new WebException ( $"{e.Message}: '{webRequest.RequestUri}'" , e , e . Status , e . Response ) ;
}
else
{
throw ;
}
}
}
}
byte [ ] data = null ;
byte [ ] data = null ;
using ( var responseStream = httpWebResponse. GetResponse Stream( ) )
using ( var responseStream = responseMessage . Content . ReadAsStream ( ) )
{
{
if ( responseStream ! = null & & responseStream ! = Stream . Null )
if ( responseStream ! = null & & responseStream ! = Stream . Null )
{
{
try
try
{
{
data = responseStream . ToBytes ( ) ;
if ( request . ResponseStream ! = null & & responseMessage . StatusCode = = HttpStatusCode . OK )
{
// A target ResponseStream was specified, write to that instead.
// But only on the OK status code, since we don't want to write failures and redirects.
responseStream . CopyTo ( request . ResponseStream ) ;
}
else
{
data = responseStream . ToBytes ( ) ;
}
}
}
catch ( Exception ex )
catch ( Exception ex )
{
{
throw new WebException ( "Failed to read complete http response" , ex , WebExceptionStatus . ReceiveFailure , httpWebResponse ) ;
throw new WebException ( "Failed to read complete http response" , ex , WebExceptionStatus . ReceiveFailure , null ) ;
}
}
}
}
}
}
return new HttpResponse ( request , new HttpHeader ( httpWebResponse . Headers ) , data , httpWebResponse . StatusCode ) ;
var headers = responseMessage . Headers . ToNameValueCollection ( ) ;
headers . Add ( responseMessage . Content . Headers . ToNameValueCollection ( ) ) ;
return new HttpResponse ( request , new HttpHeader ( responseMessage . Headers ) , data , responseMessage . StatusCode ) ;
}
}
public void DownloadFile ( string url , string fileName )
p rotected virtual System . Net . Http . HttpClient GetClient ( HttpUri uri )
{
{
try
var proxySettings = _proxySettingsProvider . GetProxySettings ( uri ) ;
{
var fileInfo = new FileInfo ( fileName ) ;
if ( fileInfo . Directory ! = null & & ! fileInfo . Directory . Exists )
{
fileInfo . Directory . Create ( ) ;
}
_logger . Debug ( "Downloading [{0}] to [{1}]" , url , fileName ) ;
var stopWatch = Stopwatch . StartNew ( ) ;
var key = proxySettings ? . Key ? ? NO_PROXY_KEY ;
var uri = new HttpUri ( url ) ;
using ( var webClient = new GZipWebClient ( ) )
return _httpClientCache . Get ( key , ( ) = > CreateHttpClient ( proxySettings ) ) ;
{
webClient . Headers . Add ( HttpRequestHeader . UserAgent , _userAgentBuilder . GetUserAgent ( ) ) ;
webClient . Proxy = GetProxy ( uri ) ;
webClient . DownloadFile ( uri . FullUri , fileName ) ;
stopWatch . Stop ( ) ;
_logger . Debug ( "Downloading Completed. took {0:0}s" , stopWatch . Elapsed . Seconds ) ;
}
}
catch ( WebException e )
{
_logger . Warn ( "Failed to get response from: {0} {1}" , url , e . Message ) ;
throw ;
}
catch ( Exception e )
{
_logger . Warn ( e , "Failed to get response from: " + url ) ;
throw ;
}
}
}
protected virtual IWebProxy GetProxy ( HttpUri uri )
protected virtual System . Net . Http . HttpClient CreateHttpClient ( HttpProxySettings proxySettings )
{
{
IWebProxy proxy = null ;
var handler = new SocketsHttpHandler ( )
{
var proxySettings = _proxySettingsProvider . GetProxySettings ( uri ) ;
AutomaticDecompression = DecompressionMethods . GZip | DecompressionMethods . Brotli ,
UseCookies = false , // sic - we don't want to use a shared cookie container
AllowAutoRedirect = false ,
Credentials = GetCredentialCache ( ) ,
PreAuthenticate = true ,
MaxConnectionsPerServer = 12 ,
ConnectCallback = onConnect ,
SslOptions = new SslClientAuthenticationOptions
{
RemoteCertificateValidationCallback = _certificateValidationService . ShouldByPassValidationError
}
} ;
if ( proxySettings ! = null )
if ( proxySettings ! = null )
{
{
proxy = _createManagedWebProxy . GetWebProxy ( proxySettings ) ;
handler. P roxy = _createManagedWebProxy . GetWebProxy ( proxySettings ) ;
}
}
return proxy ;
var client = new System . Net . Http . HttpClient ( handler )
{
Timeout = Timeout . InfiniteTimeSpan
} ;
return client ;
}
}
protected virtual void AddRequestHeaders ( HttpWebRequest webRequest , HttpHeader headers )
protected virtual void AddRequestHeaders ( Http RequestMessage webRequest , HttpHeader headers )
{
{
foreach ( var header in headers )
foreach ( var header in headers )
{
{
switch ( header . Key )
switch ( header . Key )
{
{
case "Accept" :
case "Accept" :
webRequest . Accept = header . Value ;
webRequest . Headers. Accept . ParseAdd ( header . Value ) ;
break ;
break ;
case "Connection" :
case "Connection" :
webRequest . Connection = header . Value ;
webRequest . Headers . Connection . Clear ( ) ;
webRequest . Headers . Connection . Add ( header . Value ) ;
break ;
break ;
case "Content-Length" :
case "Content-Length" :
webRequest. ContentLength = Convert . ToInt64 ( header . Value ) ;
AddContentHeader( webRequest , "Content-Length" , header . Value ) ;
break ;
break ;
case "Content-Type" :
case "Content-Type" :
webRequest. ContentType = header . Value ;
AddContentHeader( webRequest , "Content-Type" , header . Value ) ;
break ;
break ;
case "Date" :
case "Date" :
webRequest . Date = HttpHeader . ParseDateTime ( header . Value ) ;
webRequest . Headers . Remove ( "Date" ) ;
webRequest . Headers . Date = HttpHeader . ParseDateTime ( header . Value ) ;
break ;
break ;
case "Expect" :
case "Expect" :
webRequest . Expect = header . Value ;
webRequest . Headers. Expect . ParseAdd ( header . Value ) ;
break ;
break ;
case "Host" :
case "Host" :
webRequest . H ost = header . Value ;
webRequest . H eaders. H ost = header . Value ;
break ;
break ;
case "If-Modified-Since" :
case "If-Modified-Since" :
webRequest . = HttpHeader . ParseDateTime ( header . Value ) ;
webRequest . Headers. IfModifiedSince = HttpHeader . ParseDateTime ( header . Value ) ;
break ;
break ;
case "Range" :
case "Range" :
throw new NotImplementedException ( ) ;
throw new NotImplementedException ( ) ;
case "Referer" :
case "Referer" :
webRequest . Referer = header . Value ;
webRequest . Headers. Add ( "Referer" , header . Value ) ;
break ;
break ;
case "Transfer-Encoding" :
case "Transfer-Encoding" :
webRequest . TransferEncoding = header . Value ;
webRequest . Headers. TransferEncoding . ParseAdd ( header . Value ) ;
break ;
break ;
case "User-Agent" :
case "User-Agent" :
throw new NotSupportedException ( "User-Agent other than Lidarr not allowed." ) ;
throw new NotSupportedException ( "User-Agent other than Lidarr not allowed." ) ;
@ -220,5 +237,84 @@ namespace NzbDrone.Common.Http.Dispatchers
}
}
}
}
}
}
private void AddContentHeader ( HttpRequestMessage request , string header , string value )
{
var headers = request . Content ? . Headers ;
if ( headers = = null )
{
return ;
}
headers . Remove ( header ) ;
headers . Add ( header , value ) ;
}
private CredentialCache GetCredentialCache ( )
{
return _credentialCache . Get ( "credentialCache" , ( ) = > new CredentialCache ( ) ) ;
}
private static async ValueTask < Stream > onConnect ( SocketsHttpConnectionContext context , CancellationToken cancellationToken )
{
// Until .NET supports an implementation of Happy Eyeballs (https://tools.ietf.org/html/rfc8305#section-2), let's make IPv4 fallback work in a simple way.
// This issue is being tracked at https://github.com/dotnet/runtime/issues/26177 and expected to be fixed in .NET 6.
if ( useIPv6 )
{
try
{
var localToken = cancellationToken ;
if ( ! hasResolvedIPv6Availability )
{
// to make things move fast, use a very low timeout for the initial ipv6 attempt.
var quickFailCts = new CancellationTokenSource ( connection_establish_timeout ) ;
var linkedTokenSource = CancellationTokenSource . CreateLinkedTokenSource ( cancellationToken , quickFailCts . Token ) ;
localToken = linkedTokenSource . Token ;
}
return await attemptConnection ( AddressFamily . InterNetworkV6 , context , localToken ) ;
}
catch
{
// very naively fallback to ipv4 permanently for this execution based on the response of the first connection attempt.
// note that this may cause users to eventually get switched to ipv4 (on a random failure when they are switching networks, for instance)
// but in the interest of keeping this implementation simple, this is acceptable.
useIPv6 = false ;
}
finally
{
hasResolvedIPv6Availability = true ;
}
}
// fallback to IPv4.
return await attemptConnection ( AddressFamily . InterNetwork , context , cancellationToken ) ;
}
private static async ValueTask < Stream > attemptConnection ( AddressFamily addressFamily , SocketsHttpConnectionContext context , CancellationToken cancellationToken )
{
// The following socket constructor will create a dual-mode socket on systems where IPV6 is available.
var socket = new Socket ( addressFamily , SocketType . Stream , ProtocolType . Tcp )
{
// Turn off Nagle's algorithm since it degrades performance in most HttpClient scenarios.
NoDelay = true
} ;
try
{
await socket . ConnectAsync ( context . DnsEndPoint , cancellationToken ) . ConfigureAwait ( false ) ;
// The stream should take the ownership of the underlying socket,
// closing it when it's disposed.
return new NetworkStream ( socket , ownsSocket : true ) ;
}
catch
{
socket . Dispose ( ) ;
throw ;
}
}
}
}
} }
Qstick
3 years ago