using System.Net.Sockets; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Net; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.Specialized; using System.Globalization; using System.IO; using System.Linq; using System.Net; using System.Net.Cache; using System.Text; using System.Threading; using System.Threading.Tasks; using CommonIO; namespace MediaBrowser.Common.Implementations.HttpClientManager { /// /// Class HttpClientManager /// public class HttpClientManager : IHttpClient { /// /// When one request to a host times out, we'll ban all other requests for this period of time, to prevent scans from stalling /// private const int TimeoutSeconds = 30; /// /// The _logger /// private readonly ILogger _logger; /// /// The _app paths /// private readonly IApplicationPaths _appPaths; private readonly IFileSystem _fileSystem; /// /// Initializes a new instance of the class. /// /// The app paths. /// The logger. /// The file system. /// appPaths /// or /// logger public HttpClientManager(IApplicationPaths appPaths, ILogger logger, IFileSystem fileSystem) { if (appPaths == null) { throw new ArgumentNullException("appPaths"); } if (logger == null) { throw new ArgumentNullException("logger"); } _logger = logger; _fileSystem = fileSystem; _appPaths = appPaths; // http://stackoverflow.com/questions/566437/http-post-returns-the-error-417-expectation-failed-c ServicePointManager.Expect100Continue = false; // Trakt requests sometimes fail without this ServicePointManager.SecurityProtocol = SecurityProtocolType.Ssl3 | SecurityProtocolType.Tls; } /// /// Holds a dictionary of http clients by host. Use GetHttpClient(host) to retrieve or create a client for web requests. /// DON'T dispose it after use. /// /// The HTTP clients. private readonly ConcurrentDictionary _httpClients = new ConcurrentDictionary(); /// /// Gets /// /// The host. /// if set to true [enable HTTP compression]. /// HttpClient. /// host private HttpClientInfo GetHttpClient(string host, bool enableHttpCompression) { if (string.IsNullOrEmpty(host)) { throw new ArgumentNullException("host"); } HttpClientInfo client; var key = host + enableHttpCompression; if (!_httpClients.TryGetValue(key, out client)) { client = new HttpClientInfo(); _httpClients.TryAdd(key, client); } return client; } private WebRequest CreateWebRequest(string url) { try { return WebRequest.Create(url); } catch (NotSupportedException) { //Webrequest creation does fail on MONO randomly when using WebRequest.Create //the issue occurs in the GetCreator method here: http://www.oschina.net/code/explore/mono-2.8.1/mcs/class/System/System.Net/WebRequest.cs var type = Type.GetType("System.Net.HttpRequestCreator, System, Version=4.0.0.0,Culture=neutral, PublicKeyToken=b77a5c561934e089"); var creator = Activator.CreateInstance(type, nonPublic: true) as IWebRequestCreate; return creator.Create(new Uri(url)) as HttpWebRequest; } } private WebRequest GetRequest(HttpRequestOptions options, string method, bool enableHttpCompression) { var request = CreateWebRequest(options.Url); var httpWebRequest = request as HttpWebRequest; if (httpWebRequest != null) { AddRequestHeaders(httpWebRequest, options); httpWebRequest.AutomaticDecompression = enableHttpCompression ? DecompressionMethods.Deflate : DecompressionMethods.None; } request.CachePolicy = new RequestCachePolicy(RequestCacheLevel.BypassCache); if (httpWebRequest != null) { if (options.EnableKeepAlive) { httpWebRequest.KeepAlive = true; } } request.Method = method; request.Timeout = options.TimeoutMs; if (httpWebRequest != null) { if (!string.IsNullOrEmpty(options.Host)) { httpWebRequest.Host = options.Host; } if (!string.IsNullOrEmpty(options.Referer)) { httpWebRequest.Referer = options.Referer; } } //request.ServicePoint.BindIPEndPointDelegate = BindIPEndPointCallback; return request; } private static IPEndPoint BindIPEndPointCallback(ServicePoint servicePoint, IPEndPoint remoteEndPoint, int retryCount) { // Prefer local ipv4 if (remoteEndPoint.AddressFamily == AddressFamily.InterNetworkV6) { return new IPEndPoint(IPAddress.IPv6Any, 0); } return new IPEndPoint(IPAddress.Any, 0); } private void AddRequestHeaders(HttpWebRequest request, HttpRequestOptions options) { foreach (var header in options.RequestHeaders.ToList()) { if (string.Equals(header.Key, "Accept", StringComparison.OrdinalIgnoreCase)) { request.Accept = header.Value; } else if (string.Equals(header.Key, "User-Agent", StringComparison.OrdinalIgnoreCase)) { request.UserAgent = header.Value; } else { request.Headers.Set(header.Key, header.Value); } } } /// /// The _semaphoreLocks /// private readonly ConcurrentDictionary _semaphoreLocks = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); /// /// Gets the lock. /// /// The filename. /// System.Object. private SemaphoreSlim GetLock(string url) { return _semaphoreLocks.GetOrAdd(url, key => new SemaphoreSlim(1, 1)); } /// /// Gets the response internal. /// /// The options. /// Task{HttpResponseInfo}. public Task GetResponse(HttpRequestOptions options) { return SendAsync(options, "GET"); } /// /// Performs a GET request and returns the resulting stream /// /// The options. /// Task{Stream}. public async Task Get(HttpRequestOptions options) { var response = await GetResponse(options).ConfigureAwait(false); return response.Content; } /// /// Performs a GET request and returns the resulting stream /// /// The URL. /// The resource pool. /// The cancellation token. /// Task{Stream}. public Task Get(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { return Get(new HttpRequestOptions { Url = url, ResourcePool = resourcePool, CancellationToken = cancellationToken, }); } /// /// Gets the specified URL. /// /// The URL. /// The cancellation token. /// Task{Stream}. public Task Get(string url, CancellationToken cancellationToken) { return Get(url, null, cancellationToken); } /// /// send as an asynchronous operation. /// /// The options. /// The HTTP method. /// Task{HttpResponseInfo}. /// /// public async Task SendAsync(HttpRequestOptions options, string httpMethod) { HttpResponseInfo response; if (options.CacheMode == CacheMode.None) { response = await SendAsyncInternal(options, httpMethod).ConfigureAwait(false); return response; } var url = options.Url; var urlHash = url.ToLower().GetMD5().ToString("N"); var semaphore = GetLock(url); var responseCachePath = Path.Combine(_appPaths.CachePath, "httpclient", urlHash); response = await GetCachedResponse(responseCachePath, options.CacheLength, url).ConfigureAwait(false); if (response != null) { return response; } await semaphore.WaitAsync(options.CancellationToken).ConfigureAwait(false); try { response = await GetCachedResponse(responseCachePath, options.CacheLength, url).ConfigureAwait(false); if (response != null) { return response; } response = await SendAsyncInternal(options, httpMethod).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.OK) { await CacheResponse(response, responseCachePath).ConfigureAwait(false); } return response; } finally { semaphore.Release(); } } private async Task GetCachedResponse(string responseCachePath, TimeSpan cacheLength, string url) { try { if (_fileSystem.GetLastWriteTimeUtc(responseCachePath).Add(cacheLength) > DateTime.UtcNow) { using (var stream = _fileSystem.GetFileStream(responseCachePath, FileMode.Open, FileAccess.Read, FileShare.Read, true)) { var memoryStream = new MemoryStream(); await stream.CopyToAsync(memoryStream).ConfigureAwait(false); memoryStream.Position = 0; return new HttpResponseInfo { ResponseUrl = url, Content = memoryStream, StatusCode = HttpStatusCode.OK, Headers = new NameValueCollection(), ContentLength = memoryStream.Length }; } } } catch (FileNotFoundException) { } catch (DirectoryNotFoundException) { } return null; } private async Task CacheResponse(HttpResponseInfo response, string responseCachePath) { _fileSystem.CreateDirectory(Path.GetDirectoryName(responseCachePath)); using (var responseStream = response.Content) { using (var fileStream = _fileSystem.GetFileStream(responseCachePath, FileMode.Create, FileAccess.Write, FileShare.Read, true)) { var memoryStream = new MemoryStream(); await responseStream.CopyToAsync(memoryStream).ConfigureAwait(false); memoryStream.Position = 0; await memoryStream.CopyToAsync(fileStream).ConfigureAwait(false); memoryStream.Position = 0; response.Content = memoryStream; } } } private async Task SendAsyncInternal(HttpRequestOptions options, string httpMethod) { ValidateParams(options); options.CancellationToken.ThrowIfCancellationRequested(); var client = GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression); if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < TimeoutSeconds) { throw new HttpException(string.Format("Cancelling connection to {0} due to a previous timeout.", options.Url)) { IsTimedOut = true }; } var httpWebRequest = GetRequest(options, httpMethod, options.EnableHttpCompression); if (options.RequestContentBytes != null || !string.IsNullOrEmpty(options.RequestContent) || string.Equals(httpMethod, "post", StringComparison.OrdinalIgnoreCase)) { var bytes = options.RequestContentBytes ?? Encoding.UTF8.GetBytes(options.RequestContent ?? string.Empty); httpWebRequest.ContentType = options.RequestContentType ?? "application/x-www-form-urlencoded"; httpWebRequest.ContentLength = bytes.Length; httpWebRequest.GetRequestStream().Write(bytes, 0, bytes.Length); } if (options.ResourcePool != null) { await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false); } if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < TimeoutSeconds) { if (options.ResourcePool != null) { options.ResourcePool.Release(); } throw new HttpException(string.Format("Connection to {0} timed out", options.Url)) { IsTimedOut = true }; } if (options.LogRequest) { _logger.Info("HttpClientManager {0}: {1}", httpMethod.ToUpper(), options.Url); } try { options.CancellationToken.ThrowIfCancellationRequested(); if (!options.BufferContent) { var response = await GetResponseAsync(httpWebRequest, TimeSpan.FromMilliseconds(options.TimeoutMs)).ConfigureAwait(false); var httpResponse = (HttpWebResponse)response; EnsureSuccessStatusCode(client, httpResponse, options); options.CancellationToken.ThrowIfCancellationRequested(); return GetResponseInfo(httpResponse, httpResponse.GetResponseStream(), GetContentLength(httpResponse), httpResponse); } using (var response = await GetResponseAsync(httpWebRequest, TimeSpan.FromMilliseconds(options.TimeoutMs)).ConfigureAwait(false)) { var httpResponse = (HttpWebResponse)response; EnsureSuccessStatusCode(client, httpResponse, options); options.CancellationToken.ThrowIfCancellationRequested(); using (var stream = httpResponse.GetResponseStream()) { var memoryStream = new MemoryStream(); await stream.CopyToAsync(memoryStream).ConfigureAwait(false); memoryStream.Position = 0; return GetResponseInfo(httpResponse, memoryStream, memoryStream.Length, null); } } } catch (OperationCanceledException ex) { var exception = GetCancellationException(options.Url, options.CancellationToken, ex); var httpException = exception as HttpException; if (httpException != null && httpException.IsTimedOut) { client.LastTimeout = DateTime.UtcNow; } throw exception; } catch (Exception ex) { throw GetException(ex, options); } finally { if (options.ResourcePool != null) { options.ResourcePool.Release(); } } } /// /// Gets the exception. /// /// The ex. /// The options. /// HttpException. private HttpException GetException(WebException ex, HttpRequestOptions options) { _logger.ErrorException("Error getting response from " + options.Url, ex); var exception = new HttpException(ex.Message, ex); var response = ex.Response as HttpWebResponse; if (response != null) { exception.StatusCode = response.StatusCode; } return exception; } private HttpResponseInfo GetResponseInfo(HttpWebResponse httpResponse, Stream content, long? contentLength, IDisposable disposable) { return new HttpResponseInfo(disposable) { Content = content, StatusCode = httpResponse.StatusCode, ContentType = httpResponse.ContentType, Headers = new NameValueCollection(httpResponse.Headers), ContentLength = contentLength, ResponseUrl = httpResponse.ResponseUri.ToString() }; } private HttpResponseInfo GetResponseInfo(HttpWebResponse httpResponse, string tempFile, long? contentLength) { return new HttpResponseInfo { TempFilePath = tempFile, StatusCode = httpResponse.StatusCode, ContentType = httpResponse.ContentType, Headers = httpResponse.Headers, ContentLength = contentLength }; } public Task Post(HttpRequestOptions options) { return SendAsync(options, "POST"); } /// /// Performs a POST request /// /// The options. /// Params to add to the POST data. /// stream on success, null on failure public async Task Post(HttpRequestOptions options, Dictionary postData) { options.SetPostData(postData); var response = await Post(options).ConfigureAwait(false); return response.Content; } /// /// Performs a POST request /// /// The URL. /// Params to add to the POST data. /// The resource pool. /// The cancellation token. /// stream on success, null on failure public Task Post(string url, Dictionary postData, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { return Post(new HttpRequestOptions { Url = url, ResourcePool = resourcePool, CancellationToken = cancellationToken }, postData); } /// /// Downloads the contents of a given url into a temporary location /// /// The options. /// Task{System.String}. /// progress public async Task GetTempFile(HttpRequestOptions options) { var response = await GetTempFileResponse(options).ConfigureAwait(false); return response.TempFilePath; } public async Task GetTempFileResponse(HttpRequestOptions options) { ValidateParams(options); _fileSystem.CreateDirectory(_appPaths.TempDirectory); var tempFile = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid() + ".tmp"); if (options.Progress == null) { throw new ArgumentNullException("progress"); } options.CancellationToken.ThrowIfCancellationRequested(); var httpWebRequest = GetRequest(options, "GET", options.EnableHttpCompression); if (options.ResourcePool != null) { await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false); } options.Progress.Report(0); if (options.LogRequest) { _logger.Info("HttpClientManager.GetTempFileResponse url: {0}", options.Url); } try { options.CancellationToken.ThrowIfCancellationRequested(); using (var response = await httpWebRequest.GetResponseAsync().ConfigureAwait(false)) { var httpResponse = (HttpWebResponse)response; var client = GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression); EnsureSuccessStatusCode(client, httpResponse, options); options.CancellationToken.ThrowIfCancellationRequested(); var contentLength = GetContentLength(httpResponse); if (!contentLength.HasValue) { // We're not able to track progress using (var stream = httpResponse.GetResponseStream()) { using (var fs = _fileSystem.GetFileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, true)) { await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false); } } } else { using (var stream = ProgressStream.CreateReadProgressStream(httpResponse.GetResponseStream(), options.Progress.Report, contentLength.Value)) { using (var fs = _fileSystem.GetFileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, true)) { await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false); } } } options.Progress.Report(100); return GetResponseInfo(httpResponse, tempFile, contentLength); } } catch (Exception ex) { DeleteTempFile(tempFile); throw GetException(ex, options); } finally { if (options.ResourcePool != null) { options.ResourcePool.Release(); } } } private long? GetContentLength(HttpWebResponse response) { var length = response.ContentLength; if (length == 0) { return null; } return length; } protected static readonly CultureInfo UsCulture = new CultureInfo("en-US"); private Exception GetException(Exception ex, HttpRequestOptions options) { var webException = ex as WebException ?? ex.InnerException as WebException; if (webException != null) { return GetException(webException, options); } var operationCanceledException = ex as OperationCanceledException ?? ex.InnerException as OperationCanceledException; if (operationCanceledException != null) { return GetCancellationException(options.Url, options.CancellationToken, operationCanceledException); } _logger.ErrorException("Error getting response from " + options.Url, ex); return ex; } private void DeleteTempFile(string file) { try { _fileSystem.DeleteFile(file); } catch (IOException) { // Might not have been created at all. No need to worry. } } private void ValidateParams(HttpRequestOptions options) { if (string.IsNullOrEmpty(options.Url)) { throw new ArgumentNullException("options"); } } /// /// Gets the host from URL. /// /// The URL. /// System.String. private string GetHostFromUrl(string url) { var index = url.IndexOf("://", StringComparison.OrdinalIgnoreCase); if (index != -1) { url = url.Substring(index + 3); var host = url.Split(new[] { '/' }, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); if (!string.IsNullOrWhiteSpace(host)) { return host; } } return url; } /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } /// /// Releases unmanaged and - optionally - managed resources. /// /// true to release both managed and unmanaged resources; false to release only unmanaged resources. protected virtual void Dispose(bool dispose) { if (dispose) { _httpClients.Clear(); } } /// /// Throws the cancellation exception. /// /// The URL. /// The cancellation token. /// The exception. /// Exception. private Exception GetCancellationException(string url, CancellationToken cancellationToken, OperationCanceledException exception) { // If the HttpClient's timeout is reached, it will cancel the Task internally if (!cancellationToken.IsCancellationRequested) { var msg = string.Format("Connection to {0} timed out", url); _logger.Error(msg); // Throw an HttpException so that the caller doesn't think it was cancelled by user code return new HttpException(msg, exception) { IsTimedOut = true }; } return exception; } private void EnsureSuccessStatusCode(HttpClientInfo client, HttpWebResponse response, HttpRequestOptions options) { var statusCode = response.StatusCode; var isSuccessful = statusCode >= HttpStatusCode.OK && statusCode <= (HttpStatusCode)299; if (!isSuccessful) { if ((int) statusCode == 429) { client.LastTimeout = DateTime.UtcNow; } if (statusCode == HttpStatusCode.RequestEntityTooLarge) if (options.LogErrorResponseBody) { try { using (var stream = response.GetResponseStream()) { if (stream != null) { using (var reader = new StreamReader(stream)) { var msg = reader.ReadToEnd(); _logger.Error(msg); } } } } catch { } } throw new HttpException(response.StatusDescription) { StatusCode = response.StatusCode }; } } /// /// Posts the specified URL. /// /// The URL. /// The post data. /// The cancellation token. /// Task{Stream}. public Task Post(string url, Dictionary postData, CancellationToken cancellationToken) { return Post(url, postData, null, cancellationToken); } private Task GetResponseAsync(WebRequest request, TimeSpan timeout) { var taskCompletion = new TaskCompletionSource(); Task asyncTask = Task.Factory.FromAsync(request.BeginGetResponse, request.EndGetResponse, null); ThreadPool.RegisterWaitForSingleObject((asyncTask as IAsyncResult).AsyncWaitHandle, TimeoutCallback, request, timeout, true); var callback = new TaskCallback { taskCompletion = taskCompletion }; asyncTask.ContinueWith(callback.OnSuccess, TaskContinuationOptions.NotOnFaulted); // Handle errors asyncTask.ContinueWith(callback.OnError, TaskContinuationOptions.OnlyOnFaulted); return taskCompletion.Task; } private static void TimeoutCallback(object state, bool timedOut) { if (timedOut) { WebRequest request = (WebRequest)state; if (state != null) { request.Abort(); } } } private class TaskCallback { public TaskCompletionSource taskCompletion; public void OnSuccess(Task task) { taskCompletion.TrySetResult(task.Result); } public void OnError(Task task) { if (task.Exception != null) { taskCompletion.TrySetException(task.Exception); } else { taskCompletion.TrySetException(new List()); } } } } }