using MediaBrowser.Common.Configuration; using MediaBrowser.Common.IO; using MediaBrowser.Common.Net; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Net; using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; using System.Net.Http; using System.Text; using System.Threading; using System.Threading.Tasks; namespace MediaBrowser.Common.Implementations.HttpClientManager { /// /// Class HttpClientManager /// public class HttpClientManager : IHttpClient { /// /// The _logger /// private readonly ILogger _logger; /// /// The _app paths /// private readonly IApplicationPaths _appPaths; public delegate HttpClient GetHttpClientHandler(bool enableHttpCompression); private readonly GetHttpClientHandler _getHttpClientHandler; private readonly IFileSystem _fileSystem; /// /// Initializes a new instance of the class. /// /// The app paths. /// The logger. /// The get HTTP client handler. /// /// appPaths /// or /// logger /// public HttpClientManager(IApplicationPaths appPaths, ILogger logger, GetHttpClientHandler getHttpClientHandler, IFileSystem fileSystem) { if (appPaths == null) { throw new ArgumentNullException("appPaths"); } if (logger == null) { throw new ArgumentNullException("logger"); } _logger = logger; _getHttpClientHandler = getHttpClientHandler; _fileSystem = fileSystem; _appPaths = appPaths; } /// /// Holds a dictionary of http clients by host. Use GetHttpClient(host) to retrieve or create a client for web requests. /// DON'T dispose it after use. /// /// The HTTP clients. private readonly ConcurrentDictionary _httpClients = new ConcurrentDictionary(); /// /// Gets /// /// The host. /// if set to true [enable HTTP compression]. /// HttpClient. /// host private HttpClientInfo GetHttpClient(string host, bool enableHttpCompression) { if (string.IsNullOrEmpty(host)) { throw new ArgumentNullException("host"); } HttpClientInfo client; var key = host + enableHttpCompression; if (!_httpClients.TryGetValue(key, out client)) { client = new HttpClientInfo { HttpClient = _getHttpClientHandler(enableHttpCompression) }; _httpClients.TryAdd(key, client); } return client; } public async Task GetResponse(HttpRequestOptions options) { ValidateParams(options.Url, options.CancellationToken); options.CancellationToken.ThrowIfCancellationRequested(); var client = GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression); if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < 30) { throw new HttpException(string.Format("Cancelling connection to {0} due to a previous timeout.", options.Url)) { IsTimedOut = true }; } using (var message = GetHttpRequestMessage(options)) { if (options.ResourcePool != null) { await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false); } if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < 30) { if (options.ResourcePool != null) { options.ResourcePool.Release(); } throw new HttpException(string.Format("Connection to {0} timed out", options.Url)) { IsTimedOut = true }; } _logger.Info("HttpClientManager.Get url: {0}", options.Url); try { options.CancellationToken.ThrowIfCancellationRequested(); var response = await client.HttpClient.SendAsync(message, HttpCompletionOption.ResponseContentRead, options.CancellationToken).ConfigureAwait(false); EnsureSuccessStatusCode(response); options.CancellationToken.ThrowIfCancellationRequested(); return new HttpResponseInfo { Content = await response.Content.ReadAsStreamAsync().ConfigureAwait(false), StatusCode = response.StatusCode, ContentType = response.Content.Headers.ContentType.MediaType }; } catch (OperationCanceledException ex) { var exception = GetCancellationException(options.Url, options.CancellationToken, ex); var httpException = exception as HttpException; if (httpException != null && httpException.IsTimedOut) { client.LastTimeout = DateTime.UtcNow; } throw exception; } catch (HttpRequestException ex) { _logger.ErrorException("Error getting response from " + options.Url, ex); throw new HttpException(ex.Message, ex); } catch (Exception ex) { _logger.ErrorException("Error getting response from " + options.Url, ex); throw; } finally { if (options.ResourcePool != null) { options.ResourcePool.Release(); } } } } /// /// Performs a GET request and returns the resulting stream /// /// The options. /// Task{Stream}. /// /// public async Task Get(HttpRequestOptions options) { ValidateParams(options.Url, options.CancellationToken); options.CancellationToken.ThrowIfCancellationRequested(); var client = GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression); if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < 30) { throw new HttpException(string.Format("Cancelling connection to {0} due to a previous timeout.", options.Url)) { IsTimedOut = true }; } using (var message = GetHttpRequestMessage(options)) { if (options.ResourcePool != null) { await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false); } if ((DateTime.UtcNow - client.LastTimeout).TotalSeconds < 30) { if (options.ResourcePool != null) { options.ResourcePool.Release(); } throw new HttpException(string.Format("Connection to {0} timed out", options.Url)) { IsTimedOut = true }; } _logger.Info("HttpClientManager.Get url: {0}", options.Url); try { options.CancellationToken.ThrowIfCancellationRequested(); var response = await client.HttpClient.SendAsync(message, HttpCompletionOption.ResponseContentRead, options.CancellationToken).ConfigureAwait(false); EnsureSuccessStatusCode(response); options.CancellationToken.ThrowIfCancellationRequested(); return await response.Content.ReadAsStreamAsync().ConfigureAwait(false); } catch (OperationCanceledException ex) { var exception = GetCancellationException(options.Url, options.CancellationToken, ex); var httpException = exception as HttpException; if (httpException != null && httpException.IsTimedOut) { client.LastTimeout = DateTime.UtcNow; } throw exception; } catch (HttpRequestException ex) { _logger.ErrorException("Error getting response from " + options.Url, ex); throw new HttpException(ex.Message, ex); } catch (Exception ex) { _logger.ErrorException("Error getting response from " + options.Url, ex); throw; } finally { if (options.ResourcePool != null) { options.ResourcePool.Release(); } } } } /// /// Performs a GET request and returns the resulting stream /// /// The URL. /// The resource pool. /// The cancellation token. /// Task{Stream}. public Task Get(string url, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { return Get(new HttpRequestOptions { Url = url, ResourcePool = resourcePool, CancellationToken = cancellationToken, }); } /// /// Gets the specified URL. /// /// The URL. /// The cancellation token. /// Task{Stream}. public Task Get(string url, CancellationToken cancellationToken) { return Get(url, null, cancellationToken); } /// /// Performs a POST request /// /// The URL. /// Params to add to the POST data. /// The resource pool. /// The cancellation token. /// stream on success, null on failure /// postData /// public async Task Post(string url, Dictionary postData, SemaphoreSlim resourcePool, CancellationToken cancellationToken) { ValidateParams(url, cancellationToken); if (postData == null) { throw new ArgumentNullException("postData"); } cancellationToken.ThrowIfCancellationRequested(); var strings = postData.Keys.Select(key => string.Format("{0}={1}", key, postData[key])); var postContent = string.Join("&", strings.ToArray()); var content = new StringContent(postContent, Encoding.UTF8, "application/x-www-form-urlencoded"); if (resourcePool != null) { await resourcePool.WaitAsync(cancellationToken).ConfigureAwait(false); } _logger.Info("HttpClientManager.Post url: {0}", url); try { cancellationToken.ThrowIfCancellationRequested(); var msg = await GetHttpClient(GetHostFromUrl(url), true).HttpClient.PostAsync(url, content, cancellationToken).ConfigureAwait(false); EnsureSuccessStatusCode(msg); return await msg.Content.ReadAsStreamAsync().ConfigureAwait(false); } catch (OperationCanceledException ex) { throw GetCancellationException(url, cancellationToken, ex); } catch (HttpRequestException ex) { _logger.ErrorException("Error getting response from " + url, ex); throw new HttpException(ex.Message, ex); } finally { if (resourcePool != null) { resourcePool.Release(); } } } /// /// Downloads the contents of a given url into a temporary location /// /// The options. /// Task{System.String}. /// progress /// /// public async Task GetTempFile(HttpRequestOptions options) { var response = await GetTempFileResponse(options).ConfigureAwait(false); return response.TempFilePath; } public async Task GetTempFileResponse(HttpRequestOptions options) { ValidateParams(options.Url, options.CancellationToken); var tempFile = Path.Combine(_appPaths.TempDirectory, Guid.NewGuid() + ".tmp"); if (options.Progress == null) { throw new ArgumentNullException("progress"); } options.CancellationToken.ThrowIfCancellationRequested(); if (options.ResourcePool != null) { await options.ResourcePool.WaitAsync(options.CancellationToken).ConfigureAwait(false); } options.Progress.Report(0); _logger.Info("HttpClientManager.GetTempFile url: {0}, temp file: {1}", options.Url, tempFile); try { options.CancellationToken.ThrowIfCancellationRequested(); using (var message = GetHttpRequestMessage(options)) { using (var response = await GetHttpClient(GetHostFromUrl(options.Url), options.EnableHttpCompression).HttpClient.SendAsync(message, HttpCompletionOption.ResponseHeadersRead, options.CancellationToken).ConfigureAwait(false)) { EnsureSuccessStatusCode(response); options.CancellationToken.ThrowIfCancellationRequested(); var contentLength = GetContentLength(response); if (!contentLength.HasValue) { // We're not able to track progress using (var stream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) { using (var fs = _fileSystem.GetFileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, true)) { await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false); } } } else { using (var stream = ProgressStream.CreateReadProgressStream(await response.Content.ReadAsStreamAsync().ConfigureAwait(false), options.Progress.Report, contentLength.Value)) { using (var fs = _fileSystem.GetFileStream(tempFile, FileMode.Create, FileAccess.Write, FileShare.Read, true)) { await stream.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, options.CancellationToken).ConfigureAwait(false); } } } options.Progress.Report(100); return new HttpResponseInfo { TempFilePath = tempFile, StatusCode = response.StatusCode, ContentType = response.Content.Headers.ContentType.MediaType }; } } } catch (Exception ex) { throw GetTempFileException(ex, options, tempFile); } finally { if (options.ResourcePool != null) { options.ResourcePool.Release(); } } } /// /// Gets the message. /// /// The options. /// HttpResponseMessage. private HttpRequestMessage GetHttpRequestMessage(HttpRequestOptions options) { var message = new HttpRequestMessage(HttpMethod.Get, options.Url); foreach (var pair in options.RequestHeaders.ToList()) { if (!message.Headers.TryAddWithoutValidation(pair.Key, pair.Value)) { _logger.Error("Unable to add request header {0} with value {1}", pair.Key, pair.Value); } } return message; } /// /// Gets the length of the content. /// /// The response. /// System.Nullable{System.Int64}. private long? GetContentLength(HttpResponseMessage response) { IEnumerable lengthValues = null; // Seeing some InvalidOperationException here under mono try { response.Headers.TryGetValues("content-length", out lengthValues); } catch (InvalidOperationException ex) { _logger.ErrorException("Error accessing response.Headers.TryGetValues Content-Length", ex); } if (lengthValues == null) { try { response.Content.Headers.TryGetValues("content-length", out lengthValues); } catch (InvalidOperationException ex) { _logger.ErrorException("Error accessing response.Content.Headers.TryGetValues Content-Length", ex); } } if (lengthValues == null) { return null; } return long.Parse(string.Join(string.Empty, lengthValues.ToArray()), UsCulture); } protected static readonly CultureInfo UsCulture = new CultureInfo("en-US"); /// /// Handles the temp file exception. /// /// The ex. /// The options. /// The temp file. /// Task. /// private Exception GetTempFileException(Exception ex, HttpRequestOptions options, string tempFile) { var operationCanceledException = ex as OperationCanceledException; if (operationCanceledException != null) { // Cleanup DeleteTempFile(tempFile); return GetCancellationException(options.Url, options.CancellationToken, operationCanceledException); } _logger.ErrorException("Error getting response from " + options.Url, ex); var httpRequestException = ex as HttpRequestException; // Cleanup DeleteTempFile(tempFile); if (httpRequestException != null) { return new HttpException(ex.Message, ex); } return ex; } private void DeleteTempFile(string file) { try { File.Delete(file); } catch (IOException) { // Might not have been created at all. No need to worry. } } /// /// Validates the params. /// /// The URL. /// The cancellation token. /// url private void ValidateParams(string url, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(url)) { throw new ArgumentNullException("url"); } } /// /// Gets the host from URL. /// /// The URL. /// System.String. private string GetHostFromUrl(string url) { var start = url.IndexOf("://", StringComparison.OrdinalIgnoreCase) + 3; var len = url.IndexOf('/', start) - start; return url.Substring(start, len); } /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } /// /// Releases unmanaged and - optionally - managed resources. /// /// true to release both managed and unmanaged resources; false to release only unmanaged resources. protected virtual void Dispose(bool dispose) { if (dispose) { foreach (var client in _httpClients.Values.ToList()) { client.HttpClient.Dispose(); } _httpClients.Clear(); } } /// /// Throws the cancellation exception. /// /// The URL. /// The cancellation token. /// The exception. /// Exception. private Exception GetCancellationException(string url, CancellationToken cancellationToken, OperationCanceledException exception) { // If the HttpClient's timeout is reached, it will cancel the Task internally if (!cancellationToken.IsCancellationRequested) { var msg = string.Format("Connection to {0} timed out", url); _logger.Error(msg); // Throw an HttpException so that the caller doesn't think it was cancelled by user code return new HttpException(msg, exception) { IsTimedOut = true }; } return exception; } /// /// Ensures the success status code. /// /// The response. /// private void EnsureSuccessStatusCode(HttpResponseMessage response) { if (!response.IsSuccessStatusCode) { throw new HttpException(response.ReasonPhrase) { StatusCode = response.StatusCode }; } } /// /// Posts the specified URL. /// /// The URL. /// The post data. /// The cancellation token. /// Task{Stream}. public Task Post(string url, Dictionary postData, CancellationToken cancellationToken) { return Post(url, postData, null, cancellationToken); } } }