using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Linq; using System.Net; using System.Text; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.EnvironmentInfo; using NzbDrone.Common.Extensions; using NzbDrone.Common.Http.Dispatchers; using NzbDrone.Common.TPL; namespace NzbDrone.Common.Http { public interface IHttpClient { HttpResponse Execute(HttpRequest request); void DownloadFile(string url, string fileName); HttpResponse Get(HttpRequest request); HttpResponse Get(HttpRequest request) where T : new(); HttpResponse Head(HttpRequest request); HttpResponse Post(HttpRequest request); HttpResponse Post(HttpRequest request) where T : new(); } public class HttpClient : IHttpClient { private readonly Logger _logger; private readonly IRateLimitService _rateLimitService; private readonly ICached _cookieContainerCache; private readonly List _requestInterceptors; private readonly IHttpDispatcher _httpDispatcher; public HttpClient(IEnumerable requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, IHttpDispatcher httpDispatcher, Logger logger) { _logger = logger; _rateLimitService = rateLimitService; _requestInterceptors = requestInterceptors.ToList(); ServicePointManager.DefaultConnectionLimit = 12; _httpDispatcher = httpDispatcher; _cookieContainerCache = cacheManager.GetCache(typeof(HttpClient)); } public HttpClient(IEnumerable requestInterceptors, ICacheManager cacheManager, IRateLimitService rateLimitService, Logger logger) : this(requestInterceptors, cacheManager, rateLimitService, null, logger) { _httpDispatcher = new FallbackHttpDispatcher(cacheManager.GetCache(typeof(HttpClient), "curlTLSFallback"), _logger); } public HttpResponse Execute(HttpRequest request) { foreach (var interceptor in _requestInterceptors) { request = interceptor.PreRequest(request); } if (request.RateLimit != TimeSpan.Zero) { _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimit); } _logger.Trace(request); var stopWatch = Stopwatch.StartNew(); var cookies = PrepareRequestCookies(request); var response = _httpDispatcher.GetResponse(request, cookies); HandleResponseCookies(request, cookies); stopWatch.Stop(); _logger.Trace("{0} ({1:n0} ms)", response, stopWatch.ElapsedMilliseconds); foreach (var interceptor in _requestInterceptors) { response = interceptor.PostResponse(response); } if (request.LogResponseContent) { _logger.Trace("Response content ({0} bytes): {1}", response.ResponseData.Length, response.Content); } if (!RuntimeInfoBase.IsProduction && (response.StatusCode == HttpStatusCode.Moved || response.StatusCode == HttpStatusCode.MovedPermanently || response.StatusCode == HttpStatusCode.Found)) { _logger.Error("Server requested a redirect to [" + response.Headers["Location"] + "]. Update the request URL to avoid this redirect."); } if (!request.SuppressHttpError && response.HasHttpError) { _logger.Warn("HTTP Error - {0}", response); if ((int)response.StatusCode == 429) { throw new TooManyRequestsException(request, response); } else { throw new HttpException(request, response); } } return response; } private CookieContainer PrepareRequestCookies(HttpRequest request) { lock (_cookieContainerCache) { var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); if (request.Cookies.Count != 0) { foreach (var pair in request.Cookies) { persistentCookieContainer.Add(new Cookie(pair.Key, pair.Value, "/", request.Url.Host) { // Use Now rather than UtcNow to work around Mono cookie expiry bug. // See https://gist.github.com/ta264/7822b1424f72e5b4c961 Expires = DateTime.Now.AddHours(1) }); } } var requestCookies = persistentCookieContainer.GetCookies((Uri)request.Url); var cookieContainer = new CookieContainer(); cookieContainer.Add(requestCookies); return cookieContainer; } } private void HandleResponseCookies(HttpRequest request, CookieContainer cookieContainer) { if (!request.StoreResponseCookie) { return; } lock (_cookieContainerCache) { var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); var cookies = cookieContainer.GetCookies((Uri)request.Url); persistentCookieContainer.Add(cookies); } } public void DownloadFile(string url, string fileName) { try { var fileInfo = new FileInfo(fileName); if (fileInfo.Directory != null && !fileInfo.Directory.Exists) { fileInfo.Directory.Create(); } _logger.Debug("Downloading [{0}] to [{1}]", url, fileName); var stopWatch = Stopwatch.StartNew(); var webClient = new GZipWebClient(); webClient.Headers.Add(HttpRequestHeader.UserAgent, UserAgentBuilder.UserAgent); webClient.DownloadFile(url, fileName); stopWatch.Stop(); _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); } catch (WebException e) { _logger.Warn("Failed to get response from: {0} {1}", url, e.Message); throw; } catch (Exception e) { _logger.Warn(e, "Failed to get response from: " + url); throw; } } public HttpResponse Get(HttpRequest request) { request.Method = HttpMethod.GET; return Execute(request); } public HttpResponse Get(HttpRequest request) where T : new() { var response = Get(request); return new HttpResponse(response); } public HttpResponse Head(HttpRequest request) { request.Method = HttpMethod.HEAD; return Execute(request); } public HttpResponse Post(HttpRequest request) { request.Method = HttpMethod.POST; return Execute(request); } public HttpResponse Post(HttpRequest request) where T : new() { var response = Post(request); return new HttpResponse(response); } } }