diff --git a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs index f1ff75c67..67c67afd1 100644 --- a/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs +++ b/src/NzbDrone.Common.Test/Http/HttpClientFixture.cs @@ -285,19 +285,96 @@ namespace NzbDrone.Common.Test.Http response.Resource.Headers.Should().NotContainKey("Cookie"); } + [Test] + public void should_not_store_request_cookie() + { + var requestGet = new HttpRequest($"http://{_httpBinHost}/get"); + requestGet.Cookies.Add("my", "cookie"); + requestGet.AllowAutoRedirect = false; + requestGet.StoreRequestCookie = false; + requestGet.StoreResponseCookie = false; + var responseGet = Subject.Get(requestGet); + + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.AllowAutoRedirect = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().BeEmpty(); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_store_request_cookie() + { + var requestGet = new HttpRequest($"http://{_httpBinHost}/get"); + requestGet.Cookies.Add("my", "cookie"); + requestGet.AllowAutoRedirect = false; + requestGet.StoreRequestCookie.Should().BeTrue(); + requestGet.StoreResponseCookie = false; + var responseGet = Subject.Get(requestGet); + + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.AllowAutoRedirect = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_delete_request_cookie() + { + var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my"); + requestDelete.Cookies.Add("my", "cookie"); + requestDelete.AllowAutoRedirect = true; + requestDelete.StoreRequestCookie = false; + requestDelete.StoreResponseCookie = false; + + // Delete and redirect since that's the only way to check the internal temporary cookie container + var responseCookies = Subject.Get(requestDelete); + + responseCookies.Resource.Cookies.Should().BeEmpty(); + } + + [Test] + public void should_clear_request_cookie() + { + var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestSet.Cookies.Add("my", "cookie"); + requestSet.AllowAutoRedirect = false; + requestSet.StoreRequestCookie = true; + requestSet.StoreResponseCookie = false; + + var responseSet = Subject.Get(requestSet); + + var requestClear = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestClear.Cookies.Add("my", null); + requestClear.AllowAutoRedirect = false; + requestClear.StoreRequestCookie = true; + requestClear.StoreResponseCookie = false; + + var responseClear = Subject.Get(requestClear); + + responseClear.Resource.Cookies.Should().BeEmpty(); + } + [Test] public void should_not_store_response_cookie() { var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie"); requestSet.AllowAutoRedirect = false; + requestSet.StoreRequestCookie = false; + requestSet.StoreResponseCookie.Should().BeFalse(); var responseSet = Subject.Get(requestSet); - var request = new HttpRequest($"http://{_httpBinHost}/get"); + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); - var response = Subject.Get(request); + var responseCookies = Subject.Get(requestCookies); - response.Resource.Headers.Should().NotContainKey("Cookie"); + responseCookies.Resource.Cookies.Should().BeEmpty(); ExceptionVerification.IgnoreErrors(); } @@ -307,19 +384,31 @@ namespace NzbDrone.Common.Test.Http { var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie"); requestSet.AllowAutoRedirect = false; + requestSet.StoreRequestCookie = false; requestSet.StoreResponseCookie = true; var responseSet = Subject.Get(requestSet); - var request = new HttpRequest($"http://{_httpBinHost}/get"); + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); - var response = Subject.Get(request); + var responseCookies = Subject.Get(requestCookies); - response.Resource.Headers.Should().ContainKey("Cookie"); + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); - var cookie = response.Resource.Headers["Cookie"].ToString(); + ExceptionVerification.IgnoreErrors(); + } - cookie.Should().Contain("my=cookie"); + [Test] + public void should_temp_store_response_cookie() + { + var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie"); + requestSet.AllowAutoRedirect = true; + requestSet.StoreRequestCookie = false; + requestSet.StoreResponseCookie.Should().BeFalse(); + var responseSet = Subject.Get(requestSet); + + // Set and redirect since that's the only way to check the internal temporary cookie container + responseSet.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); ExceptionVerification.IgnoreErrors(); } @@ -328,21 +417,129 @@ namespace NzbDrone.Common.Test.Http public void should_overwrite_response_cookie() { var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie"); + requestSet.Cookies.Add("my", "oldcookie"); requestSet.AllowAutoRedirect = false; + requestSet.StoreRequestCookie = false; requestSet.StoreResponseCookie = true; - requestSet.Cookies["my"] = "oldcookie"; var responseSet = Subject.Get(requestSet); - var request = new HttpRequest($"http://{_httpBinHost}/get"); + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); - var response = Subject.Get(request); + var responseCookies = Subject.Get(requestCookies); - response.Resource.Headers.Should().ContainKey("Cookie"); + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); - var cookie = response.Resource.Headers["Cookie"].ToString(); + ExceptionVerification.IgnoreErrors(); + } - cookie.Should().Contain("my=cookie"); + [Test] + public void should_overwrite_temp_response_cookie() + { + var requestSet = new HttpRequest($"http://{_httpBinHost}/cookies/set?my=cookie"); + requestSet.Cookies.Add("my", "oldcookie"); + requestSet.AllowAutoRedirect = true; + requestSet.StoreRequestCookie = true; + requestSet.StoreResponseCookie = false; + + var responseSet = Subject.Get(requestSet); + + responseSet.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "oldcookie"); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_not_delete_response_cookie() + { + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.Cookies.Add("my", "cookie"); + requestCookies.AllowAutoRedirect = false; + requestCookies.StoreRequestCookie = true; + requestCookies.StoreResponseCookie = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my"); + requestDelete.AllowAutoRedirect = false; + requestDelete.StoreRequestCookie = false; + requestDelete.StoreResponseCookie = false; + + var responseDelete = Subject.Get(requestDelete); + + requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.StoreRequestCookie = false; + requestCookies.StoreResponseCookie = false; + + responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_delete_response_cookie() + { + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.Cookies.Add("my", "cookie"); + requestCookies.AllowAutoRedirect = false; + requestCookies.StoreRequestCookie = true; + requestCookies.StoreResponseCookie = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my"); + requestDelete.AllowAutoRedirect = false; + requestDelete.StoreRequestCookie = false; + requestDelete.StoreResponseCookie = true; + + var responseDelete = Subject.Get(requestDelete); + + requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.StoreRequestCookie = false; + requestCookies.StoreResponseCookie = false; + + responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().BeEmpty(); + + ExceptionVerification.IgnoreErrors(); + } + + [Test] + public void should_delete_temp_response_cookie() + { + var requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.Cookies.Add("my", "cookie"); + requestCookies.AllowAutoRedirect = false; + requestCookies.StoreRequestCookie = true; + requestCookies.StoreResponseCookie = false; + var responseCookies = Subject.Get(requestCookies); + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); + + var requestDelete = new HttpRequest($"http://{_httpBinHost}/cookies/delete?my"); + requestDelete.AllowAutoRedirect = true; + requestDelete.StoreRequestCookie = false; + requestDelete.StoreResponseCookie = false; + var responseDelete = Subject.Get(requestDelete); + + responseDelete.Resource.Cookies.Should().BeEmpty(); + + requestCookies = new HttpRequest($"http://{_httpBinHost}/cookies"); + requestCookies.StoreRequestCookie = false; + requestCookies.StoreResponseCookie = false; + + responseCookies.Resource.Cookies.Should().HaveCount(1).And.Contain("my", "cookie"); ExceptionVerification.IgnoreErrors(); } @@ -454,4 +651,9 @@ namespace NzbDrone.Common.Test.Http public string Url { get; set; } public string Data { get; set; } } + + public class HttpCookieResource + { + public Dictionary Cookies { get; set; } + } } diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index 3ebb2907a..45ff53ae3 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -52,7 +52,9 @@ namespace NzbDrone.Common.Http public HttpResponse Execute(HttpRequest request) { - var response = ExecuteRequest(request); + var cookieContainer = InitializeRequestCookies(request); + + var response = ExecuteRequest(request, cookieContainer); if (request.AllowAutoRedirect && response.HasHttpRedirect) { @@ -71,7 +73,7 @@ namespace NzbDrone.Common.Http throw new WebException($"Too many automatic redirections were attempted for {autoRedirectChain.Join(" -> ")}", WebExceptionStatus.ProtocolError); } - response = ExecuteRequest(request); + response = ExecuteRequest(request, cookieContainer); } while (response.HasHttpRedirect); } @@ -98,7 +100,7 @@ namespace NzbDrone.Common.Http return response; } - private HttpResponse ExecuteRequest(HttpRequest request) + private HttpResponse ExecuteRequest(HttpRequest request, CookieContainer cookieContainer) { foreach (var interceptor in _requestInterceptors) { @@ -114,11 +116,11 @@ namespace NzbDrone.Common.Http var stopWatch = Stopwatch.StartNew(); - var cookies = PrepareRequestCookies(request); + PrepareRequestCookies(request, cookieContainer); - var response = _httpDispatcher.GetResponse(request, cookies); + var response = _httpDispatcher.GetResponse(request, cookieContainer); - HandleResponseCookies(request, cookies); + HandleResponseCookies(response, cookieContainer); stopWatch.Stop(); @@ -137,49 +139,91 @@ namespace NzbDrone.Common.Http return response; } - private CookieContainer PrepareRequestCookies(HttpRequest request) + private CookieContainer InitializeRequestCookies(HttpRequest request) { lock (_cookieContainerCache) { - var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); + var sourceContainer = new CookieContainer(); + + var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); + var persistentCookies = presistentContainer.GetCookies((Uri)request.Url); + sourceContainer.Add(persistentCookies); if (request.Cookies.Count != 0) { foreach (var pair in request.Cookies) { - persistentCookieContainer.Add(new Cookie(pair.Key, pair.Value, "/", request.Url.Host) + Cookie cookie; + if (pair.Value == null) + { + cookie = new Cookie(pair.Key, "", "/") + { + Expires = DateTime.Now.AddDays(-1) + }; + } + else + { + cookie = new Cookie(pair.Key, pair.Value, "/") + { + // Use Now rather than UtcNow to work around Mono cookie expiry bug. + // See https://gist.github.com/ta264/7822b1424f72e5b4c961 + Expires = DateTime.Now.AddHours(1) + }; + } + + sourceContainer.Add((Uri)request.Url, cookie); + + if (request.StoreRequestCookie) { - // Use Now rather than UtcNow to work around Mono cookie expiry bug. - // See https://gist.github.com/ta264/7822b1424f72e5b4c961 - Expires = DateTime.Now.AddHours(1) - }); + presistentContainer.Add((Uri)request.Url, cookie); + } } } - var requestCookies = persistentCookieContainer.GetCookies((Uri)request.Url); - - var cookieContainer = new CookieContainer(); + return sourceContainer; + } + } - cookieContainer.Add(requestCookies); + private void PrepareRequestCookies(HttpRequest request, CookieContainer cookieContainer) + { + // Don't collect persistnet cookies for intermediate/redirected urls. + /*lock (_cookieContainerCache) + { + var presistentContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); + var persistentCookies = presistentContainer.GetCookies((Uri)request.Url); + var existingCookies = cookieContainer.GetCookies((Uri)request.Url); - return cookieContainer; - } + cookieContainer.Add(persistentCookies); + cookieContainer.Add(existingCookies); + }*/ } - private void HandleResponseCookies(HttpRequest request, CookieContainer cookieContainer) + private void HandleResponseCookies(HttpResponse response, CookieContainer cookieContainer) { - if (!request.StoreResponseCookie) + var cookieHeaders = response.GetCookieHeaders(); + if (cookieHeaders.Empty()) { return; } - lock (_cookieContainerCache) + if (response.Request.StoreResponseCookie) { - var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); - - var cookies = cookieContainer.GetCookies((Uri)request.Url); + lock (_cookieContainerCache) + { + var persistentCookieContainer = _cookieContainerCache.Get("container", () => new CookieContainer()); - persistentCookieContainer.Add(cookies); + foreach (var cookieHeader in cookieHeaders) + { + try + { + persistentCookieContainer.SetCookies((Uri)response.Request.Url, cookieHeader); + } + catch (Exception ex) + { + _logger.Debug(ex, "Invalid cookie in {0}", response.Request.Url); + } + } + } } } diff --git a/src/NzbDrone.Common/Http/HttpRequest.cs b/src/NzbDrone.Common/Http/HttpRequest.cs index 7096ca854..301890804 100644 --- a/src/NzbDrone.Common/Http/HttpRequest.cs +++ b/src/NzbDrone.Common/Http/HttpRequest.cs @@ -13,8 +13,10 @@ namespace NzbDrone.Common.Http Url = new HttpUri(url); Headers = new HttpHeader(); AllowAutoRedirect = true; + StoreRequestCookie = true; Cookies = new Dictionary(); - + + if (!RuntimeInfo.IsProduction) { AllowAutoRedirect = false; @@ -37,6 +39,7 @@ namespace NzbDrone.Common.Http public bool ConnectionKeepAlive { get; set; } public bool LogResponseContent { get; set; } public Dictionary Cookies { get; private set; } + public bool StoreRequestCookie { get; set; } public bool StoreResponseCookie { get; set; } public TimeSpan RequestTimeout { get; set; } public TimeSpan RateLimit { get; set; } diff --git a/src/NzbDrone.Common/Http/HttpResponse.cs b/src/NzbDrone.Common/Http/HttpResponse.cs index 734238cfc..e0b11b51c 100644 --- a/src/NzbDrone.Common/Http/HttpResponse.cs +++ b/src/NzbDrone.Common/Http/HttpResponse.cs @@ -55,20 +55,22 @@ namespace NzbDrone.Common.Http StatusCode == HttpStatusCode.MovedPermanently || StatusCode == HttpStatusCode.Found; + public string[] GetCookieHeaders() + { + return Headers.GetValues("Set-Cookie") ?? new string[0]; + } + public Dictionary GetCookies() { var result = new Dictionary(); - var setCookieHeaders = Headers.GetValues("Set-Cookie"); - if (setCookieHeaders != null) + var setCookieHeaders = GetCookieHeaders(); + foreach (var cookie in setCookieHeaders) { - foreach (var cookie in setCookieHeaders) + var match = RegexSetCookie.Match(cookie); + if (match.Success) { - var match = RegexSetCookie.Match(cookie); - if (match.Success) - { - result[match.Groups[1].Value] = match.Groups[2].Value; - } + result[match.Groups[1].Value] = match.Groups[2].Value; } }