From 1d6a7a18437456652a87c5f64b30c7bafd45f364 Mon Sep 17 00:00:00 2001 From: Qstick Date: Sun, 24 May 2020 15:40:36 -0400 Subject: [PATCH] Revert "Async HttpClient and list lookup" This reverts commit c8a2af867e38bee43c0546bb8effda704d5f368b. --- .../Extensions/StreamExtensions.cs | 7 +- .../Http/Dispatchers/IHttpDispatcher.cs | 3 +- .../Http/Dispatchers/ManagedHttpDispatcher.cs | 7 +- src/NzbDrone.Common/Http/HttpClient.cs | 97 ++++--------------- src/NzbDrone.Common/TPL/RateLimitService.cs | 30 ++---- .../MetadataSource/IProvideMovieInfo.cs | 3 - .../MetadataSource/ISearchForNewMovie.cs | 2 - .../MetadataSource/SkyHook/SkyHookProxy.cs | 20 +--- .../NetImport/TMDb/TMDbImportBase.cs | 4 +- .../Movies/FetchMovieListModule.cs | 19 ++-- 10 files changed, 50 insertions(+), 142 deletions(-) diff --git a/src/NzbDrone.Common/Extensions/StreamExtensions.cs b/src/NzbDrone.Common/Extensions/StreamExtensions.cs index b4ab5ad8e..8f6036baa 100644 --- a/src/NzbDrone.Common/Extensions/StreamExtensions.cs +++ b/src/NzbDrone.Common/Extensions/StreamExtensions.cs @@ -1,19 +1,18 @@ using System.IO; -using System.Threading.Tasks; namespace NzbDrone.Common.Extensions { public static class StreamExtensions { - public static async Task ToBytes(this Stream input) + public static byte[] ToBytes(this Stream input) { var buffer = new byte[16 * 1024]; using (var ms = new MemoryStream()) { int read; - while ((read = await input.ReadAsync(buffer, 0, buffer.Length)) > 0) + while ((read = input.Read(buffer, 0, buffer.Length)) > 0) { - await ms.WriteAsync(buffer, 0, read); + ms.Write(buffer, 0, read); } return ms.ToArray(); diff --git a/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs index a5565f26b..8e665ceed 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs @@ -1,10 +1,9 @@ using System.Net; -using System.Threading.Tasks; namespace NzbDrone.Common.Http.Dispatchers { public interface IHttpDispatcher { - Task GetResponseAsync(HttpRequest request, CookieContainer cookies); + HttpResponse GetResponse(HttpRequest request, CookieContainer cookies); } } diff --git a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs index 96a4f3966..96762f120 100644 --- a/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs +++ b/src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs @@ -3,7 +3,6 @@ using System.IO; using System.IO.Compression; using System.Net; using System.Reflection; -using System.Threading.Tasks; using NLog; using NLog.Fluent; using NzbDrone.Common.EnvironmentInfo; @@ -29,7 +28,7 @@ namespace NzbDrone.Common.Http.Dispatchers _logger = logger; } - public async Task GetResponseAsync(HttpRequest request, CookieContainer cookies) + public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies) { var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url); @@ -78,7 +77,7 @@ namespace NzbDrone.Common.Http.Dispatchers } } - httpWebResponse = (HttpWebResponse)await webRequest.GetResponseAsync(); + httpWebResponse = (HttpWebResponse)webRequest.GetResponse(); } catch (WebException e) { @@ -121,7 +120,7 @@ namespace NzbDrone.Common.Http.Dispatchers { try { - data = await responseStream.ToBytes(); + data = responseStream.ToBytes(); if (PlatformInfo.IsMono && httpWebResponse.ContentEncoding == "gzip") { diff --git a/src/NzbDrone.Common/Http/HttpClient.cs b/src/NzbDrone.Common/Http/HttpClient.cs index cd403034c..cce6d125f 100644 --- a/src/NzbDrone.Common/Http/HttpClient.cs +++ b/src/NzbDrone.Common/Http/HttpClient.cs @@ -4,7 +4,6 @@ using System.Diagnostics; using System.IO; using System.Linq; using System.Net; -using System.Threading.Tasks; using NLog; using NzbDrone.Common.Cache; using NzbDrone.Common.EnvironmentInfo; @@ -25,16 +24,6 @@ namespace NzbDrone.Common.Http HttpResponse Post(HttpRequest request); HttpResponse Post(HttpRequest request) where T : new(); - - Task ExecuteAsync(HttpRequest request); - Task DownloadFileAsync(string url, string fileName); - Task GetAsync(HttpRequest request); - Task> GetAsync(HttpRequest request) - where T : new(); - Task HeadAsync(HttpRequest request); - Task PostAsync(HttpRequest request); - Task> PostAsync(HttpRequest request) - where T : new(); } public class HttpClient : IHttpClient @@ -65,11 +54,11 @@ namespace NzbDrone.Common.Http _cookieContainerCache = cacheManager.GetCache(typeof(HttpClient)); } - public async Task ExecuteAsync(HttpRequest request) + public HttpResponse Execute(HttpRequest request) { var cookieContainer = InitializeRequestCookies(request); - var response = await ExecuteRequestAsync(request, cookieContainer); + var response = ExecuteRequest(request, cookieContainer); if (request.AllowAutoRedirect && response.HasHttpRedirect) { @@ -88,7 +77,7 @@ namespace NzbDrone.Common.Http throw new WebException($"Too many automatic redirections were attempted for {autoRedirectChain.Join(" -> ")}", WebExceptionStatus.ProtocolError); } - response = await ExecuteRequestAsync(request, cookieContainer); + response = ExecuteRequest(request, cookieContainer); } while (response.HasHttpRedirect); } @@ -115,12 +104,7 @@ namespace NzbDrone.Common.Http return response; } - public HttpResponse Execute(HttpRequest request) - { - return ExecuteAsync(request).GetAwaiter().GetResult(); - } - - private async Task ExecuteRequestAsync(HttpRequest request, CookieContainer cookieContainer) + private HttpResponse ExecuteRequest(HttpRequest request, CookieContainer cookieContainer) { foreach (var interceptor in _requestInterceptors) { @@ -129,7 +113,7 @@ namespace NzbDrone.Common.Http if (request.RateLimit != TimeSpan.Zero) { - await _rateLimitService.WaitAndPulseAsync(request.Url.Host, request.RateLimit); + _rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimit); } _logger.Trace(request); @@ -138,7 +122,7 @@ namespace NzbDrone.Common.Http PrepareRequestCookies(request, cookieContainer); - var response = await _httpDispatcher.GetResponseAsync(request, cookieContainer); + var response = _httpDispatcher.GetResponse(request, cookieContainer); HandleResponseCookies(response, cookieContainer); @@ -247,7 +231,7 @@ namespace NzbDrone.Common.Http } } - public async Task DownloadFileAsync(string url, string fileName) + public void DownloadFile(string url, string fileName) { try { @@ -263,7 +247,7 @@ namespace NzbDrone.Common.Http using (var webClient = new GZipWebClient()) { webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent()); - await webClient.DownloadFileTaskAsync(url, fileName); + webClient.DownloadFile(url, fileName); stopWatch.Stop(); _logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds); } @@ -271,94 +255,49 @@ namespace NzbDrone.Common.Http catch (WebException e) { _logger.Warn("Failed to get response from: {0} {1}", url, e.Message); - - if (File.Exists(fileName)) - { - File.Delete(fileName); - } - throw; } catch (Exception e) { _logger.Warn(e, "Failed to get response from: " + url); - - if (File.Exists(fileName)) - { - File.Delete(fileName); - } - throw; } } - public void DownloadFile(string url, string fileName) - { - // https://docs.microsoft.com/en-us/archive/msdn-magazine/2015/july/async-programming-brownfield-async-development#the-thread-pool-hack - Task.Run(() => DownloadFileAsync(url, fileName)).GetAwaiter().GetResult(); - } - - public Task GetAsync(HttpRequest request) - { - request.Method = HttpMethod.GET; - return ExecuteAsync(request); - } - public HttpResponse Get(HttpRequest request) { - return Task.Run(() => GetAsync(request)).GetAwaiter().GetResult(); + request.Method = HttpMethod.GET; + return Execute(request); } - public async Task> GetAsync(HttpRequest request) + public HttpResponse Get(HttpRequest request) where T : new() { - var response = await GetAsync(request); + var response = Get(request); CheckResponseContentType(response); return new HttpResponse(response); } - public HttpResponse Get(HttpRequest request) - where T : new() - { - return Task.Run(() => GetAsync(request)).GetAwaiter().GetResult(); - } - - public Task HeadAsync(HttpRequest request) - { - request.Method = HttpMethod.HEAD; - return ExecuteAsync(request); - } - public HttpResponse Head(HttpRequest request) { - return Task.Run(() => HeadAsync(request)).GetAwaiter().GetResult(); - } - - public Task PostAsync(HttpRequest request) - { - request.Method = HttpMethod.POST; - return ExecuteAsync(request); + request.Method = HttpMethod.HEAD; + return Execute(request); } public HttpResponse Post(HttpRequest request) { - return Task.Run(() => PostAsync(request)).GetAwaiter().GetResult(); + request.Method = HttpMethod.POST; + return Execute(request); } - public async Task> PostAsync(HttpRequest request) + public HttpResponse Post(HttpRequest request) where T : new() { - var response = await PostAsync(request); + var response = Post(request); CheckResponseContentType(response); return new HttpResponse(response); } - public HttpResponse Post(HttpRequest request) - where T : new() - { - return Task.Run(() => PostAsync(request)).GetAwaiter().GetResult(); - } - private void CheckResponseContentType(HttpResponse response) { if (response.Headers.ContentType != null && response.Headers.ContentType.Contains("text/html")) diff --git a/src/NzbDrone.Common/TPL/RateLimitService.cs b/src/NzbDrone.Common/TPL/RateLimitService.cs index af3bdc7b9..f0d30b4ff 100644 --- a/src/NzbDrone.Common/TPL/RateLimitService.cs +++ b/src/NzbDrone.Common/TPL/RateLimitService.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Concurrent; -using System.Threading.Tasks; using NLog; using NzbDrone.Common.Cache; @@ -9,7 +8,6 @@ namespace NzbDrone.Common.TPL public interface IRateLimitService { void WaitAndPulse(string key, TimeSpan interval); - Task WaitAndPulseAsync(string key, TimeSpan interval); } public class RateLimitService : IRateLimitService @@ -25,35 +23,19 @@ namespace NzbDrone.Common.TPL public void WaitAndPulse(string key, TimeSpan interval) { - var delay = GetDelay(key, interval); + var waitUntil = _rateLimitStore.AddOrUpdate(key, + (s) => DateTime.UtcNow + interval, + (s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval); - if (delay.TotalSeconds > 0.0) - { - _logger.Trace("Rate Limit triggered, delaying '{0}' for {1:0.000} sec", key, delay.TotalSeconds); - System.Threading.Thread.Sleep(delay); - } - } + waitUntil -= interval; - public async Task WaitAndPulseAsync(string key, TimeSpan interval) - { - var delay = GetDelay(key, interval); + var delay = waitUntil - DateTime.UtcNow; if (delay.TotalSeconds > 0.0) { _logger.Trace("Rate Limit triggered, delaying '{0}' for {1:0.000} sec", key, delay.TotalSeconds); - await Task.Delay(delay); + System.Threading.Thread.Sleep(delay); } } - - private TimeSpan GetDelay(string key, TimeSpan interval) - { - var waitUntil = _rateLimitStore.AddOrUpdate(key, - (s) => DateTime.UtcNow + interval, - (s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval); - - waitUntil -= interval; - - return waitUntil - DateTime.UtcNow; - } } } diff --git a/src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs b/src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs index fb59db871..25c109a34 100644 --- a/src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs +++ b/src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Threading.Tasks; using NzbDrone.Core.Movies; using NzbDrone.Core.Movies.Credits; @@ -11,7 +10,5 @@ namespace NzbDrone.Core.MetadataSource Movie GetMovieByImdbId(string imdbId); Tuple> GetMovieInfo(int tmdbId); HashSet GetChangedMovies(DateTime startTime); - - Task>> GetMovieInfoAsync(int tmdbId); } } diff --git a/src/NzbDrone.Core/MetadataSource/ISearchForNewMovie.cs b/src/NzbDrone.Core/MetadataSource/ISearchForNewMovie.cs index a5a9c1f06..637bdaae4 100644 --- a/src/NzbDrone.Core/MetadataSource/ISearchForNewMovie.cs +++ b/src/NzbDrone.Core/MetadataSource/ISearchForNewMovie.cs @@ -1,5 +1,4 @@ using System.Collections.Generic; -using System.Threading.Tasks; using NzbDrone.Core.Movies; namespace NzbDrone.Core.MetadataSource @@ -9,6 +8,5 @@ namespace NzbDrone.Core.MetadataSource List SearchForNewMovie(string title); Movie MapMovieToTmdbMovie(Movie movie); - Task MapMovieToTmdbMovieAsync(Movie movie); } } diff --git a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs index 09081fae9..6cba25ccd 100644 --- a/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs +++ b/src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs @@ -2,7 +2,6 @@ using System; using System.Collections.Generic; using System.Linq; using System.Net; -using System.Threading.Tasks; using NLog; using NzbDrone.Common.Cloud; using NzbDrone.Common.Extensions; @@ -15,6 +14,7 @@ using NzbDrone.Core.MetadataSource.SkyHook.Resource; using NzbDrone.Core.Movies; using NzbDrone.Core.Movies.AlternativeTitles; using NzbDrone.Core.Movies.Credits; +using NzbDrone.Core.NetImport.ImportExclusions; using NzbDrone.Core.NetImport.TMDb; using NzbDrone.Core.Parser; @@ -65,7 +65,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook return new HashSet(response.Resource.Results.Select(c => c.Id)); } - public async Task>> GetMovieInfoAsync(int tmdbId) + public Tuple> GetMovieInfo(int tmdbId) { var httpRequest = _radarrMetadata.Create() .SetSegment("route", "movie") @@ -75,7 +75,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook httpRequest.AllowAutoRedirect = true; httpRequest.SuppressHttpError = true; - var httpResponse = await _httpClient.GetAsync(httpRequest); + var httpResponse = _httpClient.Get(httpRequest); if (httpResponse.HasHttpError) { @@ -98,11 +98,6 @@ namespace NzbDrone.Core.MetadataSource.SkyHook return new Tuple>(movie, credits.ToList()); } - public Tuple> GetMovieInfo(int tmdbId) - { - return GetMovieInfoAsync(tmdbId).GetAwaiter().GetResult(); - } - public Movie GetMovieByImdbId(string imdbId) { var httpRequest = _radarrMetadata.Create() @@ -241,14 +236,14 @@ namespace NzbDrone.Core.MetadataSource.SkyHook return title; } - public async Task MapMovieToTmdbMovieAsync(Movie movie) + public Movie MapMovieToTmdbMovie(Movie movie) { try { Movie newMovie = movie; if (movie.TmdbId > 0) { - newMovie = (await GetMovieInfoAsync(movie.TmdbId)).Item1; + newMovie = GetMovieInfo(movie.TmdbId).Item1; } else if (movie.ImdbId.IsNotNullOrWhiteSpace()) { @@ -288,11 +283,6 @@ namespace NzbDrone.Core.MetadataSource.SkyHook } } - public Movie MapMovieToTmdbMovie(Movie movie) - { - return MapMovieToTmdbMovieAsync(movie).GetAwaiter().GetResult(); - } - public List SearchForNewMovie(string title) { try diff --git a/src/NzbDrone.Core/NetImport/TMDb/TMDbImportBase.cs b/src/NzbDrone.Core/NetImport/TMDb/TMDbImportBase.cs index 996655695..36875a403 100644 --- a/src/NzbDrone.Core/NetImport/TMDb/TMDbImportBase.cs +++ b/src/NzbDrone.Core/NetImport/TMDb/TMDbImportBase.cs @@ -1,5 +1,4 @@ -using System; -using NLog; +using NLog; using NzbDrone.Common.Cloud; using NzbDrone.Common.Http; using NzbDrone.Core.Configuration; @@ -12,7 +11,6 @@ namespace NzbDrone.Core.NetImport.TMDb where TSettings : TMDbSettingsBase, new() { public override NetImportType ListType => NetImportType.TMDB; - public override TimeSpan RateLimit => TimeSpan.Zero; public readonly ISearchForNewMovie _skyhookProxy; public readonly IHttpRequestBuilderFactory _requestBuilder; diff --git a/src/Radarr.Api.V3/Movies/FetchMovieListModule.cs b/src/Radarr.Api.V3/Movies/FetchMovieListModule.cs index 15e7d5630..243556dd3 100644 --- a/src/Radarr.Api.V3/Movies/FetchMovieListModule.cs +++ b/src/Radarr.Api.V3/Movies/FetchMovieListModule.cs @@ -1,6 +1,5 @@ using System.Collections.Generic; using System.Linq; -using System.Threading.Tasks; using Nancy; using NzbDrone.Common.Extensions; using NzbDrone.Core.MediaCover; @@ -28,14 +27,22 @@ namespace Radarr.Api.V3.Movies { var results = _fetchNetImport.FetchAndFilter((int)Request.Query.listId, false); - var tasks = results.Where(movie => movie.TmdbId == 0 || !movie.Images.Any() || movie.Overview.IsNullOrWhiteSpace()) - .Select(x => _movieSearch.MapMovieToTmdbMovieAsync(x)); + List realResults = new List(); - var realResults = results.Where(movie => movie.TmdbId != 0 && movie.Images.Any() && movie.Overview.IsNotNullOrWhiteSpace()).ToList(); + foreach (var movie in results) + { + var mapped = movie; - var mapped = Task.WhenAll(tasks).GetAwaiter().GetResult(); + if (movie.TmdbId == 0 || !movie.Images.Any() || movie.Overview.IsNullOrWhiteSpace()) + { + mapped = _movieSearch.MapMovieToTmdbMovie(movie); + } - realResults.AddRange(mapped.Where(x => x != null)); + if (mapped != null) + { + realResults.Add(mapped); + } + } return MapToResource(realResults); }