open-source-mirrors
/
Prowlarr
mirror of https://github.com/Prowlarr/Prowlarr
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Revert "Async HttpClient and list lookup"

Browse Source 
This reverts commit c8a2af867e.
pull/2/head
Qstick 5 years ago
parent 0479bb9cc3
commit 1d6a7a1843
10 changed files with 50 additions and 142 deletions
Show Stats Download Patch File Download Diff File

7
src/NzbDrone.Common/Extensions/StreamExtensions.cs
Unescape View File

@ -1,19 +1,18 @@
using System.IO;
using System.Threading.Tasks;
namespace NzbDrone.Common.Extensions
{
public static class StreamExtensions
{
public static async Task<byte[]> ToBytes(this Stream input)
public static byte[] ToBytes(this Stream input)
{
var buffer = new byte[16 * 1024];
using (var ms = new MemoryStream())
{
int read;
while ((read = await input.ReadAsync(buffer, 0, buffer.Length)) > 0)
while ((read = input.Read(buffer, 0, buffer.Length)) > 0)
{
await ms.WriteAsync(buffer, 0, read);
ms.Write(buffer, 0, read);
}
return ms.ToArray();

3
src/NzbDrone.Common/Http/Dispatchers/IHttpDispatcher.cs
Unescape View File

@ -1,10 +1,9 @@
using System.Net;
using System.Threading.Tasks;
namespace NzbDrone.Common.Http.Dispatchers
{
public interface IHttpDispatcher
{
Task<HttpResponse> GetResponseAsync(HttpRequest request, CookieContainer cookies);
HttpResponse GetResponse(HttpRequest request, CookieContainer cookies);
}
}

7
src/NzbDrone.Common/Http/Dispatchers/ManagedHttpDispatcher.cs
Unescape View File

@ -3,7 +3,6 @@ using System.IO;
using System.IO.Compression;
using System.Net;
using System.Reflection;
using System.Threading.Tasks;
using NLog;
using NLog.Fluent;
using NzbDrone.Common.EnvironmentInfo;
@ -29,7 +28,7 @@ namespace NzbDrone.Common.Http.Dispatchers
_logger = logger;
}
public async Task<HttpResponse> GetResponseAsync(HttpRequest request, CookieContainer cookies)
public HttpResponse GetResponse(HttpRequest request, CookieContainer cookies)
{
var webRequest = (HttpWebRequest)WebRequest.Create((Uri)request.Url);
@ -78,7 +77,7 @@ namespace NzbDrone.Common.Http.Dispatchers
}
}
httpWebResponse = (HttpWebResponse)await webRequest.GetResponseAsync();
httpWebResponse = (HttpWebResponse)webRequest.GetResponse();
}
catch (WebException e)
{
@ -121,7 +120,7 @@ namespace NzbDrone.Common.Http.Dispatchers
{
try
{
data = await responseStream.ToBytes();
data = responseStream.ToBytes();
if (PlatformInfo.IsMono && httpWebResponse.ContentEncoding == "gzip")
{

97
src/NzbDrone.Common/Http/HttpClient.cs
Unescape View File

@ -4,7 +4,6 @@ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Cache;
using NzbDrone.Common.EnvironmentInfo;
@ -25,16 +24,6 @@ namespace NzbDrone.Common.Http
HttpResponse Post(HttpRequest request);
HttpResponse<T> Post<T>(HttpRequest request)
where T : new();
Task<HttpResponse> ExecuteAsync(HttpRequest request);
Task DownloadFileAsync(string url, string fileName);
Task<HttpResponse> GetAsync(HttpRequest request);
Task<HttpResponse<T>> GetAsync<T>(HttpRequest request)
where T : new();
Task<HttpResponse> HeadAsync(HttpRequest request);
Task<HttpResponse> PostAsync(HttpRequest request);
Task<HttpResponse<T>> PostAsync<T>(HttpRequest request)
where T : new();
}
public class HttpClient : IHttpClient
@ -65,11 +54,11 @@ namespace NzbDrone.Common.Http
_cookieContainerCache = cacheManager.GetCache<CookieContainer>(typeof(HttpClient));
}
public async Task<HttpResponse> ExecuteAsync(HttpRequest request)
public HttpResponse Execute(HttpRequest request)
{
var cookieContainer = InitializeRequestCookies(request);
var response = await ExecuteRequestAsync(request, cookieContainer);
var response = ExecuteRequest(request, cookieContainer);
if (request.AllowAutoRedirect && response.HasHttpRedirect)
{
@ -88,7 +77,7 @@ namespace NzbDrone.Common.Http
throw new WebException($"Too many automatic redirections were attempted for {autoRedirectChain.Join(" -> ")}", WebExceptionStatus.ProtocolError);
}
response = await ExecuteRequestAsync(request, cookieContainer);
response = ExecuteRequest(request, cookieContainer);
}
while (response.HasHttpRedirect);
}
@ -115,12 +104,7 @@ namespace NzbDrone.Common.Http
return response;
}
public HttpResponse Execute(HttpRequest request)
{
return ExecuteAsync(request).GetAwaiter().GetResult();
}
private async Task<HttpResponse> ExecuteRequestAsync(HttpRequest request, CookieContainer cookieContainer)
private HttpResponse ExecuteRequest(HttpRequest request, CookieContainer cookieContainer)
{
foreach (var interceptor in _requestInterceptors)
{
@ -129,7 +113,7 @@ namespace NzbDrone.Common.Http
if (request.RateLimit != TimeSpan.Zero)
{
await _rateLimitService.WaitAndPulseAsync(request.Url.Host, request.RateLimit);
_rateLimitService.WaitAndPulse(request.Url.Host, request.RateLimit);
}
_logger.Trace(request);
@ -138,7 +122,7 @@ namespace NzbDrone.Common.Http
PrepareRequestCookies(request, cookieContainer);
var response = await _httpDispatcher.GetResponseAsync(request, cookieContainer);
var response = _httpDispatcher.GetResponse(request, cookieContainer);
HandleResponseCookies(response, cookieContainer);
@ -247,7 +231,7 @@ namespace NzbDrone.Common.Http
}
}
public async Task DownloadFileAsync(string url, string fileName)
public void DownloadFile(string url, string fileName)
{
try
{
@ -263,7 +247,7 @@ namespace NzbDrone.Common.Http
using (var webClient = new GZipWebClient())
{
webClient.Headers.Add(HttpRequestHeader.UserAgent, _userAgentBuilder.GetUserAgent());
await webClient.DownloadFileTaskAsync(url, fileName);
webClient.DownloadFile(url, fileName);
stopWatch.Stop();
_logger.Debug("Downloading Completed. took {0:0}s", stopWatch.Elapsed.Seconds);
}
@ -271,94 +255,49 @@ namespace NzbDrone.Common.Http
catch (WebException e)
{
_logger.Warn("Failed to get response from: {0} {1}", url, e.Message);
if (File.Exists(fileName))
{
File.Delete(fileName);
}
throw;
}
catch (Exception e)
{
_logger.Warn(e, "Failed to get response from: " + url);
if (File.Exists(fileName))
{
File.Delete(fileName);
}
throw;
}
}
public void DownloadFile(string url, string fileName)
{
// https://docs.microsoft.com/en-us/archive/msdn-magazine/2015/july/async-programming-brownfield-async-development#the-thread-pool-hack
Task.Run(() => DownloadFileAsync(url, fileName)).GetAwaiter().GetResult();
}
public Task<HttpResponse> GetAsync(HttpRequest request)
{
request.Method = HttpMethod.GET;
return ExecuteAsync(request);
}
public HttpResponse Get(HttpRequest request)
{
return Task.Run(() => GetAsync(request)).GetAwaiter().GetResult();
request.Method = HttpMethod.GET;
return Execute(request);
}
public async Task<HttpResponse<T>> GetAsync<T>(HttpRequest request)
public HttpResponse<T> Get<T>(HttpRequest request)
where T : new()
{
var response = await GetAsync(request);
var response = Get(request);
CheckResponseContentType(response);
return new HttpResponse<T>(response);
}
public HttpResponse<T> Get<T>(HttpRequest request)
where T : new()
{
return Task.Run(() => GetAsync<T>(request)).GetAwaiter().GetResult();
}
public Task<HttpResponse> HeadAsync(HttpRequest request)
{
request.Method = HttpMethod.HEAD;
return ExecuteAsync(request);
}
public HttpResponse Head(HttpRequest request)
{
return Task.Run(() => HeadAsync(request)).GetAwaiter().GetResult();
}
public Task<HttpResponse> PostAsync(HttpRequest request)
{
request.Method = HttpMethod.POST;
return ExecuteAsync(request);
request.Method = HttpMethod.HEAD;
return Execute(request);
}
public HttpResponse Post(HttpRequest request)
{
return Task.Run(() => PostAsync(request)).GetAwaiter().GetResult();
request.Method = HttpMethod.POST;
return Execute(request);
}
public async Task<HttpResponse<T>> PostAsync<T>(HttpRequest request)
public HttpResponse<T> Post<T>(HttpRequest request)
where T : new()
{
var response = await PostAsync(request);
var response = Post(request);
CheckResponseContentType(response);
return new HttpResponse<T>(response);
}
public HttpResponse<T> Post<T>(HttpRequest request)
where T : new()
{
return Task.Run(() => PostAsync<T>(request)).GetAwaiter().GetResult();
}
private void CheckResponseContentType(HttpResponse response)
{
if (response.Headers.ContentType != null && response.Headers.ContentType.Contains("text/html"))

30
src/NzbDrone.Common/TPL/RateLimitService.cs
Unescape View File

@ -1,6 +1,5 @@
using System;
using System.Collections.Concurrent;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Cache;
@ -9,7 +8,6 @@ namespace NzbDrone.Common.TPL
public interface IRateLimitService
{
void WaitAndPulse(string key, TimeSpan interval);
Task WaitAndPulseAsync(string key, TimeSpan interval);
}
public class RateLimitService : IRateLimitService
@ -25,35 +23,19 @@ namespace NzbDrone.Common.TPL
public void WaitAndPulse(string key, TimeSpan interval)
{
var delay = GetDelay(key, interval);
var waitUntil = _rateLimitStore.AddOrUpdate(key,
(s) => DateTime.UtcNow + interval,
(s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
if (delay.TotalSeconds > 0.0)
{
_logger.Trace("Rate Limit triggered, delaying '{0}' for {1:0.000} sec", key, delay.TotalSeconds);
System.Threading.Thread.Sleep(delay);
}
}
waitUntil -= interval;
public async Task WaitAndPulseAsync(string key, TimeSpan interval)
{
var delay = GetDelay(key, interval);
var delay = waitUntil - DateTime.UtcNow;
if (delay.TotalSeconds > 0.0)
{
_logger.Trace("Rate Limit triggered, delaying '{0}' for {1:0.000} sec", key, delay.TotalSeconds);
await Task.Delay(delay);
System.Threading.Thread.Sleep(delay);
}
}
private TimeSpan GetDelay(string key, TimeSpan interval)
{
var waitUntil = _rateLimitStore.AddOrUpdate(key,
(s) => DateTime.UtcNow + interval,
(s, i) => new DateTime(Math.Max(DateTime.UtcNow.Ticks, i.Ticks), DateTimeKind.Utc) + interval);
waitUntil -= interval;
return waitUntil - DateTime.UtcNow;
}
}
}

3
src/NzbDrone.Core/MetadataSource/IProvideMovieInfo.cs
Unescape View File

@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using NzbDrone.Core.Movies;
using NzbDrone.Core.Movies.Credits;
@ -11,7 +10,5 @@ namespace NzbDrone.Core.MetadataSource
Movie GetMovieByImdbId(string imdbId);
Tuple<Movie, List<Credit>> GetMovieInfo(int tmdbId);
HashSet<int> GetChangedMovies(DateTime startTime);
Task<Tuple<Movie, List<Credit>>> GetMovieInfoAsync(int tmdbId);
}
}

2
src/NzbDrone.Core/MetadataSource/ISearchForNewMovie.cs
Unescape View File

@ -1,5 +1,4 @@
using System.Collections.Generic;
using System.Threading.Tasks;
using NzbDrone.Core.Movies;
namespace NzbDrone.Core.MetadataSource
@ -9,6 +8,5 @@ namespace NzbDrone.Core.MetadataSource
List<Movie> SearchForNewMovie(string title);
Movie MapMovieToTmdbMovie(Movie movie);
Task<Movie> MapMovieToTmdbMovieAsync(Movie movie);
}
}

20
src/NzbDrone.Core/MetadataSource/SkyHook/SkyHookProxy.cs
Unescape View File

@ -2,7 +2,6 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Threading.Tasks;
using NLog;
using NzbDrone.Common.Cloud;
using NzbDrone.Common.Extensions;
@ -15,6 +14,7 @@ using NzbDrone.Core.MetadataSource.SkyHook.Resource;
using NzbDrone.Core.Movies;
using NzbDrone.Core.Movies.AlternativeTitles;
using NzbDrone.Core.Movies.Credits;
using NzbDrone.Core.NetImport.ImportExclusions;
using NzbDrone.Core.NetImport.TMDb;
using NzbDrone.Core.Parser;
@ -65,7 +65,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
return new HashSet<int>(response.Resource.Results.Select(c => c.Id));
}
public async Task<Tuple<Movie, List<Credit>>> GetMovieInfoAsync(int tmdbId)
public Tuple<Movie, List<Credit>> GetMovieInfo(int tmdbId)
{
var httpRequest = _radarrMetadata.Create()
.SetSegment("route", "movie")
@ -75,7 +75,7 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
httpRequest.AllowAutoRedirect = true;
httpRequest.SuppressHttpError = true;
var httpResponse = await _httpClient.GetAsync<MovieResource>(httpRequest);
var httpResponse = _httpClient.Get<MovieResource>(httpRequest);
if (httpResponse.HasHttpError)
{
@ -98,11 +98,6 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
return new Tuple<Movie, List<Credit>>(movie, credits.ToList());
}
public Tuple<Movie, List<Credit>> GetMovieInfo(int tmdbId)
{
return GetMovieInfoAsync(tmdbId).GetAwaiter().GetResult();
}
public Movie GetMovieByImdbId(string imdbId)
{
var httpRequest = _radarrMetadata.Create()
@ -241,14 +236,14 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
return title;
}
public async Task<Movie> MapMovieToTmdbMovieAsync(Movie movie)
public Movie MapMovieToTmdbMovie(Movie movie)
{
try
{
Movie newMovie = movie;
if (movie.TmdbId > 0)
{
newMovie = (await GetMovieInfoAsync(movie.TmdbId)).Item1;
newMovie = GetMovieInfo(movie.TmdbId).Item1;
}
else if (movie.ImdbId.IsNotNullOrWhiteSpace())
{
@ -288,11 +283,6 @@ namespace NzbDrone.Core.MetadataSource.SkyHook
}
}
public Movie MapMovieToTmdbMovie(Movie movie)
{
return MapMovieToTmdbMovieAsync(movie).GetAwaiter().GetResult();
}
public List<Movie> SearchForNewMovie(string title)
{
try

4
src/NzbDrone.Core/NetImport/TMDb/TMDbImportBase.cs
Unescape View File

@ -1,5 +1,4 @@
using System;
using NLog;
using NLog;
using NzbDrone.Common.Cloud;
using NzbDrone.Common.Http;
using NzbDrone.Core.Configuration;
@ -12,7 +11,6 @@ namespace NzbDrone.Core.NetImport.TMDb
where TSettings : TMDbSettingsBase<TSettings>, new()
{
public override NetImportType ListType => NetImportType.TMDB;
public override TimeSpan RateLimit => TimeSpan.Zero;
public readonly ISearchForNewMovie _skyhookProxy;
public readonly IHttpRequestBuilderFactory _requestBuilder;

19
src/Radarr.Api.V3/Movies/FetchMovieListModule.cs
Unescape View File

@ -1,6 +1,5 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Nancy;
using NzbDrone.Common.Extensions;
using NzbDrone.Core.MediaCover;
@ -28,14 +27,22 @@ namespace Radarr.Api.V3.Movies
{
var results = _fetchNetImport.FetchAndFilter((int)Request.Query.listId, false);
var tasks = results.Where(movie => movie.TmdbId == 0 || !movie.Images.Any() || movie.Overview.IsNullOrWhiteSpace())
.Select(x => _movieSearch.MapMovieToTmdbMovieAsync(x));
List<Movie> realResults = new List<Movie>();
var realResults = results.Where(movie => movie.TmdbId != 0 && movie.Images.Any() && movie.Overview.IsNotNullOrWhiteSpace()).ToList();
foreach (var movie in results)
{
var mapped = movie;
var mapped = Task.WhenAll(tasks).GetAwaiter().GetResult();
if (movie.TmdbId == 0 || !movie.Images.Any() || movie.Overview.IsNullOrWhiteSpace())
{
mapped = _movieSearch.MapMovieToTmdbMovie(movie);
}
realResults.AddRange(mapped.Where(x => x != null));
if (mapped != null)
{
realResults.Add(mapped);
}
}
return MapToResource(realResults);
}

Write Preview
Loading…
Cancel
Save