From 99528878b76f498f991873d57ee990f55266244d Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 02:23:44 +0200 Subject: [PATCH 1/3] OmdbProvider: Added local caching The lack of caching in the OmdbProvider could result in long library scan time. This commit adds caching for the OmdbProvider similar to the MovieDbProvider. Downloaded metadata is saved locally and only updated if the last refresh occured 3 or more days before --- .../Omdb/OmdbItemProvider.cs | 14 +++- MediaBrowser.Providers/Omdb/OmdbProvider.cs | 79 ++++++++++++++++--- .../TV/Omdb/OmdbEpisodeProvider.cs | 14 +++- 3 files changed, 86 insertions(+), 21 deletions(-) diff --git a/MediaBrowser.Providers/Omdb/OmdbItemProvider.cs b/MediaBrowser.Providers/Omdb/OmdbItemProvider.cs index a0d60c1669..7a803eb6ff 100644 --- a/MediaBrowser.Providers/Omdb/OmdbItemProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbItemProvider.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.Net; +using CommonIO; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Movies; using MediaBrowser.Controller.Entities.TV; @@ -26,13 +28,17 @@ namespace MediaBrowser.Providers.Omdb private readonly IHttpClient _httpClient; private readonly ILogger _logger; private readonly ILibraryManager _libraryManager; + private readonly IFileSystem _fileSystem; + private readonly IServerConfigurationManager _configurationManager; - public OmdbItemProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogger logger, ILibraryManager libraryManager) + public OmdbItemProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogger logger, ILibraryManager libraryManager, IFileSystem fileSystem, IServerConfigurationManager configurationManager) { _jsonSerializer = jsonSerializer; _httpClient = httpClient; _logger = logger; _libraryManager = libraryManager; + _fileSystem = fileSystem; + _configurationManager = configurationManager; } public Task> GetSearchResults(SeriesInfo searchInfo, CancellationToken cancellationToken) @@ -220,7 +226,7 @@ namespace MediaBrowser.Providers.Omdb result.Item.SetProviderId(MetadataProviders.Imdb, imdbId); result.HasMetadata = true; - await new OmdbProvider(_jsonSerializer, _httpClient).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); + await new OmdbProvider(_jsonSerializer, _httpClient, _fileSystem, _configurationManager).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); } return result; @@ -259,7 +265,7 @@ namespace MediaBrowser.Providers.Omdb result.Item.SetProviderId(MetadataProviders.Imdb, imdbId); result.HasMetadata = true; - await new OmdbProvider(_jsonSerializer, _httpClient).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); + await new OmdbProvider(_jsonSerializer, _httpClient, _fileSystem, _configurationManager).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); } return result; diff --git a/MediaBrowser.Providers/Omdb/OmdbProvider.cs b/MediaBrowser.Providers/Omdb/OmdbProvider.cs index 39da70210d..d3c895b9a0 100644 --- a/MediaBrowser.Providers/Omdb/OmdbProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbProvider.cs @@ -1,4 +1,7 @@ -using MediaBrowser.Common.Net; +using CommonIO; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Serialization; @@ -17,15 +20,19 @@ namespace MediaBrowser.Providers.Omdb { internal static readonly SemaphoreSlim ResourcePool = new SemaphoreSlim(1, 1); private readonly IJsonSerializer _jsonSerializer; + private readonly IFileSystem _fileSystem; + private readonly IServerConfigurationManager _configurationManager; private readonly IHttpClient _httpClient; private readonly CultureInfo _usCulture = new CultureInfo("en-US"); public static OmdbProvider Current; - public OmdbProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient) + public OmdbProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, IFileSystem fileSystem, IServerConfigurationManager configurationManager) { _jsonSerializer = jsonSerializer; _httpClient = httpClient; + _fileSystem = fileSystem; + _configurationManager = configurationManager; Current = this; } @@ -37,24 +44,17 @@ namespace MediaBrowser.Providers.Omdb throw new ArgumentNullException("imdbId"); } - var imdbParam = imdbId.StartsWith("tt", StringComparison.OrdinalIgnoreCase) ? imdbId : "tt" + imdbId; - - var url = string.Format("https://www.omdbapi.com/?i={0}&tomatoes=true", imdbParam); + var path = await EnsureItemInfo(imdbId, cancellationToken); - using (var stream = await _httpClient.Get(new HttpRequestOptions - { - Url = url, - ResourcePool = ResourcePool, - CancellationToken = cancellationToken + string resultString; - }).ConfigureAwait(false)) + using (Stream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 131072)) { - string resultString; - using (var reader = new StreamReader(stream, new UTF8Encoding(false))) { resultString = reader.ReadToEnd(); } + } resultString = resultString.Replace("\"N/A\"", "\"\""); @@ -130,7 +130,60 @@ namespace MediaBrowser.Providers.Omdb } ParseAdditionalMetadata(item, result); + } + + internal async Task EnsureItemInfo(string imdbId, CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(imdbId)) + { + throw new ArgumentNullException("imdbId"); + } + + var imdbParam = imdbId.StartsWith("tt", StringComparison.OrdinalIgnoreCase) ? imdbId : "tt" + imdbId; + + var path = GetDataFilePath(imdbParam); + + var fileInfo = _fileSystem.GetFileSystemInfo(path); + + if (fileInfo.Exists) + { + // If it's recent or automatic updates are enabled, don't re-download + if ((DateTime.UtcNow - _fileSystem.GetLastWriteTimeUtc(fileInfo)).TotalDays <= 3) + { + return path; + } + } + + var url = string.Format("https://www.omdbapi.com/?i={0}&tomatoes=true", imdbParam); + + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = url, + ResourcePool = ResourcePool, + CancellationToken = cancellationToken + + }).ConfigureAwait(false)) + { + var rootObject = _jsonSerializer.DeserializeFromStream(stream); + _fileSystem.CreateDirectory(Path.GetDirectoryName(path)); + _jsonSerializer.SerializeToFile(rootObject, path); + } + + return path; + } + + internal string GetDataFilePath(string imdbId) + { + if (string.IsNullOrEmpty(imdbId)) + { + throw new ArgumentNullException("imdbId"); } + + var dataPath = Path.Combine(_configurationManager.ApplicationPaths.CachePath, "omdb"); + + var filename = string.Format("{0}.json", imdbId); + + return Path.Combine(dataPath, filename); } private void ParseAdditionalMetadata(BaseItem item, RootObject result) diff --git a/MediaBrowser.Providers/TV/Omdb/OmdbEpisodeProvider.cs b/MediaBrowser.Providers/TV/Omdb/OmdbEpisodeProvider.cs index 5da1fcf279..ebbefbeb1a 100644 --- a/MediaBrowser.Providers/TV/Omdb/OmdbEpisodeProvider.cs +++ b/MediaBrowser.Providers/TV/Omdb/OmdbEpisodeProvider.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Common.Net; +using CommonIO; +using MediaBrowser.Common.Net; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities.TV; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; @@ -21,12 +23,16 @@ namespace MediaBrowser.Providers.TV private readonly IJsonSerializer _jsonSerializer; private readonly IHttpClient _httpClient; private readonly OmdbItemProvider _itemProvider; + private readonly IFileSystem _fileSystem; + private readonly IServerConfigurationManager _configurationManager; - public OmdbEpisodeProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogger logger, ILibraryManager libraryManager) + public OmdbEpisodeProvider(IJsonSerializer jsonSerializer, IHttpClient httpClient, ILogger logger, ILibraryManager libraryManager, IFileSystem fileSystem, IServerConfigurationManager configurationManager) { _jsonSerializer = jsonSerializer; _httpClient = httpClient; - _itemProvider = new OmdbItemProvider(jsonSerializer, httpClient, logger, libraryManager); + _fileSystem = fileSystem; + _configurationManager = configurationManager; + _itemProvider = new OmdbItemProvider(jsonSerializer, httpClient, logger, libraryManager, fileSystem, configurationManager); } public Task> GetSearchResults(EpisodeInfo searchInfo, CancellationToken cancellationToken) @@ -58,7 +64,7 @@ namespace MediaBrowser.Providers.TV result.Item.SetProviderId(MetadataProviders.Imdb, imdbId); result.HasMetadata = true; - await new OmdbProvider(_jsonSerializer, _httpClient).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); + await new OmdbProvider(_jsonSerializer, _httpClient, _fileSystem, _configurationManager).Fetch(result.Item, imdbId, info.MetadataLanguage, info.MetadataCountryCode, cancellationToken).ConfigureAwait(false); } return result; From f601614539559ba943bce0bbd0138d39768b2038 Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 02:35:35 +0200 Subject: [PATCH 2/3] minifix --- MediaBrowser.Providers/Omdb/OmdbProvider.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MediaBrowser.Providers/Omdb/OmdbProvider.cs b/MediaBrowser.Providers/Omdb/OmdbProvider.cs index d3c895b9a0..823a29d54e 100644 --- a/MediaBrowser.Providers/Omdb/OmdbProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbProvider.cs @@ -53,11 +53,10 @@ namespace MediaBrowser.Providers.Omdb using (var reader = new StreamReader(stream, new UTF8Encoding(false))) { resultString = reader.ReadToEnd(); + resultString = resultString.Replace("\"N/A\"", "\"\""); } } - resultString = resultString.Replace("\"N/A\"", "\"\""); - var result = _jsonSerializer.DeserializeFromString(resultString); // Only take the name and rating if the user's language is set to english, since Omdb has no localization From f7900e77570f87a4a55ff11a087e8a98ae1f5d5d Mon Sep 17 00:00:00 2001 From: softworkz Date: Sat, 4 Jun 2016 03:08:27 +0200 Subject: [PATCH 3/3] OmdbImageProvider: Check poster field of cached item to determine if an image is available --- .../Omdb/OmdbImageProvider.cs | 35 ++++++++--------- MediaBrowser.Providers/Omdb/OmdbProvider.cs | 38 +++++++++++-------- 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs b/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs index a1e038374c..e795d638d8 100644 --- a/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbImageProvider.cs @@ -7,6 +7,8 @@ using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Providers; using System.Collections.Generic; +using System.IO; +using System.Text; using System.Threading; using System.Threading.Tasks; @@ -29,22 +31,27 @@ namespace MediaBrowser.Providers.Omdb }; } - public Task> GetImages(IHasImages item, CancellationToken cancellationToken) + public async Task> GetImages(IHasImages item, CancellationToken cancellationToken) { var imdbId = item.GetProviderId(MetadataProviders.Imdb); var list = new List(); - if (!string.IsNullOrWhiteSpace(imdbId)) + if (!string.IsNullOrWhiteSpace(imdbId) && OmdbProvider.Current != null) { - list.Add(new RemoteImageInfo + OmdbProvider.RootObject rootObject = await OmdbProvider.Current.GetRootObject(imdbId, cancellationToken); + + if (!string.IsNullOrEmpty(rootObject.Poster)) { - ProviderName = Name, - Url = string.Format("https://img.omdbapi.com/?i={0}&apikey=82e83907", imdbId) - }); + list.Add(new RemoteImageInfo + { + ProviderName = Name, + Url = string.Format("https://img.omdbapi.com/?i={0}&apikey=82e83907", imdbId) + }); + } } - return Task.FromResult>(list); + return list; } public Task GetImageResponse(string url, CancellationToken cancellationToken) @@ -65,18 +72,6 @@ namespace MediaBrowser.Providers.Omdb public bool Supports(IHasImages item) { - // We'll hammer Omdb if we enable this - if (item is Person) - { - return false; - } - - // Save the http requests since we know it's not currently supported - if (item is Season || item is Episode) - { - return false; - } - // Supports images for tv movies var tvProgram = item as LiveTvProgram; if (tvProgram != null && tvProgram.IsMovie) @@ -84,7 +79,7 @@ namespace MediaBrowser.Providers.Omdb return true; } - return item is Movie || item is Trailer; + return item is Movie || item is Trailer || item is Episode; } public int Order diff --git a/MediaBrowser.Providers/Omdb/OmdbProvider.cs b/MediaBrowser.Providers/Omdb/OmdbProvider.cs index 823a29d54e..9873447bec 100644 --- a/MediaBrowser.Providers/Omdb/OmdbProvider.cs +++ b/MediaBrowser.Providers/Omdb/OmdbProvider.cs @@ -44,20 +44,7 @@ namespace MediaBrowser.Providers.Omdb throw new ArgumentNullException("imdbId"); } - var path = await EnsureItemInfo(imdbId, cancellationToken); - - string resultString; - - using (Stream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 131072)) - { - using (var reader = new StreamReader(stream, new UTF8Encoding(false))) - { - resultString = reader.ReadToEnd(); - resultString = resultString.Replace("\"N/A\"", "\"\""); - } - } - - var result = _jsonSerializer.DeserializeFromString(resultString); + var result = await GetRootObject(imdbId, cancellationToken); // Only take the name and rating if the user's language is set to english, since Omdb has no localization if (string.Equals(language, "en", StringComparison.OrdinalIgnoreCase)) @@ -131,7 +118,26 @@ namespace MediaBrowser.Providers.Omdb ParseAdditionalMetadata(item, result); } - internal async Task EnsureItemInfo(string imdbId, CancellationToken cancellationToken) + internal async Task GetRootObject(string imdbId, CancellationToken cancellationToken) + { + var path = await EnsureItemInfo(imdbId, cancellationToken); + + string resultString; + + using (Stream stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 131072)) + { + using (var reader = new StreamReader(stream, new UTF8Encoding(false))) + { + resultString = reader.ReadToEnd(); + resultString = resultString.Replace("\"N/A\"", "\"\""); + } + } + + var result = _jsonSerializer.DeserializeFromString(resultString); + return result; + } + + private async Task EnsureItemInfo(string imdbId, CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(imdbId)) { @@ -236,7 +242,7 @@ namespace MediaBrowser.Providers.Omdb return string.Equals(lang, "en", StringComparison.OrdinalIgnoreCase); } - private class RootObject + internal class RootObject { public string Title { get; set; } public string Year { get; set; }