From ed60771cdd2ab4184a7f8a7d496ab9db818a4109 Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Mon, 14 Oct 2013 13:34:54 -0400 Subject: [PATCH] fixes #587 - Support automatic tmdb updates for movies --- MediaBrowser.Mono.userprefs | 10 +- .../MediaBrowser.Providers.csproj | 1 + .../Movies/MovieDbProvider.cs | 227 ++++++++++++---- .../Movies/MovieUpdatesPrescanTask.cs | 254 ++++++++++++++++++ .../Movies/TmdbPersonProvider.cs | 14 +- .../TV/RemoteEpisodeProvider.cs | 4 +- 6 files changed, 446 insertions(+), 64 deletions(-) create mode 100644 MediaBrowser.Providers/Movies/MovieUpdatesPrescanTask.cs diff --git a/MediaBrowser.Mono.userprefs b/MediaBrowser.Mono.userprefs index a4e4b17519..9f5872eef0 100644 --- a/MediaBrowser.Mono.userprefs +++ b/MediaBrowser.Mono.userprefs @@ -1,6 +1,9 @@  - + + + + @@ -16,9 +19,8 @@ - - - + + diff --git a/MediaBrowser.Providers/MediaBrowser.Providers.csproj b/MediaBrowser.Providers/MediaBrowser.Providers.csproj index fead8e8a8e..e687e14d9d 100644 --- a/MediaBrowser.Providers/MediaBrowser.Providers.csproj +++ b/MediaBrowser.Providers/MediaBrowser.Providers.csproj @@ -59,6 +59,7 @@ + diff --git a/MediaBrowser.Providers/Movies/MovieDbProvider.cs b/MediaBrowser.Providers/Movies/MovieDbProvider.cs index 57c9b1555a..475e198835 100644 --- a/MediaBrowser.Providers/Movies/MovieDbProvider.cs +++ b/MediaBrowser.Providers/Movies/MovieDbProvider.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Common.Net; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Movies; @@ -193,7 +194,7 @@ namespace MediaBrowser.Providers.Movies protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo) { - if (HasAltMeta(item)) + if (HasAltMeta(item) && !ConfigurationManager.Configuration.EnableTmdbUpdates) return false; // Boxsets require two passes because we need the children to be refreshed @@ -205,6 +206,53 @@ namespace MediaBrowser.Providers.Movies return base.NeedsRefreshInternal(item, providerInfo); } + protected override bool NeedsRefreshBasedOnCompareDate(BaseItem item, BaseProviderInfo providerInfo) + { + var language = ConfigurationManager.Configuration.PreferredMetadataLanguage; + + var path = GetDataFilePath(item, language); + + if (!string.IsNullOrEmpty(path)) + { + var fileInfo = new FileInfo(path); + + if (fileInfo.Exists) + { + return fileInfo.LastWriteTimeUtc > providerInfo.LastRefreshed; + } + } + + return base.NeedsRefreshBasedOnCompareDate(item, providerInfo); + } + + /// + /// Gets the movie data path. + /// + /// The app paths. + /// if set to true [is box set]. + /// The TMDB id. + /// System.String. + internal static string GetMovieDataPath(IApplicationPaths appPaths, bool isBoxSet, string tmdbId) + { + var dataPath = isBoxSet ? GetBoxSetsDataPath(appPaths) : GetMoviesDataPath(appPaths); + + return Path.Combine(dataPath, tmdbId); + } + + internal static string GetMoviesDataPath(IApplicationPaths appPaths) + { + var dataPath = Path.Combine(appPaths.DataPath, "tmdb-movies"); + + return dataPath; + } + + internal static string GetBoxSetsDataPath(IApplicationPaths appPaths) + { + var dataPath = Path.Combine(appPaths.DataPath, "tmdb-collections"); + + return dataPath; + } + /// /// Fetches metadata and returns true or false indicating if any work that requires persistence was done /// @@ -216,7 +264,29 @@ namespace MediaBrowser.Providers.Movies { cancellationToken.ThrowIfCancellationRequested(); - await FetchMovieData(item, cancellationToken).ConfigureAwait(false); + var id = item.GetProviderId(MetadataProviders.Tmdb); + + if (string.IsNullOrEmpty(id)) + { + id = item.GetProviderId(MetadataProviders.Imdb); + } + + if (string.IsNullOrEmpty(id)) + { + id = await FindId(item, cancellationToken).ConfigureAwait(false); + + if (!string.IsNullOrEmpty(id)) + { + item.SetProviderId(MetadataProviders.Tmdb, id); + } + } + + if (!string.IsNullOrEmpty(id)) + { + cancellationToken.ThrowIfCancellationRequested(); + + await FetchMovieData(item, id, force, cancellationToken).ConfigureAwait(false); + } SetLastRefreshed(item, DateTime.UtcNow); return true; @@ -245,40 +315,6 @@ namespace MediaBrowser.Providers.Movies return false; } - /// - /// Fetches the movie data. - /// - /// The item. - /// - /// Task. - private async Task FetchMovieData(BaseItem item, CancellationToken cancellationToken) - { - var id = item.GetProviderId(MetadataProviders.Tmdb); - - if (string.IsNullOrEmpty(id)) - { - id = item.GetProviderId(MetadataProviders.Imdb); - } - - if (string.IsNullOrEmpty(id)) - { - id = await FindId(item, cancellationToken).ConfigureAwait(false); - } - - if (!string.IsNullOrEmpty(id)) - { - Logger.Debug("MovieDbProvider - getting movie info with id: " + id); - - cancellationToken.ThrowIfCancellationRequested(); - - await FetchMovieData(item, id, cancellationToken).ConfigureAwait(false); - } - else - { - Logger.Info("MovieDBProvider could not find " + item.Name + ". Check name on themoviedb.org."); - } - } - /// /// Parses the name. /// @@ -453,42 +489,114 @@ namespace MediaBrowser.Providers.Movies return WebUtility.UrlEncode(name); } + private readonly CultureInfo _usCulture = new CultureInfo("en-US"); + /// /// Fetches the movie data. /// /// The item. /// The id. + /// if set to true [is forced refresh]. /// The cancellation token /// Task. - protected async Task FetchMovieData(BaseItem item, string id, CancellationToken cancellationToken) + private async Task FetchMovieData(BaseItem item, string id, bool isForcedRefresh, CancellationToken cancellationToken) { - cancellationToken.ThrowIfCancellationRequested(); + // Id could be ImdbId or TmdbId + + var language = ConfigurationManager.Configuration.PreferredMetadataLanguage; - if (String.IsNullOrEmpty(id)) + var dataFilePath = GetDataFilePath(item, language); + + var hasAltMeta = HasAltMeta(item); + + var isRefreshingDueToTmdbUpdate = hasAltMeta && !isForcedRefresh; + + if (string.IsNullOrEmpty(dataFilePath) || !File.Exists(dataFilePath)) { - Logger.Info("MoviedbProvider: Ignoring " + item.Name + " because ID forced blank."); - return; + var isBoxSet = item is BoxSet; + + var mainResult = await FetchMainResult(id, isBoxSet, cancellationToken).ConfigureAwait(false); + + if (mainResult == null) return; + + var path = GetMovieDataPath(ConfigurationManager.ApplicationPaths, isBoxSet, mainResult.id.ToString(_usCulture)); + + dataFilePath = Path.Combine(path, language + ".json"); + + var directory = Path.GetDirectoryName(dataFilePath); + + Directory.CreateDirectory(directory); + + JsonSerializer.SerializeToFile(mainResult, dataFilePath); + + isRefreshingDueToTmdbUpdate = false; } - item.SetProviderId(MetadataProviders.Tmdb, id); + if (isForcedRefresh || ConfigurationManager.Configuration.EnableTmdbUpdates || !hasAltMeta) + { + dataFilePath = GetDataFilePath(item, language); - var mainResult = await FetchMainResult(item, id, cancellationToken).ConfigureAwait(false); + var mainResult = JsonSerializer.DeserializeFromFile(dataFilePath); + + ProcessMainInfo(item, mainResult, isRefreshingDueToTmdbUpdate); + } + } + + /// + /// Downloads the movie info. + /// + /// The id. + /// if set to true [is box set]. + /// The data path. + /// The cancellation token. + /// Task. + internal async Task DownloadMovieInfo(string id, bool isBoxSet, string dataPath, CancellationToken cancellationToken) + { + var language = ConfigurationManager.Configuration.PreferredMetadataLanguage; + + var mainResult = await FetchMainResult(id, isBoxSet, cancellationToken).ConfigureAwait(false); if (mainResult == null) return; - ProcessMainInfo(item, mainResult); + var dataFilePath = Path.Combine(dataPath, language + ".json"); + + Directory.CreateDirectory(dataPath); + + JsonSerializer.SerializeToFile(mainResult, dataFilePath); } /// - /// Fetches the main result. + /// Gets the data file path. /// /// The item. + /// The language. + /// System.String. + private string GetDataFilePath(BaseItem item, string language) + { + var id = item.GetProviderId(MetadataProviders.Tmdb); + + if (string.IsNullOrEmpty(id)) + { + return null; + } + + var path = GetMovieDataPath(ConfigurationManager.ApplicationPaths, item is BoxSet, id); + + path = Path.Combine(path, language + ".json"); + + return path; + } + + /// + /// Fetches the main result. + /// /// The id. + /// if set to true [is box set]. /// The cancellation token /// Task{CompleteMovieData}. - protected async Task FetchMainResult(BaseItem item, string id, CancellationToken cancellationToken) + protected async Task FetchMainResult(string id, bool isBoxSet, CancellationToken cancellationToken) { - var baseUrl = item is BoxSet ? GetBoxSetInfo3 : GetMovieInfo3; + var baseUrl = isBoxSet ? GetBoxSetInfo3 : GetMovieInfo3; string url = string.Format(baseUrl, id, ApiKey, ConfigurationManager.Configuration.PreferredMetadataLanguage); CompleteMovieData mainResult; @@ -529,7 +637,7 @@ namespace MediaBrowser.Providers.Movies if (String.IsNullOrEmpty(mainResult.overview)) { - Logger.Error("MovieDbProvider - Unable to find information for " + item.Name + " (id:" + id + ")"); + Logger.Error("MovieDbProvider - Unable to find information for (id:" + id + ")"); return null; } } @@ -542,7 +650,8 @@ namespace MediaBrowser.Providers.Movies /// /// The movie. /// The movie data. - protected virtual void ProcessMainInfo(BaseItem movie, CompleteMovieData movieData) + /// if set to true [is refreshing due to TMDB update]. + protected virtual void ProcessMainInfo(BaseItem movie, CompleteMovieData movieData, bool isRefreshingDueToTmdbUpdate) { if (movie != null && movieData != null) { @@ -580,12 +689,19 @@ namespace MediaBrowser.Providers.Movies float rating; string voteAvg = movieData.vote_average.ToString(CultureInfo.InvariantCulture); - //tmdb appears to have unified their numbers to always report "7.3" regardless of country + // tmdb appears to have unified their numbers to always report "7.3" regardless of country // so I removed the culture-specific processing here because it was not working for other countries -ebr - if (float.TryParse(voteAvg, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out rating)) + // Don't import this when responding to tmdb updates because we don't want to blow away imdb data + if (!isRefreshingDueToTmdbUpdate && float.TryParse(voteAvg, NumberStyles.AllowDecimalPoint, CultureInfo.InvariantCulture, out rating)) + { movie.CommunityRating = rating; + } - movie.VoteCount = movieData.vote_count; + // Don't import this when responding to tmdb updates because we don't want to blow away imdb data + if (!isRefreshingDueToTmdbUpdate) + { + movie.VoteCount = movieData.vote_count; + } //release date and certification are retrieved based on configured country and we fall back on US if not there and to minimun release date if still no match if (movieData.releases != null && movieData.releases.countries != null) @@ -667,8 +783,9 @@ namespace MediaBrowser.Providers.Movies } } - //genres - if (movieData.genres != null && !movie.LockedFields.Contains(MetadataFields.Genres)) + // genres + // Don't import this when responding to tmdb updates because we don't want to blow away imdb data + if (movieData.genres != null && !movie.LockedFields.Contains(MetadataFields.Genres) && !isRefreshingDueToTmdbUpdate) { movie.Genres.Clear(); diff --git a/MediaBrowser.Providers/Movies/MovieUpdatesPrescanTask.cs b/MediaBrowser.Providers/Movies/MovieUpdatesPrescanTask.cs new file mode 100644 index 0000000000..13bf19d7ee --- /dev/null +++ b/MediaBrowser.Providers/Movies/MovieUpdatesPrescanTask.cs @@ -0,0 +1,254 @@ +using MediaBrowser.Common.Net; +using MediaBrowser.Common.Progress; +using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Library; +using MediaBrowser.Model.Logging; +using MediaBrowser.Model.Serialization; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Providers.Movies +{ + public class MovieUpdatesPreScanTask : ILibraryPrescanTask + { + /// + /// The updates URL + /// + private const string UpdatesUrl = "http://api.themoviedb.org/3/movie/changes?start_date={0}&api_key={1}&page={2}"; + + /// + /// The _HTTP client + /// + private readonly IHttpClient _httpClient; + /// + /// The _logger + /// + private readonly ILogger _logger; + /// + /// The _config + /// + private readonly IServerConfigurationManager _config; + private readonly IJsonSerializer _json; + + /// + /// Initializes a new instance of the class. + /// + /// The logger. + /// The HTTP client. + /// The config. + /// The json. + public MovieUpdatesPreScanTask(ILogger logger, IHttpClient httpClient, IServerConfigurationManager config, IJsonSerializer json) + { + _logger = logger; + _httpClient = httpClient; + _config = config; + _json = json; + } + + protected readonly CultureInfo UsCulture = new CultureInfo("en-US"); + + /// + /// Runs the specified progress. + /// + /// The progress. + /// The cancellation token. + /// Task. + public async Task Run(IProgress progress, CancellationToken cancellationToken) + { + if (!_config.Configuration.EnableInternetProviders || !_config.Configuration.EnableTmdbUpdates) + { + progress.Report(100); + return; + } + + var innerProgress = new ActionableProgress(); + innerProgress.RegisterAction(pct => progress.Report(pct * .8)); + await Run(innerProgress, false, cancellationToken).ConfigureAwait(false); + + progress.Report(80); + + innerProgress = new ActionableProgress(); + innerProgress.RegisterAction(pct => progress.Report(80 + pct * .2)); + await Run(innerProgress, true, cancellationToken).ConfigureAwait(false); + + progress.Report(100); + } + + /// + /// Runs the specified progress. + /// + /// The progress. + /// if set to true [run for box sets]. + /// The cancellation token. + /// Task. + private async Task Run(IProgress progress, bool runForBoxSets, CancellationToken cancellationToken) + { + var path = runForBoxSets ? MovieDbProvider.GetBoxSetsDataPath(_config.CommonApplicationPaths) : MovieDbProvider.GetMoviesDataPath(_config.CommonApplicationPaths); + + Directory.CreateDirectory(path); + + var timestampFile = Path.Combine(path, "time.txt"); + + var timestampFileInfo = new FileInfo(timestampFile); + + // Don't check for tvdb updates anymore frequently than 24 hours + if (timestampFileInfo.Exists && (DateTime.UtcNow - timestampFileInfo.LastWriteTimeUtc).TotalDays < 1) + { + //return; + } + + // Find out the last time we queried tvdb for updates + var lastUpdateTime = timestampFileInfo.Exists ? File.ReadAllText(timestampFile, Encoding.UTF8) : string.Empty; + + var existingDirectories = Directory.EnumerateDirectories(path).Select(Path.GetFileName).ToList(); + + if (!string.IsNullOrEmpty(lastUpdateTime)) + { + long lastUpdateTicks; + + if (long.TryParse(lastUpdateTime, NumberStyles.Any, UsCulture, out lastUpdateTicks)) + { + var lastUpdateDate = new DateTime(lastUpdateTicks, DateTimeKind.Utc); + + // They only allow up to 14 days of updates + if ((DateTime.UtcNow - lastUpdateDate).TotalDays > 13) + { + lastUpdateDate = DateTime.UtcNow.AddDays(-13); + } + + var updatedIds = await GetIdsToUpdate(lastUpdateDate, 1, cancellationToken).ConfigureAwait(false); + + var existingDictionary = existingDirectories.ToDictionary(i => i, StringComparer.OrdinalIgnoreCase); + + var idsToUpdate = updatedIds.Where(i => !string.IsNullOrWhiteSpace(i) && existingDictionary.ContainsKey(i)); + + await UpdateMovies(idsToUpdate, runForBoxSets, path, progress, cancellationToken).ConfigureAwait(false); + } + } + + File.WriteAllText(timestampFile, DateTime.UtcNow.Ticks.ToString(UsCulture), Encoding.UTF8); + progress.Report(100); + } + + + /// + /// Gets the ids to update. + /// + /// The last update time. + /// The page. + /// The cancellation token. + /// Task{IEnumerable{System.String}}. + private async Task> GetIdsToUpdate(DateTime lastUpdateTime, int page, CancellationToken cancellationToken) + { + bool hasMorePages; + var list = new List(); + + // First get last time + using (var stream = await _httpClient.Get(new HttpRequestOptions + { + Url = string.Format(UpdatesUrl, lastUpdateTime.ToString("yyyy-MM-dd"), MovieDbProvider.ApiKey, page), + CancellationToken = cancellationToken, + EnableHttpCompression = true, + ResourcePool = MovieDbProvider.Current.MovieDbResourcePool, + AcceptHeader = MovieDbProvider.AcceptHeader + + }).ConfigureAwait(false)) + { + var obj = _json.DeserializeFromStream(stream); + + var data = obj.results.Select(i => i.id.ToString(UsCulture)); + + list.AddRange(data); + + hasMorePages = page < obj.total_pages; + } + + if (hasMorePages) + { + var more = await GetIdsToUpdate(lastUpdateTime, page + 1, cancellationToken).ConfigureAwait(false); + + list.AddRange(more); + } + + return list; + } + + /// + /// Updates the movies. + /// + /// The ids. + /// if set to true [is box set]. + /// The movies data path. + /// The progress. + /// The cancellation token. + /// Task. + private async Task UpdateMovies(IEnumerable ids, bool isBoxSet, string moviesDataPath, IProgress progress, CancellationToken cancellationToken) + { + var list = ids.ToList(); + var numComplete = 0; + + foreach (var id in list) + { + try + { + await UpdateMovie(id, isBoxSet, moviesDataPath, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) + { + _logger.ErrorException("Error updating tmdb movie id {0}", ex, id); + } + + numComplete++; + double percent = numComplete; + percent /= list.Count; + percent *= 100; + + progress.Report(percent); + } + } + + /// + /// Updates the movie. + /// + /// The id. + /// if set to true [is box set]. + /// The data path. + /// The cancellation token. + /// Task. + private Task UpdateMovie(string id, bool isBoxSet, string dataPath, CancellationToken cancellationToken) + { + _logger.Info("Updating movie from tmdb " + id); + + var itemDataPath = Path.Combine(dataPath, id); + + Directory.CreateDirectory(dataPath); + + return MovieDbProvider.Current.DownloadMovieInfo(id, isBoxSet, itemDataPath, cancellationToken); + } + + class Result + { + public int id { get; set; } + public bool? adult { get; set; } + } + + class RootObject + { + public List results { get; set; } + public int page { get; set; } + public int total_pages { get; set; } + public int total_results { get; set; } + + public RootObject() + { + results = new List(); + } + } + } +} diff --git a/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs b/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs index 33fb272807..fe073c6330 100644 --- a/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs +++ b/MediaBrowser.Providers/Movies/TmdbPersonProvider.cs @@ -80,6 +80,14 @@ namespace MediaBrowser.Providers.Movies } } + protected override bool NeedsRefreshInternal(BaseItem item, BaseProviderInfo providerInfo) + { + if (HasAltMeta(item) && !ConfigurationManager.Configuration.EnableTmdbUpdates) + return false; + + return base.NeedsRefreshInternal(item, providerInfo); + } + protected override bool NeedsRefreshBasedOnCompareDate(BaseItem item, BaseProviderInfo providerInfo) { var provderId = item.GetProviderId(MetadataProviders.Tmdb); @@ -187,7 +195,7 @@ namespace MediaBrowser.Providers.Movies } } - protected readonly CultureInfo UsCulture = new CultureInfo("en-US"); + private readonly CultureInfo _usCulture = new CultureInfo("en-US"); /// /// Gets the TMDB id. @@ -211,7 +219,7 @@ namespace MediaBrowser.Providers.Movies searchResult = JsonSerializer.DeserializeFromStream(json); } - return searchResult != null && searchResult.Total_Results > 0 ? searchResult.Results[0].Id.ToString(UsCulture) : null; + return searchResult != null && searchResult.Total_Results > 0 ? searchResult.Results[0].Id.ToString(_usCulture) : null; } /// @@ -311,7 +319,7 @@ namespace MediaBrowser.Providers.Movies } } - person.SetProviderId(MetadataProviders.Tmdb, searchResult.id.ToString(UsCulture)); + person.SetProviderId(MetadataProviders.Tmdb, searchResult.id.ToString(_usCulture)); } /// diff --git a/MediaBrowser.Providers/TV/RemoteEpisodeProvider.cs b/MediaBrowser.Providers/TV/RemoteEpisodeProvider.cs index be7ff192ea..e2a797a4e6 100644 --- a/MediaBrowser.Providers/TV/RemoteEpisodeProvider.cs +++ b/MediaBrowser.Providers/TV/RemoteEpisodeProvider.cs @@ -1,5 +1,4 @@ -using System.Collections.Generic; -using MediaBrowser.Common.Net; +using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.TV; @@ -9,6 +8,7 @@ using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using MediaBrowser.Model.Net; using System; +using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq;