From 0d43b06042fb1359f84afc3bd17429eb5b445b65 Mon Sep 17 00:00:00 2001 From: Claus Vium Date: Thu, 7 Feb 2019 23:56:57 +0100 Subject: [PATCH] Fix MissingEpisodeProvider (almost) --- .../TV/MissingEpisodeProvider.cs | 193 +++--------------- .../TV/SeriesMetadataService.cs | 3 +- .../TV/TheTVDB/TvDbClientManager.cs | 34 ++- 3 files changed, 67 insertions(+), 163 deletions(-) diff --git a/MediaBrowser.Providers/TV/MissingEpisodeProvider.cs b/MediaBrowser.Providers/TV/MissingEpisodeProvider.cs index 70765718c9..f2ab0d42d0 100644 --- a/MediaBrowser.Providers/TV/MissingEpisodeProvider.cs +++ b/MediaBrowser.Providers/TV/MissingEpisodeProvider.cs @@ -30,63 +30,46 @@ namespace MediaBrowser.Providers.TV private readonly IFileSystem _fileSystem; private readonly CultureInfo _usCulture = new CultureInfo("en-US"); - private readonly IXmlReaderSettingsFactory _xmlSettings; - public MissingEpisodeProvider(ILogger logger, IServerConfigurationManager config, ILibraryManager libraryManager, ILocalizationManager localization, IFileSystem fileSystem, IXmlReaderSettingsFactory xmlSettings) + public MissingEpisodeProvider(ILogger logger, IServerConfigurationManager config, ILibraryManager libraryManager, ILocalizationManager localization, IFileSystem fileSystem) { _logger = logger; _config = config; _libraryManager = libraryManager; _localization = localization; _fileSystem = fileSystem; - _xmlSettings = xmlSettings; } public async Task Run(Series series, bool addNewItems, CancellationToken cancellationToken) { - // TODO cvium fixme wtfisthisandwhydoesitrunwhenoptionisdisabled - return true; var tvdbId = series.GetProviderId(MetadataProviders.Tvdb); // Todo: Support series by imdb id - var seriesProviderIds = new Dictionary(StringComparer.OrdinalIgnoreCase); - seriesProviderIds[MetadataProviders.Tvdb.ToString()] = tvdbId; + var seriesProviderIds = new Dictionary(StringComparer.OrdinalIgnoreCase) + { + [MetadataProviders.Tvdb.ToString()] = tvdbId + }; - var episodeFiles = _fileSystem.GetFilePaths("") - .Where(i => string.Equals(Path.GetExtension(i), ".xml", StringComparison.OrdinalIgnoreCase)) - .Select(Path.GetFileNameWithoutExtension) - .Where(i => i.StartsWith("episode-", StringComparison.OrdinalIgnoreCase)) - .ToList(); + var episodes = await TvDbClientManager.Instance.GetAllEpisodesAsync(Convert.ToInt32(tvdbId), cancellationToken); - var episodeLookup = episodeFiles + var episodeLookup = episodes .Select(i => { - var parts = i.Split('-'); - - if (parts.Length == 3) - { - if (int.TryParse(parts[1], NumberStyles.Integer, _usCulture, out var seasonNumber)) - { - if (int.TryParse(parts[2], NumberStyles.Integer, _usCulture, out var episodeNumber)) - { - return new ValueTuple(seasonNumber, episodeNumber); - } - } - } - - return new ValueTuple(-1, -1); + DateTime.TryParse(i.FirstAired, out var firstAired); + return new ValueTuple( + i.AiredSeason.GetValueOrDefault(-1), i.AiredEpisodeNumber.GetValueOrDefault(-1), firstAired); }) - .Where(i => i.Item1 != -1 && i.Item2 != -1) + .Where(i => i.Item2 != -1 && i.Item2 != -1) .ToList(); var allRecursiveChildren = series.GetRecursiveChildren(); - var hasBadData = HasInvalidContent(series, allRecursiveChildren); + var hasBadData = HasInvalidContent(allRecursiveChildren); // Be conservative here to avoid creating missing episodes for ones they already have var addMissingEpisodes = !hasBadData && _libraryManager.GetLibraryOptions(series).ImportMissingEpisodes; - var anySeasonsRemoved = RemoveObsoleteOrMissingSeasons(series, allRecursiveChildren, episodeLookup); + var anySeasonsRemoved = RemoveObsoleteOrMissingSeasons(allRecursiveChildren, episodeLookup); if (anySeasonsRemoved) { @@ -94,7 +77,7 @@ namespace MediaBrowser.Providers.TV allRecursiveChildren = series.GetRecursiveChildren(); } - var anyEpisodesRemoved = RemoveObsoleteOrMissingEpisodes(series, allRecursiveChildren, episodeLookup, addMissingEpisodes); + var anyEpisodesRemoved = RemoveObsoleteOrMissingEpisodes(allRecursiveChildren, episodeLookup, addMissingEpisodes); if (anyEpisodesRemoved) { @@ -106,7 +89,7 @@ namespace MediaBrowser.Providers.TV if (addNewItems && series.IsMetadataFetcherEnabled(_libraryManager.GetLibraryOptions(series), TvdbSeriesProvider.Current.Name)) { - hasNewEpisodes = await AddMissingEpisodes(series, allRecursiveChildren, addMissingEpisodes, "", episodeLookup, cancellationToken) + hasNewEpisodes = await AddMissingEpisodes(series, allRecursiveChildren, addMissingEpisodes, episodeLookup, cancellationToken) .ConfigureAwait(false); } @@ -122,7 +105,7 @@ namespace MediaBrowser.Providers.TV /// Returns true if a series has any seasons or episodes without season or episode numbers /// If this data is missing no virtual items will be added in order to prevent possible duplicates /// - private bool HasInvalidContent(Series series, IList allItems) + private bool HasInvalidContent(IList allItems) { return allItems.OfType().Any(i => !i.IndexNumber.HasValue) || allItems.OfType().Any(i => @@ -139,31 +122,25 @@ namespace MediaBrowser.Providers.TV private const double UnairedEpisodeThresholdDays = 2; - /// - /// Adds the missing episodes. - /// - /// The series. - /// Task. - private async Task AddMissingEpisodes(Series series, + + private async Task AddMissingEpisodes( + Series series, IList allItems, bool addMissingEpisodes, - string seriesDataPath, - IEnumerable> episodeLookup, + List> episodeLookup, CancellationToken cancellationToken) { var existingEpisodes = allItems.OfType() .ToList(); - var lookup = episodeLookup as IList> ?? episodeLookup.ToList(); - - var seasonCounts = (from e in lookup + var seasonCounts = (from e in episodeLookup group e by e.Item1 into g select g) .ToDictionary(g => g.Key, g => g.Count()); var hasChanges = false; - foreach (var tuple in lookup) + foreach (var tuple in episodeLookup) { if (tuple.Item1 <= 0) { @@ -184,32 +161,14 @@ namespace MediaBrowser.Providers.TV continue; } - var airDate = GetAirDate(seriesDataPath, tuple.Item1, tuple.Item2); - - if (!airDate.HasValue) - { - continue; - } - - var now = DateTime.UtcNow; + var airDate = tuple.Item3; - now = now.AddDays(0 - UnairedEpisodeThresholdDays); + var now = DateTime.UtcNow.AddDays(0 - UnairedEpisodeThresholdDays); - if (airDate.Value < now) - { - if (addMissingEpisodes) - { - // tvdb has a lot of nearly blank episodes - _logger.LogInformation("Creating virtual missing episode {0} {1}x{2}", series.Name, tuple.Item1, tuple.Item2); - await AddEpisode(series, tuple.Item1, tuple.Item2, cancellationToken).ConfigureAwait(false); - - hasChanges = true; - } - } - else if (airDate.Value > now) + if (airDate < now && addMissingEpisodes || airDate > now) { // tvdb has a lot of nearly blank episodes - _logger.LogInformation("Creating virtual unaired episode {0} {1}x{2}", series.Name, tuple.Item1, tuple.Item2); + _logger.LogInformation("Creating virtual missing/unaired episode {0} {1}x{2}", series.Name, tuple.Item1, tuple.Item2); await AddEpisode(series, tuple.Item1, tuple.Item2, cancellationToken).ConfigureAwait(false); hasChanges = true; @@ -222,9 +181,9 @@ namespace MediaBrowser.Providers.TV /// /// Removes the virtual entry after a corresponding physical version has been added /// - private bool RemoveObsoleteOrMissingEpisodes(Series series, + private bool RemoveObsoleteOrMissingEpisodes( IList allRecursiveChildren, - IEnumerable> episodeLookup, + IEnumerable> episodeLookup, bool allowMissingEpisodes) { var existingEpisodes = allRecursiveChildren.OfType() @@ -295,12 +254,11 @@ namespace MediaBrowser.Providers.TV /// /// Removes the obsolete or missing seasons. /// - /// The series. + /// /// The episode lookup. /// Task{System.Boolean}. - private bool RemoveObsoleteOrMissingSeasons(Series series, - IList allRecursiveChildren, - IEnumerable> episodeLookup) + private bool RemoveObsoleteOrMissingSeasons(IList allRecursiveChildren, + IEnumerable<(int, int, DateTime)> episodeLookup) { var existingSeasons = allRecursiveChildren.OfType().ToList(); @@ -380,7 +338,7 @@ namespace MediaBrowser.Providers.TV season = await provider.AddSeason(series, seasonNumber, true, cancellationToken).ConfigureAwait(false); } - var name = string.Format("Episode {0}", episodeNumber.ToString(_usCulture)); + var name = $"Episode {episodeNumber.ToString(_usCulture)}"; var episode = new Episode { @@ -389,7 +347,7 @@ namespace MediaBrowser.Providers.TV ParentIndexNumber = seasonNumber, Id = _libraryManager.GetNewItemId((series.Id + seasonNumber.ToString(_usCulture) + name), typeof(Episode)), IsVirtualItem = true, - SeasonId = season == null ? Guid.Empty : season.Id, + SeasonId = season?.Id ?? Guid.Empty, SeriesId = series.Id }; @@ -407,7 +365,7 @@ namespace MediaBrowser.Providers.TV /// /// The tuple. /// Episode. - private Episode GetExistingEpisode(IList existingEpisodes, Dictionary seasonCounts, ValueTuple tuple) + private Episode GetExistingEpisode(IList existingEpisodes, Dictionary seasonCounts, ValueTuple tuple) { var s = tuple.Item1; var e = tuple.Item2; @@ -434,88 +392,5 @@ namespace MediaBrowser.Providers.TV return existingEpisodes .FirstOrDefault(i => i.ParentIndexNumber == season && i.ContainsEpisodeNumber(episode)); } - - /// - /// Gets the air date. - /// - /// The series data path. - /// The season number. - /// The episode number. - /// System.Nullable{DateTime}. - private DateTime? GetAirDate(string seriesDataPath, int seasonNumber, int episodeNumber) - { - // First open up the tvdb xml file and make sure it has valid data - var filename = string.Format("episode-{0}-{1}.xml", seasonNumber.ToString(_usCulture), episodeNumber.ToString(_usCulture)); - - var xmlPath = Path.Combine(seriesDataPath, filename); - - DateTime? airDate = null; - - using (var fileStream = _fileSystem.GetFileStream(xmlPath, FileOpenMode.Open, FileAccessMode.Read, FileShareMode.Read)) - { - // It appears the best way to filter out invalid entries is to only include those with valid air dates - using (var streamReader = new StreamReader(fileStream, Encoding.UTF8)) - { - var settings = _xmlSettings.Create(false); - - settings.CheckCharacters = false; - settings.IgnoreProcessingInstructions = true; - settings.IgnoreComments = true; - - // Use XmlReader for best performance - using (var reader = XmlReader.Create(streamReader, settings)) - { - reader.MoveToContent(); - reader.Read(); - - // Loop through each element - while (!reader.EOF && reader.ReadState == ReadState.Interactive) - { - if (reader.NodeType == XmlNodeType.Element) - { - switch (reader.Name) - { - case "EpisodeName": - { - var val = reader.ReadElementContentAsString(); - if (string.IsNullOrWhiteSpace(val)) - { - // Not valid, ignore these - return null; - } - break; - } - case "FirstAired": - { - var val = reader.ReadElementContentAsString(); - - if (!string.IsNullOrWhiteSpace(val)) - { - if (DateTime.TryParse(val, out var date)) - { - airDate = date.ToUniversalTime(); - } - } - - break; - } - default: - { - reader.Skip(); - break; - } - } - } - else - { - reader.Read(); - } - } - } - } - } - - return airDate; - } } } diff --git a/MediaBrowser.Providers/TV/SeriesMetadataService.cs b/MediaBrowser.Providers/TV/SeriesMetadataService.cs index 5f4f39d45c..b0abd7c7cb 100644 --- a/MediaBrowser.Providers/TV/SeriesMetadataService.cs +++ b/MediaBrowser.Providers/TV/SeriesMetadataService.cs @@ -36,8 +36,7 @@ namespace MediaBrowser.Providers.TV ServerConfigurationManager, LibraryManager, _localization, - FileSystem, - _xmlSettings); + FileSystem); try { diff --git a/MediaBrowser.Providers/TV/TheTVDB/TvDbClientManager.cs b/MediaBrowser.Providers/TV/TheTVDB/TvDbClientManager.cs index 7a83cfa186..c02661e374 100644 --- a/MediaBrowser.Providers/TV/TheTVDB/TvDbClientManager.cs +++ b/MediaBrowser.Providers/TV/TheTVDB/TvDbClientManager.cs @@ -1,4 +1,5 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Controller.Library; @@ -87,6 +88,24 @@ namespace MediaBrowser.Providers.TV return TryGetValue("episode" + episodeTvdbId,() => TvDbClient.Episodes.GetAsync(episodeTvdbId, cancellationToken)); } + public async Task> GetAllEpisodesAsync(int tvdbId, CancellationToken cancellationToken) + { + // Traverse all episode pages and join them together + var episodes = new List(); + var episodePage = await GetEpisodesPageAsync(tvdbId, new EpisodeQuery(), cancellationToken); + episodes.AddRange(episodePage.Data); + int next = episodePage.Links.Next.GetValueOrDefault(0); + int last = episodePage.Links.Last.GetValueOrDefault(0); + + for (var page = next; page <= last; ++page) + { + episodePage = await GetEpisodesPageAsync(tvdbId, page, new EpisodeQuery(), cancellationToken); + episodes.AddRange(episodePage.Data); + } + + return episodes; + } + public Task> GetSeriesByImdbIdAsync(string imdbId, CancellationToken cancellationToken) { return TryGetValue("series" + imdbId,() => TvDbClient.Search.SearchSeriesByImdbIdAsync(imdbId, cancellationToken)); @@ -117,10 +136,21 @@ namespace MediaBrowser.Providers.TV () => TvDbClient.Series.GetEpisodesSummaryAsync(tvdbId, cancellationToken)); } + public Task> GetEpisodesPageAsync(int tvdbId, int page, EpisodeQuery episodeQuery, CancellationToken cancellationToken) + { + // Not quite as dynamic as it could be + var cacheKey = "episodespage" + tvdbId + "page" + page; + if (episodeQuery.AiredSeason.HasValue) + { + cacheKey += "airedseason" + episodeQuery.AiredSeason.Value; + } + return TryGetValue(cacheKey, + () => TvDbClient.Series.GetEpisodesAsync(tvdbId, page, episodeQuery, cancellationToken)); + } + public Task> GetEpisodesPageAsync(int tvdbId, EpisodeQuery episodeQuery, CancellationToken cancellationToken) { - return TryGetValue("episodespage" + tvdbId + episodeQuery.AiredSeason, - () => TvDbClient.Series.GetEpisodesAsync(tvdbId, 1, episodeQuery, cancellationToken)); + return GetEpisodesPageAsync(tvdbId, 1, episodeQuery, cancellationToken); } private async Task TryGetValue(object key, Func> resultFactory)