open-source-mirrors
/
jellyfin
mirror of https://github.com/jellyfin/jellyfin
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix MissingEpisodeProvider (almost)

Browse Source
pull/776/head
Claus Vium 6 years ago
parent ced9868357
commit 0d43b06042
3 changed files with 67 additions and 163 deletions
Show Stats Download Patch File Download Diff File

193
MediaBrowser.Providers/TV/MissingEpisodeProvider.cs
Unescape View File

@ -30,63 +30,46 @@ namespace MediaBrowser.Providers.TV
private readonly IFileSystem _fileSystem; private readonly IFileSystem _fileSystem;
private readonly CultureInfo _usCulture = new CultureInfo("en-US"); private readonly CultureInfo _usCulture = new CultureInfo("en-US");
private readonly IXmlReaderSettingsFactory _xmlSettings;
public MissingEpisodeProvider(ILogger logger, IServerConfigurationManager config, ILibraryManager libraryManager, ILocalizationManager localization, IFileSystem fileSystem, IXmlReaderSettingsFactory xmlSettings) public MissingEpisodeProvider(ILogger logger, IServerConfigurationManager config, ILibraryManager libraryManager, ILocalizationManager localization, IFileSystem fileSystem)
{ {
_logger = logger; _logger = logger;
_config = config; _config = config;
_libraryManager = libraryManager; _libraryManager = libraryManager;
_localization = localization; _localization = localization;
_fileSystem = fileSystem; _fileSystem = fileSystem;
_xmlSettings = xmlSettings;
} }
public async Task<bool> Run(Series series, bool addNewItems, CancellationToken cancellationToken) public async Task<bool> Run(Series series, bool addNewItems, CancellationToken cancellationToken)
{ {
// TODO cvium fixme wtfisthisandwhydoesitrunwhenoptionisdisabled
return true;
var tvdbId = series.GetProviderId(MetadataProviders.Tvdb); var tvdbId = series.GetProviderId(MetadataProviders.Tvdb);
// Todo: Support series by imdb id // Todo: Support series by imdb id
var seriesProviderIds = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase); var seriesProviderIds = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
seriesProviderIds[MetadataProviders.Tvdb.ToString()] = tvdbId; {
[MetadataProviders.Tvdb.ToString()] = tvdbId
};
var episodeFiles = _fileSystem.GetFilePaths("") var episodes = await TvDbClientManager.Instance.GetAllEpisodesAsync(Convert.ToInt32(tvdbId), cancellationToken);
.Where(i => string.Equals(Path.GetExtension(i), ".xml", StringComparison.OrdinalIgnoreCase))
.Select(Path.GetFileNameWithoutExtension)
.Where(i => i.StartsWith("episode-", StringComparison.OrdinalIgnoreCase))
.ToList();
var episodeLookup = episodeFiles var episodeLookup = episodes
.Select(i => .Select(i =>
{ {
var parts = i.Split('-'); DateTime.TryParse(i.FirstAired, out var firstAired);
return new ValueTuple<int, int, DateTime>(
if (parts.Length == 3) i.AiredSeason.GetValueOrDefault(-1), i.AiredEpisodeNumber.GetValueOrDefault(-1), firstAired);
{
if (int.TryParse(parts[1], NumberStyles.Integer, _usCulture, out var seasonNumber))
{
if (int.TryParse(parts[2], NumberStyles.Integer, _usCulture, out var episodeNumber))
{
return new ValueTuple<int, int>(seasonNumber, episodeNumber);
}
}
}
return new ValueTuple<int, int>(-1, -1);
}) })
.Where(i => i.Item1 != -1 && i.Item2 != -1) .Where(i => i.Item2 != -1 && i.Item2 != -1)
.ToList(); .ToList();
var allRecursiveChildren = series.GetRecursiveChildren(); var allRecursiveChildren = series.GetRecursiveChildren();
var hasBadData = HasInvalidContent(series, allRecursiveChildren); var hasBadData = HasInvalidContent(allRecursiveChildren);
// Be conservative here to avoid creating missing episodes for ones they already have // Be conservative here to avoid creating missing episodes for ones they already have
var addMissingEpisodes = !hasBadData && _libraryManager.GetLibraryOptions(series).ImportMissingEpisodes; var addMissingEpisodes = !hasBadData && _libraryManager.GetLibraryOptions(series).ImportMissingEpisodes;
var anySeasonsRemoved = RemoveObsoleteOrMissingSeasons(series, allRecursiveChildren, episodeLookup); var anySeasonsRemoved = RemoveObsoleteOrMissingSeasons(allRecursiveChildren, episodeLookup);
if (anySeasonsRemoved) if (anySeasonsRemoved)
{ {
@ -94,7 +77,7 @@ namespace MediaBrowser.Providers.TV
allRecursiveChildren = series.GetRecursiveChildren(); allRecursiveChildren = series.GetRecursiveChildren();
} }
var anyEpisodesRemoved = RemoveObsoleteOrMissingEpisodes(series, allRecursiveChildren, episodeLookup, addMissingEpisodes); var anyEpisodesRemoved = RemoveObsoleteOrMissingEpisodes(allRecursiveChildren, episodeLookup, addMissingEpisodes);
if (anyEpisodesRemoved) if (anyEpisodesRemoved)
{ {
@ -106,7 +89,7 @@ namespace MediaBrowser.Providers.TV
if (addNewItems && series.IsMetadataFetcherEnabled(_libraryManager.GetLibraryOptions(series), TvdbSeriesProvider.Current.Name)) if (addNewItems && series.IsMetadataFetcherEnabled(_libraryManager.GetLibraryOptions(series), TvdbSeriesProvider.Current.Name))
{ {
hasNewEpisodes = await AddMissingEpisodes(series, allRecursiveChildren, addMissingEpisodes, "", episodeLookup, cancellationToken) hasNewEpisodes = await AddMissingEpisodes(series, allRecursiveChildren, addMissingEpisodes, episodeLookup, cancellationToken)
.ConfigureAwait(false); .ConfigureAwait(false);
} }
@ -122,7 +105,7 @@ namespace MediaBrowser.Providers.TV
/// Returns true if a series has any seasons or episodes without season or episode numbers /// Returns true if a series has any seasons or episodes without season or episode numbers
/// If this data is missing no virtual items will be added in order to prevent possible duplicates /// If this data is missing no virtual items will be added in order to prevent possible duplicates
/// </summary> /// </summary>
private bool HasInvalidContent(Series series, IList<BaseItem> allItems) private bool HasInvalidContent(IList<BaseItem> allItems)
{ {
return allItems.OfType<Season>().Any(i => !i.IndexNumber.HasValue) || return allItems.OfType<Season>().Any(i => !i.IndexNumber.HasValue) ||
allItems.OfType<Episode>().Any(i => allItems.OfType<Episode>().Any(i =>
@ -139,31 +122,25 @@ namespace MediaBrowser.Providers.TV
private const double UnairedEpisodeThresholdDays = 2; private const double UnairedEpisodeThresholdDays = 2;
/// <summary>
/// Adds the missing episodes. private async Task<bool> AddMissingEpisodes(
/// </summary> Series series,
/// <param name="series">The series.</param>
/// <returns>Task.</returns>
private async Task<bool> AddMissingEpisodes(Series series,
IList<BaseItem> allItems, IList<BaseItem> allItems,
bool addMissingEpisodes, bool addMissingEpisodes,
string seriesDataPath, List<ValueTuple<int, int, DateTime>> episodeLookup,
IEnumerable<ValueTuple<int, int>> episodeLookup,
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
var existingEpisodes = allItems.OfType<Episode>() var existingEpisodes = allItems.OfType<Episode>()
.ToList(); .ToList();
var lookup = episodeLookup as IList<ValueTuple<int, int>> ?? episodeLookup.ToList(); var seasonCounts = (from e in episodeLookup
var seasonCounts = (from e in lookup
group e by e.Item1 into g group e by e.Item1 into g
select g) select g)
.ToDictionary(g => g.Key, g => g.Count()); .ToDictionary(g => g.Key, g => g.Count());
var hasChanges = false; var hasChanges = false;
foreach (var tuple in lookup) foreach (var tuple in episodeLookup)
{ {
if (tuple.Item1 <= 0) if (tuple.Item1 <= 0)
{ {
@ -184,32 +161,14 @@ namespace MediaBrowser.Providers.TV
continue; continue;
} }
var airDate = GetAirDate(seriesDataPath, tuple.Item1, tuple.Item2); var airDate = tuple.Item3;
if (!airDate.HasValue)
{
continue;
}
var now = DateTime.UtcNow;
now = now.AddDays(0 - UnairedEpisodeThresholdDays); var now = DateTime.UtcNow.AddDays(0 - UnairedEpisodeThresholdDays);
if (airDate.Value < now) if (airDate < now && addMissingEpisodes || airDate > now)
{
if (addMissingEpisodes)
{
// tvdb has a lot of nearly blank episodes
_logger.LogInformation("Creating virtual missing episode {0} {1}x{2}", series.Name, tuple.Item1, tuple.Item2);
await AddEpisode(series, tuple.Item1, tuple.Item2, cancellationToken).ConfigureAwait(false);
hasChanges = true;
}
}
else if (airDate.Value > now)
{ {
// tvdb has a lot of nearly blank episodes // tvdb has a lot of nearly blank episodes
_logger.LogInformation("Creating virtual unaired episode {0} {1}x{2}", series.Name, tuple.Item1, tuple.Item2); _logger.LogInformation("Creating virtual missing/unaired episode {0} {1}x{2}", series.Name, tuple.Item1, tuple.Item2);
await AddEpisode(series, tuple.Item1, tuple.Item2, cancellationToken).ConfigureAwait(false); await AddEpisode(series, tuple.Item1, tuple.Item2, cancellationToken).ConfigureAwait(false);
hasChanges = true; hasChanges = true;
@ -222,9 +181,9 @@ namespace MediaBrowser.Providers.TV
/// <summary> /// <summary>
/// Removes the virtual entry after a corresponding physical version has been added /// Removes the virtual entry after a corresponding physical version has been added
/// </summary> /// </summary>
private bool RemoveObsoleteOrMissingEpisodes(Series series, private bool RemoveObsoleteOrMissingEpisodes(
IList<BaseItem> allRecursiveChildren, IList<BaseItem> allRecursiveChildren,
IEnumerable<ValueTuple<int, int>> episodeLookup, IEnumerable<ValueTuple<int, int, DateTime>> episodeLookup,
bool allowMissingEpisodes) bool allowMissingEpisodes)
{ {
var existingEpisodes = allRecursiveChildren.OfType<Episode>() var existingEpisodes = allRecursiveChildren.OfType<Episode>()
@ -295,12 +254,11 @@ namespace MediaBrowser.Providers.TV
/// <summary> /// <summary>
/// Removes the obsolete or missing seasons. /// Removes the obsolete or missing seasons.
/// </summary> /// </summary>
/// <param name="series">The series.</param> /// <param name="allRecursiveChildren"></param>
/// <param name="episodeLookup">The episode lookup.</param> /// <param name="episodeLookup">The episode lookup.</param>
/// <returns>Task{System.Boolean}.</returns> /// <returns>Task{System.Boolean}.</returns>
private bool RemoveObsoleteOrMissingSeasons(Series series, private bool RemoveObsoleteOrMissingSeasons(IList<BaseItem> allRecursiveChildren,
IList<BaseItem> allRecursiveChildren, IEnumerable<(int, int, DateTime)> episodeLookup)
IEnumerable<ValueTuple<int, int>> episodeLookup)
{ {
var existingSeasons = allRecursiveChildren.OfType<Season>().ToList(); var existingSeasons = allRecursiveChildren.OfType<Season>().ToList();
@ -380,7 +338,7 @@ namespace MediaBrowser.Providers.TV
season = await provider.AddSeason(series, seasonNumber, true, cancellationToken).ConfigureAwait(false); season = await provider.AddSeason(series, seasonNumber, true, cancellationToken).ConfigureAwait(false);
} }
var name = string.Format("Episode {0}", episodeNumber.ToString(_usCulture)); var name = $"Episode {episodeNumber.ToString(_usCulture)}";
var episode = new Episode var episode = new Episode
{ {
@ -389,7 +347,7 @@ namespace MediaBrowser.Providers.TV
ParentIndexNumber = seasonNumber, ParentIndexNumber = seasonNumber,
Id = _libraryManager.GetNewItemId((series.Id + seasonNumber.ToString(_usCulture) + name), typeof(Episode)), Id = _libraryManager.GetNewItemId((series.Id + seasonNumber.ToString(_usCulture) + name), typeof(Episode)),
IsVirtualItem = true, IsVirtualItem = true,
SeasonId = season == null ? Guid.Empty : season.Id, SeasonId = season?.Id ?? Guid.Empty,
SeriesId = series.Id SeriesId = series.Id
}; };
@ -407,7 +365,7 @@ namespace MediaBrowser.Providers.TV
/// <param name="seasonCounts"></param> /// <param name="seasonCounts"></param>
/// <param name="tuple">The tuple.</param> /// <param name="tuple">The tuple.</param>
/// <returns>Episode.</returns> /// <returns>Episode.</returns>
private Episode GetExistingEpisode(IList<Episode> existingEpisodes, Dictionary<int, int> seasonCounts, ValueTuple<int, int> tuple) private Episode GetExistingEpisode(IList<Episode> existingEpisodes, Dictionary<int, int> seasonCounts, ValueTuple<int, int, DateTime> tuple)
{ {
var s = tuple.Item1; var s = tuple.Item1;
var e = tuple.Item2; var e = tuple.Item2;
@ -434,88 +392,5 @@ namespace MediaBrowser.Providers.TV
return existingEpisodes return existingEpisodes
.FirstOrDefault(i => i.ParentIndexNumber == season && i.ContainsEpisodeNumber(episode)); .FirstOrDefault(i => i.ParentIndexNumber == season && i.ContainsEpisodeNumber(episode));
} }
/// <summary>
/// Gets the air date.
/// </summary>
/// <param name="seriesDataPath">The series data path.</param>
/// <param name="seasonNumber">The season number.</param>
/// <param name="episodeNumber">The episode number.</param>
/// <returns>System.Nullable{DateTime}.</returns>
private DateTime? GetAirDate(string seriesDataPath, int seasonNumber, int episodeNumber)
{
// First open up the tvdb xml file and make sure it has valid data
var filename = string.Format("episode-{0}-{1}.xml", seasonNumber.ToString(_usCulture), episodeNumber.ToString(_usCulture));
var xmlPath = Path.Combine(seriesDataPath, filename);
DateTime? airDate = null;
using (var fileStream = _fileSystem.GetFileStream(xmlPath, FileOpenMode.Open, FileAccessMode.Read, FileShareMode.Read))
{
// It appears the best way to filter out invalid entries is to only include those with valid air dates
using (var streamReader = new StreamReader(fileStream, Encoding.UTF8))
{
var settings = _xmlSettings.Create(false);
settings.CheckCharacters = false;
settings.IgnoreProcessingInstructions = true;
settings.IgnoreComments = true;
// Use XmlReader for best performance
using (var reader = XmlReader.Create(streamReader, settings))
{
reader.MoveToContent();
reader.Read();
// Loop through each element
while (!reader.EOF && reader.ReadState == ReadState.Interactive)
{
if (reader.NodeType == XmlNodeType.Element)
{
switch (reader.Name)
{
case "EpisodeName":
{
var val = reader.ReadElementContentAsString();
if (string.IsNullOrWhiteSpace(val))
{
// Not valid, ignore these
return null;
}
break;
}
case "FirstAired":
{
var val = reader.ReadElementContentAsString();
if (!string.IsNullOrWhiteSpace(val))
{
if (DateTime.TryParse(val, out var date))
{
airDate = date.ToUniversalTime();
}
}
break;
}
default:
{
reader.Skip();
break;
}
}
}
else
{
reader.Read();
}
}
}
}
}
return airDate;
}
} }
} }

3
MediaBrowser.Providers/TV/SeriesMetadataService.cs
Unescape View File

@ -36,8 +36,7 @@ namespace MediaBrowser.Providers.TV
ServerConfigurationManager, ServerConfigurationManager,
LibraryManager, LibraryManager,
_localization, _localization,
FileSystem, FileSystem);
_xmlSettings);
try try
{ {

34
MediaBrowser.Providers/TV/TheTVDB/TvDbClientManager.cs
Unescape View File

@ -1,4 +1,5 @@
using System; using System;
using System.Collections.Generic;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Library;
@ -87,6 +88,24 @@ namespace MediaBrowser.Providers.TV
return TryGetValue("episode" + episodeTvdbId,() => TvDbClient.Episodes.GetAsync(episodeTvdbId, cancellationToken)); return TryGetValue("episode" + episodeTvdbId,() => TvDbClient.Episodes.GetAsync(episodeTvdbId, cancellationToken));
} }
public async Task<List<EpisodeRecord>> GetAllEpisodesAsync(int tvdbId, CancellationToken cancellationToken)
{
// Traverse all episode pages and join them together
var episodes = new List<EpisodeRecord>();
var episodePage = await GetEpisodesPageAsync(tvdbId, new EpisodeQuery(), cancellationToken);
episodes.AddRange(episodePage.Data);
int next = episodePage.Links.Next.GetValueOrDefault(0);
int last = episodePage.Links.Last.GetValueOrDefault(0);
for (var page = next; page <= last; ++page)
{
episodePage = await GetEpisodesPageAsync(tvdbId, page, new EpisodeQuery(), cancellationToken);
episodes.AddRange(episodePage.Data);
}
return episodes;
}
public Task<TvDbResponse<SeriesSearchResult[]>> GetSeriesByImdbIdAsync(string imdbId, CancellationToken cancellationToken) public Task<TvDbResponse<SeriesSearchResult[]>> GetSeriesByImdbIdAsync(string imdbId, CancellationToken cancellationToken)
{ {
return TryGetValue("series" + imdbId,() => TvDbClient.Search.SearchSeriesByImdbIdAsync(imdbId, cancellationToken)); return TryGetValue("series" + imdbId,() => TvDbClient.Search.SearchSeriesByImdbIdAsync(imdbId, cancellationToken));
@ -117,10 +136,21 @@ namespace MediaBrowser.Providers.TV
() => TvDbClient.Series.GetEpisodesSummaryAsync(tvdbId, cancellationToken)); () => TvDbClient.Series.GetEpisodesSummaryAsync(tvdbId, cancellationToken));
} }
public Task<TvDbResponse<EpisodeRecord[]>> GetEpisodesPageAsync(int tvdbId, int page, EpisodeQuery episodeQuery, CancellationToken cancellationToken)
{
// Not quite as dynamic as it could be
var cacheKey = "episodespage" + tvdbId + "page" + page;
if (episodeQuery.AiredSeason.HasValue)
{
cacheKey += "airedseason" + episodeQuery.AiredSeason.Value;
}
return TryGetValue(cacheKey,
() => TvDbClient.Series.GetEpisodesAsync(tvdbId, page, episodeQuery, cancellationToken));
}
public Task<TvDbResponse<EpisodeRecord[]>> GetEpisodesPageAsync(int tvdbId, EpisodeQuery episodeQuery, CancellationToken cancellationToken) public Task<TvDbResponse<EpisodeRecord[]>> GetEpisodesPageAsync(int tvdbId, EpisodeQuery episodeQuery, CancellationToken cancellationToken)
{ {
return TryGetValue("episodespage" + tvdbId + episodeQuery.AiredSeason, return GetEpisodesPageAsync(tvdbId, 1, episodeQuery, cancellationToken);
() => TvDbClient.Series.GetEpisodesAsync(tvdbId, 1, episodeQuery, cancellationToken));
} }
private async Task<T> TryGetValue<T>(object key, Func<Task<T>> resultFactory) private async Task<T> TryGetValue<T>(object key, Func<Task<T>> resultFactory)

Write Preview
Loading…
Cancel
Save