open-source-mirrors
/
jellyfin
mirror of https://github.com/jellyfin/jellyfin
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

separate metadata refresh from validation

Browse Source
pull/702/head
Luke Pulverenti 11 years ago
parent 8b29e67e22
commit 14084fdd87
11 changed files with 195 additions and 188 deletions
Show Stats Download Patch File Download Diff File

7
MediaBrowser.Controller/Entities/Audio/MusicArtist.cs
Unescape View File

@ -1,4 +1,5 @@
using MediaBrowser.Model.Configuration;
using MediaBrowser.Controller.Providers;
using MediaBrowser.Model.Configuration;
using MediaBrowser.Model.Dto;
using MediaBrowser.Model.Entities;
using System;
@ -49,7 +50,7 @@ namespace MediaBrowser.Controller.Entities.Audio
}
private readonly Task _cachedTask = Task.FromResult(true);
protected override Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
protected override Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions)
{
if (IsAccessedByName)
{
@ -57,7 +58,7 @@ namespace MediaBrowser.Controller.Entities.Audio
return _cachedTask;
}
return base.ValidateChildrenInternal(progress, cancellationToken, recursive, forceRefreshMetadata);
return base.ValidateChildrenInternal(progress, cancellationToken, recursive, refreshChildMetadata, refreshOptions);
}
public override string GetClientTypeName()

106
MediaBrowser.Controller/Entities/BaseItem.cs
Unescape View File

@ -472,75 +472,30 @@ namespace MediaBrowser.Controller.Entities
/// <returns>List{Video}.</returns>
private IEnumerable<Trailer> LoadLocalTrailers(List<FileSystemInfo> fileSystemChildren)
{
return new List<Trailer>();
//ItemResolveArgs resolveArgs;
//try
//{
// resolveArgs = ResolveArgs;
// if (!resolveArgs.IsDirectory)
// {
// return new List<Trailer>();
// }
//}
//catch (IOException ex)
//{
// Logger.ErrorException("Error getting ResolveArgs for {0}", ex, Path);
// return new List<Trailer>();
//}
//var files = new List<FileSystemInfo>();
//var folder = resolveArgs.GetFileSystemEntryByName(TrailerFolderName);
//// Path doesn't exist. No biggie
//if (folder != null)
//{
// try
// {
// files.AddRange(new DirectoryInfo(folder.FullName).EnumerateFiles());
// }
// catch (IOException ex)
// {
// Logger.ErrorException("Error loading trailers for {0}", ex, Name);
// }
//}
//// Support xbmc trailers (-trailer suffix on video file names)
//files.AddRange(resolveArgs.FileSystemChildren.Where(i =>
//{
// try
// {
// if ((i.Attributes & FileAttributes.Directory) != FileAttributes.Directory)
// {
// if (System.IO.Path.GetFileNameWithoutExtension(i.Name).EndsWith(XbmcTrailerFileSuffix, StringComparison.OrdinalIgnoreCase) && !string.Equals(Path, i.FullName, StringComparison.OrdinalIgnoreCase))
// {
// return true;
// }
// }
// }
// catch (IOException ex)
// {
// Logger.ErrorException("Error accessing path {0}", ex, i.FullName);
// }
// return false;
//}));
//return LibraryManager.ResolvePaths<Trailer>(files, null).Select(video =>
//{
// // Try to retrieve it from the db. If we don't find it, use the resolved version
// var dbItem = LibraryManager.GetItemById(video.Id) as Trailer;
// if (dbItem != null)
// {
// video = dbItem;
// }
// return video;
//}).ToList();
var files = fileSystemChildren.OfType<DirectoryInfo>()
.Where(i => string.Equals(i.Name, TrailerFolderName, StringComparison.OrdinalIgnoreCase))
.SelectMany(i => i.EnumerateFiles("*", SearchOption.TopDirectoryOnly))
.ToList();
// Support plex/xbmc convention
files.AddRange(fileSystemChildren.OfType<FileInfo>()
.Where(i => System.IO.Path.GetFileNameWithoutExtension(i.Name).EndsWith(XbmcTrailerFileSuffix, StringComparison.OrdinalIgnoreCase) && !string.Equals(Path, i.FullName, StringComparison.OrdinalIgnoreCase))
);
return LibraryManager.ResolvePaths<Trailer>(files, null).Select(video =>
{
// Try to retrieve it from the db. If we don't find it, use the resolved version
var dbItem = LibraryManager.GetItemById(video.Id) as Trailer;
if (dbItem != null)
{
video = dbItem;
}
return video;
// Sort them so that the list can be easily compared for changes
}).OrderBy(i => i.Path).ToList();
}
/// <summary>
@ -656,19 +611,8 @@ namespace MediaBrowser.Controller.Entities
}
}
if (themeSongsChanged)
{
Logger.Debug("Theme songs have changed for {0}", Path);
options.ForceSave = true;
}
if (themeVideosChanged)
{
Logger.Debug("Theme videos have changed for {0}", Path);
options.ForceSave = true;
}
if (localTrailersChanged)
if (themeSongsChanged || themeVideosChanged || localTrailersChanged)
{
Logger.Debug("Local trailers have changed for {0}", Path);
options.ForceSave = true;
}
}

6
MediaBrowser.Controller/Entities/CollectionFolder.cs
Unescape View File

@ -7,6 +7,7 @@ using System.Linq;
using System.Runtime.Serialization;
using System.Threading;
using System.Threading.Tasks;
using MediaBrowser.Controller.Providers;
namespace MediaBrowser.Controller.Entities
{
@ -116,9 +117,10 @@ namespace MediaBrowser.Controller.Entities
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <param name="recursive">if set to <c>true</c> [recursive].</param>
/// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
/// <param name="refreshChildMetadata">if set to <c>true</c> [refresh child metadata].</param>
/// <param name="refreshOptions">The refresh options.</param>
/// <returns>Task.</returns>
protected override Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
protected override Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions)
{
CreateResolveArgs();
ResetDynamicChildren();

203
MediaBrowser.Controller/Entities/Folder.cs
Unescape View File

@ -307,7 +307,17 @@ namespace MediaBrowser.Controller.Entities
/// <param name="recursive">if set to <c>true</c> [recursive].</param>
/// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
/// <returns>Task.</returns>
public async Task ValidateChildren(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
public Task ValidateChildren(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
{
return ValidateChildrenWithCancellationSupport(progress, cancellationToken, recursive ?? true, true,
new MetadataRefreshOptions
{
ReplaceAllMetadata = forceRefreshMetadata
});
}
private async Task ValidateChildrenWithCancellationSupport(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions)
{
cancellationToken.ThrowIfCancellationRequested();
@ -327,7 +337,7 @@ namespace MediaBrowser.Controller.Entities
var linkedCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(innerCancellationTokenSource.Token, cancellationToken);
await ValidateChildrenInternal(progress, linkedCancellationTokenSource.Token, recursive, forceRefreshMetadata).ConfigureAwait(false);
await ValidateChildrenInternal(progress, linkedCancellationTokenSource.Token, recursive, refreshChildMetadata, refreshOptions).ConfigureAwait(false);
}
catch (OperationCanceledException ex)
{
@ -352,15 +362,15 @@ namespace MediaBrowser.Controller.Entities
}
/// <summary>
/// Compare our current children (presumably just read from the repo) with the current state of the file system and adjust for any changes
/// ***Currently does not contain logic to maintain items that are unavailable in the file system***
/// Validates the children internal.
/// </summary>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <param name="recursive">if set to <c>true</c> [recursive].</param>
/// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
/// <param name="refreshChildMetadata">if set to <c>true</c> [refresh child metadata].</param>
/// <param name="refreshOptions">The refresh options.</param>
/// <returns>Task.</returns>
protected async virtual Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool? recursive = null, bool forceRefreshMetadata = false)
protected async virtual Task ValidateChildrenInternal(IProgress<double> progress, CancellationToken cancellationToken, bool recursive, bool refreshChildMetadata, MetadataRefreshOptions refreshOptions)
{
var locationType = LocationType;
@ -401,33 +411,23 @@ namespace MediaBrowser.Controller.Entities
if (currentChildren.TryGetValue(child.Id, out currentChild))
{
//existing item - check if it has changed
if (currentChild.HasChanged(child))
var currentChildLocationType = currentChild.LocationType;
if (currentChildLocationType != LocationType.Remote &&
currentChildLocationType != LocationType.Virtual)
{
var currentChildLocationType = currentChild.LocationType;
if (currentChildLocationType != LocationType.Remote &&
currentChildLocationType != LocationType.Virtual)
{
currentChild.DateModified = child.DateModified;
}
currentChild.IsInMixedFolder = child.IsInMixedFolder;
validChildren.Add(currentChild);
}
else
{
validChildren.Add(currentChild);
currentChild.DateModified = child.DateModified;
}
currentChild.IsInMixedFolder = child.IsInMixedFolder;
currentChild.IsOffline = false;
}
else
{
//brand new item - needs to be added
newItems.Add(child);
validChildren.Add(child);
}
validChildren.Add(currentChild);
}
// If any items were added or removed....
@ -435,7 +435,6 @@ namespace MediaBrowser.Controller.Entities
{
// That's all the new and changed ones - now see if there are any that are missing
var itemsRemoved = currentChildren.Values.Except(validChildren).ToList();
var actualRemovals = new List<BaseItem>();
foreach (var item in itemsRemoved)
@ -450,7 +449,6 @@ namespace MediaBrowser.Controller.Entities
else if (!string.IsNullOrEmpty(item.Path) && IsPathOffline(item.Path))
{
item.IsOffline = true;
validChildren.Add(item);
}
else
@ -486,78 +484,135 @@ namespace MediaBrowser.Controller.Entities
cancellationToken.ThrowIfCancellationRequested();
await RefreshChildren(validChildren, progress, cancellationToken, recursive, forceRefreshMetadata).ConfigureAwait(false);
if (recursive)
{
await ValidateSubFolders(validChildren.OfType<Folder>().ToList(), progress, cancellationToken).ConfigureAwait(false);
}
progress.Report(20);
if (refreshChildMetadata)
{
var container = this as IMetadataContainer;
var innerProgress = new ActionableProgress<double>();
innerProgress.RegisterAction(p => progress.Report((.80 * p) + 20));
if (container != null)
{
await container.RefreshAllMetadata(refreshOptions, innerProgress, cancellationToken).ConfigureAwait(false);
}
else
{
await RefreshMetadataRecursive(refreshOptions, recursive, innerProgress, cancellationToken);
}
}
progress.Report(100);
}
/// <summary>
/// Refreshes the children.
/// </summary>
/// <param name="children">The children.</param>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <param name="recursive">if set to <c>true</c> [recursive].</param>
/// <param name="forceRefreshMetadata">if set to <c>true</c> [force refresh metadata].</param>
/// <returns>Task.</returns>
private async Task RefreshChildren(IList<BaseItem> children, IProgress<double> progress, CancellationToken cancellationToken, bool? recursive, bool forceRefreshMetadata = false)
private async Task RefreshMetadataRecursive(MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
{
var list = children;
var children = ActualChildren.ToList();
var percentages = new Dictionary<Guid, double>(list.Count);
var percentages = new Dictionary<Guid, double>(children.Count);
var tasks = new List<Task>();
foreach (var tuple in list)
foreach (var child in children)
{
if (tasks.Count > 10)
if (tasks.Count > 3)
{
await Task.WhenAll(tasks).ConfigureAwait(false);
tasks.Clear();
}
tasks.Add(RefreshChild(tuple, progress, percentages, list.Count, cancellationToken, recursive, forceRefreshMetadata));
}
cancellationToken.ThrowIfCancellationRequested();
var innerProgress = new ActionableProgress<double>();
// Avoid implicitly captured closure
var currentChild = child;
innerProgress.RegisterAction(p =>
{
lock (percentages)
{
percentages[currentChild.Id] = p / 100;
cancellationToken.ThrowIfCancellationRequested();
var percent = percentages.Values.Sum();
percent /= children.Count;
percent *= 100;
progress.Report(percent);
}
});
if (child.IsFolder)
{
await RefreshChildMetadata(child, refreshOptions, recursive, innerProgress, cancellationToken)
.ConfigureAwait(false);
}
else
{
tasks.Add(RefreshChildMetadata(child, refreshOptions, recursive, innerProgress, cancellationToken));
}
}
await Task.WhenAll(tasks).ConfigureAwait(false);
progress.Report(100);
}
private async Task RefreshChild(BaseItem item, IProgress<double> progress, Dictionary<Guid, double> percentages, int childCount, CancellationToken cancellationToken, bool? recursive, bool forceRefreshMetadata = false)
private async Task RefreshChildMetadata(BaseItem child, MetadataRefreshOptions refreshOptions, bool recursive, IProgress<double> progress, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var container = child as IMetadataContainer;
var child = item;
try
if (container != null)
{
//refresh it
await child.RefreshMetadata(new MetadataRefreshOptions
{
ReplaceAllMetadata = forceRefreshMetadata
}, cancellationToken).ConfigureAwait(false);
await container.RefreshAllMetadata(refreshOptions, progress, cancellationToken).ConfigureAwait(false);
}
catch (IOException ex)
else
{
Logger.ErrorException("Error refreshing {0}", ex, child.Path ?? child.Name);
await child.RefreshMetadata(refreshOptions, cancellationToken).ConfigureAwait(false);
if (recursive)
{
var folder = child as Folder;
if (folder != null)
{
await folder.RefreshMetadataRecursive(refreshOptions, true, progress, cancellationToken);
}
}
}
progress.Report(100);
}
/// <summary>
/// Refreshes the children.
/// </summary>
/// <param name="children">The children.</param>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
private async Task ValidateSubFolders(IList<Folder> children, IProgress<double> progress, CancellationToken cancellationToken)
{
var list = children;
var childCount = list.Count;
// Refresh children if a folder and the item changed or recursive is set to true
var refreshChildren = child.IsFolder;
var percentages = new Dictionary<Guid, double>(list.Count);
var tasks = new List<Task>();
if (refreshChildren)
foreach (var item in list)
{
// Don't refresh children if explicitly set to false
if (recursive.HasValue && recursive.Value == false)
if (tasks.Count > 10)
{
refreshChildren = false;
await Task.WhenAll(tasks).ConfigureAwait(false);
}
}
if (refreshChildren)
{
cancellationToken.ThrowIfCancellationRequested();
var child = item;
var innerProgress = new ActionableProgress<double>();
innerProgress.RegisterAction(p =>
@ -569,24 +624,16 @@ namespace MediaBrowser.Controller.Entities
var percent = percentages.Values.Sum();
percent /= childCount;
progress.Report((90 * percent) + 10);
progress.Report((10 * percent) + 10);
}
});
await ((Folder)child).ValidateChildren(innerProgress, cancellationToken, recursive, forceRefreshMetadata).ConfigureAwait(false);
tasks.Add(child.ValidateChildrenWithCancellationSupport(innerProgress, cancellationToken, true, false, null));
}
else
{
lock (percentages)
{
percentages[child.Id] = 1;
var percent = percentages.Values.Sum();
percent /= childCount;
cancellationToken.ThrowIfCancellationRequested();
progress.Report((90 * percent) + 10);
}
}
await Task.WhenAll(tasks).ConfigureAwait(false);
}
/// <summary>
@ -962,11 +1009,11 @@ namespace MediaBrowser.Controller.Entities
/// <returns>Task.</returns>
public override async Task ChangedExternally()
{
await base.ChangedExternally().ConfigureAwait(false);
var progress = new Progress<double>();
await ValidateChildren(progress, CancellationToken.None).ConfigureAwait(false);
await base.ChangedExternally().ConfigureAwait(false);
}
/// <summary>

19
MediaBrowser.Controller/Entities/IMetadataContainer.cs
Unescape View File

@ -0,0 +1,19 @@
using MediaBrowser.Controller.Providers;
using System;
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Controller.Entities
{
public interface IMetadataContainer
{
/// <summary>
/// Refreshes all metadata.
/// </summary>
/// <param name="refreshOptions">The refresh options.</param>
/// <param name="progress">The progress.</param>
/// <param name="cancellationToken">The cancellation token.</param>
/// <returns>Task.</returns>
Task RefreshAllMetadata(MetadataRefreshOptions refreshOptions, IProgress<double> progress, CancellationToken cancellationToken);
}
}

5
MediaBrowser.Controller/Entities/User.cs
Unescape View File

@ -215,8 +215,9 @@ namespace MediaBrowser.Controller.Entities
return RefreshMetadata(new MetadataRefreshOptions
{
ForceSave = true,
ReplaceAllMetadata = true
ReplaceAllMetadata = true,
ImageRefreshMode = ImageRefreshMode.FullRefresh,
MetadataRefreshMode = MetadataRefreshMode.FullRefresh
}, CancellationToken.None);
}

1
MediaBrowser.Controller/MediaBrowser.Controller.csproj
Unescape View File

@ -103,6 +103,7 @@
<Compile Include="Entities\IItemByName.cs" />
<Compile Include="Entities\ILibraryItem.cs" />
<Compile Include="Entities\ImageSourceInfo.cs" />
<Compile Include="Entities\IMetadataContainer.cs" />
<Compile Include="Entities\LinkedChild.cs" />
<Compile Include="Entities\MusicVideo.cs" />
<Compile Include="Entities\IHasAwards.cs" />

5
MediaBrowser.Providers/Manager/MetadataService.cs
Unescape View File

@ -197,6 +197,11 @@ namespace MediaBrowser.Providers.Manager
.Where(i => i.HasChanged(currentItem, currentItem.DateLastSaved))
.ToList();
if (providersWithChanges.Count > 0)
{
var b = true;
}
// If local providers are the only ones with changes, then just run those
if (providersWithChanges.All(i => i is ILocalMetadataProvider))
{

5
MediaBrowser.Providers/TV/SeriesMetadataService.cs
Unescape View File

@ -51,12 +51,15 @@ namespace MediaBrowser.Providers.TV
var dateLastEpisodeAdded = item.DateLastEpisodeAdded;
item.DateLastEpisodeAdded = episodes.Select(i => i.DateCreated)
item.DateLastEpisodeAdded = episodes
.Where(i => i.LocationType != LocationType.Virtual)
.Select(i => i.DateCreated)
.OrderByDescending(i => i)
.FirstOrDefault();
if (dateLastEpisodeAdded != item.DateLastEpisodeAdded)
{
Logger.Debug("DateLastEpisodeAdded changed for {0}", item.Path);
updateType = updateType | ItemUpdateType.MetadataImport;
}

25
MediaBrowser.Providers/TV/TvdbSeriesProvider.cs
Unescape View File

@ -1088,30 +1088,15 @@ namespace MediaBrowser.Providers.TV
{
var seriesDataPath = GetSeriesDataPath(_config.ApplicationPaths, seriesId);
try
{
var files = new DirectoryInfo(seriesDataPath).EnumerateFiles("*.xml", SearchOption.TopDirectoryOnly)
.ToList();
var seriesXmlFilename = item.GetPreferredMetadataLanguage() + ".xml";
var seriesXmlFilename = item.GetPreferredMetadataLanguage() + ".xml";
var seriesFile = files.FirstOrDefault(i => string.Equals(seriesXmlFilename, i.Name, StringComparison.OrdinalIgnoreCase));
if (seriesFile != null && seriesFile.Exists && _fileSystem.GetLastWriteTimeUtc(seriesFile) > date)
{
return true;
}
var filePath = Path.Combine(seriesDataPath, seriesXmlFilename);
var actorsXml = files.FirstOrDefault(i => string.Equals("actors.xml", i.Name, StringComparison.OrdinalIgnoreCase));
var seriesFile = new FileInfo(filePath);
if (actorsXml != null && actorsXml.Exists && _fileSystem.GetLastWriteTimeUtc(actorsXml) > date)
{
return true;
}
}
catch (DirectoryNotFoundException)
if (seriesFile.Exists && _fileSystem.GetLastWriteTimeUtc(seriesFile) > date)
{
// Don't blow up
return true;
}
}

1
MediaBrowser.Server.Implementations/Library/LibraryManager.cs
Unescape View File

@ -1110,7 +1110,6 @@ namespace MediaBrowser.Server.Implementations.Library
cancellationToken.ThrowIfCancellationRequested();
await userRootFolder.ValidateChildren(new Progress<double>(), cancellationToken, recursive: false).ConfigureAwait(false);
var b = true;
}
/// <summary>

Write Preview
Loading…
Cancel
Save