using MediaBrowser.Common.Extensions; using MediaBrowser.Common.IO; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Logging; using System; using System.Collections.Generic; using System.Linq; using System.Threading; using System.Threading.Tasks; namespace MediaBrowser.Providers.Manager { public abstract class MetadataService : IMetadataService where TItemType : IHasMetadata, IHasLookupInfo, new() where TIdType : ItemLookupInfo, new() { protected readonly IServerConfigurationManager ServerConfigurationManager; protected readonly ILogger Logger; protected readonly IProviderManager ProviderManager; protected readonly IProviderRepository ProviderRepo; protected readonly IFileSystem FileSystem; protected MetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, IProviderRepository providerRepo, IFileSystem fileSystem) { ServerConfigurationManager = serverConfigurationManager; Logger = logger; ProviderManager = providerManager; ProviderRepo = providerRepo; FileSystem = fileSystem; } /// /// Saves the provider result. /// /// The item. /// The result. /// Task. protected Task SaveProviderResult(TItemType item, MetadataStatus result) { result.ItemId = item.Id; result.ItemName = item.Name; result.ItemType = item.GetType().Name; var series = item as IHasSeries; result.SeriesName = series == null ? null : series.SeriesName; return ProviderRepo.SaveMetadataStatus(result, CancellationToken.None); } /// /// Gets the last result. /// /// The item. /// ProviderResult. protected MetadataStatus GetLastResult(IHasMetadata item) { if (item.DateLastSaved == default(DateTime)) { return new MetadataStatus { ItemId = item.Id }; } return ProviderRepo.GetMetadataStatus(item.Id) ?? new MetadataStatus { ItemId = item.Id }; } public async Task RefreshMetadata(IHasMetadata item, MetadataRefreshOptions refreshOptions, CancellationToken cancellationToken) { if (refreshOptions.DirectoryService == null) { refreshOptions.DirectoryService = new DirectoryService(Logger); } var itemOfType = (TItemType)item; var config = ProviderManager.GetMetadataOptions(item); var updateType = ItemUpdateType.None; var refreshResult = GetLastResult(item); refreshResult.LastErrorMessage = string.Empty; refreshResult.LastStatus = ProviderRefreshStatus.Success; var itemImageProvider = new ItemImageProvider(Logger, ProviderManager, ServerConfigurationManager, FileSystem); var localImagesFailed = false; var allImageProviders = ((ProviderManager)ProviderManager).GetImageProviders(item).ToList(); // Start by validating images try { // Always validate images and check for new locally stored ones. if (itemImageProvider.ValidateImages(item, allImageProviders.OfType(), refreshOptions.DirectoryService)) { updateType = updateType | ItemUpdateType.ImageUpdate; } } catch (Exception ex) { localImagesFailed = true; Logger.ErrorException("Error validating images for {0}", ex, item.Path ?? item.Name); refreshResult.AddStatus(ProviderRefreshStatus.Failure, ex.Message); } // Next run metadata providers if (refreshOptions.MetadataRefreshMode != MetadataRefreshMode.None) { var providers = GetProviders(item, refreshResult.DateLastMetadataRefresh.HasValue, refreshOptions) .ToList(); if (providers.Count > 0 || !refreshResult.DateLastMetadataRefresh.HasValue) { if (item.BeforeMetadataRefresh()) { updateType = updateType | ItemUpdateType.MetadataImport; } } if (providers.Count > 0) { var id = await CreateInitialLookupInfo(itemOfType, cancellationToken).ConfigureAwait(false); var result = await RefreshWithProviders(itemOfType, id, refreshOptions, providers, itemImageProvider, cancellationToken).ConfigureAwait(false); updateType = updateType | result.UpdateType; refreshResult.AddStatus(result.Status, result.ErrorMessage); refreshResult.SetDateLastMetadataRefresh(DateTime.UtcNow); refreshResult.AddImageProvidersRefreshed(result.Providers); MergeIdentities(itemOfType, id); } } // Next run remote image providers, but only if local image providers didn't throw an exception if (!localImagesFailed && refreshOptions.ImageRefreshMode != ImageRefreshMode.ValidationOnly) { var providers = GetNonLocalImageProviders(item, allImageProviders, refreshResult.DateLastImagesRefresh, refreshOptions).ToList(); if (providers.Count > 0) { var result = await itemImageProvider.RefreshImages(itemOfType, providers, refreshOptions, config, cancellationToken).ConfigureAwait(false); updateType = updateType | result.UpdateType; refreshResult.AddStatus(result.Status, result.ErrorMessage); refreshResult.SetDateLastImagesRefresh(DateTime.UtcNow); refreshResult.AddImageProvidersRefreshed(result.Providers); } } updateType = updateType | BeforeSave(itemOfType); var providersHadChanges = updateType > ItemUpdateType.None; // Save if changes were made, or it's never been saved before if (refreshOptions.ForceSave || providersHadChanges || item.DateLastSaved == default(DateTime)) { // Save to database await SaveItem(itemOfType, updateType, cancellationToken); } if (providersHadChanges || refreshResult.IsDirty) { await SaveProviderResult(itemOfType, refreshResult).ConfigureAwait(false); } } private void MergeIdentities(TItemType item, TIdType id) { var hasIdentity = id as IHasIdentities; if (hasIdentity != null) { item.Identities = hasIdentity.Identities.ToList(); } } /// /// Befores the save. /// /// The item. /// ItemUpdateType. protected virtual ItemUpdateType BeforeSave(TItemType item) { return ItemUpdateType.None; } /// /// Gets the providers. /// /// The item. /// if set to true [has refreshed metadata]. /// The options. /// IEnumerable{`0}. protected virtual IEnumerable GetProviders(IHasMetadata item, bool hasRefreshedMetadata, MetadataRefreshOptions options) { // Get providers to refresh var providers = ((ProviderManager)ProviderManager).GetMetadataProviders(item).ToList(); // Run all if either of these flags are true var runAllProviders = options.ReplaceAllMetadata || options.MetadataRefreshMode == MetadataRefreshMode.FullRefresh || !hasRefreshedMetadata; if (!runAllProviders) { // Avoid implicitly captured closure var currentItem = item; var providersWithChanges = providers.OfType() .Where(i => HasChanged(currentItem, i, currentItem.DateLastSaved, options.DirectoryService)) .Cast>() .ToList(); if (providersWithChanges.Count == 0) { providers = new List>(); } else { providers = providers.Where(i => { // If any provider reports a change, always run local ones as well if (i is ILocalMetadataProvider) { return true; } var anyRemoteProvidersChanged = providersWithChanges.OfType() .Any(); // If any remote providers changed, run them all so that priorities can be honored if (i is IRemoteMetadataProvider) { return anyRemoteProvidersChanged; } // Run custom providers if they report a change or any remote providers change return anyRemoteProvidersChanged || providersWithChanges.Contains(i); }).ToList(); } } return providers; } protected virtual IEnumerable GetNonLocalImageProviders(IHasMetadata item, IEnumerable allImageProviders, DateTime? dateLastImageRefresh, ImageRefreshOptions options) { // Get providers to refresh var providers = allImageProviders.Where(i => !(i is ILocalImageProvider)).ToList(); // Run all if either of these flags are true var runAllProviders = options.ImageRefreshMode == ImageRefreshMode.FullRefresh || !dateLastImageRefresh.HasValue; if (!runAllProviders) { providers = providers.OfType() .Where(i => HasChanged(item, i, dateLastImageRefresh.Value, options.DirectoryService)) .Cast() .ToList(); } return providers; } protected Task SaveItem(TItemType item, ItemUpdateType reason, CancellationToken cancellationToken) { return item.UpdateToRepository(reason, cancellationToken); } public bool CanRefresh(IHasMetadata item) { return item is TItemType; } protected virtual async Task RefreshWithProviders(TItemType item, TIdType id, MetadataRefreshOptions options, List providers, ItemImageProvider imageService, CancellationToken cancellationToken) { var refreshResult = new RefreshResult { UpdateType = ItemUpdateType.None, Providers = providers.Select(i => i.GetType().FullName.GetMD5()).ToList() }; var customProviders = providers.OfType>().ToList(); foreach (var provider in customProviders.Where(i => i is IPreRefreshProvider)) { await RunCustomProvider(provider, item, options, refreshResult, cancellationToken).ConfigureAwait(false); } var temp = CreateNew(); temp.Path = item.Path; var successfulProviderCount = 0; var failedProviderCount = 0; // If replacing all metadata, run internet providers first if (options.ReplaceAllMetadata) { var remoteResult = await ExecuteRemoteProviders(item, temp, id, providers.OfType>(), cancellationToken) .ConfigureAwait(false); refreshResult.UpdateType = refreshResult.UpdateType | remoteResult.UpdateType; refreshResult.Status = remoteResult.Status; refreshResult.ErrorMessage = remoteResult.ErrorMessage; successfulProviderCount += remoteResult.Successes; failedProviderCount += remoteResult.Failures; } var hasLocalMetadata = false; foreach (var provider in providers.OfType>()) { var providerName = provider.GetType().Name; Logger.Debug("Running {0} for {1}", providerName, item.Path ?? item.Name); var itemInfo = new ItemInfo { Path = item.Path, IsInMixedFolder = item.IsInMixedFolder }; try { var localItem = await provider.GetMetadata(itemInfo, cancellationToken).ConfigureAwait(false); if (localItem.HasMetadata) { if (imageService.MergeImages(item, localItem.Images)) { refreshResult.UpdateType = refreshResult.UpdateType | ItemUpdateType.ImageUpdate; } MergeData(localItem.Item, temp, new List(), !options.ReplaceAllMetadata, true); refreshResult.UpdateType = refreshResult.UpdateType | ItemUpdateType.MetadataImport; // Only one local provider allowed per item hasLocalMetadata = true; successfulProviderCount++; break; } Logger.Debug("{0} returned no metadata for {1}", providerName, item.Path ?? item.Name); } catch (OperationCanceledException) { throw; } catch (Exception ex) { failedProviderCount++; Logger.ErrorException("Error in {0}", ex, provider.Name); // If a local provider fails, consider that a failure refreshResult.Status = ProviderRefreshStatus.Failure; refreshResult.ErrorMessage = ex.Message; if (options.MetadataRefreshMode != MetadataRefreshMode.FullRefresh) { // If the local provider fails don't continue with remote providers because the user's saved metadata could be lost return refreshResult; } } } // Local metadata is king - if any is found don't run remote providers if (!options.ReplaceAllMetadata && (!hasLocalMetadata || options.MetadataRefreshMode == MetadataRefreshMode.FullRefresh)) { var remoteResult = await ExecuteRemoteProviders(item, temp, id, providers.OfType>(), cancellationToken) .ConfigureAwait(false); refreshResult.UpdateType = refreshResult.UpdateType | remoteResult.UpdateType; if (remoteResult.Status != ProviderRefreshStatus.Success) { refreshResult.Status = remoteResult.Status; refreshResult.ErrorMessage = remoteResult.ErrorMessage; } successfulProviderCount += remoteResult.Successes; } if (refreshResult.UpdateType > ItemUpdateType.None) { MergeData(temp, item, item.LockedFields, true, true); } var isUnidentified = failedProviderCount > 0 && successfulProviderCount == 0; if (item.IsUnidentified != isUnidentified) { item.IsUnidentified = isUnidentified; refreshResult.UpdateType = refreshResult.UpdateType | ItemUpdateType.MetadataImport; } foreach (var provider in customProviders.Where(i => !(i is IPreRefreshProvider))) { await RunCustomProvider(provider, item, options, refreshResult, cancellationToken).ConfigureAwait(false); } return refreshResult; } private async Task RunCustomProvider(ICustomMetadataProvider provider, TItemType item, MetadataRefreshOptions options, RefreshResult refreshResult, CancellationToken cancellationToken) { Logger.Debug("Running {0} for {1}", provider.GetType().Name, item.Path ?? item.Name); try { refreshResult.UpdateType = refreshResult.UpdateType | await provider.FetchAsync(item, options, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { throw; } catch (Exception ex) { refreshResult.Status = ProviderRefreshStatus.CompletedWithErrors; refreshResult.ErrorMessage = ex.Message; Logger.ErrorException("Error in {0}", ex, provider.Name); } } protected virtual TItemType CreateNew() { return new TItemType(); } private async Task ExecuteRemoteProviders(TItemType item, TItemType temp, TIdType id, IEnumerable> providers, CancellationToken cancellationToken) { var refreshResult = new RefreshResult(); foreach (var provider in providers) { var providerName = provider.GetType().Name; Logger.Debug("Running {0} for {1}", providerName, item.Path ?? item.Name); if (id != null) { MergeNewData(temp, id); } try { var result = await provider.GetMetadata(id, cancellationToken).ConfigureAwait(false); if (result.HasMetadata) { MergeData(result.Item, temp, new List(), false, false); refreshResult.UpdateType = refreshResult.UpdateType | ItemUpdateType.MetadataDownload; refreshResult.Successes++; } else { refreshResult.Failures++; Logger.Debug("{0} returned no metadata for {1}", providerName, item.Path ?? item.Name); } } catch (OperationCanceledException) { throw; } catch (Exception ex) { refreshResult.Failures++; refreshResult.Status = ProviderRefreshStatus.CompletedWithErrors; refreshResult.ErrorMessage = ex.Message; Logger.ErrorException("Error in {0}", ex, provider.Name); } } if (refreshResult.Successes > 0) { AfterRemoteRefresh(temp); } return refreshResult; } protected virtual void AfterRemoteRefresh(TItemType item) { } private async Task CreateInitialLookupInfo(TItemType item, CancellationToken cancellationToken) { var info = item.GetLookupInfo(); var hasIdentity = info as IHasIdentities; if (hasIdentity != null) { try { await hasIdentity.FindIdentities(ProviderManager, cancellationToken).ConfigureAwait(false); } catch (Exception ex) { Logger.ErrorException("Error in identity providers", ex); } } return info; } private void MergeNewData(TItemType source, TIdType lookupInfo) { // Copy new provider id's that may have been obtained foreach (var providerId in source.ProviderIds) { var key = providerId.Key; // Don't replace existing Id's. if (!lookupInfo.ProviderIds.ContainsKey(key)) { lookupInfo.ProviderIds[key] = providerId.Value; } } } protected abstract void MergeData(TItemType source, TItemType target, List lockedFields, bool replaceData, bool mergeMetadataSettings); public virtual int Order { get { return 0; } } private bool HasChanged(IHasMetadata item, IHasChangeMonitor changeMonitor, DateTime date, IDirectoryService directoryService) { try { return changeMonitor.HasChanged(item, directoryService, date); } catch (Exception ex) { Logger.ErrorException("Error in {0}.HasChanged", ex, changeMonitor.GetType().Name); return false; } } } public class RefreshResult { public ItemUpdateType UpdateType { get; set; } public ProviderRefreshStatus Status { get; set; } public string ErrorMessage { get; set; } public List Providers { get; set; } public int Successes { get; set; } public int Failures { get; set; } } }