using MediaBrowser.Common.Extensions;
using MediaBrowser.Common.IO;
using MediaBrowser.Common.Net;
using MediaBrowser.Controller.Configuration;
using MediaBrowser.Controller.Entities;
using MediaBrowser.Controller.IO;
using MediaBrowser.Controller.Providers;
using MediaBrowser.Model.Logging;
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace MediaBrowser.Server.Implementations.Providers
{
///
/// Class ProviderManager
///
public class ProviderManager : IProviderManager
{
///
/// The remote image cache
///
private readonly FileSystemRepository _remoteImageCache;
///
/// The currently running metadata providers
///
private readonly ConcurrentDictionary> _currentlyRunningProviders =
new ConcurrentDictionary>();
///
/// The _logger
///
private readonly ILogger _logger;
///
/// The _HTTP client
///
private readonly IHttpClient _httpClient;
///
/// The _directory watchers
///
private readonly IDirectoryWatchers _directoryWatchers;
///
/// Gets or sets the configuration manager.
///
/// The configuration manager.
private IServerConfigurationManager ConfigurationManager { get; set; }
///
/// Gets the list of currently registered metadata prvoiders
///
/// The metadata providers enumerable.
private BaseMetadataProvider[] MetadataProviders { get; set; }
///
/// Initializes a new instance of the class.
///
/// The HTTP client.
/// The configuration manager.
/// The directory watchers.
/// The log manager.
public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, IDirectoryWatchers directoryWatchers, ILogManager logManager)
{
_logger = logManager.GetLogger("ProviderManager");
_httpClient = httpClient;
ConfigurationManager = configurationManager;
_directoryWatchers = directoryWatchers;
_remoteImageCache = new FileSystemRepository(configurationManager.ApplicationPaths.DownloadedImagesDataPath);
configurationManager.ConfigurationUpdated += configurationManager_ConfigurationUpdated;
}
///
/// Handles the ConfigurationUpdated event of the configurationManager control.
///
/// The source of the event.
/// The instance containing the event data.
void configurationManager_ConfigurationUpdated(object sender, EventArgs e)
{
// Validate currently executing providers, in the background
Task.Run(() => ValidateCurrentlyRunningProviders());
}
///
/// Adds the metadata providers.
///
/// The providers.
public void AddMetadataProviders(IEnumerable providers)
{
MetadataProviders = providers.OrderBy(e => e.Priority).ToArray();
}
///
/// The _supported providers key
///
private readonly Guid _supportedProvidersKey = "SupportedProviders".GetMD5();
///
/// Runs all metadata providers for an entity, and returns true or false indicating if at least one was refreshed and requires persistence
///
/// The item.
/// The cancellation token.
/// if set to true [force].
/// if set to true [allow slow providers].
/// Task{System.Boolean}.
public async Task ExecuteMetadataProviders(BaseItem item, CancellationToken cancellationToken, bool force = false, bool allowSlowProviders = true)
{
// Allow providers of the same priority to execute in parallel
MetadataProviderPriority? currentPriority = null;
var currentTasks = new List>();
var result = false;
cancellationToken.ThrowIfCancellationRequested();
// Determine if supported providers have changed
var supportedProviders = MetadataProviders.Where(p => p.Supports(item)).ToList();
BaseProviderInfo supportedProvidersInfo;
var supportedProvidersValue = string.Join("+", supportedProviders.Select(i => i.GetType().Name));
var providersChanged = false;
item.ProviderData.TryGetValue(_supportedProvidersKey, out supportedProvidersInfo);
if (supportedProvidersInfo != null)
{
// Force refresh if the supported providers have changed
providersChanged = force = force || !string.Equals(supportedProvidersInfo.FileSystemStamp, supportedProvidersValue);
// If providers have changed, clear provider info and update the supported providers hash
if (providersChanged)
{
_logger.Debug("Providers changed for {0}. Clearing and forcing refresh.", item.Name);
item.ProviderData.Clear();
}
}
if (providersChanged)
{
supportedProvidersInfo.FileSystemStamp = supportedProvidersValue;
}
if (force) item.ClearMetaValues();
// Run the normal providers sequentially in order of priority
foreach (var provider in supportedProviders)
{
cancellationToken.ThrowIfCancellationRequested();
// Skip if internet providers are currently disabled
if (provider.RequiresInternet && !ConfigurationManager.Configuration.EnableInternetProviders)
{
continue;
}
// Skip if is slow and we aren't allowing slow ones
if (provider.IsSlow && !allowSlowProviders)
{
continue;
}
// Skip if internet provider and this type is not allowed
if (provider.RequiresInternet && ConfigurationManager.Configuration.EnableInternetProviders && ConfigurationManager.Configuration.InternetProviderExcludeTypes.Contains(item.GetType().Name, StringComparer.OrdinalIgnoreCase))
{
continue;
}
// When a new priority is reached, await the ones that are currently running and clear the list
if (currentPriority.HasValue && currentPriority.Value != provider.Priority && currentTasks.Count > 0)
{
var results = await Task.WhenAll(currentTasks).ConfigureAwait(false);
result |= results.Contains(true);
currentTasks.Clear();
}
// Put this check below the await because the needs refresh of the next tier of providers may depend on the previous ones running
// This is the case for the fan art provider which depends on the movie and tv providers having run before them
if (!force && !provider.NeedsRefresh(item))
{
continue;
}
currentTasks.Add(FetchAsync(provider, item, force, cancellationToken));
currentPriority = provider.Priority;
}
if (currentTasks.Count > 0)
{
var results = await Task.WhenAll(currentTasks).ConfigureAwait(false);
result |= results.Contains(true);
}
if (providersChanged)
{
item.ProviderData[_supportedProvidersKey] = supportedProvidersInfo;
}
return result || providersChanged;
}
///
/// Fetches metadata and returns true or false indicating if any work that requires persistence was done
///
/// The provider.
/// The item.
/// if set to true [force].
/// The cancellation token.
/// Task{System.Boolean}.
///
private async Task FetchAsync(BaseMetadataProvider provider, BaseItem item, bool force, CancellationToken cancellationToken)
{
if (item == null)
{
throw new ArgumentNullException();
}
cancellationToken.ThrowIfCancellationRequested();
_logger.Debug("Running {0} for {1}", provider.GetType().Name, item.Path ?? item.Name ?? "--Unknown--");
// This provides the ability to cancel just this one provider
var innerCancellationTokenSource = new CancellationTokenSource();
OnProviderRefreshBeginning(provider, item, innerCancellationTokenSource);
try
{
return await provider.FetchAsync(item, force, CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, innerCancellationTokenSource.Token).Token).ConfigureAwait(false);
}
catch (OperationCanceledException ex)
{
_logger.Debug("{0} canceled for {1}", provider.GetType().Name, item.Name);
// If the outer cancellation token is the one that caused the cancellation, throw it
if (cancellationToken.IsCancellationRequested && ex.CancellationToken == cancellationToken)
{
throw;
}
return false;
}
catch (Exception ex)
{
_logger.ErrorException("{0} failed refreshing {1}", ex, provider.GetType().Name, item.Name);
provider.SetLastRefreshed(item, DateTime.UtcNow, ProviderRefreshStatus.Failure);
return true;
}
finally
{
innerCancellationTokenSource.Dispose();
OnProviderRefreshCompleted(provider, item);
}
}
///
/// Notifies the kernal that a provider has begun refreshing
///
/// The provider.
/// The item.
/// The cancellation token source.
public void OnProviderRefreshBeginning(BaseMetadataProvider provider, BaseItem item, CancellationTokenSource cancellationTokenSource)
{
var key = item.Id + provider.GetType().Name;
Tuple current;
if (_currentlyRunningProviders.TryGetValue(key, out current))
{
try
{
current.Item3.Cancel();
}
catch (ObjectDisposedException)
{
}
}
var tuple = new Tuple(provider, item, cancellationTokenSource);
_currentlyRunningProviders.AddOrUpdate(key, tuple, (k, v) => tuple);
}
///
/// Notifies the kernal that a provider has completed refreshing
///
/// The provider.
/// The item.
public void OnProviderRefreshCompleted(BaseMetadataProvider provider, BaseItem item)
{
var key = item.Id + provider.GetType().Name;
Tuple current;
if (_currentlyRunningProviders.TryRemove(key, out current))
{
current.Item3.Dispose();
}
}
///
/// Validates the currently running providers and cancels any that should not be run due to configuration changes
///
private void ValidateCurrentlyRunningProviders()
{
var enableInternetProviders = ConfigurationManager.Configuration.EnableInternetProviders;
var internetProviderExcludeTypes = ConfigurationManager.Configuration.InternetProviderExcludeTypes;
foreach (var tuple in _currentlyRunningProviders.Values
.Where(p => p.Item1.RequiresInternet && (!enableInternetProviders || internetProviderExcludeTypes.Contains(p.Item2.GetType().Name, StringComparer.OrdinalIgnoreCase)))
.ToList())
{
tuple.Item3.Cancel();
}
}
///
/// Downloads the and save image.
///
/// The item.
/// The source.
/// Name of the target.
/// if set to true [save locally].
/// The resource pool.
/// The cancellation token.
/// Task{System.String}.
/// item
public async Task DownloadAndSaveImage(BaseItem item, string source, string targetName, bool saveLocally, SemaphoreSlim resourcePool, CancellationToken cancellationToken)
{
if (item == null)
{
throw new ArgumentNullException("item");
}
if (string.IsNullOrEmpty(source))
{
throw new ArgumentNullException("source");
}
if (string.IsNullOrEmpty(targetName))
{
throw new ArgumentNullException("targetName");
}
if (resourcePool == null)
{
throw new ArgumentNullException("resourcePool");
}
//download and save locally
var localPath = (saveLocally && item.MetaLocation != null) ?
Path.Combine(item.MetaLocation, targetName) :
_remoteImageCache.GetResourcePath(item.GetType().FullName + item.Path.ToLower(), targetName);
var img = await _httpClient.Get(source, resourcePool, cancellationToken).ConfigureAwait(false);
if (saveLocally) // queue to media directories
{
await SaveToLibraryFilesystem(item, localPath, img, cancellationToken).ConfigureAwait(false);
}
else
{
// we can write directly here because it won't affect the watchers
try
{
using (var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
{
await img.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
_logger.ErrorException("Error downloading and saving image " + localPath, e);
throw;
}
finally
{
img.Dispose();
}
}
return localPath;
}
///
/// Saves to library filesystem.
///
/// The item.
/// The path.
/// The data to save.
/// The cancellation token.
/// Task.
///
public async Task SaveToLibraryFilesystem(BaseItem item, string path, Stream dataToSave, CancellationToken cancellationToken)
{
if (item == null)
{
throw new ArgumentNullException();
}
if (string.IsNullOrEmpty(path))
{
throw new ArgumentNullException();
}
if (dataToSave == null)
{
throw new ArgumentNullException();
}
if (cancellationToken == null)
{
throw new ArgumentNullException();
}
if (cancellationToken.IsCancellationRequested)
{
dataToSave.Dispose();
cancellationToken.ThrowIfCancellationRequested();
}
//Tell the watchers to ignore
_directoryWatchers.TemporarilyIgnore(path);
if (dataToSave.CanSeek)
{
dataToSave.Position = 0;
}
if (!(dataToSave is MemoryStream || dataToSave is FileStream))
{
var ms = new MemoryStream();
using (var input = dataToSave)
{
await input.CopyToAsync(ms).ConfigureAwait(false);
}
ms.Position = 0;
dataToSave = ms;
}
try
{
using (var fs = new FileStream(path, FileMode.Create, FileAccess.Write, FileShare.Read, StreamDefaults.DefaultFileStreamBufferSize, FileOptions.Asynchronous))
{
using (var input = dataToSave)
{
await input.CopyToAsync(fs, StreamDefaults.DefaultCopyToBufferSize, cancellationToken).ConfigureAwait(false);
}
}
// If this is ever used for something other than metadata we can add a file type param
item.ResolveArgs.AddMetadataFile(path);
}
finally
{
//Remove the ignore
_directoryWatchers.RemoveTempIgnore(path);
}
}
}
}