From 81d5e9f8087227591b2be068e822342b17ef3d7a Mon Sep 17 00:00:00 2001 From: Luke Pulverenti Date: Wed, 29 Jan 2014 00:17:58 -0500 Subject: [PATCH] persist provider results --- MediaBrowser.Controller/Entities/BaseItem.cs | 7 +- .../MediaBrowser.Controller.csproj | 4 +- .../Persistence/IItemRepository.cs | 17 -- .../Providers/IMetadataProvider.cs | 28 +-- .../Providers/IProviderRepository.cs | 48 ++++ MediaBrowser.Controller/Providers/ItemId.cs | 35 +++ .../Providers/MetadataStatus.cs | 122 ++++++++++ .../Providers/ProviderResult.cs | 60 ----- .../GameGenres/GameGenreMetadataService.cs | 4 +- .../Genres/GenreMetadataService.cs | 4 +- .../Manager/ItemImageProvider.cs | 11 +- .../Manager/MetadataService.cs | 51 +++-- .../Manager/ProviderManager.cs | 12 +- .../MusicGenres/MusicGenreMetadataService.cs | 4 +- .../People/PersonMetadataService.cs | 4 +- .../Studios/StudioMetadataService.cs | 4 +- .../IO/LibraryMonitor.cs | 25 +-- .../Persistence/SqliteItemRepository.cs | 22 -- .../SqliteProviderInfoRepository.cs | 212 ++++++++++++++++-- .../ApplicationHost.cs | 9 +- 20 files changed, 477 insertions(+), 206 deletions(-) create mode 100644 MediaBrowser.Controller/Providers/IProviderRepository.cs create mode 100644 MediaBrowser.Controller/Providers/ItemId.cs create mode 100644 MediaBrowser.Controller/Providers/MetadataStatus.cs delete mode 100644 MediaBrowser.Controller/Providers/ProviderResult.cs diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index a50a0f4674..06ebe89055 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -364,11 +364,16 @@ namespace MediaBrowser.Controller.Entities } } + private string _forcedSortName; /// /// Gets or sets the name of the forced sort. /// /// The name of the forced sort. - public string ForcedSortName { get; set; } + public string ForcedSortName + { + get { return _forcedSortName; } + set { _forcedSortName = value; _sortName = null; } + } private string _sortName; /// diff --git a/MediaBrowser.Controller/MediaBrowser.Controller.csproj b/MediaBrowser.Controller/MediaBrowser.Controller.csproj index 45297ef3d9..ee8bb27619 100644 --- a/MediaBrowser.Controller/MediaBrowser.Controller.csproj +++ b/MediaBrowser.Controller/MediaBrowser.Controller.csproj @@ -146,13 +146,15 @@ + + - + diff --git a/MediaBrowser.Controller/Persistence/IItemRepository.cs b/MediaBrowser.Controller/Persistence/IItemRepository.cs index 3affe48e74..3a5cb4e870 100644 --- a/MediaBrowser.Controller/Persistence/IItemRepository.cs +++ b/MediaBrowser.Controller/Persistence/IItemRepository.cs @@ -1,5 +1,4 @@ using MediaBrowser.Controller.Entities; -using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Entities; using System; using System.Collections.Generic; @@ -112,22 +111,6 @@ namespace MediaBrowser.Controller.Persistence /// The cancellation token. /// Task. Task SaveMediaStreams(Guid id, IEnumerable streams, CancellationToken cancellationToken); - - /// - /// Gets the provider history. - /// - /// The item identifier. - /// IEnumerable{BaseProviderInfo}. - IEnumerable GetProviderHistory(Guid itemId); - - /// - /// Saves the provider history. - /// - /// The identifier. - /// The history. - /// The cancellation token. - /// Task. - Task SaveProviderHistory(Guid id, IEnumerable history, CancellationToken cancellationToken); } } diff --git a/MediaBrowser.Controller/Providers/IMetadataProvider.cs b/MediaBrowser.Controller/Providers/IMetadataProvider.cs index 97249e26d9..843ba263bc 100644 --- a/MediaBrowser.Controller/Providers/IMetadataProvider.cs +++ b/MediaBrowser.Controller/Providers/IMetadataProvider.cs @@ -1,6 +1,4 @@ -using MediaBrowser.Model.Entities; -using System; -using System.Collections.Generic; +using System; using System.Threading; using System.Threading.Tasks; @@ -52,21 +50,14 @@ namespace MediaBrowser.Controller.Providers public interface IHasChangeMonitor { /// - /// Determines whether the specified date has changed. + /// Determines whether the specified item has changed. /// /// The item. /// The date. - /// true if the specified date has changed; otherwise, false. + /// true if the specified item has changed; otherwise, false. bool HasChanged(IHasMetadata item, DateTime date); } - public enum MetadataProviderType - { - Embedded = 0, - Local = 1, - Remote = 2 - } - public class MetadataResult where T : IHasMetadata { @@ -74,17 +65,4 @@ namespace MediaBrowser.Controller.Providers public T Item { get; set; } } - public class ItemId : IHasProviderIds - { - public string Name { get; set; } - public string MetadataLanguage { get; set; } - public string MetadataCountryCode { get; set; } - - public Dictionary ProviderIds { get; set; } - - public ItemId() - { - ProviderIds = new Dictionary(StringComparer.OrdinalIgnoreCase); - } - } } diff --git a/MediaBrowser.Controller/Providers/IProviderRepository.cs b/MediaBrowser.Controller/Providers/IProviderRepository.cs new file mode 100644 index 0000000000..1c0ad2cd73 --- /dev/null +++ b/MediaBrowser.Controller/Providers/IProviderRepository.cs @@ -0,0 +1,48 @@ +using MediaBrowser.Controller.Persistence; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace MediaBrowser.Controller.Providers +{ + public interface IProviderRepository : IRepository + { + /// + /// Gets the provider history. + /// + /// The item identifier. + /// IEnumerable{BaseProviderInfo}. + IEnumerable GetProviderHistory(Guid itemId); + + /// + /// Saves the provider history. + /// + /// The identifier. + /// The history. + /// The cancellation token. + /// Task. + Task SaveProviderHistory(Guid id, IEnumerable history, CancellationToken cancellationToken); + + /// + /// Gets the metadata status. + /// + /// The item identifier. + /// MetadataStatus. + MetadataStatus GetMetadataStatus(Guid itemId); + + /// + /// Saves the metadata status. + /// + /// The status. + /// The cancellation token. + /// Task. + Task SaveMetadataStatus(MetadataStatus status, CancellationToken cancellationToken); + + /// + /// Initializes this instance. + /// + /// Task. + Task Initialize(); + } +} diff --git a/MediaBrowser.Controller/Providers/ItemId.cs b/MediaBrowser.Controller/Providers/ItemId.cs new file mode 100644 index 0000000000..1116eb8b5d --- /dev/null +++ b/MediaBrowser.Controller/Providers/ItemId.cs @@ -0,0 +1,35 @@ +using MediaBrowser.Model.Entities; +using System; +using System.Collections.Generic; + +namespace MediaBrowser.Controller.Providers +{ + public class ItemId : IHasProviderIds + { + /// + /// Gets or sets the name. + /// + /// The name. + public string Name { get; set; } + /// + /// Gets or sets the metadata language. + /// + /// The metadata language. + public string MetadataLanguage { get; set; } + /// + /// Gets or sets the metadata country code. + /// + /// The metadata country code. + public string MetadataCountryCode { get; set; } + /// + /// Gets or sets the provider ids. + /// + /// The provider ids. + public Dictionary ProviderIds { get; set; } + + public ItemId() + { + ProviderIds = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + } +} diff --git a/MediaBrowser.Controller/Providers/MetadataStatus.cs b/MediaBrowser.Controller/Providers/MetadataStatus.cs new file mode 100644 index 0000000000..834d8ec35f --- /dev/null +++ b/MediaBrowser.Controller/Providers/MetadataStatus.cs @@ -0,0 +1,122 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using MediaBrowser.Common.Extensions; + +namespace MediaBrowser.Controller.Providers +{ + public class MetadataStatus + { + /// + /// Gets or sets the item identifier. + /// + /// The item identifier. + public Guid ItemId { get; set; } + + /// + /// Gets or sets the date last metadata refresh. + /// + /// The date last metadata refresh. + public DateTime? DateLastMetadataRefresh { get; set; } + + /// + /// Gets or sets the date last images refresh. + /// + /// The date last images refresh. + public DateTime? DateLastImagesRefresh { get; set; } + + /// + /// Gets or sets the last result. + /// + /// The last result. + public ProviderRefreshStatus LastStatus { get; set; } + + /// + /// Gets or sets the last result error message. + /// + /// The last result error message. + public string LastErrorMessage { get; set; } + + /// + /// Gets or sets the providers refreshed. + /// + /// The providers refreshed. + public List MetadataProvidersRefreshed { get; set; } + public List ImageProvidersRefreshed { get; set; } + + public void AddStatus(ProviderRefreshStatus status, string errorMessage) + { + if (LastStatus != status) + { + IsDirty = true; + } + + if (string.IsNullOrEmpty(LastErrorMessage)) + { + LastErrorMessage = errorMessage; + } + if (LastStatus == ProviderRefreshStatus.Success) + { + LastStatus = status; + } + } + + public MetadataStatus() + { + LastStatus = ProviderRefreshStatus.Success; + + MetadataProvidersRefreshed = new List(); + ImageProvidersRefreshed = new List(); + } + + public bool IsDirty { get; private set; } + + public void SetDateLastMetadataRefresh(DateTime date) + { + if (date != (DateLastMetadataRefresh ?? DateTime.MinValue)) + { + IsDirty = true; + } + + DateLastMetadataRefresh = date; + } + + public void SetDateLastImagesRefresh(DateTime date) + { + if (date != (DateLastImagesRefresh ?? DateTime.MinValue)) + { + IsDirty = true; + } + + DateLastImagesRefresh = date; + } + + public void AddImageProvidersRefreshed(List providerIds) + { + var count = ImageProvidersRefreshed.Count; + + providerIds.AddRange(ImageProvidersRefreshed); + + ImageProvidersRefreshed = providerIds.Distinct().ToList(); + + if (ImageProvidersRefreshed.Count != count) + { + IsDirty = true; + } + } + + public void AddMetadataProvidersRefreshed(List providerIds) + { + var count = MetadataProvidersRefreshed.Count; + + providerIds.AddRange(MetadataProvidersRefreshed); + + MetadataProvidersRefreshed = providerIds.Distinct().ToList(); + + if (MetadataProvidersRefreshed.Count != count) + { + IsDirty = true; + } + } + } +} diff --git a/MediaBrowser.Controller/Providers/ProviderResult.cs b/MediaBrowser.Controller/Providers/ProviderResult.cs deleted file mode 100644 index 2486faeedc..0000000000 --- a/MediaBrowser.Controller/Providers/ProviderResult.cs +++ /dev/null @@ -1,60 +0,0 @@ -using System; - -namespace MediaBrowser.Controller.Providers -{ - public class ProviderResult - { - /// - /// Gets or sets the item identifier. - /// - /// The item identifier. - public Guid ItemId { get; set; } - - /// - /// Gets or sets a value indicating whether this instance has refreshed metadata. - /// - /// true if this instance has refreshed metadata; otherwise, false. - public bool HasRefreshedMetadata { get; set; } - - /// - /// Gets or sets a value indicating whether this instance has refreshed images. - /// - /// true if this instance has refreshed images; otherwise, false. - public bool HasRefreshedImages { get; set; } - - /// - /// Gets or sets the date last refreshed. - /// - /// The date last refreshed. - public DateTime DateLastRefreshed { get; set; } - - /// - /// Gets or sets the last result. - /// - /// The last result. - public ProviderRefreshStatus Status { get; set; } - - /// - /// Gets or sets the last result error message. - /// - /// The last result error message. - public string ErrorMessage { get; set; } - - public void AddStatus(ProviderRefreshStatus status, string errorMessage) - { - if (string.IsNullOrEmpty(ErrorMessage)) - { - ErrorMessage = errorMessage; - } - if (Status == ProviderRefreshStatus.Success) - { - Status = status; - } - } - - public ProviderResult() - { - Status = ProviderRefreshStatus.Success; - } - } -} diff --git a/MediaBrowser.Providers/GameGenres/GameGenreMetadataService.cs b/MediaBrowser.Providers/GameGenres/GameGenreMetadataService.cs index 7625cdd5a7..389e2a2755 100644 --- a/MediaBrowser.Providers/GameGenres/GameGenreMetadataService.cs +++ b/MediaBrowser.Providers/GameGenres/GameGenreMetadataService.cs @@ -15,8 +15,8 @@ namespace MediaBrowser.Providers.GameGenres { private readonly ILibraryManager _libraryManager; - public GameGenreMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, ILibraryManager libraryManager) - : base(serverConfigurationManager, logger, providerManager) + public GameGenreMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, IProviderRepository providerRepo, ILibraryManager libraryManager) + : base(serverConfigurationManager, logger, providerManager, providerRepo) { _libraryManager = libraryManager; } diff --git a/MediaBrowser.Providers/Genres/GenreMetadataService.cs b/MediaBrowser.Providers/Genres/GenreMetadataService.cs index 7a8ffa5d44..83253a190f 100644 --- a/MediaBrowser.Providers/Genres/GenreMetadataService.cs +++ b/MediaBrowser.Providers/Genres/GenreMetadataService.cs @@ -15,8 +15,8 @@ namespace MediaBrowser.Providers.Genres { private readonly ILibraryManager _libraryManager; - public GenreMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, ILibraryManager libraryManager) - : base(serverConfigurationManager, logger, providerManager) + public GenreMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, IProviderRepository providerRepo, ILibraryManager libraryManager) + : base(serverConfigurationManager, logger, providerManager, providerRepo) { _libraryManager = libraryManager; } diff --git a/MediaBrowser.Providers/Manager/ItemImageProvider.cs b/MediaBrowser.Providers/Manager/ItemImageProvider.cs index 5358506f0e..d455a08dbe 100644 --- a/MediaBrowser.Providers/Manager/ItemImageProvider.cs +++ b/MediaBrowser.Providers/Manager/ItemImageProvider.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Controller.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; @@ -51,16 +52,24 @@ namespace MediaBrowser.Providers.Manager var providers = GetImageProviders(item, imageProviders).ToList(); + var providerIds = new List(); + foreach (var provider in providers.OfType()) { await RefreshFromProvider(item, provider, options, result, cancellationToken).ConfigureAwait(false); + + providerIds.Add(provider.GetType().FullName.GetMD5()); } foreach (var provider in providers.OfType()) { await RefreshFromProvider(item, provider, result, cancellationToken).ConfigureAwait(false); + + providerIds.Add(provider.GetType().FullName.GetMD5()); } + result.Providers = providerIds; + return result; } diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index 5dde3098ac..7916d7e86d 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -1,4 +1,5 @@ -using MediaBrowser.Controller.Configuration; +using MediaBrowser.Common.Extensions; +using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.Providers; @@ -18,16 +19,18 @@ namespace MediaBrowser.Providers.Manager protected readonly IServerConfigurationManager ServerConfigurationManager; protected readonly ILogger Logger; protected readonly IProviderManager ProviderManager; + private readonly IProviderRepository _providerRepo; private IMetadataProvider[] _providers = { }; private IImageProvider[] _imageProviders = { }; - protected MetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager) + protected MetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, IProviderRepository providerRepo) { ServerConfigurationManager = serverConfigurationManager; Logger = logger; ProviderManager = providerManager; + _providerRepo = providerRepo; } /// @@ -48,9 +51,9 @@ namespace MediaBrowser.Providers.Manager /// /// The result. /// Task. - protected Task SaveProviderResult(ProviderResult result) + protected Task SaveProviderResult(MetadataStatus result) { - return Task.FromResult(true); + return _providerRepo.SaveMetadataStatus(result, CancellationToken.None); } /// @@ -58,12 +61,9 @@ namespace MediaBrowser.Providers.Manager /// /// The item identifier. /// ProviderResult. - protected ProviderResult GetLastResult(Guid itemId) + protected MetadataStatus GetLastResult(Guid itemId) { - return new ProviderResult - { - ItemId = itemId - }; + return _providerRepo.GetMetadataStatus(itemId) ?? new MetadataStatus { ItemId = itemId }; } public async Task RefreshMetadata(IHasMetadata item, MetadataRefreshOptions options, CancellationToken cancellationToken) @@ -72,7 +72,9 @@ namespace MediaBrowser.Providers.Manager var updateType = ItemUpdateType.Unspecified; var lastResult = GetLastResult(item.Id); - var refreshResult = new ProviderResult { ItemId = item.Id }; + var refreshResult = lastResult; + refreshResult.LastErrorMessage = string.Empty; + refreshResult.LastStatus = ProviderRefreshStatus.Success; var imageProviders = GetImageProviders(item).ToList(); var itemImageProvider = new ItemImageProvider(Logger, ProviderManager, ServerConfigurationManager); @@ -97,7 +99,7 @@ namespace MediaBrowser.Providers.Manager // Next run metadata providers if (options.MetadataRefreshMode != MetadataRefreshMode.None) { - var providers = GetProviders(item, lastResult.HasRefreshedMetadata, options).ToList(); + var providers = GetProviders(item, lastResult.DateLastMetadataRefresh.HasValue, options).ToList(); if (providers.Count > 0) { @@ -105,22 +107,23 @@ namespace MediaBrowser.Providers.Manager updateType = updateType | result.UpdateType; refreshResult.AddStatus(result.Status, result.ErrorMessage); + refreshResult.SetDateLastMetadataRefresh(DateTime.UtcNow); + refreshResult.AddImageProvidersRefreshed(result.Providers); } - - refreshResult.HasRefreshedMetadata = true; } // Next run remote image providers, but only if local image providers didn't throw an exception if (!localImagesFailed) { - if ((options.ImageRefreshMode == MetadataRefreshMode.EnsureMetadata && !lastResult.HasRefreshedImages) || + if ((options.ImageRefreshMode == MetadataRefreshMode.EnsureMetadata && !lastResult.DateLastImagesRefresh.HasValue) || options.ImageRefreshMode == MetadataRefreshMode.FullRefresh) { - var imagesReult = await itemImageProvider.RefreshImages(itemOfType, imageProviders, options, cancellationToken).ConfigureAwait(false); + var result = await itemImageProvider.RefreshImages(itemOfType, imageProviders, options, cancellationToken).ConfigureAwait(false); - updateType = updateType | imagesReult.UpdateType; - refreshResult.AddStatus(imagesReult.Status, imagesReult.ErrorMessage); - refreshResult.HasRefreshedImages = true; + updateType = updateType | result.UpdateType; + refreshResult.AddStatus(result.Status, result.ErrorMessage); + refreshResult.SetDateLastImagesRefresh(DateTime.UtcNow); + refreshResult.AddImageProvidersRefreshed(result.Providers); } } @@ -137,9 +140,8 @@ namespace MediaBrowser.Providers.Manager await SaveItem(itemOfType, updateType, cancellationToken); } - if (providersHadChanges) + if (providersHadChanges || refreshResult.IsDirty) { - refreshResult.DateLastRefreshed = DateTime.UtcNow; await SaveProviderResult(refreshResult).ConfigureAwait(false); } } @@ -165,7 +167,7 @@ namespace MediaBrowser.Providers.Manager var currentItem = item; var providersWithChanges = providers.OfType() - .Where(i => i.HasChanged(currentItem, item.DateLastSaved)) + .Where(i => i.HasChanged(currentItem, currentItem.DateLastSaved)) .ToList(); // If local providers are the only ones with changes, then just run those @@ -219,7 +221,11 @@ namespace MediaBrowser.Providers.Manager protected virtual async Task RefreshWithProviders(TItemType item, MetadataRefreshOptions options, List providers, CancellationToken cancellationToken) { - var refreshResult = new RefreshResult { UpdateType = ItemUpdateType.Unspecified }; + var refreshResult = new RefreshResult + { + UpdateType = ItemUpdateType.Unspecified, + Providers = providers.Select(i => i.GetType().FullName.GetMD5()).ToList() + }; var temp = new TItemType(); @@ -347,5 +353,6 @@ namespace MediaBrowser.Providers.Manager public ItemUpdateType UpdateType { get; set; } public ProviderRefreshStatus Status { get; set; } public string ErrorMessage { get; set; } + public List Providers { get; set; } } } diff --git a/MediaBrowser.Providers/Manager/ProviderManager.cs b/MediaBrowser.Providers/Manager/ProviderManager.cs index faad156698..3696bd02f5 100644 --- a/MediaBrowser.Providers/Manager/ProviderManager.cs +++ b/MediaBrowser.Providers/Manager/ProviderManager.cs @@ -55,7 +55,7 @@ namespace MediaBrowser.Providers.Manager private readonly IFileSystem _fileSystem; - private readonly IItemRepository _itemRepo; + private readonly IProviderRepository _providerRepo; private IMetadataService[] _metadataServices = { }; @@ -67,15 +67,15 @@ namespace MediaBrowser.Providers.Manager /// The directory watchers. /// The log manager. /// The file system. - /// The item repo. - public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, ILibraryMonitor libraryMonitor, ILogManager logManager, IFileSystem fileSystem, IItemRepository itemRepo) + /// The provider repo. + public ProviderManager(IHttpClient httpClient, IServerConfigurationManager configurationManager, ILibraryMonitor libraryMonitor, ILogManager logManager, IFileSystem fileSystem, IProviderRepository providerRepo) { _logger = logManager.GetLogger("ProviderManager"); _httpClient = httpClient; ConfigurationManager = configurationManager; _libraryMonitor = libraryMonitor; _fileSystem = fileSystem; - _itemRepo = itemRepo; + _providerRepo = providerRepo; } /// @@ -136,7 +136,7 @@ namespace MediaBrowser.Providers.Manager var providerHistories = item.DateLastSaved == default(DateTime) ? new List() : - _itemRepo.GetProviderHistory(item.Id).ToList(); + _providerRepo.GetProviderHistory(item.Id).ToList(); // Run the normal providers sequentially in order of priority foreach (var provider in MetadataProviders) @@ -201,7 +201,7 @@ namespace MediaBrowser.Providers.Manager if (result.HasValue || force) { - await _itemRepo.SaveProviderHistory(item.Id, providerHistories, cancellationToken); + await _providerRepo.SaveProviderHistory(item.Id, providerHistories, cancellationToken); } return result; diff --git a/MediaBrowser.Providers/MusicGenres/MusicGenreMetadataService.cs b/MediaBrowser.Providers/MusicGenres/MusicGenreMetadataService.cs index bc4a99fc3f..b88ca92bc6 100644 --- a/MediaBrowser.Providers/MusicGenres/MusicGenreMetadataService.cs +++ b/MediaBrowser.Providers/MusicGenres/MusicGenreMetadataService.cs @@ -15,8 +15,8 @@ namespace MediaBrowser.Providers.MusicGenres { private readonly ILibraryManager _libraryManager; - public MusicGenreMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, ILibraryManager libraryManager) - : base(serverConfigurationManager, logger, providerManager) + public MusicGenreMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, IProviderRepository providerRepo, ILibraryManager libraryManager) + : base(serverConfigurationManager, logger, providerManager, providerRepo) { _libraryManager = libraryManager; } diff --git a/MediaBrowser.Providers/People/PersonMetadataService.cs b/MediaBrowser.Providers/People/PersonMetadataService.cs index 577158aa8b..e040139348 100644 --- a/MediaBrowser.Providers/People/PersonMetadataService.cs +++ b/MediaBrowser.Providers/People/PersonMetadataService.cs @@ -15,8 +15,8 @@ namespace MediaBrowser.Providers.People { private readonly ILibraryManager _libraryManager; - public PersonMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, ILibraryManager libraryManager) - : base(serverConfigurationManager, logger, providerManager) + public PersonMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, IProviderRepository providerRepo, ILibraryManager libraryManager) + : base(serverConfigurationManager, logger, providerManager, providerRepo) { _libraryManager = libraryManager; } diff --git a/MediaBrowser.Providers/Studios/StudioMetadataService.cs b/MediaBrowser.Providers/Studios/StudioMetadataService.cs index 1a74d8317f..1a35b94b37 100644 --- a/MediaBrowser.Providers/Studios/StudioMetadataService.cs +++ b/MediaBrowser.Providers/Studios/StudioMetadataService.cs @@ -15,8 +15,8 @@ namespace MediaBrowser.Providers.Studios { private readonly ILibraryManager _libraryManager; - public StudioMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, ILibraryManager libraryManager) - : base(serverConfigurationManager, logger, providerManager) + public StudioMetadataService(IServerConfigurationManager serverConfigurationManager, ILogger logger, IProviderManager providerManager, IProviderRepository providerRepo, ILibraryManager libraryManager) + : base(serverConfigurationManager, logger, providerManager, providerRepo) { _libraryManager = libraryManager; } diff --git a/MediaBrowser.Server.Implementations/IO/LibraryMonitor.cs b/MediaBrowser.Server.Implementations/IO/LibraryMonitor.cs index 22ea668f48..0716a3d83f 100644 --- a/MediaBrowser.Server.Implementations/IO/LibraryMonitor.cs +++ b/MediaBrowser.Server.Implementations/IO/LibraryMonitor.cs @@ -56,21 +56,6 @@ namespace MediaBrowser.Server.Implementations.IO _tempIgnoredPaths[path] = path; } - /// - /// Removes the temp ignore. - /// - /// The path. - private async void RemoveTempIgnore(string path) - { - // This is an arbitraty amount of time, but delay it because file system writes often trigger events after RemoveTempIgnore has been called. - // Seeing long delays in some situations, especially over the network. - // Seeing delays up to 40 seconds, but not going to ignore changes for that long. - await Task.Delay(1500).ConfigureAwait(false); - - string val; - _tempIgnoredPaths.TryRemove(path, out val); - } - public void ReportFileSystemChangeBeginning(string path) { if (string.IsNullOrEmpty(path)) @@ -81,14 +66,20 @@ namespace MediaBrowser.Server.Implementations.IO TemporarilyIgnore(path); } - public void ReportFileSystemChangeComplete(string path, bool refreshPath) + public async void ReportFileSystemChangeComplete(string path, bool refreshPath) { if (string.IsNullOrEmpty(path)) { throw new ArgumentNullException("path"); } - RemoveTempIgnore(path); + // This is an arbitraty amount of time, but delay it because file system writes often trigger events after RemoveTempIgnore has been called. + // Seeing long delays in some situations, especially over the network. + // Seeing delays up to 40 seconds, but not going to ignore changes for that long. + await Task.Delay(1500).ConfigureAwait(false); + + string val; + _tempIgnoredPaths.TryRemove(path, out val); if (refreshPath) { diff --git a/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs b/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs index 200898a623..6b463bbdfc 100644 --- a/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs +++ b/MediaBrowser.Server.Implementations/Persistence/SqliteItemRepository.cs @@ -58,7 +58,6 @@ namespace MediaBrowser.Server.Implementations.Persistence private SqliteChapterRepository _chapterRepository; private SqliteMediaStreamsRepository _mediaStreamsRepository; - private SqliteProviderInfoRepository _providerInfoRepository; private IDbCommand _deleteChildrenCommand; private IDbCommand _saveChildrenCommand; @@ -99,10 +98,6 @@ namespace MediaBrowser.Server.Implementations.Persistence var mediaStreamsDbFile = Path.Combine(_appPaths.DataPath, "mediainfo.db"); var mediaStreamsConnection = SqliteExtensions.ConnectToDb(mediaStreamsDbFile, _logger).Result; _mediaStreamsRepository = new SqliteMediaStreamsRepository(mediaStreamsConnection, logManager); - - var providerInfosDbFile = Path.Combine(_appPaths.DataPath, "providerinfo.db"); - var providerInfoConnection = SqliteExtensions.ConnectToDb(providerInfosDbFile, _logger).Result; - _providerInfoRepository = new SqliteProviderInfoRepository(providerInfoConnection, logManager); } /// @@ -134,7 +129,6 @@ namespace MediaBrowser.Server.Implementations.Persistence PrepareStatements(); _mediaStreamsRepository.Initialize(); - _providerInfoRepository.Initialize(); _chapterRepository.Initialize(); _shrinkMemoryTimer = new SqliteShrinkMemoryTimer(_connection, _writeLock, _logger); @@ -436,12 +430,6 @@ namespace MediaBrowser.Server.Implementations.Persistence _mediaStreamsRepository.Dispose(); _mediaStreamsRepository = null; } - - if (_providerInfoRepository != null) - { - _providerInfoRepository.Dispose(); - _providerInfoRepository = null; - } } } catch (Exception ex) @@ -556,15 +544,5 @@ namespace MediaBrowser.Server.Implementations.Persistence { return _mediaStreamsRepository.SaveMediaStreams(id, streams, cancellationToken); } - - public IEnumerable GetProviderHistory(Guid itemId) - { - return _providerInfoRepository.GetBaseProviderInfos(itemId); - } - - public Task SaveProviderHistory(Guid id, IEnumerable history, CancellationToken cancellationToken) - { - return _providerInfoRepository.SaveProviderInfos(id, history, cancellationToken); - } } } \ No newline at end of file diff --git a/MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs b/MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs index 9971c74605..8a82c062dd 100644 --- a/MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs +++ b/MediaBrowser.Server.Implementations/Persistence/SqliteProviderInfoRepository.cs @@ -1,4 +1,6 @@ -using MediaBrowser.Controller.Providers; +using System.IO; +using MediaBrowser.Common.Configuration; +using MediaBrowser.Controller.Providers; using MediaBrowser.Model.Logging; using System; using System.Collections.Generic; @@ -9,7 +11,7 @@ using System.Threading.Tasks; namespace MediaBrowser.Server.Implementations.Persistence { - class SqliteProviderInfoRepository + public class SqliteProviderInfoRepository : IProviderRepository { private IDbConnection _connection; @@ -17,32 +19,47 @@ namespace MediaBrowser.Server.Implementations.Persistence private IDbCommand _deleteInfosCommand; private IDbCommand _saveInfoCommand; + private IDbCommand _saveStatusCommand; + private readonly IApplicationPaths _appPaths; - public SqliteProviderInfoRepository(IDbConnection connection, ILogManager logManager) + public SqliteProviderInfoRepository(IApplicationPaths appPaths, ILogManager logManager) { - _connection = connection; - + _appPaths = appPaths; _logger = logManager.GetLogger(GetType().Name); } private SqliteShrinkMemoryTimer _shrinkMemoryTimer; - + + /// + /// Gets the name of the repository + /// + /// The name. + public string Name + { + get + { + return "SQLite"; + } + } + /// /// Opens the connection to the database /// /// Task. - public void Initialize() + public async Task Initialize() { - var createTableCommand - = "create table if not exists providerinfos "; + var dbFile = Path.Combine(_appPaths.DataPath, "providerinfo.db"); - createTableCommand += "(ItemId GUID, ProviderId GUID, ProviderVersion TEXT, FileStamp GUID, LastRefreshStatus TEXT, LastRefreshed datetime, PRIMARY KEY (ItemId, ProviderId))"; + _connection = await SqliteExtensions.ConnectToDb(dbFile, _logger).ConfigureAwait(false); string[] queries = { - createTableCommand, + "create table if not exists providerinfos (ItemId GUID, ProviderId GUID, ProviderVersion TEXT, FileStamp GUID, LastRefreshStatus TEXT, LastRefreshed datetime, PRIMARY KEY (ItemId, ProviderId))", "create index if not exists idx_providerinfos on providerinfos(ItemId, ProviderId)", + "create table if not exists MetadataStatus (ItemId GUID PRIMARY KEY, DateLastMetadataRefresh datetime, DateLastImagesRefresh datetime, LastStatus TEXT, LastErrorMessage TEXT, MetadataProvidersRefreshed TEXT, ImageProvidersRefreshed TEXT)", + "create index if not exists idx_MetadataStatus on MetadataStatus(ItemId)", + //pragmas "pragma temp_store = memory", @@ -56,7 +73,7 @@ namespace MediaBrowser.Server.Implementations.Persistence _shrinkMemoryTimer = new SqliteShrinkMemoryTimer(_connection, _writeLock, _logger); } - private static readonly string[] SaveColumns = + private static readonly string[] SaveHistoryColumns = { "ItemId", "ProviderId", @@ -66,7 +83,18 @@ namespace MediaBrowser.Server.Implementations.Persistence "LastRefreshed" }; - private readonly string[] _selectColumns = SaveColumns.Skip(1).ToArray(); + private readonly string[] _historySelectColumns = SaveHistoryColumns.Skip(1).ToArray(); + + private static readonly string[] StatusColumns = + { + "ItemId", + "DateLastMetadataRefresh", + "DateLastImagesRefresh", + "LastStatus", + "LastErrorMessage", + "MetadataProvidersRefreshed", + "ImageProvidersRefreshed" + }; /// /// The _write lock @@ -85,16 +113,27 @@ namespace MediaBrowser.Server.Implementations.Persistence _saveInfoCommand = _connection.CreateCommand(); _saveInfoCommand.CommandText = string.Format("replace into providerinfos ({0}) values ({1})", - string.Join(",", SaveColumns), - string.Join(",", SaveColumns.Select(i => "@" + i).ToArray())); + string.Join(",", SaveHistoryColumns), + string.Join(",", SaveHistoryColumns.Select(i => "@" + i).ToArray())); - foreach (var col in SaveColumns) + foreach (var col in SaveHistoryColumns) { _saveInfoCommand.Parameters.Add(_saveInfoCommand, "@" + col); } + + _saveStatusCommand = _connection.CreateCommand(); + + _saveStatusCommand.CommandText = string.Format("replace into MetadataStatus ({0}) values ({1})", + string.Join(",", StatusColumns), + string.Join(",", StatusColumns.Select(i => "@" + i).ToArray())); + + foreach (var col in StatusColumns) + { + _saveStatusCommand.Parameters.Add(_saveStatusCommand, "@" + col); + } } - public IEnumerable GetBaseProviderInfos(Guid itemId) + public IEnumerable GetProviderHistory(Guid itemId) { if (itemId == Guid.Empty) { @@ -103,7 +142,7 @@ namespace MediaBrowser.Server.Implementations.Persistence using (var cmd = _connection.CreateCommand()) { - var cmdText = "select " + string.Join(",", _selectColumns) + " from providerinfos where"; + var cmdText = "select " + string.Join(",", _historySelectColumns) + " from providerinfos where"; cmdText += " ItemId=@ItemId"; cmd.Parameters.Add(cmd, "@ItemId", DbType.Guid).Value = itemId; @@ -121,10 +160,10 @@ namespace MediaBrowser.Server.Implementations.Persistence } /// - /// Gets the chapter. + /// Gets the base provider information. /// /// The reader. - /// ChapterInfo. + /// BaseProviderInfo. private BaseProviderInfo GetBaseProviderInfo(IDataReader reader) { var item = new BaseProviderInfo @@ -144,7 +183,7 @@ namespace MediaBrowser.Server.Implementations.Persistence return item; } - public async Task SaveProviderInfos(Guid id, IEnumerable infos, CancellationToken cancellationToken) + public async Task SaveProviderHistory(Guid id, IEnumerable infos, CancellationToken cancellationToken) { if (id == Guid.Empty) { @@ -166,7 +205,6 @@ namespace MediaBrowser.Server.Implementations.Persistence { transaction = _connection.BeginTransaction(); - // First delete chapters _deleteInfosCommand.GetParameter(0).Value = id; _deleteInfosCommand.Transaction = transaction; @@ -221,6 +259,136 @@ namespace MediaBrowser.Server.Implementations.Persistence } } + public MetadataStatus GetMetadataStatus(Guid itemId) + { + if (itemId == Guid.Empty) + { + throw new ArgumentNullException("itemId"); + } + + using (var cmd = _connection.CreateCommand()) + { + var cmdText = "select " + string.Join(",", StatusColumns) + " from MetadataStatus where"; + + cmdText += " ItemId=@ItemId"; + cmd.Parameters.Add(cmd, "@ItemId", DbType.Guid).Value = itemId; + + cmd.CommandText = cmdText; + + using (var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess | CommandBehavior.SingleResult | CommandBehavior.SingleRow)) + { + while (reader.Read()) + { + return GetStatus(reader); + } + + return null; + } + } + } + + private MetadataStatus GetStatus(IDataReader reader) + { + var result = new MetadataStatus + { + ItemId = reader.GetGuid(0) + }; + + if (!reader.IsDBNull(1)) + { + result.DateLastMetadataRefresh = reader.GetDateTime(1).ToUniversalTime(); + } + + if (!reader.IsDBNull(2)) + { + result.DateLastImagesRefresh = reader.GetDateTime(2).ToUniversalTime(); + } + + if (!reader.IsDBNull(3)) + { + result.LastStatus = (ProviderRefreshStatus)Enum.Parse(typeof(ProviderRefreshStatus), reader.GetString(3), true); + } + + if (!reader.IsDBNull(4)) + { + result.LastErrorMessage = reader.GetString(4); + } + + if (!reader.IsDBNull(5)) + { + result.MetadataProvidersRefreshed = reader.GetString(5).Split('|').Where(i => !string.IsNullOrEmpty(i)).Select(i => new Guid(i)).ToList(); + } + + if (!reader.IsDBNull(6)) + { + result.ImageProvidersRefreshed = reader.GetString(6).Split('|').Where(i => !string.IsNullOrEmpty(i)).Select(i => new Guid(i)).ToList(); + } + + return result; + } + + public async Task SaveMetadataStatus(MetadataStatus status, CancellationToken cancellationToken) + { + if (status == null) + { + throw new ArgumentNullException("status"); + } + + cancellationToken.ThrowIfCancellationRequested(); + + await _writeLock.WaitAsync(cancellationToken).ConfigureAwait(false); + + IDbTransaction transaction = null; + + try + { + transaction = _connection.BeginTransaction(); + + _saveStatusCommand.GetParameter(0).Value = status.ItemId; + _saveStatusCommand.GetParameter(1).Value = status.DateLastMetadataRefresh; + _saveStatusCommand.GetParameter(2).Value = status.DateLastImagesRefresh; + _saveStatusCommand.GetParameter(3).Value = status.LastStatus.ToString(); + _saveStatusCommand.GetParameter(4).Value = status.LastErrorMessage; + _saveStatusCommand.GetParameter(5).Value = string.Join("|", status.MetadataProvidersRefreshed.ToArray()); + _saveStatusCommand.GetParameter(6).Value = string.Join("|", status.ImageProvidersRefreshed.ToArray()); + + _saveStatusCommand.Transaction = transaction; + + _saveStatusCommand.ExecuteNonQuery(); + + transaction.Commit(); + } + catch (OperationCanceledException) + { + if (transaction != null) + { + transaction.Rollback(); + } + + throw; + } + catch (Exception e) + { + _logger.ErrorException("Failed to save provider info:", e); + + if (transaction != null) + { + transaction.Rollback(); + } + + throw; + } + finally + { + if (transaction != null) + { + transaction.Dispose(); + } + + _writeLock.Release(); + } + } + /// /// Performs application-defined tasks associated with freeing, releasing, or resetting unmanaged resources. /// diff --git a/MediaBrowser.ServerApplication/ApplicationHost.cs b/MediaBrowser.ServerApplication/ApplicationHost.cs index bf652d8cb6..4b15ca8d03 100644 --- a/MediaBrowser.ServerApplication/ApplicationHost.cs +++ b/MediaBrowser.ServerApplication/ApplicationHost.cs @@ -14,7 +14,6 @@ using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.FileOrganization; -using MediaBrowser.Controller.IO; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.LiveTv; using MediaBrowser.Controller.Localization; @@ -173,6 +172,7 @@ namespace MediaBrowser.ServerApplication internal IItemRepository ItemRepository { get; set; } private INotificationsRepository NotificationsRepository { get; set; } private IFileOrganizationRepository FileOrganizationRepository { get; set; } + private IProviderRepository ProviderRepository { get; set; } /// /// Initializes a new instance of the class. @@ -267,6 +267,9 @@ namespace MediaBrowser.ServerApplication ItemRepository = new SqliteItemRepository(ApplicationPaths, JsonSerializer, LogManager); RegisterSingleInstance(ItemRepository); + ProviderRepository = new SqliteProviderInfoRepository(ApplicationPaths, LogManager); + RegisterSingleInstance(ProviderRepository); + FileOrganizationRepository = await GetFileOrganizationRepository().ConfigureAwait(false); RegisterSingleInstance(FileOrganizationRepository); @@ -279,7 +282,7 @@ namespace MediaBrowser.ServerApplication LibraryMonitor = new LibraryMonitor(LogManager, TaskManager, LibraryManager, ServerConfigurationManager, FileSystemManager); RegisterSingleInstance(LibraryMonitor); - ProviderManager = new ProviderManager(HttpClient, ServerConfigurationManager, LibraryMonitor, LogManager, FileSystemManager, ItemRepository); + ProviderManager = new ProviderManager(HttpClient, ServerConfigurationManager, LibraryMonitor, LogManager, FileSystemManager, ProviderRepository); RegisterSingleInstance(ProviderManager); RegisterSingleInstance(() => new SearchEngine(LogManager, LibraryManager, UserManager)); @@ -427,6 +430,8 @@ namespace MediaBrowser.ServerApplication { await ItemRepository.Initialize().ConfigureAwait(false); + await ProviderRepository.Initialize().ConfigureAwait(false); + ((LibraryManager)LibraryManager).ItemRepository = ItemRepository; }