From 3c5bbeb80ccde3a97d766cdd7d99b514b20ac85d Mon Sep 17 00:00:00 2001 From: cvium Date: Thu, 10 Dec 2020 13:38:33 +0100 Subject: [PATCH 1/4] Remove ImageFetcherPostScanTask --- .../Library/ImageFetcherPostScanTask.cs | 130 ------------------ .../Library/LibraryManager.cs | 10 +- .../Library/ILibraryManager.cs | 2 +- .../Manager/MetadataService.cs | 17 ++- 4 files changed, 14 insertions(+), 145 deletions(-) delete mode 100644 Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs diff --git a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs b/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs deleted file mode 100644 index d4e790c9a6..0000000000 --- a/Emby.Server.Implementations/Library/ImageFetcherPostScanTask.cs +++ /dev/null @@ -1,130 +0,0 @@ -using System; -using System.Collections.Concurrent; -using System.Globalization; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; -using Jellyfin.Data.Events; -using MediaBrowser.Controller.Entities; -using MediaBrowser.Controller.Library; -using MediaBrowser.Controller.Providers; -using MediaBrowser.Model.Net; -using Microsoft.Extensions.Logging; - -namespace Emby.Server.Implementations.Library -{ - /// - /// A library post scan/refresh task for pre-fetching remote images. - /// - public class ImageFetcherPostScanTask : ILibraryPostScanTask - { - private readonly ILibraryManager _libraryManager; - private readonly IProviderManager _providerManager; - private readonly ILogger _logger; - private readonly SemaphoreSlim _imageFetcherLock; - - private ConcurrentDictionary _queuedItems; - - /// - /// Initializes a new instance of the class. - /// - /// An instance of . - /// An instance of . - /// An instance of . - public ImageFetcherPostScanTask( - ILibraryManager libraryManager, - IProviderManager providerManager, - ILogger logger) - { - _libraryManager = libraryManager; - _providerManager = providerManager; - _logger = logger; - _queuedItems = new ConcurrentDictionary(); - _imageFetcherLock = new SemaphoreSlim(1, 1); - _libraryManager.ItemAdded += OnLibraryManagerItemAddedOrUpdated; - _libraryManager.ItemUpdated += OnLibraryManagerItemAddedOrUpdated; - _providerManager.RefreshCompleted += OnProviderManagerRefreshCompleted; - } - - /// - public async Task Run(IProgress progress, CancellationToken cancellationToken) - { - // Sometimes a library scan will cause this to run twice if there's an item refresh going on. - await _imageFetcherLock.WaitAsync(cancellationToken).ConfigureAwait(false); - - try - { - var now = DateTime.UtcNow; - var itemGuids = _queuedItems.Keys.ToList(); - - for (var i = 0; i < itemGuids.Count; i++) - { - if (!_queuedItems.TryGetValue(itemGuids[i], out var queuedItem)) - { - continue; - } - - var itemId = queuedItem.item.Id.ToString("N", CultureInfo.InvariantCulture); - var itemType = queuedItem.item.GetType(); - _logger.LogDebug( - "Updating remote images for item {ItemId} with media type {ItemMediaType}", - itemId, - itemType); - try - { - await _libraryManager.UpdateImagesAsync(queuedItem.item, queuedItem.updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false); - } - catch (Exception ex) - { - _logger.LogError(ex, "Failed to fetch images for {Type} item with id {ItemId}", itemType, itemId); - } - - _queuedItems.TryRemove(queuedItem.item.Id, out _); - } - - if (itemGuids.Count > 0) - { - _logger.LogInformation( - "Finished updating/pre-fetching {NumberOfImages} images. Elapsed time: {TimeElapsed}s.", - itemGuids.Count.ToString(CultureInfo.InvariantCulture), - (DateTime.UtcNow - now).TotalSeconds.ToString(CultureInfo.InvariantCulture)); - } - else - { - _logger.LogDebug("No images were updated."); - } - } - finally - { - _imageFetcherLock.Release(); - } - } - - private void OnLibraryManagerItemAddedOrUpdated(object sender, ItemChangeEventArgs itemChangeEventArgs) - { - if (!_queuedItems.ContainsKey(itemChangeEventArgs.Item.Id) && itemChangeEventArgs.Item.ImageInfos.Length > 0) - { - _queuedItems.AddOrUpdate( - itemChangeEventArgs.Item.Id, - (itemChangeEventArgs.Item, itemChangeEventArgs.UpdateReason), - (key, existingValue) => existingValue); - } - } - - private void OnProviderManagerRefreshCompleted(object sender, GenericEventArgs e) - { - if (!_queuedItems.ContainsKey(e.Argument.Id) && e.Argument.ImageInfos.Length > 0) - { - _queuedItems.AddOrUpdate( - e.Argument.Id, - (e.Argument, ItemUpdateType.None), - (key, existingValue) => existingValue); - } - - // The RefreshCompleted event is a bit awkward in that it seems to _only_ be fired on - // the item that was refreshed regardless of children refreshes. So we take it as a signal - // that the refresh is entirely completed. - Run(null, CancellationToken.None).GetAwaiter().GetResult(); - } - } -} diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 5b926b0f4f..47184b7892 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1955,9 +1955,9 @@ namespace Emby.Server.Implementations.Library } /// - public Task UpdateItemsAsync(IReadOnlyList items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) + public async Task UpdateItemsAsync(IReadOnlyList items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) { - RunMetadataSavers(items, updateReason); + await RunMetadataSavers(items, updateReason).ConfigureAwait(false); _itemRepository.SaveItems(items, cancellationToken); @@ -1988,15 +1988,13 @@ namespace Emby.Server.Implementations.Library } } } - - return Task.CompletedTask; } /// public Task UpdateItemAsync(BaseItem item, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) => UpdateItemsAsync(new[] { item }, parent, updateReason, cancellationToken); - public void RunMetadataSavers(IReadOnlyList items, ItemUpdateType updateReason) + public async Task RunMetadataSavers(IReadOnlyList items, ItemUpdateType updateReason) { foreach (var item in items) { @@ -2006,6 +2004,8 @@ namespace Emby.Server.Implementations.Library } item.DateLastSaved = DateTime.UtcNow; + + await UpdateImagesAsync(item, updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false); } } diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index 24b101694d..ff80952f76 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -571,7 +571,7 @@ namespace MediaBrowser.Controller.Library string videoPath, string[] files); - void RunMetadataSavers(IReadOnlyList items, ItemUpdateType updateReason); + Task RunMetadataSavers(IReadOnlyList items, ItemUpdateType updateReason); BaseItem GetParentItem(string parentId, Guid? userId); diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index 6dbce3067e..e78eae4fd1 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -232,6 +232,7 @@ namespace MediaBrowser.Providers.Manager private Task SavePeopleMetadataAsync(List people, LibraryOptions libraryOptions, CancellationToken cancellationToken) { var personsToSave = new List(); + var personsToSaveWithImages = new List(); foreach (var person in people) { @@ -239,14 +240,13 @@ namespace MediaBrowser.Providers.Manager if (person.ProviderIds.Count > 0 || !string.IsNullOrWhiteSpace(person.ImageUrl)) { - var saveEntity = false; var personEntity = LibraryManager.GetPerson(person.Name); foreach (var id in person.ProviderIds) { if (!string.Equals(personEntity.GetProviderId(id.Key), id.Value, StringComparison.OrdinalIgnoreCase)) { personEntity.SetProviderId(id.Key, id.Value); - saveEntity = true; + personsToSave.Add(personEntity); } } @@ -260,18 +260,17 @@ namespace MediaBrowser.Providers.Manager }, 0); - saveEntity = true; - } - - if (saveEntity) - { - personsToSave.Add(personEntity); + personsToSaveWithImages.Add(personEntity); } } } + // This is a little ugly, but it saves a lot of I/O with the db by doing this in bulk. + // To avoid updating images for no reason, we differentiate between the two item update types. LibraryManager.RunMetadataSavers(personsToSave, ItemUpdateType.MetadataDownload); - LibraryManager.CreateItems(personsToSave, null, CancellationToken.None); + LibraryManager.RunMetadataSavers(personsToSaveWithImages, ItemUpdateType.ImageUpdate); + + LibraryManager.CreateItems(personsToSave.Concat(personsToSaveWithImages).ToList(), null, CancellationToken.None); return Task.CompletedTask; } From dd238937f4739291dd4eab337d8be00194951761 Mon Sep 17 00:00:00 2001 From: cvium Date: Thu, 10 Dec 2020 13:44:58 +0100 Subject: [PATCH 2/4] Remember to await the async task --- MediaBrowser.Providers/Manager/MetadataService.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index e78eae4fd1..a1cd0b2eac 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -229,7 +229,7 @@ namespace MediaBrowser.Providers.Manager await result.Item.UpdateToRepositoryAsync(reason, cancellationToken).ConfigureAwait(false); } - private Task SavePeopleMetadataAsync(List people, LibraryOptions libraryOptions, CancellationToken cancellationToken) + private async Task SavePeopleMetadataAsync(List people, LibraryOptions libraryOptions, CancellationToken cancellationToken) { var personsToSave = new List(); var personsToSaveWithImages = new List(); @@ -267,11 +267,10 @@ namespace MediaBrowser.Providers.Manager // This is a little ugly, but it saves a lot of I/O with the db by doing this in bulk. // To avoid updating images for no reason, we differentiate between the two item update types. - LibraryManager.RunMetadataSavers(personsToSave, ItemUpdateType.MetadataDownload); - LibraryManager.RunMetadataSavers(personsToSaveWithImages, ItemUpdateType.ImageUpdate); + await LibraryManager.RunMetadataSavers(personsToSave, ItemUpdateType.MetadataDownload).ConfigureAwait(false); + await LibraryManager.RunMetadataSavers(personsToSaveWithImages, ItemUpdateType.ImageUpdate).ConfigureAwait(false); LibraryManager.CreateItems(personsToSave.Concat(personsToSaveWithImages).ToList(), null, CancellationToken.None); - return Task.CompletedTask; } protected virtual Task AfterMetadataRefresh(TItemType item, MetadataRefreshOptions refreshOptions, CancellationToken cancellationToken) From 3f6e6c48393f0cabadb10f95cd3b98b8a1cdd7b3 Mon Sep 17 00:00:00 2001 From: cvium Date: Thu, 10 Dec 2020 14:47:47 +0100 Subject: [PATCH 3/4] Simplify --- .../Library/LibraryManager.cs | 21 +++++++++---------- .../Library/ILibraryManager.cs | 2 +- .../Manager/MetadataService.cs | 21 +++++++++++-------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 47184b7892..db27862ce7 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -42,7 +42,6 @@ using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; using MediaBrowser.Model.Library; -using MediaBrowser.Model.Net; using MediaBrowser.Model.Querying; using MediaBrowser.Model.Tasks; using MediaBrowser.Providers.MediaInfo; @@ -1957,7 +1956,10 @@ namespace Emby.Server.Implementations.Library /// public async Task UpdateItemsAsync(IReadOnlyList items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) { - await RunMetadataSavers(items, updateReason).ConfigureAwait(false); + foreach (var item in items) + { + await RunMetadataSavers(item, updateReason).ConfigureAwait(false); + } _itemRepository.SaveItems(items, cancellationToken); @@ -1994,19 +1996,16 @@ namespace Emby.Server.Implementations.Library public Task UpdateItemAsync(BaseItem item, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken) => UpdateItemsAsync(new[] { item }, parent, updateReason, cancellationToken); - public async Task RunMetadataSavers(IReadOnlyList items, ItemUpdateType updateReason) + public Task RunMetadataSavers(BaseItem item, ItemUpdateType updateReason) { - foreach (var item in items) + if (item.IsFileProtocol) { - if (item.IsFileProtocol) - { - ProviderManager.SaveMetadata(item, updateReason); - } + ProviderManager.SaveMetadata(item, updateReason); + } - item.DateLastSaved = DateTime.UtcNow; + item.DateLastSaved = DateTime.UtcNow; - await UpdateImagesAsync(item, updateReason >= ItemUpdateType.ImageUpdate).ConfigureAwait(false); - } + return UpdateImagesAsync(item, updateReason >= ItemUpdateType.ImageUpdate); } /// diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index ff80952f76..6700761fcc 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -571,7 +571,7 @@ namespace MediaBrowser.Controller.Library string videoPath, string[] files); - Task RunMetadataSavers(IReadOnlyList items, ItemUpdateType updateReason); + Task RunMetadataSavers(BaseItem item, ItemUpdateType updateReason); BaseItem GetParentItem(string parentId, Guid? userId); diff --git a/MediaBrowser.Providers/Manager/MetadataService.cs b/MediaBrowser.Providers/Manager/MetadataService.cs index a1cd0b2eac..8b3ca17ca1 100644 --- a/MediaBrowser.Providers/Manager/MetadataService.cs +++ b/MediaBrowser.Providers/Manager/MetadataService.cs @@ -232,7 +232,6 @@ namespace MediaBrowser.Providers.Manager private async Task SavePeopleMetadataAsync(List people, LibraryOptions libraryOptions, CancellationToken cancellationToken) { var personsToSave = new List(); - var personsToSaveWithImages = new List(); foreach (var person in people) { @@ -240,13 +239,15 @@ namespace MediaBrowser.Providers.Manager if (person.ProviderIds.Count > 0 || !string.IsNullOrWhiteSpace(person.ImageUrl)) { + var itemUpdateType = ItemUpdateType.MetadataDownload; + var saveEntity = false; var personEntity = LibraryManager.GetPerson(person.Name); foreach (var id in person.ProviderIds) { if (!string.Equals(personEntity.GetProviderId(id.Key), id.Value, StringComparison.OrdinalIgnoreCase)) { personEntity.SetProviderId(id.Key, id.Value); - personsToSave.Add(personEntity); + saveEntity = true; } } @@ -260,17 +261,19 @@ namespace MediaBrowser.Providers.Manager }, 0); - personsToSaveWithImages.Add(personEntity); + saveEntity = true; + itemUpdateType = ItemUpdateType.ImageUpdate; + } + + if (saveEntity) + { + personsToSave.Add(personEntity); + await LibraryManager.RunMetadataSavers(personEntity, itemUpdateType).ConfigureAwait(false); } } } - // This is a little ugly, but it saves a lot of I/O with the db by doing this in bulk. - // To avoid updating images for no reason, we differentiate between the two item update types. - await LibraryManager.RunMetadataSavers(personsToSave, ItemUpdateType.MetadataDownload).ConfigureAwait(false); - await LibraryManager.RunMetadataSavers(personsToSaveWithImages, ItemUpdateType.ImageUpdate).ConfigureAwait(false); - - LibraryManager.CreateItems(personsToSave.Concat(personsToSaveWithImages).ToList(), null, CancellationToken.None); + LibraryManager.CreateItems(personsToSave, null, CancellationToken.None); } protected virtual Task AfterMetadataRefresh(TItemType item, MetadataRefreshOptions refreshOptions, CancellationToken cancellationToken) From 15a3d8d62680fb7eec911ca65ac1cd1ef12be507 Mon Sep 17 00:00:00 2001 From: cvium Date: Thu, 10 Dec 2020 18:02:12 +0100 Subject: [PATCH 4/4] Forgot to revert some other changes, dunno if needed --- MediaBrowser.Controller/Entities/BaseItem.cs | 1 + MediaBrowser.Controller/Entities/Folder.cs | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index d8fad3bfbb..1b25fbdbb7 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -1385,6 +1385,7 @@ namespace MediaBrowser.Controller.Entities new List(); var ownedItemsChanged = await RefreshedOwnedItems(options, files, cancellationToken).ConfigureAwait(false); + await LibraryManager.UpdateImagesAsync(this).ConfigureAwait(false); // ensure all image properties in DB are fresh if (ownedItemsChanged) { diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index 23f4c00c10..57d04ddfac 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -354,6 +354,11 @@ namespace MediaBrowser.Controller.Entities { await currentChild.UpdateToRepositoryAsync(ItemUpdateType.MetadataImport, cancellationToken).ConfigureAwait(false); } + else + { + // metadata is up-to-date; make sure DB has correct images dimensions and hash + await LibraryManager.UpdateImagesAsync(currentChild).ConfigureAwait(false); + } continue; }