From b9fc0d26287e46017515e4ac3e569ca2c60f622f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Higueras?= Date: Mon, 23 Mar 2020 20:05:49 +0100 Subject: [PATCH 01/24] Add BlurHash support to backend --- Emby.Drawing/ImageProcessor.cs | 4 ++ Emby.Drawing/NullImageEncoder.cs | 6 +++ .../Data/SqliteItemRepository.cs | 17 ++++++- Emby.Server.Implementations/Dto/DtoService.cs | 7 +++ .../Library/LibraryManager.cs | 30 +++++++++++- .../Jellyfin.Drawing.Skia.csproj | 2 + Jellyfin.Drawing.Skia/SkiaEncoder.cs | 48 ++++++++++++++++++- MediaBrowser.Api/Images/ImageService.cs | 7 ++- .../Drawing/IImageEncoder.cs | 7 +++ .../Drawing/IImageProcessor.cs | 7 +++ MediaBrowser.Controller/Entities/BaseItem.cs | 1 + .../Entities/ItemImageInfo.cs | 6 +++ MediaBrowser.Model/Dto/BaseItemDto.cs | 6 +++ MediaBrowser.Model/Dto/ImageInfo.cs | 6 +++ 14 files changed, 149 insertions(+), 5 deletions(-) diff --git a/Emby.Drawing/ImageProcessor.cs b/Emby.Drawing/ImageProcessor.cs index 0b3bbe29ef..1237b603b6 100644 --- a/Emby.Drawing/ImageProcessor.cs +++ b/Emby.Drawing/ImageProcessor.cs @@ -313,6 +313,10 @@ namespace Emby.Drawing public ImageDimensions GetImageDimensions(string path) => _imageEncoder.GetImageSize(path); + /// + public string GetImageHash(string path) + => _imageEncoder.GetImageHash(path); + /// public string GetImageCacheTag(BaseItem item, ItemImageInfo image) => (item.Path + image.DateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture); diff --git a/Emby.Drawing/NullImageEncoder.cs b/Emby.Drawing/NullImageEncoder.cs index 5af7f16225..fa89b4c638 100644 --- a/Emby.Drawing/NullImageEncoder.cs +++ b/Emby.Drawing/NullImageEncoder.cs @@ -42,5 +42,11 @@ namespace Emby.Drawing { throw new NotImplementedException(); } + + /// + public string GetImageHash(string inputPath) + { + throw new NotImplementedException(); + } } } diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs index ca5cd6fdd5..5a43a138bd 100644 --- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs @@ -1144,12 +1144,18 @@ namespace Emby.Server.Implementations.Data var delimeter = "*"; var path = image.Path; + var hash = image.Hash; if (path == null) { path = string.Empty; } + if (hash == null) + { + hash = string.Empty; + } + return GetPathToSave(path) + delimeter + image.DateModified.Ticks.ToString(CultureInfo.InvariantCulture) + @@ -1158,7 +1164,11 @@ namespace Emby.Server.Implementations.Data delimeter + image.Width.ToString(CultureInfo.InvariantCulture) + delimeter + - image.Height.ToString(CultureInfo.InvariantCulture); + image.Height.ToString(CultureInfo.InvariantCulture) + + delimeter + + // Replace delimiters with other characters. + // This can be removed when we migrate to a proper DB. + hash.Replace('*', '/').Replace('|', '\\'); } public ItemImageInfo ItemImageInfoFromValueString(string value) @@ -1192,6 +1202,11 @@ namespace Emby.Server.Implementations.Data image.Width = width; image.Height = height; } + + if (parts.Length >= 6) + { + image.Hash = parts[5].Replace('/', '*').Replace('\\', '|'); + } } return image; diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index c4b65d2654..a34a3a192f 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -718,6 +718,7 @@ namespace Emby.Server.Implementations.Dto if (options.EnableImages) { dto.ImageTags = new Dictionary(); + dto.ImageHashes = new Dictionary(); // Prevent implicitly captured closure var currentItem = item; @@ -732,6 +733,12 @@ namespace Emby.Server.Implementations.Dto { dto.ImageTags[image.Type] = tag; } + + var hash = image.Hash; + if (hash != null && hash.Length > 0) + { + dto.ImageHashes[tag] = image.Hash; + } } } } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 0b86b2db7e..bc35b04106 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -21,6 +21,7 @@ using MediaBrowser.Common.Extensions; using MediaBrowser.Common.Progress; using MediaBrowser.Controller; using MediaBrowser.Controller.Configuration; +using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Dto; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Entities.Audio; @@ -35,6 +36,7 @@ using MediaBrowser.Controller.Resolvers; using MediaBrowser.Controller.Sorting; using MediaBrowser.Model.Configuration; using MediaBrowser.Model.Dlna; +using MediaBrowser.Model.Drawing; using MediaBrowser.Model.Dto; using MediaBrowser.Model.Entities; using MediaBrowser.Model.IO; @@ -109,6 +111,18 @@ namespace Emby.Server.Implementations.Library /// The comparers. private IBaseItemComparer[] Comparers { get; set; } + /// + /// Gets or sets the active item repository + /// + /// The item repository. + public IItemRepository ItemRepository { get; set; } + + /// + /// Gets or sets the active image processor + /// + /// The image processor. + public IImageProcessor ImageProcessor { get; set; } + /// /// Occurs when [item added]. /// @@ -1817,7 +1831,19 @@ namespace Emby.Server.Implementations.Library public void UpdateImages(BaseItem item) { - _itemRepository.SaveImages(item); + item.ImageInfos + .Where(i => (i.Width == 0 || i.Height == 0)) + .ToList() + .ForEach(x => + { + string blurhash = ImageProcessor.GetImageHash(x.Path); + ImageDimensions size = ImageProcessor.GetImageDimensions(item, x, true); + x.Width = size.Width; + x.Height = size.Height; + x.Hash = blurhash; + }); + + ItemRepository.SaveImages(item); RegisterItem(item); } @@ -1839,7 +1865,7 @@ namespace Emby.Server.Implementations.Library item.DateLastSaved = DateTime.UtcNow; - RegisterItem(item); + UpdateImages(item); } _itemRepository.SaveItems(itemsList, cancellationToken); diff --git a/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj b/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj index a6e1f490ad..9f0e3a0042 100644 --- a/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj +++ b/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj @@ -21,6 +21,8 @@ + + diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 5c7462ee29..1d73078638 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -1,7 +1,11 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Blurhash.Core; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Extensions; @@ -15,7 +19,7 @@ namespace Jellyfin.Drawing.Skia /// /// Image encoder that uses to manipulate images. /// - public class SkiaEncoder : IImageEncoder + public class SkiaEncoder : CoreEncoder, IImageEncoder { private static readonly HashSet _transparentImageTypes = new HashSet(StringComparer.OrdinalIgnoreCase) { ".png", ".gif", ".webp" }; @@ -229,6 +233,48 @@ namespace Jellyfin.Drawing.Skia } } + /// + /// The path is null. + /// The path is not valid. + /// The file at the specified path could not be used to generate a codec. + [SuppressMessage("Microsoft.Performance", "CA1814:PreferJaggedArraysOverMultidimensional")] + public string GetImageHash(string path) + { + if (path == null) + { + throw new ArgumentNullException(nameof(path)); + } + + if (!File.Exists(path)) + { + throw new FileNotFoundException("File not found", path); + } + + using (var bitmap = GetBitmap(path, false, false, null)) + { + if (bitmap == null) + { + throw new ArgumentOutOfRangeException($"Skia unable to read image {path}"); + } + + var width = bitmap.Width; + var height = bitmap.Height; + var pixels = new Pixel[width, height]; + Parallel.ForEach(Enumerable.Range(0, height), y => + { + for (var x = 0; x < width; x++) + { + var color = bitmap.GetPixel(x, y); + pixels[x, y].Red = MathUtils.SRgbToLinear(color.Red); + pixels[x, y].Green = MathUtils.SRgbToLinear(color.Green); + pixels[x, y].Blue = MathUtils.SRgbToLinear(color.Blue); + } + }); + + return CoreEncode(pixels, 4, 4); + } + } + private static bool HasDiacritics(string text) => !string.Equals(text, text.RemoveDiacritics(), StringComparison.Ordinal); diff --git a/MediaBrowser.Api/Images/ImageService.cs b/MediaBrowser.Api/Images/ImageService.cs index 2e9b3e6cb4..ecfe2ed763 100644 --- a/MediaBrowser.Api/Images/ImageService.cs +++ b/MediaBrowser.Api/Images/ImageService.cs @@ -323,6 +323,7 @@ namespace MediaBrowser.Api.Images { int? width = null; int? height = null; + string? blurhash = null; long length = 0; try @@ -332,7 +333,10 @@ namespace MediaBrowser.Api.Images var fileInfo = _fileSystem.GetFileInfo(info.Path); length = fileInfo.Length; - ImageDimensions size = _imageProcessor.GetImageDimensions(item, info); + blurhash = _imageProcessor.GetImageHash(info.Path); + info.Hash = blurhash; // TODO: this doesn't seem like the right thing to do + + ImageDimensions size = _imageProcessor.GetImageDimensions(item, info, true); _libraryManager.UpdateImages(item); width = size.Width; height = size.Height; @@ -358,6 +362,7 @@ namespace MediaBrowser.Api.Images ImageType = info.Type, ImageTag = _imageProcessor.GetImageCacheTag(item, info), Size = length, + Hash = blurhash, Width = width, Height = height }; diff --git a/MediaBrowser.Controller/Drawing/IImageEncoder.cs b/MediaBrowser.Controller/Drawing/IImageEncoder.cs index 88e67b6486..1d3f0d3b44 100644 --- a/MediaBrowser.Controller/Drawing/IImageEncoder.cs +++ b/MediaBrowser.Controller/Drawing/IImageEncoder.cs @@ -43,6 +43,13 @@ namespace MediaBrowser.Controller.Drawing /// The image dimensions. ImageDimensions GetImageSize(string path); + /// + /// Get the blurhash of an image. + /// + /// The filepath of the image. + /// The blurhash. + string GetImageHash(string path); + /// /// Encode an image. /// diff --git a/MediaBrowser.Controller/Drawing/IImageProcessor.cs b/MediaBrowser.Controller/Drawing/IImageProcessor.cs index 36c746624e..be5906cbc9 100644 --- a/MediaBrowser.Controller/Drawing/IImageProcessor.cs +++ b/MediaBrowser.Controller/Drawing/IImageProcessor.cs @@ -32,6 +32,13 @@ namespace MediaBrowser.Controller.Drawing /// ImageDimensions ImageDimensions GetImageDimensions(string path); + /// + /// Gets the blurhash of the image. + /// + /// Path to the image file. + /// BlurHash + String GetImageHash(string path); + /// /// Gets the dimensions of the image. /// diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index 7ed8fa7671..e5f6ea09d6 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -2222,6 +2222,7 @@ namespace MediaBrowser.Controller.Entities existingImage.DateModified = image.DateModified; existingImage.Width = image.Width; existingImage.Height = image.Height; + existingImage.Hash = image.Hash; } else { diff --git a/MediaBrowser.Controller/Entities/ItemImageInfo.cs b/MediaBrowser.Controller/Entities/ItemImageInfo.cs index fc46dec2ef..ba02971073 100644 --- a/MediaBrowser.Controller/Entities/ItemImageInfo.cs +++ b/MediaBrowser.Controller/Entities/ItemImageInfo.cs @@ -28,6 +28,12 @@ namespace MediaBrowser.Controller.Entities public int Height { get; set; } + /// + /// Gets or sets the blurhash. + /// + /// The blurhash. + public string Hash { get; set; } + [JsonIgnore] public bool IsLocalFile => Path == null || !Path.StartsWith("http", StringComparison.OrdinalIgnoreCase); } diff --git a/MediaBrowser.Model/Dto/BaseItemDto.cs b/MediaBrowser.Model/Dto/BaseItemDto.cs index 607355d8d3..8c6c9683a4 100644 --- a/MediaBrowser.Model/Dto/BaseItemDto.cs +++ b/MediaBrowser.Model/Dto/BaseItemDto.cs @@ -510,6 +510,12 @@ namespace MediaBrowser.Model.Dto /// The series thumb image tag. public string SeriesThumbImageTag { get; set; } + /// + /// Gets or sets the blurhash for the image tags. + /// + /// The blurhashes. + public Dictionary ImageHashes { get; set; } + /// /// Gets or sets the series studio. /// diff --git a/MediaBrowser.Model/Dto/ImageInfo.cs b/MediaBrowser.Model/Dto/ImageInfo.cs index 57942ac23b..39bdc09ed8 100644 --- a/MediaBrowser.Model/Dto/ImageInfo.cs +++ b/MediaBrowser.Model/Dto/ImageInfo.cs @@ -30,6 +30,12 @@ namespace MediaBrowser.Model.Dto /// The path. public string Path { get; set; } + /// + /// Gets or sets the blurhash. + /// + /// The blurhash. + public string Hash { get; set; } + /// /// Gets or sets the height. /// From fe480caf5486a21d7ef152009e5c3e08364e0f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jes=C3=BAs=20Higueras?= Date: Wed, 25 Mar 2020 17:26:53 +0100 Subject: [PATCH 02/24] Add endpoint to update all items in library --- .../Library/LibraryManager.cs | 28 +++++++++++++++++++ MediaBrowser.Api/ItemUpdateService.cs | 6 ++++ .../Library/ILibraryManager.cs | 1 + 3 files changed, 35 insertions(+) diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index bc35b04106..0a97e007cd 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1911,6 +1911,34 @@ namespace Emby.Server.Implementations.Library UpdateItems(new[] { item }, parent, updateReason, cancellationToken); } + /// + /// Updates everything in the database. + /// + public void UpdateAll() + { + Task.Run(() => + { + var items = ItemRepository.GetItemList(new InternalItemsQuery { + Recursive = true + }); + foreach (var item in items) + { + _logger.LogDebug("Updating item {Name} ({ItemId})", + item.Name, + item.Id); + try + { + item.UpdateToRepository(ItemUpdateType.MetadataEdit, CancellationToken.None); + } + catch (Exception ex) + { + _logger.LogError(ex, "Updating item {ItemId} failed", item.Id); + } + } + _logger.LogDebug("All items have been updated"); + }); + } + /// /// Reports the item removed. /// diff --git a/MediaBrowser.Api/ItemUpdateService.cs b/MediaBrowser.Api/ItemUpdateService.cs index 2db6d717aa..808627b41a 100644 --- a/MediaBrowser.Api/ItemUpdateService.cs +++ b/MediaBrowser.Api/ItemUpdateService.cs @@ -198,6 +198,12 @@ namespace MediaBrowser.Api public void Post(UpdateItem request) { + if (request.ItemId == "*") + { + // Special case: Refresh everything in database. Probably not a great idea to run often. + _libraryManager.UpdateAll(); + return; + } var item = _libraryManager.GetItemById(request.ItemId); var newLockData = request.LockData ?? false; diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index 2e1c97f674..559a5415fb 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -196,6 +196,7 @@ namespace MediaBrowser.Controller.Library /// void UpdateItems(IEnumerable items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken); void UpdateItem(BaseItem item, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken); + void UpdateAll(); /// /// Retrieves the item. From 02da312f8aaf9975f31291fd65687f637e38530c Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 19 May 2020 00:22:52 +0300 Subject: [PATCH 03/24] Fix compilation after rebase --- Emby.Server.Implementations/Library/LibraryManager.cs | 2 +- MediaBrowser.Api/Images/ImageService.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 0a97e007cd..c31fd5bf98 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1837,7 +1837,7 @@ namespace Emby.Server.Implementations.Library .ForEach(x => { string blurhash = ImageProcessor.GetImageHash(x.Path); - ImageDimensions size = ImageProcessor.GetImageDimensions(item, x, true); + ImageDimensions size = ImageProcessor.GetImageDimensions(item, x); x.Width = size.Width; x.Height = size.Height; x.Hash = blurhash; diff --git a/MediaBrowser.Api/Images/ImageService.cs b/MediaBrowser.Api/Images/ImageService.cs index ecfe2ed763..09b99781bf 100644 --- a/MediaBrowser.Api/Images/ImageService.cs +++ b/MediaBrowser.Api/Images/ImageService.cs @@ -336,7 +336,7 @@ namespace MediaBrowser.Api.Images blurhash = _imageProcessor.GetImageHash(info.Path); info.Hash = blurhash; // TODO: this doesn't seem like the right thing to do - ImageDimensions size = _imageProcessor.GetImageDimensions(item, info, true); + ImageDimensions size = _imageProcessor.GetImageDimensions(item, info); _libraryManager.UpdateImages(item); width = size.Width; height = size.Height; From bfb644d5f50fff9fd8ba5fe4504c73ec5be851af Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 19 May 2020 13:45:39 +0300 Subject: [PATCH 04/24] Fix nullref exception --- .vscode/tasks.json | 13 ++++++++++++- .../Library/LibraryManager.cs | 10 ++-------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index ac517e10c6..7475617c93 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -10,6 +10,17 @@ "${workspaceFolder}/Jellyfin.Server/Jellyfin.Server.csproj" ], "problemMatcher": "$msCompile" + }, + { + "label": "api tests", + "command": "dotnet", + "type": "process", + "args": [ + "test", + "${workspaceFolder}/tests/MediaBrowser.Api.Tests/MediaBrowser.Api.Tests.csproj" + ], + "problemMatcher": "$msCompile" } + ] -} \ No newline at end of file +} diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index c31fd5bf98..e1cb282ccb 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -111,12 +111,6 @@ namespace Emby.Server.Implementations.Library /// The comparers. private IBaseItemComparer[] Comparers { get; set; } - /// - /// Gets or sets the active item repository - /// - /// The item repository. - public IItemRepository ItemRepository { get; set; } - /// /// Gets or sets the active image processor /// @@ -1843,7 +1837,7 @@ namespace Emby.Server.Implementations.Library x.Hash = blurhash; }); - ItemRepository.SaveImages(item); + _itemRepository.SaveImages(item); RegisterItem(item); } @@ -1918,7 +1912,7 @@ namespace Emby.Server.Implementations.Library { Task.Run(() => { - var items = ItemRepository.GetItemList(new InternalItemsQuery { + var items = _itemRepository.GetItemList(new InternalItemsQuery { Recursive = true }); foreach (var item in items) From f18293bf762c86581153ab8d9b1b6267421178a9 Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 19 May 2020 13:46:00 +0300 Subject: [PATCH 05/24] Switch to BlurHashSharp lib which should be faster --- .../Jellyfin.Drawing.Skia.csproj | 4 +-- Jellyfin.Drawing.Skia/SkiaEncoder.cs | 28 ++----------------- 2 files changed, 5 insertions(+), 27 deletions(-) diff --git a/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj b/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj index 9f0e3a0042..221346b688 100644 --- a/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj +++ b/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj @@ -18,11 +18,11 @@ + + - - diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 1d73078638..d2da0cf17e 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -5,7 +5,7 @@ using System.Globalization; using System.IO; using System.Linq; using System.Threading.Tasks; -using Blurhash.Core; +using BlurHashSharp.SkiaSharp; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Drawing; using MediaBrowser.Controller.Extensions; @@ -19,7 +19,7 @@ namespace Jellyfin.Drawing.Skia /// /// Image encoder that uses to manipulate images. /// - public class SkiaEncoder : CoreEncoder, IImageEncoder + public class SkiaEncoder : IImageEncoder { private static readonly HashSet _transparentImageTypes = new HashSet(StringComparer.OrdinalIgnoreCase) { ".png", ".gif", ".webp" }; @@ -250,29 +250,7 @@ namespace Jellyfin.Drawing.Skia throw new FileNotFoundException("File not found", path); } - using (var bitmap = GetBitmap(path, false, false, null)) - { - if (bitmap == null) - { - throw new ArgumentOutOfRangeException($"Skia unable to read image {path}"); - } - - var width = bitmap.Width; - var height = bitmap.Height; - var pixels = new Pixel[width, height]; - Parallel.ForEach(Enumerable.Range(0, height), y => - { - for (var x = 0; x < width; x++) - { - var color = bitmap.GetPixel(x, y); - pixels[x, y].Red = MathUtils.SRgbToLinear(color.Red); - pixels[x, y].Green = MathUtils.SRgbToLinear(color.Green); - pixels[x, y].Blue = MathUtils.SRgbToLinear(color.Blue); - } - }); - - return CoreEncode(pixels, 4, 4); - } + return BlurHashSharp.SkiaSharp.BlurHashEncoder.Encode(4, 4, path); } private static bool HasDiacritics(string text) From a226a4ee03d974615a6fa26b936a93458a255b70 Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 19 May 2020 14:50:14 +0300 Subject: [PATCH 06/24] Compute hash only when one is not computed in DB, small optimizations here and there --- .../Data/SqliteItemRepository.cs | 16 +---- .../Library/LibraryManager.cs | 59 +++++++------------ MediaBrowser.Api/Images/ImageService.cs | 22 ++++--- MediaBrowser.Api/ItemUpdateService.cs | 6 -- .../Library/ILibraryManager.cs | 2 +- 5 files changed, 37 insertions(+), 68 deletions(-) diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs index 5a43a138bd..10eb96b10c 100644 --- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs @@ -1141,20 +1141,10 @@ namespace Emby.Server.Implementations.Data public string ToValueString(ItemImageInfo image) { - var delimeter = "*"; + const string delimeter = "*"; - var path = image.Path; - var hash = image.Hash; - - if (path == null) - { - path = string.Empty; - } - - if (hash == null) - { - hash = string.Empty; - } + var path = image.Path ?? string.Empty; + var hash = image.Hash ?? string.Empty; return GetPathToSave(path) + delimeter + diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index e1cb282ccb..c48664a316 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1825,17 +1825,26 @@ namespace Emby.Server.Implementations.Library public void UpdateImages(BaseItem item) { - item.ImageInfos - .Where(i => (i.Width == 0 || i.Height == 0)) - .ToList() - .ForEach(x => - { - string blurhash = ImageProcessor.GetImageHash(x.Path); - ImageDimensions size = ImageProcessor.GetImageDimensions(item, x); - x.Width = size.Width; - x.Height = size.Height; - x.Hash = blurhash; - }); + if (item == null) + { + throw new ArgumentNullException(nameof(item)); + } + + var outdated = item.ImageInfos + .Where(i => (i.Width == 0 || i.Height == 0 || string.IsNullOrEmpty(i.Hash))) + .ToList(); + if (outdated.Count == 0) + { + return; + } + + outdated.ForEach(img => + { + ImageDimensions size = ImageProcessor.GetImageDimensions(item, img); + img.Width = size.Width; + img.Height = size.Height; + img.Hash = ImageProcessor.GetImageHash(img.Path); + }); _itemRepository.SaveImages(item); @@ -1905,34 +1914,6 @@ namespace Emby.Server.Implementations.Library UpdateItems(new[] { item }, parent, updateReason, cancellationToken); } - /// - /// Updates everything in the database. - /// - public void UpdateAll() - { - Task.Run(() => - { - var items = _itemRepository.GetItemList(new InternalItemsQuery { - Recursive = true - }); - foreach (var item in items) - { - _logger.LogDebug("Updating item {Name} ({ItemId})", - item.Name, - item.Id); - try - { - item.UpdateToRepository(ItemUpdateType.MetadataEdit, CancellationToken.None); - } - catch (Exception ex) - { - _logger.LogError(ex, "Updating item {ItemId} failed", item.Id); - } - } - _logger.LogDebug("All items have been updated"); - }); - } - /// /// Reports the item removed. /// diff --git a/MediaBrowser.Api/Images/ImageService.cs b/MediaBrowser.Api/Images/ImageService.cs index 09b99781bf..559db550b8 100644 --- a/MediaBrowser.Api/Images/ImageService.cs +++ b/MediaBrowser.Api/Images/ImageService.cs @@ -3,6 +3,7 @@ using System.Collections.Generic; using System.Globalization; using System.IO; using System.Linq; +using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; using MediaBrowser.Common.Extensions; @@ -280,9 +281,16 @@ namespace MediaBrowser.Api.Images public List GetItemImageInfos(BaseItem item) { var list = new List(); - var itemImages = item.ImageInfos; + if (itemImages.Length == 0) + { + // short-circuit + return list; + } + + _libraryManager.UpdateImages(item); // this makes sure dimensions and hashes are correct + foreach (var image in itemImages) { if (!item.AllowsMultipleImages(image.Type)) @@ -323,7 +331,7 @@ namespace MediaBrowser.Api.Images { int? width = null; int? height = null; - string? blurhash = null; + string blurhash = null; long length = 0; try @@ -333,13 +341,9 @@ namespace MediaBrowser.Api.Images var fileInfo = _fileSystem.GetFileInfo(info.Path); length = fileInfo.Length; - blurhash = _imageProcessor.GetImageHash(info.Path); - info.Hash = blurhash; // TODO: this doesn't seem like the right thing to do - - ImageDimensions size = _imageProcessor.GetImageDimensions(item, info); - _libraryManager.UpdateImages(item); - width = size.Width; - height = size.Height; + blurhash = info.Hash; + width = info.Width; + height = info.Height; if (width <= 0 || height <= 0) { diff --git a/MediaBrowser.Api/ItemUpdateService.cs b/MediaBrowser.Api/ItemUpdateService.cs index 808627b41a..2db6d717aa 100644 --- a/MediaBrowser.Api/ItemUpdateService.cs +++ b/MediaBrowser.Api/ItemUpdateService.cs @@ -198,12 +198,6 @@ namespace MediaBrowser.Api public void Post(UpdateItem request) { - if (request.ItemId == "*") - { - // Special case: Refresh everything in database. Probably not a great idea to run often. - _libraryManager.UpdateAll(); - return; - } var item = _libraryManager.GetItemById(request.ItemId); var newLockData = request.LockData ?? false; diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index 559a5415fb..81160efece 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -195,8 +195,8 @@ namespace MediaBrowser.Controller.Library /// Updates the item. /// void UpdateItems(IEnumerable items, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken); + void UpdateItem(BaseItem item, BaseItem parent, ItemUpdateType updateReason, CancellationToken cancellationToken); - void UpdateAll(); /// /// Retrieves the item. From 186b7f303cd6f95ca64e020c2838dfe2028ea54c Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 19 May 2020 14:56:52 +0300 Subject: [PATCH 07/24] More small optimizations --- Emby.Server.Implementations/Dto/DtoService.cs | 5 ++--- Emby.Server.Implementations/Library/LibraryManager.cs | 3 ++- Jellyfin.Drawing.Skia/SkiaEncoder.cs | 6 +----- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index a34a3a192f..07105786b9 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -722,8 +722,7 @@ namespace Emby.Server.Implementations.Dto // Prevent implicitly captured closure var currentItem = item; - foreach (var image in currentItem.ImageInfos.Where(i => !currentItem.AllowsMultipleImages(i.Type)) - .ToList()) + foreach (var image in currentItem.ImageInfos.Where(i => !currentItem.AllowsMultipleImages(i.Type))) { if (options.GetImageLimit(image.Type) > 0) { @@ -735,7 +734,7 @@ namespace Emby.Server.Implementations.Dto } var hash = image.Hash; - if (hash != null && hash.Length > 0) + if (!string.IsNullOrEmpty(hash)) { dto.ImageHashes[tag] = image.Hash; } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index c48664a316..9f412b725e 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1831,10 +1831,11 @@ namespace Emby.Server.Implementations.Library } var outdated = item.ImageInfos - .Where(i => (i.Width == 0 || i.Height == 0 || string.IsNullOrEmpty(i.Hash))) + .Where(i => (i.IsLocalFile && (i.Width == 0 || i.Height == 0 || string.IsNullOrEmpty(i.Hash)))) .ToList(); if (outdated.Count == 0) { + RegisterItem(item); return; } diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index d2da0cf17e..99091ea57e 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -1,10 +1,7 @@ using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.IO; -using System.Linq; -using System.Threading.Tasks; using BlurHashSharp.SkiaSharp; using MediaBrowser.Common.Configuration; using MediaBrowser.Controller.Drawing; @@ -237,7 +234,6 @@ namespace Jellyfin.Drawing.Skia /// The path is null. /// The path is not valid. /// The file at the specified path could not be used to generate a codec. - [SuppressMessage("Microsoft.Performance", "CA1814:PreferJaggedArraysOverMultidimensional")] public string GetImageHash(string path) { if (path == null) @@ -250,7 +246,7 @@ namespace Jellyfin.Drawing.Skia throw new FileNotFoundException("File not found", path); } - return BlurHashSharp.SkiaSharp.BlurHashEncoder.Encode(4, 4, path); + return BlurHashEncoder.Encode(4, 4, path); } private static bool HasDiacritics(string text) From 2b1ae7ac5834054866aa485b18f85f1793f7b8b4 Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 19 May 2020 15:42:50 +0300 Subject: [PATCH 08/24] Fix code smells --- Emby.Drawing/NullImageEncoder.cs | 2 +- .../Drawing/IImageProcessor.cs | 15 ++++++++------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Emby.Drawing/NullImageEncoder.cs b/Emby.Drawing/NullImageEncoder.cs index fa89b4c638..54de7212a3 100644 --- a/Emby.Drawing/NullImageEncoder.cs +++ b/Emby.Drawing/NullImageEncoder.cs @@ -44,7 +44,7 @@ namespace Emby.Drawing } /// - public string GetImageHash(string inputPath) + public string GetImageHash(string path) { throw new NotImplementedException(); } diff --git a/MediaBrowser.Controller/Drawing/IImageProcessor.cs b/MediaBrowser.Controller/Drawing/IImageProcessor.cs index be5906cbc9..e38eaf7606 100644 --- a/MediaBrowser.Controller/Drawing/IImageProcessor.cs +++ b/MediaBrowser.Controller/Drawing/IImageProcessor.cs @@ -32,13 +32,6 @@ namespace MediaBrowser.Controller.Drawing /// ImageDimensions ImageDimensions GetImageDimensions(string path); - /// - /// Gets the blurhash of the image. - /// - /// Path to the image file. - /// BlurHash - String GetImageHash(string path); - /// /// Gets the dimensions of the image. /// @@ -47,6 +40,13 @@ namespace MediaBrowser.Controller.Drawing /// ImageDimensions ImageDimensions GetImageDimensions(BaseItem item, ItemImageInfo info); + /// + /// Gets the blurhash of the image. + /// + /// Path to the image file. + /// BlurHash + String GetImageHash(string path); + /// /// Gets the image cache tag. /// @@ -54,6 +54,7 @@ namespace MediaBrowser.Controller.Drawing /// The image. /// Guid. string GetImageCacheTag(BaseItem item, ItemImageInfo image); + string GetImageCacheTag(BaseItem item, ChapterInfo info); /// From 3fb4c1356c22c8be03855a161140c21eb395086b Mon Sep 17 00:00:00 2001 From: Vasily Date: Wed, 20 May 2020 23:50:17 +0300 Subject: [PATCH 09/24] Make blurhash be computed during regular scans if it was not already computed --- Jellyfin.Drawing.Skia/SkiaEncoder.cs | 2 ++ MediaBrowser.Controller/Entities/BaseItem.cs | 1 + MediaBrowser.Controller/Entities/Folder.cs | 5 +++++ 3 files changed, 8 insertions(+) diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 99091ea57e..7ab0a54dfd 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -246,6 +246,8 @@ namespace Jellyfin.Drawing.Skia throw new FileNotFoundException("File not found", path); } + // Use 4 vertical and 4 horizontal components of DCT of the image. + // See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components return BlurHashEncoder.Encode(4, 4, path); } diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index e5f6ea09d6..035ab1dd94 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -1374,6 +1374,7 @@ namespace MediaBrowser.Controller.Entities new List(); var ownedItemsChanged = await RefreshedOwnedItems(options, files, cancellationToken).ConfigureAwait(false); + LibraryManager.UpdateImages(this); // ensure all image properties in DB are fresh if (ownedItemsChanged) { diff --git a/MediaBrowser.Controller/Entities/Folder.cs b/MediaBrowser.Controller/Entities/Folder.cs index a468e0c35a..29040f92e6 100644 --- a/MediaBrowser.Controller/Entities/Folder.cs +++ b/MediaBrowser.Controller/Entities/Folder.cs @@ -341,6 +341,11 @@ namespace MediaBrowser.Controller.Entities { currentChild.UpdateToRepository(ItemUpdateType.MetadataImport, cancellationToken); } + else + { + // metadata is up-to-date; make sure DB has correct images dimensions and hash + LibraryManager.UpdateImages(currentChild); + } continue; } From 8b517e9beffd5cf7b1e7ed2b82b0bdf63fe60f03 Mon Sep 17 00:00:00 2001 From: Vasily Date: Thu, 21 May 2020 00:03:22 +0300 Subject: [PATCH 10/24] Fix nullref for imageProcessor in LibraryManager --- .../Library/LibraryManager.cs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 9f412b725e..75350ac2a8 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -69,6 +69,8 @@ namespace Emby.Server.Implementations.Library private readonly IFileSystem _fileSystem; private readonly IItemRepository _itemRepository; private readonly ConcurrentDictionary _libraryItemsCache; + private readonly IImageProcessor _imageProcessor; + private NamingOptions _namingOptions; private string[] _videoFileExtensions; @@ -111,12 +113,6 @@ namespace Emby.Server.Implementations.Library /// The comparers. private IBaseItemComparer[] Comparers { get; set; } - /// - /// Gets or sets the active image processor - /// - /// The image processor. - public IImageProcessor ImageProcessor { get; set; } - /// /// Occurs when [item added]. /// @@ -155,7 +151,8 @@ namespace Emby.Server.Implementations.Library Lazy providerManagerFactory, Lazy userviewManagerFactory, IMediaEncoder mediaEncoder, - IItemRepository itemRepository) + IItemRepository itemRepository, + IImageProcessor imageProcessor) { _appHost = appHost; _logger = logger; @@ -169,6 +166,7 @@ namespace Emby.Server.Implementations.Library _userviewManagerFactory = userviewManagerFactory; _mediaEncoder = mediaEncoder; _itemRepository = itemRepository; + _imageProcessor = imageProcessor; _libraryItemsCache = new ConcurrentDictionary(); @@ -1841,10 +1839,10 @@ namespace Emby.Server.Implementations.Library outdated.ForEach(img => { - ImageDimensions size = ImageProcessor.GetImageDimensions(item, img); + ImageDimensions size = _imageProcessor.GetImageDimensions(item, img); img.Width = size.Width; img.Height = size.Height; - img.Hash = ImageProcessor.GetImageHash(img.Path); + img.Hash = _imageProcessor.GetImageHash(img.Path); }); _itemRepository.SaveImages(item); From 1f83a212886bae879c47b4b4f5b1eb25a28e2ad3 Mon Sep 17 00:00:00 2001 From: Vasily Date: Thu, 21 May 2020 01:43:19 +0300 Subject: [PATCH 11/24] Rename Hash to BlurHash in all properties and methods for clarity --- Emby.Drawing/ImageProcessor.cs | 2 +- Emby.Server.Implementations/Data/SqliteItemRepository.cs | 4 ++-- Emby.Server.Implementations/Dto/DtoService.cs | 6 +++--- Emby.Server.Implementations/Library/LibraryManager.cs | 4 ++-- MediaBrowser.Api/Images/ImageService.cs | 2 +- MediaBrowser.Controller/Drawing/IImageProcessor.cs | 2 +- MediaBrowser.Controller/Entities/BaseItem.cs | 2 +- MediaBrowser.Controller/Entities/ItemImageInfo.cs | 2 +- MediaBrowser.Model/Dto/BaseItemDto.cs | 2 +- 9 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Emby.Drawing/ImageProcessor.cs b/Emby.Drawing/ImageProcessor.cs index 1237b603b6..35da6f6359 100644 --- a/Emby.Drawing/ImageProcessor.cs +++ b/Emby.Drawing/ImageProcessor.cs @@ -314,7 +314,7 @@ namespace Emby.Drawing => _imageEncoder.GetImageSize(path); /// - public string GetImageHash(string path) + public string GetImageBlurHash(string path) => _imageEncoder.GetImageHash(path); /// diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs index 10eb96b10c..dd60dd2228 100644 --- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs @@ -1144,7 +1144,7 @@ namespace Emby.Server.Implementations.Data const string delimeter = "*"; var path = image.Path ?? string.Empty; - var hash = image.Hash ?? string.Empty; + var hash = image.BlurHash ?? string.Empty; return GetPathToSave(path) + delimeter + @@ -1195,7 +1195,7 @@ namespace Emby.Server.Implementations.Data if (parts.Length >= 6) { - image.Hash = parts[5].Replace('/', '*').Replace('\\', '|'); + image.BlurHash = parts[5].Replace('/', '*').Replace('\\', '|'); } } diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index 07105786b9..593e7be7db 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -718,7 +718,7 @@ namespace Emby.Server.Implementations.Dto if (options.EnableImages) { dto.ImageTags = new Dictionary(); - dto.ImageHashes = new Dictionary(); + dto.ImageBlurHashes = new Dictionary(); // Prevent implicitly captured closure var currentItem = item; @@ -733,10 +733,10 @@ namespace Emby.Server.Implementations.Dto dto.ImageTags[image.Type] = tag; } - var hash = image.Hash; + var hash = image.BlurHash; if (!string.IsNullOrEmpty(hash)) { - dto.ImageHashes[tag] = image.Hash; + dto.ImageBlurHashes[tag] = image.BlurHash; } } } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 75350ac2a8..579fb7eddf 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1829,7 +1829,7 @@ namespace Emby.Server.Implementations.Library } var outdated = item.ImageInfos - .Where(i => (i.IsLocalFile && (i.Width == 0 || i.Height == 0 || string.IsNullOrEmpty(i.Hash)))) + .Where(i => (i.IsLocalFile && (i.Width == 0 || i.Height == 0 || string.IsNullOrEmpty(i.BlurHash)))) .ToList(); if (outdated.Count == 0) { @@ -1842,7 +1842,7 @@ namespace Emby.Server.Implementations.Library ImageDimensions size = _imageProcessor.GetImageDimensions(item, img); img.Width = size.Width; img.Height = size.Height; - img.Hash = _imageProcessor.GetImageHash(img.Path); + img.BlurHash = _imageProcessor.GetImageBlurHash(img.Path); }); _itemRepository.SaveImages(item); diff --git a/MediaBrowser.Api/Images/ImageService.cs b/MediaBrowser.Api/Images/ImageService.cs index 559db550b8..d0846bfc3f 100644 --- a/MediaBrowser.Api/Images/ImageService.cs +++ b/MediaBrowser.Api/Images/ImageService.cs @@ -341,7 +341,7 @@ namespace MediaBrowser.Api.Images var fileInfo = _fileSystem.GetFileInfo(info.Path); length = fileInfo.Length; - blurhash = info.Hash; + blurhash = info.BlurHash; width = info.Width; height = info.Height; diff --git a/MediaBrowser.Controller/Drawing/IImageProcessor.cs b/MediaBrowser.Controller/Drawing/IImageProcessor.cs index e38eaf7606..8800fdf990 100644 --- a/MediaBrowser.Controller/Drawing/IImageProcessor.cs +++ b/MediaBrowser.Controller/Drawing/IImageProcessor.cs @@ -45,7 +45,7 @@ namespace MediaBrowser.Controller.Drawing /// /// Path to the image file. /// BlurHash - String GetImageHash(string path); + string GetImageBlurHash(string path); /// /// Gets the image cache tag. diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index 035ab1dd94..07aeb69db9 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -2223,7 +2223,7 @@ namespace MediaBrowser.Controller.Entities existingImage.DateModified = image.DateModified; existingImage.Width = image.Width; existingImage.Height = image.Height; - existingImage.Hash = image.Hash; + existingImage.BlurHash = image.BlurHash; } else { diff --git a/MediaBrowser.Controller/Entities/ItemImageInfo.cs b/MediaBrowser.Controller/Entities/ItemImageInfo.cs index ba02971073..12f5db2e0b 100644 --- a/MediaBrowser.Controller/Entities/ItemImageInfo.cs +++ b/MediaBrowser.Controller/Entities/ItemImageInfo.cs @@ -32,7 +32,7 @@ namespace MediaBrowser.Controller.Entities /// Gets or sets the blurhash. /// /// The blurhash. - public string Hash { get; set; } + public string BlurHash { get; set; } [JsonIgnore] public bool IsLocalFile => Path == null || !Path.StartsWith("http", StringComparison.OrdinalIgnoreCase); diff --git a/MediaBrowser.Model/Dto/BaseItemDto.cs b/MediaBrowser.Model/Dto/BaseItemDto.cs index 8c6c9683a4..df84dcf123 100644 --- a/MediaBrowser.Model/Dto/BaseItemDto.cs +++ b/MediaBrowser.Model/Dto/BaseItemDto.cs @@ -514,7 +514,7 @@ namespace MediaBrowser.Model.Dto /// Gets or sets the blurhash for the image tags. /// /// The blurhashes. - public Dictionary ImageHashes { get; set; } + public Dictionary ImageBlurHashes { get; set; } /// /// Gets or sets the series studio. From 29443e36817e4866cc58e8397c1233b05624284a Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 26 May 2020 00:50:29 +0300 Subject: [PATCH 12/24] Apply suggestions from code review Co-authored-by: dkanada --- .vscode/tasks.json | 1 - Emby.Server.Implementations/Library/LibraryManager.cs | 1 - MediaBrowser.Controller/Drawing/IImageEncoder.cs | 2 +- MediaBrowser.Model/Dto/BaseItemDto.cs | 2 +- 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 7475617c93..2289fd9914 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -21,6 +21,5 @@ ], "problemMatcher": "$msCompile" } - ] } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 579fb7eddf..e63776bfff 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -71,7 +71,6 @@ namespace Emby.Server.Implementations.Library private readonly ConcurrentDictionary _libraryItemsCache; private readonly IImageProcessor _imageProcessor; - private NamingOptions _namingOptions; private string[] _videoFileExtensions; diff --git a/MediaBrowser.Controller/Drawing/IImageEncoder.cs b/MediaBrowser.Controller/Drawing/IImageEncoder.cs index 1d3f0d3b44..4baec62044 100644 --- a/MediaBrowser.Controller/Drawing/IImageEncoder.cs +++ b/MediaBrowser.Controller/Drawing/IImageEncoder.cs @@ -44,7 +44,7 @@ namespace MediaBrowser.Controller.Drawing ImageDimensions GetImageSize(string path); /// - /// Get the blurhash of an image. + /// Gets the blurhash of an image. /// /// The filepath of the image. /// The blurhash. diff --git a/MediaBrowser.Model/Dto/BaseItemDto.cs b/MediaBrowser.Model/Dto/BaseItemDto.cs index df84dcf123..6213206ffc 100644 --- a/MediaBrowser.Model/Dto/BaseItemDto.cs +++ b/MediaBrowser.Model/Dto/BaseItemDto.cs @@ -511,7 +511,7 @@ namespace MediaBrowser.Model.Dto public string SeriesThumbImageTag { get; set; } /// - /// Gets or sets the blurhash for the image tags. + /// Gets or sets the blurhashes for the image tags. /// /// The blurhashes. public Dictionary ImageBlurHashes { get; set; } From 279f0da98031d60b81bb634ca11a2d548cb0f646 Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 26 May 2020 00:52:13 +0300 Subject: [PATCH 13/24] Rename ImageInfo.Hash to ImageInfo.BlurHash --- MediaBrowser.Api/Images/ImageService.cs | 2 +- MediaBrowser.Model/Dto/ImageInfo.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/MediaBrowser.Api/Images/ImageService.cs b/MediaBrowser.Api/Images/ImageService.cs index d0846bfc3f..89fe726350 100644 --- a/MediaBrowser.Api/Images/ImageService.cs +++ b/MediaBrowser.Api/Images/ImageService.cs @@ -366,7 +366,7 @@ namespace MediaBrowser.Api.Images ImageType = info.Type, ImageTag = _imageProcessor.GetImageCacheTag(item, info), Size = length, - Hash = blurhash, + BlurHash = blurhash, Width = width, Height = height }; diff --git a/MediaBrowser.Model/Dto/ImageInfo.cs b/MediaBrowser.Model/Dto/ImageInfo.cs index 39bdc09ed8..664ea332e8 100644 --- a/MediaBrowser.Model/Dto/ImageInfo.cs +++ b/MediaBrowser.Model/Dto/ImageInfo.cs @@ -34,7 +34,7 @@ namespace MediaBrowser.Model.Dto /// Gets or sets the blurhash. /// /// The blurhash. - public string Hash { get; set; } + public string BlurHash { get; set; } /// /// Gets or sets the height. From 0f32b0ffad03817f34b02a4fc7371e2a0a67e63a Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 26 May 2020 01:04:40 +0300 Subject: [PATCH 14/24] Change image blurhash mapping to "image type to blurhash" --- Emby.Server.Implementations/Dto/DtoService.cs | 4 ++-- MediaBrowser.Model/Dto/BaseItemDto.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index 593e7be7db..77a734ebff 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -718,7 +718,7 @@ namespace Emby.Server.Implementations.Dto if (options.EnableImages) { dto.ImageTags = new Dictionary(); - dto.ImageBlurHashes = new Dictionary(); + dto.ImageBlurHashes = new Dictionary(); // Prevent implicitly captured closure var currentItem = item; @@ -736,7 +736,7 @@ namespace Emby.Server.Implementations.Dto var hash = image.BlurHash; if (!string.IsNullOrEmpty(hash)) { - dto.ImageBlurHashes[tag] = image.BlurHash; + dto.ImageBlurHashes[image.Type] = image.BlurHash; } } } diff --git a/MediaBrowser.Model/Dto/BaseItemDto.cs b/MediaBrowser.Model/Dto/BaseItemDto.cs index 6213206ffc..734bcaf81b 100644 --- a/MediaBrowser.Model/Dto/BaseItemDto.cs +++ b/MediaBrowser.Model/Dto/BaseItemDto.cs @@ -514,7 +514,7 @@ namespace MediaBrowser.Model.Dto /// Gets or sets the blurhashes for the image tags. /// /// The blurhashes. - public Dictionary ImageBlurHashes { get; set; } + public Dictionary ImageBlurHashes { get; set; } /// /// Gets or sets the series studio. From f575415e0b611cbfb4139e9e9c816adcb000442e Mon Sep 17 00:00:00 2001 From: Vasily Date: Tue, 26 May 2020 02:31:40 +0300 Subject: [PATCH 15/24] Pick blurhash sizes depending on image aspect ratio --- Jellyfin.Drawing.Skia/SkiaEncoder.cs | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 7ab0a54dfd..ccd501214e 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -241,14 +241,20 @@ namespace Jellyfin.Drawing.Skia throw new ArgumentNullException(nameof(path)); } - if (!File.Exists(path)) + var dims = GetImageSize(path); + if (dims.Width <= 0 || dims.Height <= 0) { - throw new FileNotFoundException("File not found", path); + // empty image does not have any blurhash + return string.Empty; } - // Use 4 vertical and 4 horizontal components of DCT of the image. + // We want tiles to be as close to square as possible, and to *mostly* keep under 16 tiles for performance. + // One tile is (width / xComp) x (height / yComp) pixels, which means that ideally yComp = xComp * height / width. // See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components - return BlurHashEncoder.Encode(4, 4, path); + float xComp = MathF.Sqrt(16.0f * dims.Width / dims.Height); + float yComp = xComp * dims.Height / dims.Width; + + return BlurHashEncoder.Encode(Math.Min((int)xComp + 1, 9), Math.Min((int)yComp + 1, 9), path); } private static bool HasDiacritics(string text) From 6c9dc0418961a673f9e5dcfb36f66d076381a92e Mon Sep 17 00:00:00 2001 From: Vasily Date: Wed, 27 May 2020 15:01:03 +0300 Subject: [PATCH 16/24] Handle errors during blurhash generation so it does not fail the scan --- Emby.Server.Implementations/Library/LibraryManager.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index e63776bfff..903c0b3cfb 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1841,7 +1841,15 @@ namespace Emby.Server.Implementations.Library ImageDimensions size = _imageProcessor.GetImageDimensions(item, img); img.Width = size.Width; img.Height = size.Height; - img.BlurHash = _imageProcessor.GetImageBlurHash(img.Path); + try + { + img.BlurHash = _imageProcessor.GetImageBlurHash(img.Path); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot compute blurhash for {0}", img.Path); + img.BlurHash = string.Empty; + } }); _itemRepository.SaveImages(item); From 2482bcb3b1ba9ea6e861709704ce1f184fcc0d9c Mon Sep 17 00:00:00 2001 From: Vasily Date: Wed, 27 May 2020 16:27:27 +0300 Subject: [PATCH 17/24] Add blurhashes to ImageBlurHashes for all images --- Emby.Server.Implementations/Dto/DtoService.cs | 98 +++++++++++++++---- MediaBrowser.Model/Dto/BaseItemDto.cs | 3 +- 2 files changed, 80 insertions(+), 21 deletions(-) diff --git a/Emby.Server.Implementations/Dto/DtoService.cs b/Emby.Server.Implementations/Dto/DtoService.cs index 77a734ebff..38c4f940d7 100644 --- a/Emby.Server.Implementations/Dto/DtoService.cs +++ b/Emby.Server.Implementations/Dto/DtoService.cs @@ -605,7 +605,7 @@ namespace Emby.Server.Implementations.Dto if (dictionary.TryGetValue(person.Name, out Person entity)) { - baseItemPerson.PrimaryImageTag = GetImageCacheTag(entity, ImageType.Primary); + baseItemPerson.PrimaryImageTag = GetTagAndFillBlurhash(dto, entity, ImageType.Primary); baseItemPerson.Id = entity.Id.ToString("N", CultureInfo.InvariantCulture); list.Add(baseItemPerson); } @@ -654,6 +654,70 @@ namespace Emby.Server.Implementations.Dto return _libraryManager.GetGenreId(name); } + private string GetTagAndFillBlurhash(BaseItemDto dto, BaseItem item, ImageType imageType, int imageIndex = 0) + { + var image = item.GetImageInfo(imageType, imageIndex); + if (image != null) + { + return GetTagAndFillBlurhash(dto, item, image); + } + + return null; + } + + private string GetTagAndFillBlurhash(BaseItemDto dto, BaseItem item, ItemImageInfo image) + { + var tag = GetImageCacheTag(item, image); + if (!string.IsNullOrEmpty(image.BlurHash)) + { + if (dto.ImageBlurHashes == null) + { + dto.ImageBlurHashes = new Dictionary>(); + } + + if (!dto.ImageBlurHashes.ContainsKey(image.Type)) + { + dto.ImageBlurHashes[image.Type] = new Dictionary(); + } + + dto.ImageBlurHashes[image.Type][tag] = image.BlurHash; + } + + return tag; + } + + private string[] GetTagsAndFillBlurhashes(BaseItemDto dto, BaseItem item, ImageType imageType, int limit) + { + return GetTagsAndFillBlurhashes(dto, item, imageType, item.GetImages(imageType).Take(limit).ToList()); + } + + private string[] GetTagsAndFillBlurhashes(BaseItemDto dto, BaseItem item, ImageType imageType, List images) + { + var tags = GetImageTags(item, images); + var hashes = new Dictionary(); + for (int i = 0; i < images.Count; i++) + { + var img = images[i]; + if (!string.IsNullOrEmpty(img.BlurHash)) + { + var tag = tags[i]; + hashes[tag] = img.BlurHash; + } + } + + if (hashes.Count > 0) + { + if (dto.ImageBlurHashes == null) + { + dto.ImageBlurHashes = new Dictionary>(); + } + + dto.ImageBlurHashes[imageType] = hashes; + } + + return tags; + } + /// /// Sets simple property values on a DTOBaseItem /// @@ -674,8 +738,8 @@ namespace Emby.Server.Implementations.Dto dto.LockData = item.IsLocked; dto.ForcedSortName = item.ForcedSortName; } - dto.Container = item.Container; + dto.Container = item.Container; dto.EndDate = item.EndDate; if (options.ContainsField(ItemFields.ExternalUrls)) @@ -694,10 +758,12 @@ namespace Emby.Server.Implementations.Dto dto.AspectRatio = hasAspectRatio.AspectRatio; } + dto.ImageBlurHashes = new Dictionary>(); + var backdropLimit = options.GetImageLimit(ImageType.Backdrop); if (backdropLimit > 0) { - dto.BackdropImageTags = GetImageTags(item, item.GetImages(ImageType.Backdrop).Take(backdropLimit).ToList()); + dto.BackdropImageTags = GetTagsAndFillBlurhashes(dto, item, ImageType.Backdrop, backdropLimit); } if (options.ContainsField(ItemFields.ScreenshotImageTags)) @@ -705,7 +771,7 @@ namespace Emby.Server.Implementations.Dto var screenshotLimit = options.GetImageLimit(ImageType.Screenshot); if (screenshotLimit > 0) { - dto.ScreenshotImageTags = GetImageTags(item, item.GetImages(ImageType.Screenshot).Take(screenshotLimit).ToList()); + dto.ScreenshotImageTags = GetTagsAndFillBlurhashes(dto, item, ImageType.Screenshot, screenshotLimit); } } @@ -718,7 +784,6 @@ namespace Emby.Server.Implementations.Dto if (options.EnableImages) { dto.ImageTags = new Dictionary(); - dto.ImageBlurHashes = new Dictionary(); // Prevent implicitly captured closure var currentItem = item; @@ -726,18 +791,12 @@ namespace Emby.Server.Implementations.Dto { if (options.GetImageLimit(image.Type) > 0) { - var tag = GetImageCacheTag(item, image); + var tag = GetTagAndFillBlurhash(dto, item, image); if (tag != null) { dto.ImageTags[image.Type] = tag; } - - var hash = image.BlurHash; - if (!string.IsNullOrEmpty(hash)) - { - dto.ImageBlurHashes[image.Type] = image.BlurHash; - } } } } @@ -877,8 +936,7 @@ namespace Emby.Server.Implementations.Dto if (albumParent != null) { dto.AlbumId = albumParent.Id; - - dto.AlbumPrimaryImageTag = GetImageCacheTag(albumParent, ImageType.Primary); + dto.AlbumPrimaryImageTag = GetTagAndFillBlurhash(dto, albumParent, ImageType.Primary); } //if (options.ContainsField(ItemFields.MediaSourceCount)) @@ -1105,7 +1163,7 @@ namespace Emby.Server.Implementations.Dto episodeSeries = episodeSeries ?? episode.Series; if (episodeSeries != null) { - dto.SeriesPrimaryImageTag = GetImageCacheTag(episodeSeries, ImageType.Primary); + dto.SeriesPrimaryImageTag = GetTagAndFillBlurhash(dto, episodeSeries, ImageType.Primary); } } @@ -1151,7 +1209,7 @@ namespace Emby.Server.Implementations.Dto series = series ?? season.Series; if (series != null) { - dto.SeriesPrimaryImageTag = GetImageCacheTag(series, ImageType.Primary); + dto.SeriesPrimaryImageTag = GetTagAndFillBlurhash(dto, series, ImageType.Primary); } } } @@ -1281,7 +1339,7 @@ namespace Emby.Server.Implementations.Dto if (image != null) { dto.ParentLogoItemId = GetDtoId(parent); - dto.ParentLogoImageTag = GetImageCacheTag(parent, image); + dto.ParentLogoImageTag = GetTagAndFillBlurhash(dto, parent, image); } } if (artLimit > 0 && !(imageTags != null && imageTags.ContainsKey(ImageType.Art)) && dto.ParentArtItemId == null) @@ -1291,7 +1349,7 @@ namespace Emby.Server.Implementations.Dto if (image != null) { dto.ParentArtItemId = GetDtoId(parent); - dto.ParentArtImageTag = GetImageCacheTag(parent, image); + dto.ParentArtImageTag = GetTagAndFillBlurhash(dto, parent, image); } } if (thumbLimit > 0 && !(imageTags != null && imageTags.ContainsKey(ImageType.Thumb)) && (dto.ParentThumbItemId == null || parent is Series) && !(parent is ICollectionFolder) && !(parent is UserView)) @@ -1301,7 +1359,7 @@ namespace Emby.Server.Implementations.Dto if (image != null) { dto.ParentThumbItemId = GetDtoId(parent); - dto.ParentThumbImageTag = GetImageCacheTag(parent, image); + dto.ParentThumbImageTag = GetTagAndFillBlurhash(dto, parent, image); } } if (backdropLimit > 0 && !((dto.BackdropImageTags != null && dto.BackdropImageTags.Length > 0) || (dto.ParentBackdropImageTags != null && dto.ParentBackdropImageTags.Length > 0))) @@ -1311,7 +1369,7 @@ namespace Emby.Server.Implementations.Dto if (images.Count > 0) { dto.ParentBackdropItemId = GetDtoId(parent); - dto.ParentBackdropImageTags = GetImageTags(parent, images); + dto.ParentBackdropImageTags = GetTagsAndFillBlurhashes(dto, parent, ImageType.Backdrop, images); } } diff --git a/MediaBrowser.Model/Dto/BaseItemDto.cs b/MediaBrowser.Model/Dto/BaseItemDto.cs index 734bcaf81b..fc1cb01db7 100644 --- a/MediaBrowser.Model/Dto/BaseItemDto.cs +++ b/MediaBrowser.Model/Dto/BaseItemDto.cs @@ -512,9 +512,10 @@ namespace MediaBrowser.Model.Dto /// /// Gets or sets the blurhashes for the image tags. + /// Maps image type to dictionary mapping image tag to blurhash value. /// /// The blurhashes. - public Dictionary ImageBlurHashes { get; set; } + public Dictionary> ImageBlurHashes { get; set; } /// /// Gets or sets the series studio. From edcfcadcd327affb308b0c0eb5cfbb1416e27cae Mon Sep 17 00:00:00 2001 From: Vasily Date: Wed, 27 May 2020 17:00:59 +0300 Subject: [PATCH 18/24] Make sure blurhash is recomputed if image changed or metadata refresh toggled --- .../Library/LibraryManager.cs | 47 +++++++++++++++---- .../Library/ILibraryManager.cs | 2 +- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 903c0b3cfb..84bcd1bc16 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1820,23 +1820,44 @@ namespace Emby.Server.Implementations.Library } } - public void UpdateImages(BaseItem item) + private bool ImageNeedsRefresh(ItemImageInfo image) + { + if (image.Path != null && image.IsLocalFile) + { + if (image.Width == 0 || image.Height == 0 || string.IsNullOrEmpty(image.BlurHash)) + { + return true; + } + + try + { + return _fileSystem.GetLastWriteTimeUtc(image.Path) != image.DateModified; + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot get file info for {0}", image.Path); + return false; + } + } + + return false; + } + + public void UpdateImages(BaseItem item, bool forceUpdate = false) { if (item == null) { throw new ArgumentNullException(nameof(item)); } - var outdated = item.ImageInfos - .Where(i => (i.IsLocalFile && (i.Width == 0 || i.Height == 0 || string.IsNullOrEmpty(i.BlurHash)))) - .ToList(); - if (outdated.Count == 0) + var outdated = forceUpdate ? item.ImageInfos : item.ImageInfos.Where(ImageNeedsRefresh).ToArray(); + if (outdated.Length == 0) { RegisterItem(item); return; } - outdated.ForEach(img => + foreach (var img in outdated) { ImageDimensions size = _imageProcessor.GetImageDimensions(item, img); img.Width = size.Width; @@ -1850,10 +1871,18 @@ namespace Emby.Server.Implementations.Library _logger.LogError(ex, "Cannot compute blurhash for {0}", img.Path); img.BlurHash = string.Empty; } - }); - _itemRepository.SaveImages(item); + try + { + img.DateModified = _fileSystem.GetLastWriteTimeUtc(img.Path); + } + catch (Exception ex) + { + _logger.LogError(ex, "Cannot update DateModified for {0}", img.Path); + } + } + _itemRepository.SaveImages(item); RegisterItem(item); } @@ -1874,7 +1903,7 @@ namespace Emby.Server.Implementations.Library item.DateLastSaved = DateTime.UtcNow; - UpdateImages(item); + UpdateImages(item, updateReason >= ItemUpdateType.ImageUpdate); } _itemRepository.SaveItems(itemsList, cancellationToken); diff --git a/MediaBrowser.Controller/Library/ILibraryManager.cs b/MediaBrowser.Controller/Library/ILibraryManager.cs index 81160efece..916e4fda77 100644 --- a/MediaBrowser.Controller/Library/ILibraryManager.cs +++ b/MediaBrowser.Controller/Library/ILibraryManager.cs @@ -118,7 +118,7 @@ namespace MediaBrowser.Controller.Library /// void QueueLibraryScan(); - void UpdateImages(BaseItem item); + void UpdateImages(BaseItem item, bool forceUpdate = false); /// /// Gets the default view. From f30b07130fc21247e3c6edd61d004a9286c5340c Mon Sep 17 00:00:00 2001 From: Vasily Date: Wed, 27 May 2020 19:29:57 +0300 Subject: [PATCH 19/24] Workaround a bug in BlurHashSharp --- Jellyfin.Drawing.Skia/SkiaEncoder.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index ccd501214e..85461ca4ef 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -251,10 +251,18 @@ namespace Jellyfin.Drawing.Skia // We want tiles to be as close to square as possible, and to *mostly* keep under 16 tiles for performance. // One tile is (width / xComp) x (height / yComp) pixels, which means that ideally yComp = xComp * height / width. // See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components - float xComp = MathF.Sqrt(16.0f * dims.Width / dims.Height); - float yComp = xComp * dims.Height / dims.Width; + float xCompF = MathF.Sqrt(16.0f * dims.Width / dims.Height); + float yCompF = xCompF * dims.Height / dims.Width; - return BlurHashEncoder.Encode(Math.Min((int)xComp + 1, 9), Math.Min((int)yComp + 1, 9), path); + int xComp = Math.Min((int)xCompF + 1, 9); + int yComp = Math.Min((int)yCompF + 1, 9); + + // FIXME: current lib is bugged for xComp != yComp + // remove when https://github.com/Bond-009/BlurHashSharp/pull/1 is merged + int tmp = Math.Max(xComp, yComp); + xComp = yComp = tmp; + + return BlurHashEncoder.Encode(xComp, yComp, path); } private static bool HasDiacritics(string text) From a76cee7a953a77794ca9824cdff581a350fcf0ae Mon Sep 17 00:00:00 2001 From: Vasily Date: Thu, 28 May 2020 17:23:16 +0300 Subject: [PATCH 20/24] Update BlurHashSharp to 1.0.1, remove workaround --- Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj | 2 +- Jellyfin.Drawing.Skia/SkiaEncoder.cs | 5 ----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj b/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj index 221346b688..d3fdb5de4b 100644 --- a/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj +++ b/Jellyfin.Drawing.Skia/Jellyfin.Drawing.Skia.csproj @@ -18,7 +18,7 @@ - + diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 85461ca4ef..7f0da2c9ee 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -257,11 +257,6 @@ namespace Jellyfin.Drawing.Skia int xComp = Math.Min((int)xCompF + 1, 9); int yComp = Math.Min((int)yCompF + 1, 9); - // FIXME: current lib is bugged for xComp != yComp - // remove when https://github.com/Bond-009/BlurHashSharp/pull/1 is merged - int tmp = Math.Max(xComp, yComp); - xComp = yComp = tmp; - return BlurHashEncoder.Encode(xComp, yComp, path); } From ed791dee46b8f3b72fbdb68a3382622b4337e254 Mon Sep 17 00:00:00 2001 From: Vasily Date: Thu, 28 May 2020 17:30:11 +0300 Subject: [PATCH 21/24] Do not compute dimensions or blurhash for remote images --- Emby.Server.Implementations/Library/LibraryManager.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 84bcd1bc16..6cbca7d02d 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1850,7 +1850,7 @@ namespace Emby.Server.Implementations.Library throw new ArgumentNullException(nameof(item)); } - var outdated = forceUpdate ? item.ImageInfos : item.ImageInfos.Where(ImageNeedsRefresh).ToArray(); + var outdated = forceUpdate ? item.ImageInfos.Where(i => i.IsLocalFile).ToArray() : item.ImageInfos.Where(ImageNeedsRefresh).ToArray(); if (outdated.Length == 0) { RegisterItem(item); From 9208acd5ae26d41af6791d7dd7322d682a8b80b6 Mon Sep 17 00:00:00 2001 From: Vasily Date: Thu, 28 May 2020 17:55:29 +0300 Subject: [PATCH 22/24] Convert non-local image to local before computing blurhash --- .../Library/LibraryManager.cs | 40 ++++++++++++++----- MediaBrowser.Controller/Entities/BaseItem.cs | 40 +++++++++++++++++++ 2 files changed, 70 insertions(+), 10 deletions(-) diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index 6cbca7d02d..bb3e3dd117 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1840,7 +1840,7 @@ namespace Emby.Server.Implementations.Library } } - return false; + return image.Path != null && !image.IsLocalFile; } public void UpdateImages(BaseItem item, bool forceUpdate = false) @@ -1850,7 +1850,7 @@ namespace Emby.Server.Implementations.Library throw new ArgumentNullException(nameof(item)); } - var outdated = forceUpdate ? item.ImageInfos.Where(i => i.IsLocalFile).ToArray() : item.ImageInfos.Where(ImageNeedsRefresh).ToArray(); + var outdated = forceUpdate ? item.ImageInfos.Where(i => i.Path != null).ToArray() : item.ImageInfos.Where(ImageNeedsRefresh).ToArray(); if (outdated.Length == 0) { RegisterItem(item); @@ -1859,26 +1859,46 @@ namespace Emby.Server.Implementations.Library foreach (var img in outdated) { - ImageDimensions size = _imageProcessor.GetImageDimensions(item, img); - img.Width = size.Width; - img.Height = size.Height; + var image = img; + if (!img.IsLocalFile) + { + try + { + var index = item.GetImageIndex(img); + image = ConvertImageToLocal(item, img, index).ConfigureAwait(false).GetAwaiter().GetResult(); + } + catch (ArgumentException) + { + _logger.LogWarning("Cannot get image index for {0}", img.Path); + continue; + } + catch (InvalidOperationException) + { + _logger.LogWarning("Cannot fetch image from {0}", img.Path); + continue; + } + } + + ImageDimensions size = _imageProcessor.GetImageDimensions(item, image); + image.Width = size.Width; + image.Height = size.Height; try { - img.BlurHash = _imageProcessor.GetImageBlurHash(img.Path); + image.BlurHash = _imageProcessor.GetImageBlurHash(image.Path); } catch (Exception ex) { - _logger.LogError(ex, "Cannot compute blurhash for {0}", img.Path); - img.BlurHash = string.Empty; + _logger.LogError(ex, "Cannot compute blurhash for {0}", image.Path); + image.BlurHash = string.Empty; } try { - img.DateModified = _fileSystem.GetLastWriteTimeUtc(img.Path); + image.DateModified = _fileSystem.GetLastWriteTimeUtc(image.Path); } catch (Exception ex) { - _logger.LogError(ex, "Cannot update DateModified for {0}", img.Path); + _logger.LogError(ex, "Cannot update DateModified for {0}", image.Path); } } diff --git a/MediaBrowser.Controller/Entities/BaseItem.cs b/MediaBrowser.Controller/Entities/BaseItem.cs index 07aeb69db9..f4b71d8bf7 100644 --- a/MediaBrowser.Controller/Entities/BaseItem.cs +++ b/MediaBrowser.Controller/Entities/BaseItem.cs @@ -2375,6 +2375,46 @@ namespace MediaBrowser.Controller.Entities .ElementAtOrDefault(imageIndex); } + /// + /// Computes image index for given image or raises if no matching image found. + /// + /// Image to compute index for. + /// Image index cannot be computed as no matching image found. + /// + /// Image index. + public int GetImageIndex(ItemImageInfo image) + { + if (image == null) + { + throw new ArgumentNullException(nameof(image)); + } + + if (image.Type == ImageType.Chapter) + { + var chapters = ItemRepository.GetChapters(this); + for (var i = 0; i < chapters.Count; i++) + { + if (chapters[i].ImagePath == image.Path) + { + return i; + } + } + + throw new ArgumentException("No chapter index found for image path", image.Path); + } + + var images = GetImages(image.Type).ToArray(); + for (var i = 0; i < images.Length; i++) + { + if (images[i].Path == image.Path) + { + return i; + } + } + + throw new ArgumentException("No image index found for image path", image.Path); + } + public IEnumerable GetImages(ImageType imageType) { if (imageType == ImageType.Chapter) From 58f099c0e2fec23d861bc9bdb76ac0d6d8e239b7 Mon Sep 17 00:00:00 2001 From: Vasily Date: Thu, 28 May 2020 19:12:08 +0300 Subject: [PATCH 23/24] Fix naming per code review --- .../Data/SqliteItemRepository.cs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Emby.Server.Implementations/Data/SqliteItemRepository.cs b/Emby.Server.Implementations/Data/SqliteItemRepository.cs index dd60dd2228..608801ac3d 100644 --- a/Emby.Server.Implementations/Data/SqliteItemRepository.cs +++ b/Emby.Server.Implementations/Data/SqliteItemRepository.cs @@ -1141,21 +1141,21 @@ namespace Emby.Server.Implementations.Data public string ToValueString(ItemImageInfo image) { - const string delimeter = "*"; + const string Delimeter = "*"; var path = image.Path ?? string.Empty; var hash = image.BlurHash ?? string.Empty; return GetPathToSave(path) + - delimeter + + Delimeter + image.DateModified.Ticks.ToString(CultureInfo.InvariantCulture) + - delimeter + + Delimeter + image.Type + - delimeter + + Delimeter + image.Width.ToString(CultureInfo.InvariantCulture) + - delimeter + + Delimeter + image.Height.ToString(CultureInfo.InvariantCulture) + - delimeter + + Delimeter + // Replace delimiters with other characters. // This can be removed when we migrate to a proper DB. hash.Replace('*', '/').Replace('|', '\\'); From 26eef1bbf823e6f9fc22b11d95a17b1370b21842 Mon Sep 17 00:00:00 2001 From: Vasily Date: Mon, 1 Jun 2020 18:12:49 +0300 Subject: [PATCH 24/24] Move logic of computing Blurhash components to ImageProcessor Also rename last few instances of GetImageHash to GetImageBlurHash for clarity --- Emby.Drawing/ImageProcessor.cs | 19 ++++++++++++++++++- Emby.Drawing/NullImageEncoder.cs | 2 +- .../Library/LibraryManager.cs | 1 + Jellyfin.Drawing.Skia/SkiaEncoder.cs | 18 +----------------- .../Drawing/IImageEncoder.cs | 4 +++- 5 files changed, 24 insertions(+), 20 deletions(-) diff --git a/Emby.Drawing/ImageProcessor.cs b/Emby.Drawing/ImageProcessor.cs index 35da6f6359..89bb3068b9 100644 --- a/Emby.Drawing/ImageProcessor.cs +++ b/Emby.Drawing/ImageProcessor.cs @@ -315,7 +315,24 @@ namespace Emby.Drawing /// public string GetImageBlurHash(string path) - => _imageEncoder.GetImageHash(path); + { + var size = GetImageDimensions(path); + if (size.Width <= 0 || size.Height <= 0) + { + return string.Empty; + } + + // We want tiles to be as close to square as possible, and to *mostly* keep under 16 tiles for performance. + // One tile is (width / xComp) x (height / yComp) pixels, which means that ideally yComp = xComp * height / width. + // See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components + float xCompF = MathF.Sqrt(16.0f * size.Width / size.Height); + float yCompF = xCompF * size.Height / size.Width; + + int xComp = Math.Min((int)xCompF + 1, 9); + int yComp = Math.Min((int)yCompF + 1, 9); + + return _imageEncoder.GetImageBlurHash(xComp, yComp, path); + } /// public string GetImageCacheTag(BaseItem item, ItemImageInfo image) diff --git a/Emby.Drawing/NullImageEncoder.cs b/Emby.Drawing/NullImageEncoder.cs index 54de7212a3..bbb5c17162 100644 --- a/Emby.Drawing/NullImageEncoder.cs +++ b/Emby.Drawing/NullImageEncoder.cs @@ -44,7 +44,7 @@ namespace Emby.Drawing } /// - public string GetImageHash(string path) + public string GetImageBlurHash(int xComp, int yComp, string path) { throw new NotImplementedException(); } diff --git a/Emby.Server.Implementations/Library/LibraryManager.cs b/Emby.Server.Implementations/Library/LibraryManager.cs index bb3e3dd117..15362182cf 100644 --- a/Emby.Server.Implementations/Library/LibraryManager.cs +++ b/Emby.Server.Implementations/Library/LibraryManager.cs @@ -1882,6 +1882,7 @@ namespace Emby.Server.Implementations.Library ImageDimensions size = _imageProcessor.GetImageDimensions(item, image); image.Width = size.Width; image.Height = size.Height; + try { image.BlurHash = _imageProcessor.GetImageBlurHash(image.Path); diff --git a/Jellyfin.Drawing.Skia/SkiaEncoder.cs b/Jellyfin.Drawing.Skia/SkiaEncoder.cs index 7f0da2c9ee..dae0e94d45 100644 --- a/Jellyfin.Drawing.Skia/SkiaEncoder.cs +++ b/Jellyfin.Drawing.Skia/SkiaEncoder.cs @@ -234,29 +234,13 @@ namespace Jellyfin.Drawing.Skia /// The path is null. /// The path is not valid. /// The file at the specified path could not be used to generate a codec. - public string GetImageHash(string path) + public string GetImageBlurHash(int xComp, int yComp, string path) { if (path == null) { throw new ArgumentNullException(nameof(path)); } - var dims = GetImageSize(path); - if (dims.Width <= 0 || dims.Height <= 0) - { - // empty image does not have any blurhash - return string.Empty; - } - - // We want tiles to be as close to square as possible, and to *mostly* keep under 16 tiles for performance. - // One tile is (width / xComp) x (height / yComp) pixels, which means that ideally yComp = xComp * height / width. - // See more at https://github.com/woltapp/blurhash/#how-do-i-pick-the-number-of-x-and-y-components - float xCompF = MathF.Sqrt(16.0f * dims.Width / dims.Height); - float yCompF = xCompF * dims.Height / dims.Width; - - int xComp = Math.Min((int)xCompF + 1, 9); - int yComp = Math.Min((int)yCompF + 1, 9); - return BlurHashEncoder.Encode(xComp, yComp, path); } diff --git a/MediaBrowser.Controller/Drawing/IImageEncoder.cs b/MediaBrowser.Controller/Drawing/IImageEncoder.cs index 4baec62044..e09ccd204a 100644 --- a/MediaBrowser.Controller/Drawing/IImageEncoder.cs +++ b/MediaBrowser.Controller/Drawing/IImageEncoder.cs @@ -46,9 +46,11 @@ namespace MediaBrowser.Controller.Drawing /// /// Gets the blurhash of an image. /// + /// Amount of X components of DCT to take. + /// Amount of Y components of DCT to take. /// The filepath of the image. /// The blurhash. - string GetImageHash(string path); + string GetImageBlurHash(int xComp, int yComp, string path); /// /// Encode an image.