pull/11816/merge
Tim Eisele 3 weeks ago committed by GitHub
commit 8defbbc785
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194

@ -75,6 +75,7 @@
<PackageVersion Include="Swashbuckle.AspNetCore.ReDoc" Version="6.5.0" />
<PackageVersion Include="Swashbuckle.AspNetCore" Version="6.2.3" />
<PackageVersion Include="System.Globalization" Version="4.3.0" />
<PackageVersion Include="System.IO.Hashing" Version="9.0.3" />
<PackageVersion Include="System.Linq.Async" Version="6.0.1" />
<PackageVersion Include="System.Text.Encoding.CodePages" Version="9.0.3" />
<PackageVersion Include="System.Text.Json" Version="9.0.3" />

@ -1908,7 +1908,7 @@ namespace Emby.Server.Implementations.Library
{
if (image.Path is not null && image.IsLocalFile)
{
if (image.Width == 0 || image.Height == 0 || string.IsNullOrEmpty(image.BlurHash))
if (image.Width == 0 || image.Height == 0 || string.IsNullOrEmpty(image.BlurHash) || string.IsNullOrEmpty(image.FileHash))
{
return true;
}
@ -1932,9 +1932,11 @@ namespace Emby.Server.Implementations.Library
{
ArgumentNullException.ThrowIfNull(item);
var imagesWithPaths = item.ImageInfos.Where(i => i.Path is not null).ToArray();
var outdated = forceUpdate
? item.ImageInfos.Where(i => i.Path is not null).ToArray()
? imagesWithPaths
: item.ImageInfos.Where(ImageNeedsRefresh).ToArray();
// Skip image processing if current or live tv source
if (outdated.Length == 0 || item.SourceType != SourceType.Library)
{
@ -1994,6 +1996,16 @@ namespace Emby.Server.Implementations.Library
image.BlurHash = string.Empty;
}
try
{
image.FileHash = _imageProcessor.GetImageFileHash(image.Path);
}
catch (Exception ex)
{
_logger.LogError(ex, "Cannot compute file hash for {ImagePath}", image.Path);
image.FileHash = string.Empty;
}
try
{
image.DateModified = _fileSystem.GetLastWriteTimeUtc(image.Path);
@ -2004,6 +2016,42 @@ namespace Emby.Server.Implementations.Library
}
}
// Remove duplicate images
var metadataPath = item.GetInternalMetadataPath();
var distinctImages = imagesWithPaths
// Prefer external images
.OrderBy(i => !i.Path.StartsWith(metadataPath, StringComparison.Ordinal))
.DistinctBy(i => i.FileHash);
var imagesToRemove = imagesWithPaths.Where(i => !string.IsNullOrEmpty(i.FileHash)).Except(distinctImages).ToList();
if (imagesToRemove.Count > 0)
{
foreach (var image in imagesToRemove)
{
_logger.LogDebug("Removing {ImagePath} due to duplication", image.Path);
item.RemoveImage(image);
List<string> failedFiles = [];
if (image.IsLocalFile)
{
try
{
_fileSystem.DeleteFile(image.Path);
}
catch (Exception ex)
{
if (ex is IOException || ex is UnauthorizedAccessException || ex is DirectoryNotFoundException)
{
_logger.LogDebug("Removing {ImagePath} failed due to {Exception}", image.Path, ex.Message);
}
else
{
throw;
}
}
}
}
}
_itemRepository.SaveImages(item);
RegisterItem(item);
}

@ -1136,6 +1136,7 @@ public sealed class BaseItemRepository
Id = Guid.NewGuid(),
Path = e.Path,
Blurhash = e.BlurHash is null ? null : Encoding.UTF8.GetBytes(e.BlurHash),
FileHash = e.FileHash is null ? null : Encoding.UTF8.GetBytes(e.FileHash),
DateModified = e.DateModified,
Height = e.Height,
Width = e.Width,

@ -47,16 +47,10 @@ public class ChapterRepository : IChapterRepository
public ChapterInfo? GetChapter(Guid baseItemId, int index)
{
using var context = _dbProvider.CreateDbContext();
var chapter = context.Chapters.AsNoTracking()
.Select(e => new
{
chapter = e,
baseItemPath = e.Item.Path
})
.FirstOrDefault(e => e.chapter.ItemId.Equals(baseItemId) && e.chapter.ChapterIndex == index);
var chapter = context.Chapters.AsNoTracking().FirstOrDefault(e => e.ItemId.Equals(baseItemId) && e.ChapterIndex == index);
if (chapter is not null)
{
return Map(chapter.chapter, chapter.baseItemPath!);
return Map(chapter);
}
return null;
@ -67,13 +61,7 @@ public class ChapterRepository : IChapterRepository
{
using var context = _dbProvider.CreateDbContext();
return context.Chapters.AsNoTracking().Where(e => e.ItemId.Equals(baseItemId))
.Select(e => new
{
chapter = e,
baseItemPath = e.Item.Path
})
.AsEnumerable()
.Select(e => Map(e.chapter, e.baseItemPath!))
.Select(e => Map(e))
.ToArray();
}
@ -109,16 +97,17 @@ public class ChapterRepository : IChapterRepository
};
}
private ChapterInfo Map(Chapter chapterInfo, string baseItemPath)
private ChapterInfo Map(Chapter chapter)
{
var chapterEntity = new ChapterInfo()
var chapterInfo = new ChapterInfo()
{
StartPositionTicks = chapterInfo.StartPositionTicks,
ImageDateModified = chapterInfo.ImageDateModified.GetValueOrDefault(),
ImagePath = chapterInfo.ImagePath,
Name = chapterInfo.Name,
StartPositionTicks = chapter.StartPositionTicks,
ImageDateModified = chapter.ImageDateModified.GetValueOrDefault(),
ImagePath = chapter.ImagePath,
Name = chapter.Name,
};
chapterEntity.ImageTag = _imageProcessor.GetImageCacheTag(baseItemPath, chapterEntity.ImageDateModified);
return chapterEntity;
chapterInfo.ImageTag = _imageProcessor.GetImageCacheTag(chapter.Item, chapterInfo);
return chapterInfo;
}
}

@ -58,13 +58,20 @@ namespace MediaBrowser.Controller.Drawing
/// <returns>BlurHash.</returns>
string GetImageBlurHash(string path, ImageDimensions imageDimensions);
/// <summary>
/// Gets the file hash of the image.
/// </summary>
/// <param name="path">Path to the image file.</param>
/// <returns>FileHash.</returns>
string GetImageFileHash(string path);
/// <summary>
/// Gets the image cache tag.
/// </summary>
/// <param name="baseItemPath">The items basePath.</param>
/// <param name="imageDateModified">The image last modification date.</param>
/// <param name="item">The item.</param>
/// <param name="image">The image.</param>
/// <returns>Guid.</returns>
string? GetImageCacheTag(string baseItemPath, DateTime imageDateModified);
string? GetImageCacheTag(BaseItemDto item, ChapterInfo image);
/// <summary>
/// Gets the image cache tag.
@ -72,7 +79,7 @@ namespace MediaBrowser.Controller.Drawing
/// <param name="item">The item.</param>
/// <param name="image">The image.</param>
/// <returns>Guid.</returns>
string? GetImageCacheTag(BaseItemDto item, ChapterInfo image);
string? GetImageCacheTag(BaseItemEntity item, ChapterInfo image);
/// <summary>
/// Gets the image cache tag.

@ -1925,6 +1925,7 @@ namespace MediaBrowser.Controller.Entities
existingImage.Width = image.Width;
existingImage.Height = image.Height;
existingImage.BlurHash = image.BlurHash;
existingImage.FileHash = image.FileHash;
}
}

@ -36,6 +36,12 @@ namespace MediaBrowser.Controller.Entities
/// <value>The blurhash.</value>
public string? BlurHash { get; set; }
/// <summary>
/// Gets or sets the file hash.
/// </summary>
/// <value>The file hash.</value>
public string? FileHash { get; set; }
[JsonIgnore]
public bool IsLocalFile => !Path.StartsWith("http", StringComparison.OrdinalIgnoreCase);
}

@ -1,4 +1,5 @@
#pragma warning disable CA2227
#pragma warning disable CA1819 // Properties should not return arrays
#pragma warning disable CA2227 // Collection properties should be read only
using System;
@ -39,12 +40,15 @@ public class BaseItemImageInfo
/// </summary>
public int Height { get; set; }
#pragma warning disable CA1819 // Properties should not return arrays
/// <summary>
/// Gets or Sets the blurhash.
/// </summary>
public byte[]? Blurhash { get; set; }
#pragma warning restore CA1819
/// <summary>
/// Gets or Sets the file hash.
/// </summary>
public byte[]? FileHash { get; set; }
/// <summary>
/// Gets or Sets the reference id to the BaseItem.

@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace Jellyfin.Server.Implementations.Migrations
{
/// <inheritdoc />
public partial class AddImageFileHash : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<byte[]>(
name: "FileHash",
table: "BaseItemImageInfos",
type: "BLOB",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "FileHash",
table: "BaseItemImageInfos");
}
}
}

@ -407,6 +407,9 @@ namespace Jellyfin.Server.Implementations.Migrations
b.Property<DateTime>("DateModified")
.HasColumnType("TEXT");
b.Property<byte[]>("FileHash")
.HasColumnType("BLOB");
b.Property<int>("Height")
.HasColumnType("INTEGER");

@ -2,11 +2,11 @@ using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.IO.Hashing;
using System.Linq;
using System.Net.Mime;
using System.Reflection.Metadata.Ecma335;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using AsyncKeyedLock;
using Jellyfin.Database.Implementations.Entities;
@ -40,7 +40,6 @@ public sealed class ImageProcessor : IImageProcessor, IDisposable
private readonly IFileSystem _fileSystem;
private readonly IServerApplicationPaths _appPaths;
private readonly IImageEncoder _imageEncoder;
private readonly AsyncNonKeyedLocker _parallelEncodingLimit;
private bool _disposed;
@ -406,30 +405,66 @@ public sealed class ImageProcessor : IImageProcessor, IDisposable
}
/// <inheritdoc />
public string GetImageCacheTag(string baseItemPath, DateTime imageDateModified)
=> (baseItemPath + imageDateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture);
public string GetImageFileHash(string path)
{
var fileInfo = new FileInfo(path);
if (fileInfo.Exists)
{
using (FileStream fileStream = fileInfo.Open(FileMode.Open, FileAccess.Read))
{
fileStream.Position = 0;
var hasher = new XxHash3();
hasher.Append(fileStream);
return Convert.ToHexString(hasher.GetCurrentHash());
}
}
return string.Empty;
}
/// <inheritdoc />
public string GetImageCacheTag(BaseItem item, ItemImageInfo image)
=> GetImageCacheTag(item.Path, image.DateModified);
{
var fileHash = image.FileHash;
if (!string.IsNullOrEmpty(fileHash))
{
return fileHash;
}
/// <inheritdoc />
public string GetImageCacheTag(BaseItemDto item, ItemImageInfo image)
=> GetImageCacheTag(item.Path, image.DateModified);
return (item.Path + image.DateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture);
}
/// <inheritdoc />
public string? GetImageCacheTag(BaseItemDto item, ChapterInfo chapter)
public string? GetImageCacheTag(BaseItem item, ChapterInfo chapter)
{
if (chapter.ImagePath is null)
{
return null;
}
return GetImageCacheTag(item.Path, chapter.ImageDateModified);
return GetImageCacheTag(item, new ItemImageInfo
{
Path = chapter.ImagePath,
Type = ImageType.Chapter,
DateModified = chapter.ImageDateModified
});
}
/// <inheritdoc />
public string GetImageCacheTag(BaseItemDto item, ItemImageInfo image)
{
var fileHash = image.FileHash;
if (!string.IsNullOrEmpty(fileHash))
{
return fileHash;
}
return (item.Path + image.DateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture);
}
/// <inheritdoc />
public string? GetImageCacheTag(BaseItem item, ChapterInfo chapter)
public string? GetImageCacheTag(BaseItemDto item, ChapterInfo chapter)
{
if (chapter.ImagePath is null)
{
@ -444,15 +479,27 @@ public sealed class ImageProcessor : IImageProcessor, IDisposable
});
}
/// <inheritdoc />
public string? GetImageCacheTag(BaseItemEntity item, ChapterInfo chapter)
{
if (chapter.ImagePath is null)
{
return null;
}
return (item.Path + chapter.ImageDateModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture);
}
/// <inheritdoc />
public string? GetImageCacheTag(User user)
{
if (user.ProfileImage is null)
var profileImage = user.ProfileImage;
if (profileImage is null)
{
return null;
}
return GetImageCacheTag(user.ProfileImage.Path, user.ProfileImage.LastModified);
return (profileImage.Path + profileImage.LastModified.Ticks).GetMD5().ToString("N", CultureInfo.InvariantCulture);
}
private Task<(string Path, DateTime DateModified)> GetSupportedImage(string originalImagePath, DateTime dateModified)

@ -23,6 +23,7 @@
<ItemGroup>
<PackageReference Include="AsyncKeyedLock" />
<PackageReference Include="System.IO.Hashing" />
</ItemGroup>
</Project>

@ -47,65 +47,80 @@ namespace Jellyfin.Server.Implementations.Tests.Data
public static TheoryData<string, ItemImageInfo> ItemImageInfoFromValueString_Valid_TestData()
{
var data = new TheoryData<string, ItemImageInfo>();
data.Add(
"/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg*637452096478512963*Primary*1920*1080*WjQbtJtSO8nhNZ%L_Io#R/oaS6o}-;adXAoIn7j[%hW9s:WGw[nN",
new ItemImageInfo
var data = new TheoryData<string, ItemImageInfo>
{
{
Path = "/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg",
Type = ImageType.Primary,
DateModified = new DateTime(637452096478512963, DateTimeKind.Utc),
Width = 1920,
Height = 1080,
BlurHash = "WjQbtJtSO8nhNZ%L_Io#R*oaS6o}-;adXAoIn7j[%hW9s:WGw[nN"
});
data.Add(
"https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg*0*Primary*0*0",
new ItemImageInfo
"/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg*637452096478512963*Primary*1920*1080*WjQbtJtSO8nhNZ%L_Io#R/oaS6o}-;adXAoIn7j[%hW9s:WGw[nN*938c2cc0dcc05f2b68c4287040cfcf71",
new ItemImageInfo
{
Path = "/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg",
Type = ImageType.Primary,
DateModified = new DateTime(637452096478512963, DateTimeKind.Utc),
Width = 1920,
Height = 1080,
BlurHash = "WjQbtJtSO8nhNZ%L_Io#R*oaS6o}-;adXAoIn7j[%hW9s:WGw[nN",
FileHash = "938c2cc0dcc05f2b68c4287040cfcf71"
}
},
{
Path = "https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg",
Type = ImageType.Primary,
});
data.Add(
"https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg*0*Primary",
new ItemImageInfo
"https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg*0*Primary*0*0",
new ItemImageInfo
{
Path = "https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg",
Type = ImageType.Primary,
}
},
{
Path = "https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg",
Type = ImageType.Primary,
});
data.Add(
"https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg*0*Primary*600",
new ItemImageInfo
"https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg*0*Primary",
new ItemImageInfo
{
Path = "https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg",
Type = ImageType.Primary,
}
},
{
Path = "https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg",
Type = ImageType.Primary,
});
data.Add(
"%MetadataPath%/library/68/68578562b96c80a7ebd530848801f645/poster.jpg*637264380567586027*Primary*600*336",
new ItemImageInfo
"https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg*0*Primary*600",
new ItemImageInfo
{
Path = "https://image.tmdb.org/t/p/original/zhB5CHEgqqh4wnEqDNJLfWXJlcL.jpg",
Type = ImageType.Primary,
}
},
{
"%MetadataPath%/library/68/68578562b96c80a7ebd530848801f645/poster.jpg*637264380567586027*Primary*600*336",
new ItemImageInfo
{
Path = "/meta/data/path/library/68/68578562b96c80a7ebd530848801f645/poster.jpg",
Type = ImageType.Primary,
DateModified = new DateTime(637264380567586027, DateTimeKind.Utc),
Width = 600,
Height = 336
}
},
{
Path = "/meta/data/path/library/68/68578562b96c80a7ebd530848801f645/poster.jpg",
Type = ImageType.Primary,
DateModified = new DateTime(637264380567586027, DateTimeKind.Utc),
Width = 600,
Height = 336
});
"%MetadataPath%/library/68/68578562b96c80a7ebd530848801f645/poster.jpg*637264380567586027*Primary*600*336**938c2cc0dcc05f2b68c4287040cfcf71",
new ItemImageInfo
{
Path = "/meta/data/path/library/68/68578562b96c80a7ebd530848801f645/poster.jpg",
Type = ImageType.Primary,
DateModified = new DateTime(637264380567586027, DateTimeKind.Utc),
Width = 600,
Height = 336,
FileHash = "938c2cc0dcc05f2b68c4287040cfcf71"
}
}
};
return data;
}
public static TheoryData<string, ItemImageInfo[]> DeserializeImages_Valid_TestData()
{
var data = new TheoryData<string, ItemImageInfo[]>();
data.Add(
"/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg*637452096478512963*Primary*1920*1080*WjQbtJtSO8nhNZ%L_Io#R/oaS6o}-;adXAoIn7j[%hW9s:WGw[nN",
new ItemImageInfo[]
var data = new TheoryData<string, ItemImageInfo[]>
{
{
"/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg*637452096478512963*Primary*1920*1080*WjQbtJtSO8nhNZ%L_Io#R/oaS6o}-;adXAoIn7j[%hW9s:WGw[nN",
[
new ItemImageInfo()
{
Path = "/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg",
@ -115,12 +130,11 @@ namespace Jellyfin.Server.Implementations.Tests.Data
Height = 1080,
BlurHash = "WjQbtJtSO8nhNZ%L_Io#R*oaS6o}-;adXAoIn7j[%hW9s:WGw[nN"
}
});
data.Add(
"%MetadataPath%/library/2a/2a27372f1e9bc757b1db99721bbeae1e/poster.jpg*637261226720645297*Primary*0*0|%MetadataPath%/library/2a/2a27372f1e9bc757b1db99721bbeae1e/logo.png*637261226720805297*Logo*0*0|%MetadataPath%/library/2a/2a27372f1e9bc757b1db99721bbeae1e/landscape.jpg*637261226721285297*Thumb*0*0|%MetadataPath%/library/2a/2a27372f1e9bc757b1db99721bbeae1e/backdrop.jpg*637261226721685297*Backdrop*0*0",
new ItemImageInfo[]
]
},
{
"%MetadataPath%/library/2a/2a27372f1e9bc757b1db99721bbeae1e/poster.jpg*637261226720645297*Primary*0*0|%MetadataPath%/library/2a/2a27372f1e9bc757b1db99721bbeae1e/logo.png*637261226720805297*Logo*0*0|%MetadataPath%/library/2a/2a27372f1e9bc757b1db99721bbeae1e/landscape.jpg*637261226721285297*Thumb*0*0|%MetadataPath%/library/2a/2a27372f1e9bc757b1db99721bbeae1e/backdrop.jpg*637261226721685297*Backdrop*0*0",
[
new ItemImageInfo()
{
Path = "/meta/data/path/library/2a/2a27372f1e9bc757b1db99721bbeae1e/poster.jpg",
@ -145,22 +159,24 @@ namespace Jellyfin.Server.Implementations.Tests.Data
Type = ImageType.Backdrop,
DateModified = new DateTime(637261226721685297, DateTimeKind.Utc),
}
});
]
}
};
return data;
}
public static TheoryData<string, ItemImageInfo[]> DeserializeImages_ValidAndInvalid_TestData()
{
var data = new TheoryData<string, ItemImageInfo[]>();
data.Add(
string.Empty,
Array.Empty<ItemImageInfo>());
data.Add(
"/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg*637452096478512963*Primary*1920*1080*WjQbtJtSO8nhNZ%L_Io#R/oaS6o}-;adXAoIn7j[%hW9s:WGw[nN|test|1234||ss",
new ItemImageInfo[]
var data = new TheoryData<string, ItemImageInfo[]>
{
{
string.Empty,
Array.Empty<ItemImageInfo>()
},
{
"/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg*637452096478512963*Primary*1920*1080*WjQbtJtSO8nhNZ%L_Io#R/oaS6o}-;adXAoIn7j[%hW9s:WGw[nN|test|1234||ss",
[
new()
{
Path = "/mnt/series/Family Guy/Season 1/Family Guy - S01E01-thumb.jpg",
@ -170,18 +186,20 @@ namespace Jellyfin.Server.Implementations.Tests.Data
Height = 1080,
BlurHash = "WjQbtJtSO8nhNZ%L_Io#R*oaS6o}-;adXAoIn7j[%hW9s:WGw[nN"
}
});
data.Add(
"|",
Array.Empty<ItemImageInfo>());
]
},
{
"|",
Array.Empty<ItemImageInfo>()
}
};
return data;
}
private sealed class ProviderIdsExtensionsTestsObject : IHasProviderIds
{
public Dictionary<string, string> ProviderIds { get; set; } = new Dictionary<string, string>();
public Dictionary<string, string> ProviderIds { get; set; } = [];
}
}
}

@ -1,5 +1,4 @@
using Jellyfin.Database.Providers.Sqlite.Migrations;
using Jellyfin.Server.Implementations.Migrations;
using Microsoft.EntityFrameworkCore;
using Xunit;

Loading…
Cancel
Save