Fixed Duplicate returns on grouping

Fixed UserDataKey not stored

Emby.Server.Implementations/Library/UserDataManager.cs

@@ -6,6 +6,7 @@ using System.Globalization;
 using System.Linq;
 using System.Threading;
 using Jellyfin.Data.Entities;
+using Jellyfin.Extensions;
 using Jellyfin.Server.Implementations;
 using MediaBrowser.Controller.Configuration;
 using MediaBrowser.Controller.Dto;
@@ -65,7 +66,15 @@ namespace Emby.Server.Implementations.Library
             foreach (var key in keys)
             {
                 userData.Key = key;
-                repository.UserData.Add(Map(userData, user.Id));
+                var userDataEntry = Map(userData, user.Id, item.Id);
+                if (repository.UserData.Any(f => f.ItemId == item.Id && f.UserId == user.Id && f.CustomDataKey == key))
+                {
+                    repository.UserData.Attach(userDataEntry).State = EntityState.Modified;
+                }
+                else
+                {
+                    repository.UserData.Add(userDataEntry);
+                }
             }
 
             repository.SaveChanges();
@@ -131,11 +140,12 @@ namespace Emby.Server.Implementations.Library
             SaveUserData(user, item, userData, reason, CancellationToken.None);
         }
 
-        private UserData Map(UserItemData dto, Guid userId)
+        private UserData Map(UserItemData dto, Guid userId, Guid itemId)
         {
             return new UserData()
             {
-                ItemId = Guid.Parse(dto.Key),
+                ItemId = itemId,
+                CustomDataKey = dto.Key,
                 Item = null!,
                 User = null!,
                 AudioStreamIndex = dto.AudioStreamIndex,
@@ -155,7 +165,7 @@ namespace Emby.Server.Implementations.Library
         {
             return new UserItemData()
             {
-                Key = dto.ItemId.ToString("D"),
+                Key = dto.CustomDataKey!,
                 AudioStreamIndex = dto.AudioStreamIndex,
                 IsFavorite = dto.IsFavorite,
                 LastPlayedDate = dto.LastPlayedDate,
@@ -175,7 +185,10 @@ namespace Emby.Server.Implementations.Library
 
             if (data is null)
             {
-                return null;
+                return new UserItemData()
+                {
+                    Key = keys[0],
+                };
             }
 
             return _userData.GetOrAdd(cacheKey, data);
@@ -184,13 +197,9 @@ namespace Emby.Server.Implementations.Library
         private UserItemData? GetUserDataInternal(Guid userId, List<string> keys)
         {
             var key = keys.FirstOrDefault();
-            if (key is null || !Guid.TryParse(key, out var itemId))
-            {
-                return null;
-            }
 
             using var context = _repository.CreateDbContext();
-            var userData = context.UserData.AsNoTracking().FirstOrDefault(e => e.ItemId == itemId && e.UserId.Equals(userId));
+            var userData = context.UserData.AsNoTracking().FirstOrDefault(e => e.CustomDataKey == key && e.UserId.Equals(userId));
 
             if (userData is not null)
             {
@@ -236,7 +245,7 @@ namespace Emby.Server.Implementations.Library
                 return null;
             }
 
-            var dto = GetUserItemDataDto(userData);
+            var dto = GetUserItemDataDto(userData, item.Id);
 
             item.FillUserDataDtoValues(dto, userData, itemDto, user, options);
             return dto;
@@ -246,9 +255,10 @@ namespace Emby.Server.Implementations.Library
         /// Converts a UserItemData to a DTOUserItemData.
         /// </summary>
         /// <param name="data">The data.</param>
+        /// <param name="itemId">The the reference key to an Item.</param>
         /// <returns>DtoUserItemData.</returns>
         /// <exception cref="ArgumentNullException"><paramref name="data"/> is <c>null</c>.</exception>
-        private UserItemDataDto GetUserItemDataDto(UserItemData data)
+        private UserItemDataDto GetUserItemDataDto(UserItemData data, Guid itemId)
         {
             ArgumentNullException.ThrowIfNull(data);
 
@@ -261,6 +271,7 @@ namespace Emby.Server.Implementations.Library
                 Rating = data.Rating,
                 Played = data.Played,
                 LastPlayedDate = data.LastPlayedDate,
+                ItemId = itemId,
                 Key = data.Key
             };
         }

Jellyfin.Data/Entities/UserData.cs

@@ -8,6 +8,12 @@ namespace Jellyfin.Data.Entities;
 /// </summary>
 public class UserData
 {
+    /// <summary>
+    /// Gets or sets the custom data key.
+    /// </summary>
+    /// <value>The rating.</value>
+    public required string CustomDataKey { get; set; }
+
     /// <summary>
     /// Gets or sets the users 0-10 rating.
     /// </summary>

Jellyfin.Server.Implementations/Item/BaseItemRepository.cs

@@ -116,22 +116,23 @@ public sealed class BaseItemRepository(
 
         using var context = dbProvider.CreateDbContext();
         var dbQuery = TranslateQuery(context.BaseItems.AsNoTracking(), context, filter);
-        // .DistinctBy(e => e.Id);
 
         var enableGroupByPresentationUniqueKey = EnableGroupByPresentationUniqueKey(filter);
         if (enableGroupByPresentationUniqueKey && filter.GroupBySeriesPresentationUniqueKey)
         {
-            dbQuery = dbQuery.GroupBy(e => new { e.PresentationUniqueKey, e.SeriesPresentationUniqueKey }).SelectMany(e => e);
+            dbQuery = dbQuery.GroupBy(e => new { e.PresentationUniqueKey, e.SeriesPresentationUniqueKey }).Select(e => e.First());
         }
-
+        else if (enableGroupByPresentationUniqueKey)
-        if (enableGroupByPresentationUniqueKey)
         {
-            dbQuery = dbQuery.GroupBy(e => e.PresentationUniqueKey).SelectMany(e => e);
+            dbQuery = dbQuery.GroupBy(e => e.PresentationUniqueKey).Select(e => e.First());
         }
-
+        else if (filter.GroupBySeriesPresentationUniqueKey)
-        if (filter.GroupBySeriesPresentationUniqueKey)
+        {
+            dbQuery = dbQuery.GroupBy(e => e.SeriesPresentationUniqueKey).Select(e => e.First());
+        }
+        else
         {
-            dbQuery = dbQuery.GroupBy(e => e.SeriesPresentationUniqueKey).SelectMany(e => e);
+            dbQuery = dbQuery.Distinct();
         }
 
         dbQuery = ApplyOrder(dbQuery, filter);
@@ -225,9 +226,15 @@ public sealed class BaseItemRepository(
         IQueryable<BaseItemEntity> dbQuery = context.BaseItems.AsNoTracking()
             .Include(e => e.TrailerTypes)
             .Include(e => e.Provider)
-            .Include(e => e.Images)
             .Include(e => e.LockedFields);
+
+        if (filter.DtoOptions.EnableImages)
+        {
+            dbQuery = dbQuery.Include(e => e.Images);
+        }
+
         dbQuery = TranslateQuery(dbQuery, context, filter);
+        dbQuery = dbQuery.Distinct();
         // .DistinctBy(e => e.Id);
         if (filter.EnableTotalRecordCount)
         {
@@ -266,10 +273,34 @@ public sealed class BaseItemRepository(
         IQueryable<BaseItemEntity> dbQuery = context.BaseItems.AsNoTracking()
             .Include(e => e.TrailerTypes)
             .Include(e => e.Provider)
-            .Include(e => e.Images)
             .Include(e => e.LockedFields);
+
+        if (filter.DtoOptions.EnableImages)
+        {
+            dbQuery = dbQuery.Include(e => e.Images);
+        }
+
         dbQuery = TranslateQuery(dbQuery, context, filter);
         dbQuery = ApplyOrder(dbQuery, filter);
+
+        var enableGroupByPresentationUniqueKey = EnableGroupByPresentationUniqueKey(filter);
+        if (enableGroupByPresentationUniqueKey && filter.GroupBySeriesPresentationUniqueKey)
+        {
+            dbQuery = dbQuery.GroupBy(e => new { e.PresentationUniqueKey, e.SeriesPresentationUniqueKey }).Select(e => e.First());
+        }
+        else if (enableGroupByPresentationUniqueKey)
+        {
+            dbQuery = dbQuery.GroupBy(e => e.PresentationUniqueKey).Select(e => e.First());
+        }
+        else if (filter.GroupBySeriesPresentationUniqueKey)
+        {
+            dbQuery = dbQuery.GroupBy(e => e.SeriesPresentationUniqueKey).Select(e => e.First());
+        }
+        else
+        {
+            dbQuery = dbQuery.Distinct();
+        }
+
         if (filter.Limit.HasValue || filter.StartIndex.HasValue)
         {
             var offset = filter.StartIndex ?? 0;
@@ -1330,7 +1361,7 @@ public sealed class BaseItemRepository(
             .Include(e => e.TrailerTypes)
             .Include(e => e.Provider)
             .Include(e => e.Images)
-            .Include(e => e.LockedFields).AsNoTracking().FirstOrDefault(e => e.Id == id);
+            .Include(e => e.LockedFields).AsNoTracking().AsSingleQuery().FirstOrDefault(e => e.Id == id);
         if (item is null)
         {
             return null;

Jellyfin.Server.Implementations/Migrations/20241111131257_AddedCustomDataKey.cs

@@ -0,0 +1,28 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Jellyfin.Server.Implementations.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddedCustomDataKey : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<string>(
+                name: "CustomDataKey",
+                table: "UserData",
+                type: "TEXT",
+                nullable: true);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "CustomDataKey",
+                table: "UserData");
+        }
+    }
+}

Jellyfin.Server.Implementations/Migrations/20241111135439_AddedCustomDataKeyKey.cs

@@ -0,0 +1,54 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace Jellyfin.Server.Implementations.Migrations
+{
+    /// <inheritdoc />
+    public partial class AddedCustomDataKeyKey : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropPrimaryKey(
+                name: "PK_UserData",
+                table: "UserData");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "CustomDataKey",
+                table: "UserData",
+                type: "TEXT",
+                nullable: false,
+                defaultValue: string.Empty,
+                oldClrType: typeof(string),
+                oldType: "TEXT",
+                oldNullable: true);
+
+            migrationBuilder.AddPrimaryKey(
+                name: "PK_UserData",
+                table: "UserData",
+                columns: new[] { "ItemId", "UserId", "CustomDataKey" });
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropPrimaryKey(
+                name: "PK_UserData",
+                table: "UserData");
+
+            migrationBuilder.AlterColumn<string>(
+                name: "CustomDataKey",
+                table: "UserData",
+                type: "TEXT",
+                nullable: true,
+                oldClrType: typeof(string),
+                oldType: "TEXT");
+
+            migrationBuilder.AddPrimaryKey(
+                name: "PK_UserData",
+                table: "UserData",
+                columns: new[] { "ItemId", "UserId" });
+        }
+    }
+}

Jellyfin.Server.Implementations/Migrations/JellyfinDbModelSnapshot.cs

@@ -1276,6 +1276,9 @@ namespace Jellyfin.Server.Implementations.Migrations
                     b.Property<Guid>("UserId")
                         .HasColumnType("TEXT");
 
+                    b.Property<string>("CustomDataKey")
+                        .HasColumnType("TEXT");
+
                     b.Property<int?>("AudioStreamIndex")
                         .HasColumnType("INTEGER");
 
@@ -1303,7 +1306,7 @@ namespace Jellyfin.Server.Implementations.Migrations
                     b.Property<int?>("SubtitleStreamIndex")
                         .HasColumnType("INTEGER");
 
-                    b.HasKey("ItemId", "UserId");
+                    b.HasKey("ItemId", "UserId", "CustomDataKey");
 
                     b.HasIndex("UserId");
 

Jellyfin.Server.Implementations/ModelConfiguration/UserDataConfiguration.cs

@@ -13,7 +13,7 @@ public class UserDataConfiguration : IEntityTypeConfiguration<UserData>
     /// <inheritdoc/>
     public void Configure(EntityTypeBuilder<UserData> builder)
     {
-        builder.HasKey(d => new { d.ItemId, d.UserId });
+        builder.HasKey(d => new { d.ItemId, d.UserId, d.CustomDataKey });
         builder.HasIndex(d => new { d.ItemId, d.UserId, d.Played });
         builder.HasIndex(d => new { d.ItemId, d.UserId, d.PlaybackPositionTicks });
         builder.HasIndex(d => new { d.ItemId, d.UserId, d.IsFavorite });

Jellyfin.Server/Migrations/Routines/MigrateLibraryDb.cs

@@ -107,20 +107,20 @@ public class MigrateLibraryDb : IMigrationRoutine
         foreach (var entity in queryResult)
         {
             var userData = GetUserData(users, entity);
-            if (userData.Data is null)
+            if (userData is null)
             {
                 _logger.LogError("Was not able to migrate user data with key {0}", entity.GetString(0));
                 continue;
             }
 
-            if (!legacyBaseItemWithUserKeys.TryGetValue(userData.LegacyUserDataKey!, out var refItem))
+            if (!legacyBaseItemWithUserKeys.TryGetValue(userData.CustomDataKey!, out var refItem))
             {
                 _logger.LogError("Was not able to migrate user data with key {0} because it does not reference a valid BaseItem.", entity.GetString(0));
                 continue;
             }
 
-            userData.Data.ItemId = refItem.Id;
+            userData.ItemId = refItem.Id;
-            dbContext.UserData.Add(userData.Data);
+            dbContext.UserData.Add(userData);
         }
 
         _logger.LogInformation("Try saving {0} UserData entries.", dbContext.UserData.Local.Count);
@@ -289,7 +289,7 @@ public class MigrateLibraryDb : IMigrationRoutine
         }
     }
 
-    private (UserData? Data, string? LegacyUserDataKey) GetUserData(ImmutableArray<User> users, SqliteDataReader dto)
+    private UserData? GetUserData(ImmutableArray<User> users, SqliteDataReader dto)
     {
         var indexOfUser = dto.GetInt32(1);
         var user = users.ElementAtOrDefault(indexOfUser - 1);
@@ -297,14 +297,15 @@ public class MigrateLibraryDb : IMigrationRoutine
         if (user is null)
         {
             _logger.LogError("Tried to find user with index '{Idx}' but there are only '{MaxIdx}' users.", indexOfUser, users.Length);
-            return (null, null);
+            return null;
         }
 
         var oldKey = dto.GetString(0);
 
-        return (new UserData()
+        return new UserData()
         {
             ItemId = Guid.NewGuid(),
+            CustomDataKey = oldKey,
             UserId = user.Id,
             Rating = dto.IsDBNull(2) ? null : dto.GetDouble(2),
             Played = dto.GetBoolean(3),
@@ -317,7 +318,7 @@ public class MigrateLibraryDb : IMigrationRoutine
             Likes = null,
             User = null!,
             Item = null!
-        }, oldKey);
+        };
     }
 
     private AncestorId GetAncestorId(SqliteDataReader reader)

MediaBrowser.Controller/Entities/BaseItem.cs

@@ -1825,7 +1825,10 @@ namespace MediaBrowser.Controller.Entities
         {
             ArgumentNullException.ThrowIfNull(user);
 
-            var data = UserDataManager.GetUserData(user, this);
+            var data = UserDataManager.GetUserData(user, this) ?? new UserItemData()
+            {
+                Key = GetUserDataKeys().First(),
+            };
 
             if (datePlayed.HasValue)
             {