feat(i18n): transliterate item name before sorting (#11172)

Directory.Packages.props

@@ -19,6 +19,7 @@
     <PackageVersion Include="EFCoreSecondLevelCacheInterceptor" Version="4.2.3" />
     <PackageVersion Include="FsCheck.Xunit" Version="2.16.6" />
     <PackageVersion Include="HarfBuzzSharp.NativeAssets.Linux" Version="7.3.0.1" />
+    <PackageVersion Include="ICU4N.Transliterator" Version="60.1.0-alpha.356" />
     <PackageVersion Include="IDisposableAnalyzers" Version="4.0.7" />
     <PackageVersion Include="Jellyfin.XmlTv" Version="10.8.0" />
     <PackageVersion Include="libse" Version="3.6.13" />
@@ -88,4 +89,4 @@
     <PackageVersion Include="Xunit.SkippableFact" Version="1.4.13" />
     <PackageVersion Include="xunit" Version="2.7.0" />
   </ItemGroup>
-</Project>
+</Project>

MediaBrowser.Controller/Entities/BaseItem.cs

@@ -964,7 +964,13 @@ namespace MediaBrowser.Controller.Entities
             AppendChunk(builder, isDigitChunk, name.Slice(chunkStart));
 
             // logger.LogDebug("ModifySortChunks Start: {0} End: {1}", name, builder.ToString());
-            return builder.ToString().RemoveDiacritics();
+            var result = builder.ToString().RemoveDiacritics();
+            if (!result.All(char.IsAscii))
+            {
+                result = result.Transliterated();
+            }
+
+            return result;
         }
 
         public BaseItem GetParent()

src/Jellyfin.Extensions/Jellyfin.Extensions.csproj

@@ -29,6 +29,7 @@
 
   <ItemGroup>
     <PackageReference Include="Diacritics" />
+    <PackageReference Include="ICU4N.Transliterator" />
   </ItemGroup>
 
 </Project>

src/Jellyfin.Extensions/StringExtensions.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Text.RegularExpressions;
+using ICU4N.Text;
 
 namespace Jellyfin.Extensions
 {
@@ -8,6 +9,9 @@ namespace Jellyfin.Extensions
     /// </summary>
     public static partial class StringExtensions
     {
+        private static readonly Lazy<Transliterator> _transliterator = new(() => Transliterator.GetInstance(
+            "Any-Latin; Latin-Ascii; Lower; NFD; [:Nonspacing Mark:] Remove; [:Punctuation:] Remove;"));
+
         // Matches non-conforming unicode chars
         // https://mnaoumov.wordpress.com/2014/06/14/stripping-invalid-characters-from-utf-16-strings/
 
@@ -96,5 +100,15 @@ namespace Jellyfin.Extensions
 
             return haystack[(pos + 1)..];
         }
+
+        /// <summary>
+        /// Returns a transliterated string which only contain ascii characters.
+        /// </summary>
+        /// <param name="text">The string to act on.</param>
+        /// <returns>The transliterated string.</returns>
+        public static string Transliterated(this string text)
+        {
+            return _transliterator.Value.Transliterate(text);
+        }
     }
 }