From 8025f72a555cf38626375087f88c5a39dc0a5887 Mon Sep 17 00:00:00 2001 From: gnattu Date: Mon, 22 Apr 2024 06:19:17 +0800 Subject: [PATCH 1/2] feat: add audio remux to UniversalAudioController Signed-off-by: gnattu --- .../Controllers/UniversalAudioController.cs | 16 +++-- Jellyfin.Api/Helpers/DynamicHlsHelper.cs | 8 +++ .../Transcoding/TranscodeManager.cs | 5 ++ MediaBrowser.Model/Dlna/StreamBuilder.cs | 63 ++++++++++++++++++- 4 files changed, 86 insertions(+), 6 deletions(-) diff --git a/Jellyfin.Api/Controllers/UniversalAudioController.cs b/Jellyfin.Api/Controllers/UniversalAudioController.cs index 1d4adae067..3115c4750a 100644 --- a/Jellyfin.Api/Controllers/UniversalAudioController.cs +++ b/Jellyfin.Api/Controllers/UniversalAudioController.cs @@ -17,6 +17,7 @@ using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Streaming; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.MediaInfo; +using MediaBrowser.Model.Session; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; @@ -137,6 +138,8 @@ public class UniversalAudioController : BaseJellyfinApiController // set device specific data foreach (var sourceInfo in info.MediaSources) { + sourceInfo.TranscodingContainer = transcodingContainer; + sourceInfo.TranscodingSubProtocol = transcodingProtocol ?? sourceInfo.TranscodingSubProtocol; _mediaInfoHelper.SetDeviceSpecificData( item, sourceInfo, @@ -171,27 +174,30 @@ public class UniversalAudioController : BaseJellyfinApiController return Redirect(mediaSource.Path); } + // This one is currently very misleading as the SupportsDirectStream is always false + // The definition of DirectStream also seems changed during development + // It used to mean HTTP direct streaming, but now HLS is used even for DirectStream var isStatic = mediaSource.SupportsDirectStream; - if (!isStatic && mediaSource.TranscodingSubProtocol == MediaStreamProtocol.hls) + if (mediaSource.TranscodingSubProtocol == MediaStreamProtocol.hls) { // hls segment container can only be mpegts or fmp4 per ffmpeg documentation // ffmpeg option -> file extension // mpegts -> ts // fmp4 -> mp4 - // TODO: remove this when we switch back to the segment muxer var supportedHlsContainers = new[] { "ts", "mp4" }; + // fallback to mpegts if device reports some weird value unsupported by hls + var segmentContainer = Array.Exists(supportedHlsContainers, element => element == transcodingContainer) ? transcodingContainer : "ts"; var dynamicHlsRequestDto = new HlsAudioRequestDto { Id = itemId, Container = ".m3u8", Static = isStatic, PlaySessionId = info.PlaySessionId, - // fallback to mpegts if device reports some weird value unsupported by hls - SegmentContainer = Array.Exists(supportedHlsContainers, element => element == transcodingContainer) ? transcodingContainer : "ts", + SegmentContainer = segmentContainer, MediaSourceId = mediaSourceId, DeviceId = deviceId, - AudioCodec = audioCodec, + AudioCodec = mediaSource.TranscodeReasons == TranscodeReason.ContainerNotSupported ? "copy" : audioCodec, EnableAutoStreamCopy = true, AllowAudioStreamCopy = true, AllowVideoStreamCopy = true, diff --git a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs index f8d89119a4..8df4caea5a 100644 --- a/Jellyfin.Api/Helpers/DynamicHlsHelper.cs +++ b/Jellyfin.Api/Helpers/DynamicHlsHelper.cs @@ -151,6 +151,14 @@ public class DynamicHlsHelper var queryString = _httpContextAccessor.HttpContext.Request.QueryString.ToString(); + // from universal audio service, need to override the AudioCodec when the actual request differs from original query + if (!string.Equals(state.OutputAudioCodec, _httpContextAccessor.HttpContext.Request.Query["AudioCodec"].ToString(), StringComparison.OrdinalIgnoreCase)) + { + var newQuery = Microsoft.AspNetCore.WebUtilities.QueryHelpers.ParseQuery(_httpContextAccessor.HttpContext.Request.QueryString.ToString()); + newQuery["AudioCodec"] = state.OutputAudioCodec; + queryString = Microsoft.AspNetCore.WebUtilities.QueryHelpers.AddQueryString(string.Empty, newQuery); + } + // from universal audio service if (!string.IsNullOrWhiteSpace(state.Request.SegmentContainer) && !queryString.Contains("SegmentContainer", StringComparison.OrdinalIgnoreCase)) diff --git a/MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs b/MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs index ea5dbf7f7f..e45e8507a1 100644 --- a/MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs +++ b/MediaBrowser.MediaEncoding/Transcoding/TranscodeManager.cs @@ -479,6 +479,11 @@ public sealed class TranscodeManager : ITranscodeManager, IDisposable : "FFmpeg.DirectStream-"; } + if (state.VideoRequest is null && EncodingHelper.IsCopyCodec(state.OutputAudioCodec)) + { + logFilePrefix = "FFmpeg.Remux-"; + } + var logFilePath = Path.Combine( _serverConfigurationManager.ApplicationPaths.LogDirectoryPath, $"{logFilePrefix}{DateTime.Now:yyyy-MM-dd_HH-mm-ss}_{state.Request.MediaSourceId}_{Guid.NewGuid().ToString()[..8]}.log"); diff --git a/MediaBrowser.Model/Dlna/StreamBuilder.cs b/MediaBrowser.Model/Dlna/StreamBuilder.cs index 55d1c3d51a..f9fe7d351a 100644 --- a/MediaBrowser.Model/Dlna/StreamBuilder.cs +++ b/MediaBrowser.Model/Dlna/StreamBuilder.cs @@ -108,7 +108,7 @@ namespace MediaBrowser.Model.Dlna var inputAudioSampleRate = audioStream?.SampleRate; var inputAudioBitDepth = audioStream?.BitDepth; - if (directPlayMethod.HasValue) + if (directPlayMethod is PlayMethod.DirectPlay) { var profile = options.Profile; var audioFailureConditions = GetProfileConditionsForAudio(profile.CodecProfiles, item.Container, audioStream?.Codec, inputAudioChannels, inputAudioBitrate, inputAudioSampleRate, inputAudioBitDepth, true); @@ -124,6 +124,41 @@ namespace MediaBrowser.Model.Dlna } } + if (directPlayMethod is PlayMethod.DirectStream) + { + var remuxContainer = item.TranscodingContainer ?? "ts"; + bool codeIsSupported; + if (item.TranscodingSubProtocol == MediaStreamProtocol.hls) + { + // Enforce HLS audio codec restrictions + if (string.Equals(remuxContainer, "mp4", StringComparison.OrdinalIgnoreCase)) + { + codeIsSupported = _supportedHlsAudioCodecsMp4.Contains(directPlayInfo.Profile?.AudioCodec ?? directPlayInfo.Profile?.Container); + } + else + { + codeIsSupported = _supportedHlsAudioCodecsTs.Contains(directPlayInfo.Profile?.AudioCodec ?? directPlayInfo.Profile?.Container); + } + } + else + { + // Let's assume the client has given a correct container for http + codeIsSupported = true; + } + + if (codeIsSupported) + { + playlistItem.PlayMethod = directPlayMethod.Value; + playlistItem.Container = remuxContainer; + playlistItem.TranscodeReasons = transcodeReasons; + playlistItem.SubProtocol = item.TranscodingSubProtocol; + return playlistItem; + } + + transcodeReasons |= TranscodeReason.AudioCodecNotSupported; + playlistItem.TranscodeReasons = transcodeReasons; + } + TranscodingProfile? transcodingProfile = null; foreach (var tcProfile in options.Profile.TranscodingProfiles) { @@ -387,6 +422,14 @@ namespace MediaBrowser.Model.Dlna item.Path ?? "Unknown path", audioStream.Codec ?? "Unknown codec"); + var directStreamProfile = options.Profile.DirectPlayProfiles + .FirstOrDefault(x => x.Type == DlnaProfileType.Audio && IsAudioDirectStreamSupported(x, item, audioStream)); + + if (directStreamProfile is not null) + { + return (directStreamProfile, PlayMethod.DirectStream, TranscodeReason.ContainerNotSupported); + } + return (null, null, GetTranscodeReasonsFromDirectPlayProfile(item, null, audioStream, options.Profile.DirectPlayProfiles)); } @@ -2129,5 +2172,23 @@ namespace MediaBrowser.Model.Dlna return true; } + + private static bool IsAudioDirectStreamSupported(DirectPlayProfile profile, MediaSourceInfo item, MediaStream audioStream) + { + // Check container type, this should NOT be supported + if (!profile.SupportsContainer(item.Container)) + { + // Check audio codec, we cannot use the SupportsAudioCodec here + // Because that one assumes empty container supports all codec, which is just useless + string? audioCodec = audioStream?.Codec; + if (string.Equals(profile.AudioCodec, audioCodec, StringComparison.OrdinalIgnoreCase) || + string.Equals(profile.Container, audioCodec, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } } } From 6695baf8ec45b265cab9571f7aebf3f208796169 Mon Sep 17 00:00:00 2001 From: gnattu Date: Mon, 22 Apr 2024 22:31:41 +0800 Subject: [PATCH 2/2] Allow clients to send audio container override for HLS This will improve flexibility due to overcome the complex compatibility situation of HLS Signed-off-by: gnattu --- Jellyfin.Api/Controllers/UniversalAudioController.cs | 3 ++- MediaBrowser.Model/Dlna/StreamBuilder.cs | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/Jellyfin.Api/Controllers/UniversalAudioController.cs b/Jellyfin.Api/Controllers/UniversalAudioController.cs index 3115c4750a..1305d14176 100644 --- a/Jellyfin.Api/Controllers/UniversalAudioController.cs +++ b/Jellyfin.Api/Controllers/UniversalAudioController.cs @@ -187,7 +187,8 @@ public class UniversalAudioController : BaseJellyfinApiController var supportedHlsContainers = new[] { "ts", "mp4" }; // fallback to mpegts if device reports some weird value unsupported by hls - var segmentContainer = Array.Exists(supportedHlsContainers, element => element == transcodingContainer) ? transcodingContainer : "ts"; + var requestedSegmentContainer = Array.Exists(supportedHlsContainers, element => element == transcodingContainer) ? transcodingContainer : "ts"; + var segmentContainer = Array.Exists(supportedHlsContainers, element => element == mediaSource.TranscodingContainer) ? mediaSource.TranscodingContainer : requestedSegmentContainer; var dynamicHlsRequestDto = new HlsAudioRequestDto { Id = itemId, diff --git a/MediaBrowser.Model/Dlna/StreamBuilder.cs b/MediaBrowser.Model/Dlna/StreamBuilder.cs index f9fe7d351a..20d9cd8d53 100644 --- a/MediaBrowser.Model/Dlna/StreamBuilder.cs +++ b/MediaBrowser.Model/Dlna/StreamBuilder.cs @@ -127,6 +127,10 @@ namespace MediaBrowser.Model.Dlna if (directPlayMethod is PlayMethod.DirectStream) { var remuxContainer = item.TranscodingContainer ?? "ts"; + var supportedHlsContainers = new[] { "ts", "mp4" }; + // If the container specified for the profile is an HLS supported container, use that container instead, overriding the preference + // The client should be responsible to ensure this container is compatible + remuxContainer = Array.Exists(supportedHlsContainers, element => element == directPlayInfo.Profile?.Container) ? directPlayInfo.Profile?.Container : remuxContainer; bool codeIsSupported; if (item.TranscodingSubProtocol == MediaStreamProtocol.hls) { @@ -152,6 +156,7 @@ namespace MediaBrowser.Model.Dlna playlistItem.Container = remuxContainer; playlistItem.TranscodeReasons = transcodeReasons; playlistItem.SubProtocol = item.TranscodingSubProtocol; + item.TranscodingContainer = remuxContainer; return playlistItem; }