using System; using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.Globalization; using System.Linq; using System.Threading.Tasks; using Jellyfin.Api.Attributes; using Jellyfin.Api.Helpers; using Jellyfin.Api.ModelBinders; using Jellyfin.Api.Models.StreamingDtos; using Jellyfin.Data.Enums; using Jellyfin.Extensions; using MediaBrowser.Common.Extensions; using MediaBrowser.Controller.Entities; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Controller.Streaming; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.MediaInfo; using MediaBrowser.Model.Session; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; namespace Jellyfin.Api.Controllers; /// <summary> /// The universal audio controller. /// </summary> [Route("")] public class UniversalAudioController : BaseJellyfinApiController { private readonly ILibraryManager _libraryManager; private readonly ILogger<UniversalAudioController> _logger; private readonly MediaInfoHelper _mediaInfoHelper; private readonly AudioHelper _audioHelper; private readonly DynamicHlsHelper _dynamicHlsHelper; private readonly IUserManager _userManager; /// <summary> /// Initializes a new instance of the <see cref="UniversalAudioController"/> class. /// </summary> /// <param name="libraryManager">Instance of the <see cref="ILibraryManager"/> interface.</param> /// <param name="logger">Instance of the <see cref="ILogger{UniversalAudioController}"/> interface.</param> /// <param name="mediaInfoHelper">Instance of <see cref="MediaInfoHelper"/>.</param> /// <param name="audioHelper">Instance of <see cref="AudioHelper"/>.</param> /// <param name="dynamicHlsHelper">Instance of <see cref="DynamicHlsHelper"/>.</param> /// <param name="userManager">Instance of the <see cref="IUserManager"/> interface.</param> public UniversalAudioController( ILibraryManager libraryManager, ILogger<UniversalAudioController> logger, MediaInfoHelper mediaInfoHelper, AudioHelper audioHelper, DynamicHlsHelper dynamicHlsHelper, IUserManager userManager) { _libraryManager = libraryManager; _logger = logger; _mediaInfoHelper = mediaInfoHelper; _audioHelper = audioHelper; _dynamicHlsHelper = dynamicHlsHelper; _userManager = userManager; } /// <summary> /// Gets an audio stream. /// </summary> /// <param name="itemId">The item id.</param> /// <param name="container">Optional. The audio container.</param> /// <param name="mediaSourceId">The media version id, if playing an alternate version.</param> /// <param name="deviceId">The device id of the client requesting. Used to stop encoding processes when needed.</param> /// <param name="userId">Optional. The user id.</param> /// <param name="audioCodec">Optional. The audio codec to transcode to.</param> /// <param name="maxAudioChannels">Optional. The maximum number of audio channels.</param> /// <param name="transcodingAudioChannels">Optional. The number of how many audio channels to transcode to.</param> /// <param name="maxStreamingBitrate">Optional. The maximum streaming bitrate.</param> /// <param name="audioBitRate">Optional. Specify an audio bitrate to encode to, e.g. 128000. If omitted this will be left to encoder defaults.</param> /// <param name="startTimeTicks">Optional. Specify a starting offset, in ticks. 1 tick = 10000 ms.</param> /// <param name="transcodingContainer">Optional. The container to transcode to.</param> /// <param name="transcodingProtocol">Optional. The transcoding protocol.</param> /// <param name="maxAudioSampleRate">Optional. The maximum audio sample rate.</param> /// <param name="maxAudioBitDepth">Optional. The maximum audio bit depth.</param> /// <param name="enableRemoteMedia">Optional. Whether to enable remote media.</param> /// <param name="enableAudioVbrEncoding">Optional. Whether to enable Audio Encoding.</param> /// <param name="breakOnNonKeyFrames">Optional. Whether to break on non key frames.</param> /// <param name="enableRedirection">Whether to enable redirection. Defaults to true.</param> /// <response code="200">Audio stream returned.</response> /// <response code="302">Redirected to remote audio stream.</response> /// <response code="404">Item not found.</response> /// <returns>A <see cref="Task"/> containing the audio file.</returns> [HttpGet("Audio/{itemId}/universal")] [HttpHead("Audio/{itemId}/universal", Name = "HeadUniversalAudioStream")] [Authorize] [ProducesResponseType(StatusCodes.Status200OK)] [ProducesResponseType(StatusCodes.Status302Found)] [ProducesResponseType(StatusCodes.Status404NotFound)] [ProducesAudioFile] public async Task<ActionResult> GetUniversalAudioStream( [FromRoute, Required] Guid itemId, [FromQuery, ModelBinder(typeof(CommaDelimitedArrayModelBinder))] string[] container, [FromQuery] string? mediaSourceId, [FromQuery] string? deviceId, [FromQuery] Guid? userId, [FromQuery] [RegularExpression(EncodingHelper.ValidationRegex)] string? audioCodec, [FromQuery] int? maxAudioChannels, [FromQuery] int? transcodingAudioChannels, [FromQuery] int? maxStreamingBitrate, [FromQuery] int? audioBitRate, [FromQuery] long? startTimeTicks, [FromQuery] [RegularExpression(EncodingHelper.ValidationRegex)] string? transcodingContainer, [FromQuery] MediaStreamProtocol? transcodingProtocol, [FromQuery] int? maxAudioSampleRate, [FromQuery] int? maxAudioBitDepth, [FromQuery] bool? enableRemoteMedia, [FromQuery] bool enableAudioVbrEncoding = true, [FromQuery] bool breakOnNonKeyFrames = false, [FromQuery] bool enableRedirection = true) { userId = RequestHelpers.GetUserId(User, userId); var user = userId.IsNullOrEmpty() ? null : _userManager.GetUserById(userId.Value); var item = _libraryManager.GetItemById<BaseItem>(itemId, user); if (item is null) { return NotFound(); } var deviceProfile = GetDeviceProfile(container, transcodingContainer, audioCodec, transcodingProtocol, breakOnNonKeyFrames, transcodingAudioChannels, maxAudioSampleRate, maxAudioBitDepth, maxAudioChannels); _logger.LogInformation("GetPostedPlaybackInfo profile: {@Profile}", deviceProfile); var info = await _mediaInfoHelper.GetPlaybackInfo( item, user, mediaSourceId) .ConfigureAwait(false); // set device specific data foreach (var sourceInfo in info.MediaSources) { sourceInfo.TranscodingContainer = transcodingContainer; sourceInfo.TranscodingSubProtocol = transcodingProtocol ?? sourceInfo.TranscodingSubProtocol; _mediaInfoHelper.SetDeviceSpecificData( item, sourceInfo, deviceProfile, User, maxStreamingBitrate ?? deviceProfile.MaxStreamingBitrate, startTimeTicks ?? 0, mediaSourceId ?? string.Empty, null, null, maxAudioChannels, info.PlaySessionId!, userId ?? Guid.Empty, true, true, true, true, true, Request.HttpContext.GetNormalizedRemoteIP()); } _mediaInfoHelper.SortMediaSources(info, maxStreamingBitrate); foreach (var source in info.MediaSources) { _mediaInfoHelper.NormalizeMediaSourceContainer(source, deviceProfile, DlnaProfileType.Video); } var mediaSource = info.MediaSources[0]; if (mediaSource.SupportsDirectPlay && mediaSource.Protocol == MediaProtocol.Http && enableRedirection && mediaSource.IsRemote && enableRemoteMedia.HasValue && enableRemoteMedia.Value) { return Redirect(mediaSource.Path); } // This one is currently very misleading as the SupportsDirectStream actually means "can direct play" // The definition of DirectStream also seems changed during development var isStatic = mediaSource.SupportsDirectStream; if (!isStatic && mediaSource.TranscodingSubProtocol == MediaStreamProtocol.hls) { // hls segment container can only be mpegts or fmp4 per ffmpeg documentation // ffmpeg option -> file extension // mpegts -> ts // fmp4 -> mp4 var supportedHlsContainers = new[] { "ts", "mp4" }; // fallback to mpegts if device reports some weird value unsupported by hls var requestedSegmentContainer = Array.Exists( supportedHlsContainers, element => string.Equals(element, transcodingContainer, StringComparison.OrdinalIgnoreCase)) ? transcodingContainer : "ts"; var segmentContainer = Array.Exists( supportedHlsContainers, element => string.Equals(element, mediaSource.TranscodingContainer, StringComparison.OrdinalIgnoreCase)) ? mediaSource.TranscodingContainer : requestedSegmentContainer; var dynamicHlsRequestDto = new HlsAudioRequestDto { Id = itemId, Container = ".m3u8", Static = isStatic, PlaySessionId = info.PlaySessionId, SegmentContainer = segmentContainer, MediaSourceId = mediaSourceId, DeviceId = deviceId, AudioCodec = mediaSource.TranscodeReasons == TranscodeReason.ContainerNotSupported ? "copy" : audioCodec, EnableAutoStreamCopy = true, AllowAudioStreamCopy = true, AllowVideoStreamCopy = true, BreakOnNonKeyFrames = breakOnNonKeyFrames, AudioSampleRate = maxAudioSampleRate, MaxAudioChannels = maxAudioChannels, MaxAudioBitDepth = maxAudioBitDepth, AudioBitRate = audioBitRate ?? maxStreamingBitrate, StartTimeTicks = startTimeTicks, SubtitleMethod = SubtitleDeliveryMethod.Hls, RequireAvc = false, DeInterlace = false, RequireNonAnamorphic = false, EnableMpegtsM2TsMode = false, TranscodeReasons = mediaSource.TranscodeReasons == 0 ? null : mediaSource.TranscodeReasons.ToString(), Context = EncodingContext.Static, StreamOptions = new Dictionary<string, string>(), EnableAdaptiveBitrateStreaming = true, EnableAudioVbrEncoding = enableAudioVbrEncoding }; return await _dynamicHlsHelper.GetMasterHlsPlaylist(TranscodingJobType.Hls, dynamicHlsRequestDto, true) .ConfigureAwait(false); } var audioStreamingDto = new StreamingRequestDto { Id = itemId, Container = isStatic ? null : ("." + mediaSource.TranscodingContainer), Static = isStatic, PlaySessionId = info.PlaySessionId, MediaSourceId = mediaSourceId, DeviceId = deviceId, AudioCodec = audioCodec, EnableAutoStreamCopy = true, AllowAudioStreamCopy = true, AllowVideoStreamCopy = true, BreakOnNonKeyFrames = breakOnNonKeyFrames, AudioSampleRate = maxAudioSampleRate, MaxAudioChannels = maxAudioChannels, AudioBitRate = isStatic ? null : (audioBitRate ?? maxStreamingBitrate), MaxAudioBitDepth = maxAudioBitDepth, AudioChannels = maxAudioChannels, CopyTimestamps = true, StartTimeTicks = startTimeTicks, SubtitleMethod = SubtitleDeliveryMethod.Embed, TranscodeReasons = mediaSource.TranscodeReasons == 0 ? null : mediaSource.TranscodeReasons.ToString(), Context = EncodingContext.Static }; return await _audioHelper.GetAudioStream(TranscodingJobType.Progressive, audioStreamingDto).ConfigureAwait(false); } private DeviceProfile GetDeviceProfile( string[] containers, string? transcodingContainer, string? audioCodec, MediaStreamProtocol? transcodingProtocol, bool? breakOnNonKeyFrames, int? transcodingAudioChannels, int? maxAudioSampleRate, int? maxAudioBitDepth, int? maxAudioChannels) { var deviceProfile = new DeviceProfile(); int len = containers.Length; var directPlayProfiles = new DirectPlayProfile[len]; for (int i = 0; i < len; i++) { var parts = containers[i].Split('|', StringSplitOptions.RemoveEmptyEntries); var audioCodecs = parts.Length == 1 ? null : string.Join(',', parts.Skip(1)); directPlayProfiles[i] = new DirectPlayProfile { Type = DlnaProfileType.Audio, Container = parts[0], AudioCodec = audioCodecs }; } deviceProfile.DirectPlayProfiles = directPlayProfiles; deviceProfile.TranscodingProfiles = new[] { new TranscodingProfile { Type = DlnaProfileType.Audio, Context = EncodingContext.Streaming, Container = transcodingContainer ?? "mp3", AudioCodec = audioCodec ?? "mp3", Protocol = transcodingProtocol ?? MediaStreamProtocol.http, BreakOnNonKeyFrames = breakOnNonKeyFrames ?? false, MaxAudioChannels = transcodingAudioChannels?.ToString(CultureInfo.InvariantCulture) } }; var codecProfiles = new List<CodecProfile>(); var conditions = new List<ProfileCondition>(); if (maxAudioSampleRate.HasValue) { // codec profile conditions.Add( new ProfileCondition { Condition = ProfileConditionType.LessThanEqual, IsRequired = false, Property = ProfileConditionValue.AudioSampleRate, Value = maxAudioSampleRate.Value.ToString(CultureInfo.InvariantCulture) }); } if (maxAudioBitDepth.HasValue) { // codec profile conditions.Add( new ProfileCondition { Condition = ProfileConditionType.LessThanEqual, IsRequired = false, Property = ProfileConditionValue.AudioBitDepth, Value = maxAudioBitDepth.Value.ToString(CultureInfo.InvariantCulture) }); } if (maxAudioChannels.HasValue) { // codec profile conditions.Add( new ProfileCondition { Condition = ProfileConditionType.LessThanEqual, IsRequired = false, Property = ProfileConditionValue.AudioChannels, Value = maxAudioChannels.Value.ToString(CultureInfo.InvariantCulture) }); } if (conditions.Count > 0) { // codec profile codecProfiles.Add( new CodecProfile { Type = CodecType.Audio, Container = string.Join(',', containers), Conditions = conditions.ToArray() }); } deviceProfile.CodecProfiles = codecProfiles.ToArray(); return deviceProfile; } }