using System; using System.Collections.Generic; using System.Globalization; using System.Linq; using System.Net; using System.Security.Claims; using System.Text; using System.Threading; using System.Threading.Tasks; using Jellyfin.Api.Extensions; using Jellyfin.Api.Models.StreamingDtos; using Jellyfin.Data.Enums; using Jellyfin.Extensions; using MediaBrowser.Common.Configuration; using MediaBrowser.Common.Extensions; using MediaBrowser.Common.Net; using MediaBrowser.Controller.Configuration; using MediaBrowser.Controller.Devices; using MediaBrowser.Controller.Dlna; using MediaBrowser.Controller.Library; using MediaBrowser.Controller.MediaEncoding; using MediaBrowser.Model.Dlna; using MediaBrowser.Model.Entities; using MediaBrowser.Model.Net; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using Microsoft.Net.Http.Headers; namespace Jellyfin.Api.Helpers; /// /// Dynamic hls helper. /// public class DynamicHlsHelper { private readonly ILibraryManager _libraryManager; private readonly IUserManager _userManager; private readonly IDlnaManager _dlnaManager; private readonly IMediaSourceManager _mediaSourceManager; private readonly IServerConfigurationManager _serverConfigurationManager; private readonly IMediaEncoder _mediaEncoder; private readonly IDeviceManager _deviceManager; private readonly TranscodingJobHelper _transcodingJobHelper; private readonly INetworkManager _networkManager; private readonly ILogger _logger; private readonly IHttpContextAccessor _httpContextAccessor; private readonly EncodingHelper _encodingHelper; /// /// Initializes a new instance of the class. /// /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of . /// Instance of the interface. /// Instance of the interface. /// Instance of the interface. /// Instance of . public DynamicHlsHelper( ILibraryManager libraryManager, IUserManager userManager, IDlnaManager dlnaManager, IMediaSourceManager mediaSourceManager, IServerConfigurationManager serverConfigurationManager, IMediaEncoder mediaEncoder, IDeviceManager deviceManager, TranscodingJobHelper transcodingJobHelper, INetworkManager networkManager, ILogger logger, IHttpContextAccessor httpContextAccessor, EncodingHelper encodingHelper) { _libraryManager = libraryManager; _userManager = userManager; _dlnaManager = dlnaManager; _mediaSourceManager = mediaSourceManager; _serverConfigurationManager = serverConfigurationManager; _mediaEncoder = mediaEncoder; _deviceManager = deviceManager; _transcodingJobHelper = transcodingJobHelper; _networkManager = networkManager; _logger = logger; _httpContextAccessor = httpContextAccessor; _encodingHelper = encodingHelper; } /// /// Get master hls playlist. /// /// Transcoding job type. /// Streaming request dto. /// Enable adaptive bitrate streaming. /// A containing the resulting . public async Task GetMasterHlsPlaylist( TranscodingJobType transcodingJobType, StreamingRequestDto streamingRequest, bool enableAdaptiveBitrateStreaming) { var isHeadRequest = _httpContextAccessor.HttpContext?.Request.Method == WebRequestMethods.Http.Head; // CTS lifecycle is managed internally. var cancellationTokenSource = new CancellationTokenSource(); return await GetMasterPlaylistInternal( streamingRequest, isHeadRequest, enableAdaptiveBitrateStreaming, transcodingJobType, cancellationTokenSource).ConfigureAwait(false); } private async Task GetMasterPlaylistInternal( StreamingRequestDto streamingRequest, bool isHeadRequest, bool enableAdaptiveBitrateStreaming, TranscodingJobType transcodingJobType, CancellationTokenSource cancellationTokenSource) { if (_httpContextAccessor.HttpContext is null) { throw new ResourceNotFoundException(nameof(_httpContextAccessor.HttpContext)); } using var state = await StreamingHelpers.GetStreamingState( streamingRequest, _httpContextAccessor.HttpContext, _mediaSourceManager, _userManager, _libraryManager, _serverConfigurationManager, _mediaEncoder, _encodingHelper, _dlnaManager, _deviceManager, _transcodingJobHelper, transcodingJobType, cancellationTokenSource.Token) .ConfigureAwait(false); _httpContextAccessor.HttpContext.Response.Headers.Add(HeaderNames.Expires, "0"); if (isHeadRequest) { return new FileContentResult(Array.Empty(), MimeTypes.GetMimeType("playlist.m3u8")); } var totalBitrate = (state.OutputAudioBitrate ?? 0) + (state.OutputVideoBitrate ?? 0); var builder = new StringBuilder(); builder.AppendLine("#EXTM3U"); var isLiveStream = state.IsSegmentedLiveStream; var queryString = _httpContextAccessor.HttpContext.Request.QueryString.ToString(); // from universal audio service if (!string.IsNullOrWhiteSpace(state.Request.SegmentContainer) && !queryString.Contains("SegmentContainer", StringComparison.OrdinalIgnoreCase)) { queryString += "&SegmentContainer=" + state.Request.SegmentContainer; } // from universal audio service if (!string.IsNullOrWhiteSpace(state.Request.TranscodeReasons) && !queryString.Contains("TranscodeReasons=", StringComparison.OrdinalIgnoreCase)) { queryString += "&TranscodeReasons=" + state.Request.TranscodeReasons; } // Main stream var playlistUrl = isLiveStream ? "live.m3u8" : "main.m3u8"; playlistUrl += queryString; var subtitleStreams = state.MediaSource .MediaStreams .Where(i => i.IsTextSubtitleStream) .ToList(); var subtitleGroup = subtitleStreams.Count > 0 && (state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Hls || state.VideoRequest!.EnableSubtitlesInManifest) ? "subs" : null; // If we're burning in subtitles then don't add additional subs to the manifest if (state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode) { subtitleGroup = null; } if (!string.IsNullOrWhiteSpace(subtitleGroup)) { AddSubtitles(state, subtitleStreams, builder, _httpContextAccessor.HttpContext.User); } var basicPlaylist = AppendPlaylist(builder, state, playlistUrl, totalBitrate, subtitleGroup); if (state.VideoStream is not null && state.VideoRequest is not null) { // Provide a workaround for the case issue between flac and fLaC. var flacWaPlaylist = ApplyFlacCaseWorkaround(state, basicPlaylist.ToString()); if (!string.IsNullOrEmpty(flacWaPlaylist)) { builder.Append(flacWaPlaylist); } var encodingOptions = _serverConfigurationManager.GetEncodingOptions(); // Provide SDR HEVC entrance for backward compatibility. if (encodingOptions.AllowHevcEncoding && !encodingOptions.AllowAv1Encoding && EncodingHelper.IsCopyCodec(state.OutputVideoCodec) && state.VideoStream.VideoRange == VideoRange.HDR && string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase)) { var requestedVideoProfiles = state.GetRequestedProfiles("hevc"); if (requestedVideoProfiles is not null && requestedVideoProfiles.Length > 0) { // Force HEVC Main Profile and disable video stream copy. state.OutputVideoCodec = "hevc"; var sdrVideoUrl = ReplaceProfile(playlistUrl, "hevc", string.Join(',', requestedVideoProfiles), "main"); sdrVideoUrl += "&AllowVideoStreamCopy=false"; var sdrOutputVideoBitrate = _encodingHelper.GetVideoBitrateParamValue(state.VideoRequest, state.VideoStream, state.OutputVideoCodec); var sdrOutputAudioBitrate = 0; if (EncodingHelper.LosslessAudioCodecs.Contains(state.VideoRequest.AudioCodec, StringComparison.OrdinalIgnoreCase)) { sdrOutputAudioBitrate = state.AudioStream.BitRate ?? 0; } else { sdrOutputAudioBitrate = _encodingHelper.GetAudioBitrateParam(state.VideoRequest, state.AudioStream, state.OutputAudioChannels) ?? 0; } var sdrTotalBitrate = sdrOutputAudioBitrate + sdrOutputVideoBitrate; var sdrPlaylist = AppendPlaylist(builder, state, sdrVideoUrl, sdrTotalBitrate, subtitleGroup); // Provide a workaround for the case issue between flac and fLaC. flacWaPlaylist = ApplyFlacCaseWorkaround(state, sdrPlaylist.ToString()); if (!string.IsNullOrEmpty(flacWaPlaylist)) { builder.Append(flacWaPlaylist); } // Restore the video codec state.OutputVideoCodec = "copy"; } } // Provide Level 5.0 entrance for backward compatibility. // e.g. Apple A10 chips refuse the master playlist containing SDR HEVC Main Level 5.1 video, // but in fact it is capable of playing videos up to Level 6.1. if (encodingOptions.AllowHevcEncoding && !encodingOptions.AllowAv1Encoding && EncodingHelper.IsCopyCodec(state.OutputVideoCodec) && state.VideoStream.Level.HasValue && state.VideoStream.Level > 150 && state.VideoStream.VideoRange == VideoRange.SDR && string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase)) { var playlistCodecsField = new StringBuilder(); AppendPlaylistCodecsField(playlistCodecsField, state); // Force the video level to 5.0. var originalLevel = state.VideoStream.Level; state.VideoStream.Level = 150; var newPlaylistCodecsField = new StringBuilder(); AppendPlaylistCodecsField(newPlaylistCodecsField, state); // Restore the video level. state.VideoStream.Level = originalLevel; var newPlaylist = ReplacePlaylistCodecsField(basicPlaylist, playlistCodecsField, newPlaylistCodecsField); builder.Append(newPlaylist); // Provide a workaround for the case issue between flac and fLaC. flacWaPlaylist = ApplyFlacCaseWorkaround(state, newPlaylist); if (!string.IsNullOrEmpty(flacWaPlaylist)) { builder.Append(flacWaPlaylist); } } } if (EnableAdaptiveBitrateStreaming(state, isLiveStream, enableAdaptiveBitrateStreaming, _httpContextAccessor.HttpContext.GetNormalizedRemoteIP())) { var requestedVideoBitrate = state.VideoRequest is null ? 0 : state.VideoRequest.VideoBitRate ?? 0; // By default, vary by just 200k var variation = GetBitrateVariation(totalBitrate); var newBitrate = totalBitrate - variation; var variantUrl = ReplaceVideoBitrate(playlistUrl, requestedVideoBitrate, requestedVideoBitrate - variation); AppendPlaylist(builder, state, variantUrl, newBitrate, subtitleGroup); variation *= 2; newBitrate = totalBitrate - variation; variantUrl = ReplaceVideoBitrate(playlistUrl, requestedVideoBitrate, requestedVideoBitrate - variation); AppendPlaylist(builder, state, variantUrl, newBitrate, subtitleGroup); } return new FileContentResult(Encoding.UTF8.GetBytes(builder.ToString()), MimeTypes.GetMimeType("playlist.m3u8")); } private StringBuilder AppendPlaylist(StringBuilder builder, StreamState state, string url, int bitrate, string? subtitleGroup) { var playlistBuilder = new StringBuilder(); playlistBuilder.Append("#EXT-X-STREAM-INF:BANDWIDTH=") .Append(bitrate.ToString(CultureInfo.InvariantCulture)) .Append(",AVERAGE-BANDWIDTH=") .Append(bitrate.ToString(CultureInfo.InvariantCulture)); AppendPlaylistVideoRangeField(playlistBuilder, state); AppendPlaylistCodecsField(playlistBuilder, state); AppendPlaylistResolutionField(playlistBuilder, state); AppendPlaylistFramerateField(playlistBuilder, state); if (!string.IsNullOrWhiteSpace(subtitleGroup)) { playlistBuilder.Append(",SUBTITLES=\"") .Append(subtitleGroup) .Append('"'); } playlistBuilder.Append(Environment.NewLine); playlistBuilder.AppendLine(url); builder.Append(playlistBuilder); return playlistBuilder; } /// /// Appends a VIDEO-RANGE field containing the range of the output video stream. /// /// /// StringBuilder to append the field to. /// StreamState of the current stream. private void AppendPlaylistVideoRangeField(StringBuilder builder, StreamState state) { if (state.VideoStream is not null && state.VideoStream.VideoRange != VideoRange.Unknown) { var videoRange = state.VideoStream.VideoRange; if (EncodingHelper.IsCopyCodec(state.OutputVideoCodec)) { if (videoRange == VideoRange.SDR) { builder.Append(",VIDEO-RANGE=SDR"); } if (videoRange == VideoRange.HDR) { builder.Append(",VIDEO-RANGE=PQ"); } } else { // Currently we only encode to SDR. builder.Append(",VIDEO-RANGE=SDR"); } } } /// /// Appends a CODECS field containing formatted strings of /// the active streams output video and audio codecs. /// /// /// /// /// StringBuilder to append the field to. /// StreamState of the current stream. private void AppendPlaylistCodecsField(StringBuilder builder, StreamState state) { // Video string videoCodecs = string.Empty; int? videoCodecLevel = GetOutputVideoCodecLevel(state); if (!string.IsNullOrEmpty(state.ActualOutputVideoCodec) && videoCodecLevel.HasValue) { videoCodecs = GetPlaylistVideoCodecs(state, state.ActualOutputVideoCodec, videoCodecLevel.Value); } // Audio string audioCodecs = string.Empty; if (!string.IsNullOrEmpty(state.ActualOutputAudioCodec)) { audioCodecs = GetPlaylistAudioCodecs(state); } StringBuilder codecs = new StringBuilder(); codecs.Append(videoCodecs); if (!string.IsNullOrEmpty(videoCodecs) && !string.IsNullOrEmpty(audioCodecs)) { codecs.Append(','); } codecs.Append(audioCodecs); if (codecs.Length > 1) { builder.Append(",CODECS=\"") .Append(codecs) .Append('"'); } } /// /// Appends a RESOLUTION field containing the resolution of the output stream. /// /// /// StringBuilder to append the field to. /// StreamState of the current stream. private void AppendPlaylistResolutionField(StringBuilder builder, StreamState state) { if (state.OutputWidth.HasValue && state.OutputHeight.HasValue) { builder.Append(",RESOLUTION=") .Append(state.OutputWidth.GetValueOrDefault()) .Append('x') .Append(state.OutputHeight.GetValueOrDefault()); } } /// /// Appends a FRAME-RATE field containing the framerate of the output stream. /// /// /// StringBuilder to append the field to. /// StreamState of the current stream. private void AppendPlaylistFramerateField(StringBuilder builder, StreamState state) { double? framerate = null; if (state.TargetFramerate.HasValue) { framerate = Math.Round(state.TargetFramerate.GetValueOrDefault(), 3); } else if (state.VideoStream?.RealFrameRate is not null) { framerate = Math.Round(state.VideoStream.RealFrameRate.GetValueOrDefault(), 3); } if (framerate.HasValue) { builder.Append(",FRAME-RATE=") .Append(framerate.Value.ToString(CultureInfo.InvariantCulture)); } } private bool EnableAdaptiveBitrateStreaming(StreamState state, bool isLiveStream, bool enableAdaptiveBitrateStreaming, IPAddress ipAddress) { // Within the local network this will likely do more harm than good. if (_networkManager.IsInLocalNetwork(ipAddress)) { return false; } if (!enableAdaptiveBitrateStreaming) { return false; } if (isLiveStream || string.IsNullOrWhiteSpace(state.MediaPath)) { // Opening live streams is so slow it's not even worth it return false; } if (EncodingHelper.IsCopyCodec(state.OutputVideoCodec)) { return false; } if (EncodingHelper.IsCopyCodec(state.OutputAudioCodec)) { return false; } if (!state.IsOutputVideo) { return false; } // Having problems in android return false; // return state.VideoRequest.VideoBitRate.HasValue; } private void AddSubtitles(StreamState state, IEnumerable subtitles, StringBuilder builder, ClaimsPrincipal user) { if (state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Drop) { return; } var selectedIndex = state.SubtitleStream is null || state.SubtitleDeliveryMethod != SubtitleDeliveryMethod.Hls ? (int?)null : state.SubtitleStream.Index; const string Format = "#EXT-X-MEDIA:TYPE=SUBTITLES,GROUP-ID=\"subs\",NAME=\"{0}\",DEFAULT={1},FORCED={2},AUTOSELECT=YES,URI=\"{3}\",LANGUAGE=\"{4}\""; foreach (var stream in subtitles) { var name = stream.DisplayTitle; var isDefault = selectedIndex.HasValue && selectedIndex.Value == stream.Index; var isForced = stream.IsForced; var url = string.Format( CultureInfo.InvariantCulture, "{0}/Subtitles/{1}/subtitles.m3u8?SegmentLength={2}&api_key={3}", state.Request.MediaSourceId, stream.Index.ToString(CultureInfo.InvariantCulture), 30.ToString(CultureInfo.InvariantCulture), user.GetToken()); var line = string.Format( CultureInfo.InvariantCulture, Format, name, isDefault ? "YES" : "NO", isForced ? "YES" : "NO", url, stream.Language ?? "Unknown"); builder.AppendLine(line); } } /// /// Get the H.26X level of the output video stream. /// /// StreamState of the current stream. /// H.26X level of the output video stream. private int? GetOutputVideoCodecLevel(StreamState state) { string levelString = string.Empty; if (EncodingHelper.IsCopyCodec(state.OutputVideoCodec) && state.VideoStream is not null && state.VideoStream.Level.HasValue) { levelString = state.VideoStream.Level.ToString() ?? string.Empty; } else { if (string.Equals(state.ActualOutputVideoCodec, "h264", StringComparison.OrdinalIgnoreCase)) { levelString = state.GetRequestedLevel(state.ActualOutputVideoCodec) ?? "41"; levelString = EncodingHelper.NormalizeTranscodingLevel(state, levelString); } if (string.Equals(state.ActualOutputVideoCodec, "h265", StringComparison.OrdinalIgnoreCase) || string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase)) { levelString = state.GetRequestedLevel("h265") ?? state.GetRequestedLevel("hevc") ?? "120"; levelString = EncodingHelper.NormalizeTranscodingLevel(state, levelString); } if (string.Equals(state.ActualOutputVideoCodec, "av1", StringComparison.OrdinalIgnoreCase)) { levelString = state.GetRequestedLevel("av1") ?? "19"; levelString = EncodingHelper.NormalizeTranscodingLevel(state, levelString); } } if (int.TryParse(levelString, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedLevel)) { return parsedLevel; } return null; } /// /// Get the profile of the output video stream. /// /// StreamState of the current stream. /// Video codec. /// Profile of the output video stream. private string GetOutputVideoCodecProfile(StreamState state, string codec) { string profileString = string.Empty; if (EncodingHelper.IsCopyCodec(state.OutputVideoCodec) && !string.IsNullOrEmpty(state.VideoStream.Profile)) { profileString = state.VideoStream.Profile; } else if (!string.IsNullOrEmpty(codec)) { profileString = state.GetRequestedProfiles(codec).FirstOrDefault() ?? string.Empty; if (string.Equals(state.ActualOutputVideoCodec, "h264", StringComparison.OrdinalIgnoreCase)) { profileString ??= "high"; } if (string.Equals(state.ActualOutputVideoCodec, "h265", StringComparison.OrdinalIgnoreCase) || string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase) || string.Equals(state.ActualOutputVideoCodec, "av1", StringComparison.OrdinalIgnoreCase)) { profileString ??= "main"; } } return profileString; } /// /// Gets a formatted string of the output audio codec, for use in the CODECS field. /// /// /// /// StreamState of the current stream. /// Formatted audio codec string. private string GetPlaylistAudioCodecs(StreamState state) { if (string.Equals(state.ActualOutputAudioCodec, "aac", StringComparison.OrdinalIgnoreCase)) { string? profile = state.GetRequestedProfiles("aac").FirstOrDefault(); return HlsCodecStringHelpers.GetAACString(profile); } if (string.Equals(state.ActualOutputAudioCodec, "mp3", StringComparison.OrdinalIgnoreCase)) { return HlsCodecStringHelpers.GetMP3String(); } if (string.Equals(state.ActualOutputAudioCodec, "ac3", StringComparison.OrdinalIgnoreCase)) { return HlsCodecStringHelpers.GetAC3String(); } if (string.Equals(state.ActualOutputAudioCodec, "eac3", StringComparison.OrdinalIgnoreCase)) { return HlsCodecStringHelpers.GetEAC3String(); } if (string.Equals(state.ActualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase)) { return HlsCodecStringHelpers.GetFLACString(); } if (string.Equals(state.ActualOutputAudioCodec, "alac", StringComparison.OrdinalIgnoreCase)) { return HlsCodecStringHelpers.GetALACString(); } if (string.Equals(state.ActualOutputAudioCodec, "opus", StringComparison.OrdinalIgnoreCase)) { return HlsCodecStringHelpers.GetOPUSString(); } return string.Empty; } /// /// Gets a formatted string of the output video codec, for use in the CODECS field. /// /// /// /// StreamState of the current stream. /// Video codec. /// Video level. /// Formatted video codec string. private string GetPlaylistVideoCodecs(StreamState state, string codec, int level) { if (level == 0) { // This is 0 when there's no requested level in the device profile // and the source is not encoded in H.26X or AV1 _logger.LogError("Got invalid level when building CODECS field for HLS master playlist"); return string.Empty; } if (string.Equals(codec, "h264", StringComparison.OrdinalIgnoreCase)) { string profile = GetOutputVideoCodecProfile(state, "h264"); return HlsCodecStringHelpers.GetH264String(profile, level); } if (string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)) { string profile = GetOutputVideoCodecProfile(state, "hevc"); return HlsCodecStringHelpers.GetH265String(profile, level); } if (string.Equals(codec, "av1", StringComparison.OrdinalIgnoreCase)) { string profile = GetOutputVideoCodecProfile(state, "av1"); // Currently we only transcode to 8 bits AV1 int bitDepth = 8; if (EncodingHelper.IsCopyCodec(state.OutputVideoCodec) && state.VideoStream is not null && state.VideoStream.BitDepth.HasValue) { bitDepth = state.VideoStream.BitDepth.Value; } return HlsCodecStringHelpers.GetAv1String(profile, level, false, bitDepth); } return string.Empty; } private int GetBitrateVariation(int bitrate) { // By default, vary by just 50k var variation = 50000; if (bitrate >= 10000000) { variation = 2000000; } else if (bitrate >= 5000000) { variation = 1500000; } else if (bitrate >= 3000000) { variation = 1000000; } else if (bitrate >= 2000000) { variation = 500000; } else if (bitrate >= 1000000) { variation = 300000; } else if (bitrate >= 600000) { variation = 200000; } else if (bitrate >= 400000) { variation = 100000; } return variation; } private string ReplaceVideoBitrate(string url, int oldValue, int newValue) { return url.Replace( "videobitrate=" + oldValue.ToString(CultureInfo.InvariantCulture), "videobitrate=" + newValue.ToString(CultureInfo.InvariantCulture), StringComparison.OrdinalIgnoreCase); } private string ReplaceProfile(string url, string codec, string oldValue, string newValue) { string profileStr = codec + "-profile="; return url.Replace( profileStr + oldValue, profileStr + newValue, StringComparison.OrdinalIgnoreCase); } private string ReplacePlaylistCodecsField(StringBuilder playlist, StringBuilder oldValue, StringBuilder newValue) { var oldPlaylist = playlist.ToString(); return oldPlaylist.Replace( oldValue.ToString(), newValue.ToString(), StringComparison.Ordinal); } private string ApplyFlacCaseWorkaround(StreamState state, string srcPlaylist) { if (!string.Equals(state.ActualOutputAudioCodec, "flac", StringComparison.OrdinalIgnoreCase)) { return string.Empty; } var newPlaylist = srcPlaylist.Replace(",flac\"", ",fLaC\"", StringComparison.Ordinal); return newPlaylist.Contains(",fLaC\"", StringComparison.Ordinal) ? newPlaylist : string.Empty; } }