diff --git a/Jellyfin.Api/Controllers/DynamicHlsController.cs b/Jellyfin.Api/Controllers/DynamicHlsController.cs index caa3d23681..6ef3a2ff97 100644 --- a/Jellyfin.Api/Controllers/DynamicHlsController.cs +++ b/Jellyfin.Api/Controllers/DynamicHlsController.cs @@ -39,7 +39,8 @@ namespace Jellyfin.Api.Controllers [Authorize(Policy = Policies.DefaultAuthorization)] public class DynamicHlsController : BaseJellyfinApiController { - private const string DefaultEncoderPreset = "veryfast"; + private const string DefaultVodEncoderPreset = "veryfast"; + private const string DefaultEventEncoderPreset = "superfast"; private const TranscodingJobType TranscodingJobType = MediaBrowser.Controller.MediaEncoding.TranscodingJobType.Hls; private readonly ILibraryManager _libraryManager; @@ -105,6 +106,253 @@ namespace Jellyfin.Api.Controllers _encodingOptions = serverConfigurationManager.GetEncodingOptions(); } + /// + /// Gets a hls live stream. + /// + /// The item id. + /// The audio container. + /// Optional. If true, the original file will be streamed statically without any encoding. Use either no url extension or the original file extension. true/false. + /// The streaming parameters. + /// The tag. + /// Optional. The dlna device profile id to utilize. + /// The play session id. + /// The segment container. + /// The segment lenght. + /// The minimum number of segments. + /// The media version id, if playing an alternate version. + /// The device id of the client requesting. Used to stop encoding processes when needed. + /// Optional. Specify a audio codec to encode to, e.g. mp3. If omitted the server will auto-select using the url's extension. Options: aac, mp3, vorbis, wma. + /// Whether or not to allow automatic stream copy if requested values match the original source. Defaults to true. + /// Whether or not to allow copying of the video stream url. + /// Whether or not to allow copying of the audio stream url. + /// Optional. Whether to break on non key frames. + /// Optional. Specify a specific audio sample rate, e.g. 44100. + /// Optional. The maximum audio bit depth. + /// Optional. Specify an audio bitrate to encode to, e.g. 128000. If omitted this will be left to encoder defaults. + /// Optional. Specify a specific number of audio channels to encode to, e.g. 2. + /// Optional. Specify a maximum number of audio channels to encode to, e.g. 2. + /// Optional. Specify a specific an encoder profile (varies by encoder), e.g. main, baseline, high. + /// Optional. Specify a level for the encoder profile (varies by encoder), e.g. 3, 3.1. + /// Optional. A specific video framerate to encode to, e.g. 23.976. Generally this should be omitted unless the device has specific requirements. + /// Optional. A specific maximum video framerate to encode to, e.g. 23.976. Generally this should be omitted unless the device has specific requirements. + /// Whether or not to copy timestamps when transcoding with an offset. Defaults to false. + /// Optional. Specify a starting offset, in ticks. 1 tick = 10000 ms. + /// Optional. The fixed horizontal resolution of the encoded video. + /// Optional. The fixed vertical resolution of the encoded video. + /// Optional. Specify a video bitrate to encode to, e.g. 500000. If omitted this will be left to encoder defaults. + /// Optional. The index of the subtitle stream to use. If omitted no subtitles will be used. + /// Optional. Specify the subtitle delivery method. + /// Optional. + /// Optional. The maximum video bit depth. + /// Optional. Whether to require avc. + /// Optional. Whether to deinterlace the video. + /// Optional. Whether to require a non anamorphic stream. + /// Optional. The maximum number of audio channels to transcode. + /// Optional. The limit of how many cpu cores to use. + /// The live stream id. + /// Optional. Whether to enable the MpegtsM2Ts mode. + /// Optional. Specify a video codec to encode to, e.g. h264. If omitted the server will auto-select using the url's extension. Options: h265, h264, mpeg4, theora, vp8, vp9, vpx (deprecated), wmv. + /// Optional. Specify a subtitle codec to encode to. + /// Optional. The transcoding reason. + /// Optional. The index of the audio stream to use. If omitted the first audio stream will be used. + /// Optional. The index of the video stream to use. If omitted the first video stream will be used. + /// Optional. The . + /// Optional. The streaming options. + /// Optional. The max width. + /// Optional. The max height. + /// Optional. Whether to enable subtitles in the manifest. + /// Hls live stream retrieved. + /// A containing the hls file. + [HttpGet("Videos/{itemId}/live.m3u8")] + [ProducesResponseType(StatusCodes.Status200OK)] + [ProducesPlaylistFile] + public async Task GetLiveHlsStream( + [FromRoute, Required] Guid itemId, + [FromQuery] string? container, + [FromQuery] bool? @static, + [FromQuery] string? @params, + [FromQuery] string? tag, + [FromQuery] string? deviceProfileId, + [FromQuery] string? playSessionId, + [FromQuery] string? segmentContainer, + [FromQuery] int? segmentLength, + [FromQuery] int? minSegments, + [FromQuery] string? mediaSourceId, + [FromQuery] string? deviceId, + [FromQuery] string? audioCodec, + [FromQuery] bool? enableAutoStreamCopy, + [FromQuery] bool? allowVideoStreamCopy, + [FromQuery] bool? allowAudioStreamCopy, + [FromQuery] bool? breakOnNonKeyFrames, + [FromQuery] int? audioSampleRate, + [FromQuery] int? maxAudioBitDepth, + [FromQuery] int? audioBitRate, + [FromQuery] int? audioChannels, + [FromQuery] int? maxAudioChannels, + [FromQuery] string? profile, + [FromQuery] string? level, + [FromQuery] float? framerate, + [FromQuery] float? maxFramerate, + [FromQuery] bool? copyTimestamps, + [FromQuery] long? startTimeTicks, + [FromQuery] int? width, + [FromQuery] int? height, + [FromQuery] int? videoBitRate, + [FromQuery] int? subtitleStreamIndex, + [FromQuery] SubtitleDeliveryMethod? subtitleMethod, + [FromQuery] int? maxRefFrames, + [FromQuery] int? maxVideoBitDepth, + [FromQuery] bool? requireAvc, + [FromQuery] bool? deInterlace, + [FromQuery] bool? requireNonAnamorphic, + [FromQuery] int? transcodingMaxAudioChannels, + [FromQuery] int? cpuCoreLimit, + [FromQuery] string? liveStreamId, + [FromQuery] bool? enableMpegtsM2TsMode, + [FromQuery] string? videoCodec, + [FromQuery] string? subtitleCodec, + [FromQuery] string? transcodeReasons, + [FromQuery] int? audioStreamIndex, + [FromQuery] int? videoStreamIndex, + [FromQuery] EncodingContext? context, + [FromQuery] Dictionary streamOptions, + [FromQuery] int? maxWidth, + [FromQuery] int? maxHeight, + [FromQuery] bool? enableSubtitlesInManifest) + { + VideoRequestDto streamingRequest = new VideoRequestDto + { + Id = itemId, + Container = container, + Static = @static ?? false, + Params = @params, + Tag = tag, + DeviceProfileId = deviceProfileId, + PlaySessionId = playSessionId, + SegmentContainer = segmentContainer, + SegmentLength = segmentLength, + MinSegments = minSegments, + MediaSourceId = mediaSourceId, + DeviceId = deviceId, + AudioCodec = audioCodec, + EnableAutoStreamCopy = enableAutoStreamCopy ?? true, + AllowAudioStreamCopy = allowAudioStreamCopy ?? true, + AllowVideoStreamCopy = allowVideoStreamCopy ?? true, + BreakOnNonKeyFrames = breakOnNonKeyFrames ?? false, + AudioSampleRate = audioSampleRate, + MaxAudioChannels = maxAudioChannels, + AudioBitRate = audioBitRate, + MaxAudioBitDepth = maxAudioBitDepth, + AudioChannels = audioChannels, + Profile = profile, + Level = level, + Framerate = framerate, + MaxFramerate = maxFramerate, + CopyTimestamps = copyTimestamps ?? false, + StartTimeTicks = startTimeTicks, + Width = width, + Height = height, + VideoBitRate = videoBitRate, + SubtitleStreamIndex = subtitleStreamIndex, + SubtitleMethod = subtitleMethod ?? SubtitleDeliveryMethod.Encode, + MaxRefFrames = maxRefFrames, + MaxVideoBitDepth = maxVideoBitDepth, + RequireAvc = requireAvc ?? false, + DeInterlace = deInterlace ?? false, + RequireNonAnamorphic = requireNonAnamorphic ?? false, + TranscodingMaxAudioChannels = transcodingMaxAudioChannels, + CpuCoreLimit = cpuCoreLimit, + LiveStreamId = liveStreamId, + EnableMpegtsM2TsMode = enableMpegtsM2TsMode ?? false, + VideoCodec = videoCodec, + SubtitleCodec = subtitleCodec, + TranscodeReasons = transcodeReasons, + AudioStreamIndex = audioStreamIndex, + VideoStreamIndex = videoStreamIndex, + Context = context ?? EncodingContext.Streaming, + StreamOptions = streamOptions, + MaxHeight = maxHeight, + MaxWidth = maxWidth, + EnableSubtitlesInManifest = enableSubtitlesInManifest ?? true + }; + + // CTS lifecycle is managed internally. + var cancellationTokenSource = new CancellationTokenSource(); + // Due to CTS.Token calling ThrowIfDisposed (https://github.com/dotnet/runtime/issues/29970) we have to "cache" the token + // since it gets disposed when ffmpeg exits + var cancellationToken = cancellationTokenSource.Token; + using var state = await StreamingHelpers.GetStreamingState( + streamingRequest, + Request, + _authContext, + _mediaSourceManager, + _userManager, + _libraryManager, + _serverConfigurationManager, + _mediaEncoder, + _encodingHelper, + _dlnaManager, + _deviceManager, + _transcodingJobHelper, + TranscodingJobType, + cancellationToken) + .ConfigureAwait(false); + + TranscodingJobDto? job = null; + var playlistPath = Path.ChangeExtension(state.OutputFilePath, ".m3u8"); + + if (!System.IO.File.Exists(playlistPath)) + { + var transcodingLock = _transcodingJobHelper.GetTranscodingLock(playlistPath); + await transcodingLock.WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + if (!System.IO.File.Exists(playlistPath)) + { + // If the playlist doesn't already exist, startup ffmpeg + try + { + job = await _transcodingJobHelper.StartFfMpeg( + state, + playlistPath, + GetCommandLineArguments(playlistPath, state, true, 0), + Request, + TranscodingJobType, + cancellationTokenSource) + .ConfigureAwait(false); + job.IsLiveOutput = true; + } + catch + { + state.Dispose(); + throw; + } + + minSegments = state.MinSegments; + if (minSegments > 0) + { + await HlsHelpers.WaitForMinimumSegmentCount(playlistPath, minSegments, _logger, cancellationToken).ConfigureAwait(false); + } + } + } + finally + { + transcodingLock.Release(); + } + } + + job ??= _transcodingJobHelper.OnTranscodeBeginRequest(playlistPath, TranscodingJobType); + + if (job != null) + { + _transcodingJobHelper.OnTranscodeEndRequest(job); + } + + var playlistText = HlsHelpers.GetLivePlaylistText(playlistPath, state); + + return Content(playlistText, MimeTypes.GetMimeType("playlist.m3u8")); + } + /// /// Gets a video hls playlist stream. /// @@ -1149,7 +1397,7 @@ namespace Jellyfin.Api.Controllers .AppendLine("#EXT-X-MEDIA-SEQUENCE:0"); var index = 0; - var segmentExtension = GetSegmentFileExtension(streamingRequest.SegmentContainer); + var segmentExtension = EncodingHelper.GetSegmentFileExtension(streamingRequest.SegmentContainer); var queryString = Request.QueryString; if (isHlsInFmp4) @@ -1214,7 +1462,7 @@ namespace Jellyfin.Api.Controllers var segmentPath = GetSegmentPath(state, playlistPath, segmentId); - var segmentExtension = GetSegmentFileExtension(state.Request.SegmentContainer); + var segmentExtension = EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer); TranscodingJobDto? job; @@ -1286,7 +1534,7 @@ namespace Jellyfin.Api.Controllers job = await _transcodingJobHelper.StartFfMpeg( state, playlistPath, - GetCommandLineArguments(playlistPath, state, true, segmentId), + GetCommandLineArguments(playlistPath, state, false, segmentId), Request, TranscodingJobType, cancellationTokenSource).ConfigureAwait(false); @@ -1346,7 +1594,7 @@ namespace Jellyfin.Api.Controllers return segments; } - private string GetCommandLineArguments(string outputPath, StreamState state, bool isEncoding, int startNumber) + private string GetCommandLineArguments(string outputPath, StreamState state, bool isEventPlaylist, int startNumber) { var videoCodec = _encodingHelper.GetVideoEncoder(state, _encodingOptions); var threads = EncodingHelper.GetNumberOfThreads(state, _encodingOptions, videoCodec); @@ -1361,15 +1609,13 @@ namespace Jellyfin.Api.Controllers state.BaseRequest.BreakOnNonKeyFrames = false; } - // If isEncoding is true we're actually starting ffmpeg - var startNumberParam = isEncoding ? startNumber.ToString(CultureInfo.InvariantCulture) : "0"; var inputModifier = _encodingHelper.GetInputModifier(state, _encodingOptions); var mapArgs = state.IsOutputVideo ? _encodingHelper.GetMapArgs(state) : string.Empty; var directory = Path.GetDirectoryName(outputPath) ?? throw new ArgumentException($"Provided path ({outputPath}) is not valid.", nameof(outputPath)); var outputFileNameWithoutExtension = Path.GetFileNameWithoutExtension(outputPath); var outputPrefix = Path.Combine(directory, outputFileNameWithoutExtension); - var outputExtension = GetSegmentFileExtension(state.Request.SegmentContainer); + var outputExtension = EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer); var outputTsArg = outputPrefix + "%d" + outputExtension; var segmentFormat = outputExtension.TrimStart('.'); @@ -1398,19 +1644,30 @@ namespace Jellyfin.Api.Controllers ? _encodingOptions.MaxMuxingQueueSize.ToString(CultureInfo.InvariantCulture) : "128"; + var baseUrlParam = string.Empty; + if (isEventPlaylist) + { + baseUrlParam = string.Format( + CultureInfo.InvariantCulture, + " -hls_base_url \"hls/{0}/\"", + Path.GetFileNameWithoutExtension(outputPath)); + } + return string.Format( CultureInfo.InvariantCulture, - "{0} {1} -map_metadata -1 -map_chapters -1 -threads {2} {3} {4} {5} -copyts -avoid_negative_ts disabled -max_muxing_queue_size {6} -f hls -max_delay 5000000 -hls_time {7} -hls_segment_type {8} -start_number {9} -hls_segment_filename \"{10}\" -hls_playlist_type vod -hls_list_size 0 -y \"{11}\"", + "{0} {1} -map_metadata -1 -map_chapters -1 -threads {2} {3} {4} {5} -copyts -avoid_negative_ts disabled -max_muxing_queue_size {6} -f hls -max_delay 5000000 -hls_time {7} -hls_segment_type {8} -start_number {9}{10} -hls_segment_filename \"{12}\" -hls_playlist_type {11} -hls_list_size 0 -y \"{13}\"", inputModifier, _encodingHelper.GetInputArgument(state, _encodingOptions), threads, mapArgs, - GetVideoArguments(state, startNumber), + GetVideoArguments(state, startNumber, isEventPlaylist), GetAudioArguments(state), maxMuxingQueueSize, state.SegmentLength.ToString(CultureInfo.InvariantCulture), segmentFormat, - startNumberParam, + startNumber.ToString(CultureInfo.InvariantCulture), + baseUrlParam, + isEventPlaylist ? "event" : "vod", outputTsArg, outputPath).Trim(); } @@ -1505,8 +1762,9 @@ namespace Jellyfin.Api.Controllers /// /// The . /// The first number in the hls sequence. + /// Whether the playlist is EVENT or VOD. /// The command line arguments for video transcoding. - private string GetVideoArguments(StreamState state, int startNumber) + private string GetVideoArguments(StreamState state, int startNumber, bool isEventPlaylist) { if (state.VideoStream == null) { @@ -1539,6 +1797,7 @@ namespace Jellyfin.Api.Controllers // See if we can save come cpu cycles by avoiding encoding. if (EncodingHelper.IsCopyCodec(codec)) { + // If h264_mp4toannexb is ever added, do not use it for live tv. if (state.VideoStream != null && !string.Equals(state.VideoStream.NalLengthSize, "0", StringComparison.OrdinalIgnoreCase)) { string bitStreamArgs = EncodingHelper.GetBitStreamArgs(state.VideoStream); @@ -1549,15 +1808,13 @@ namespace Jellyfin.Api.Controllers } args += " -start_at_zero"; - - // args += " -flags -global_header"; } else { - args += _encodingHelper.GetVideoQualityParam(state, codec, _encodingOptions, DefaultEncoderPreset); + args += _encodingHelper.GetVideoQualityParam(state, codec, _encodingOptions, isEventPlaylist ? DefaultEventEncoderPreset : DefaultVodEncoderPreset); // Set the key frame params for video encoding to match the hls segment time. - args += _encodingHelper.GetHlsVideoKeyFrameArguments(state, codec, state.SegmentLength, false, startNumber); + args += _encodingHelper.GetHlsVideoKeyFrameArguments(state, codec, state.SegmentLength, isEventPlaylist, startNumber); // Currenly b-frames in libx265 breaks the FMP4-HLS playback on iOS, disable it for now. if (string.Equals(codec, "libx265", StringComparison.OrdinalIgnoreCase)) @@ -1567,27 +1824,25 @@ namespace Jellyfin.Api.Controllers // args += " -mixed-refs 0 -refs 3 -x264opts b_pyramid=0:weightb=0:weightp=0"; - var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; - - if (hasGraphicalSubs) - { - // Graphical subs overlay and resolution params. - args += _encodingHelper.GetGraphicalSubtitleParam(state, _encodingOptions, codec); - } - else - { - // Resolution params. - args += _encodingHelper.GetOutputSizeParam(state, _encodingOptions, codec); - } + // video processing filters. + args += _encodingHelper.GetVideoProcessingFilterParam(state, _encodingOptions, codec); // -start_at_zero is necessary to use with -ss when seeking, // otherwise the target position cannot be determined. - if (!(state.SubtitleStream != null && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream)) + if (state.SubtitleStream != null) { - args += " -start_at_zero"; + // Disable start_at_zero for external graphical subs + if (!(state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream)) + { + args += " -start_at_zero"; + } } + } - // args += " -flags -global_header"; + // TODO why was this not enabled for VOD? + if (isEventPlaylist) + { + args += " -flags -global_header"; } if (!string.IsNullOrEmpty(state.OutputVideoSync)) @@ -1600,22 +1855,12 @@ namespace Jellyfin.Api.Controllers return args; } - private string GetSegmentFileExtension(string? segmentContainer) - { - if (!string.IsNullOrWhiteSpace(segmentContainer)) - { - return "." + segmentContainer; - } - - return ".ts"; - } - private string GetSegmentPath(StreamState state, string playlist, int index) { var folder = Path.GetDirectoryName(playlist) ?? throw new ArgumentException($"Provided path ({playlist}) is not valid.", nameof(playlist)); var filename = Path.GetFileNameWithoutExtension(playlist); - return Path.Combine(folder, filename + index.ToString(CultureInfo.InvariantCulture) + GetSegmentFileExtension(state.Request.SegmentContainer)); + return Path.Combine(folder, filename + index.ToString(CultureInfo.InvariantCulture) + EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer)); } private async Task GetSegmentResult( diff --git a/Jellyfin.Api/Controllers/VideoHlsController.cs b/Jellyfin.Api/Controllers/VideoHlsController.cs deleted file mode 100644 index 36643e464c..0000000000 --- a/Jellyfin.Api/Controllers/VideoHlsController.cs +++ /dev/null @@ -1,586 +0,0 @@ -using System; -using System.Collections.Generic; -using System.ComponentModel.DataAnnotations; -using System.Globalization; -using System.IO; -using System.Threading; -using System.Threading.Tasks; -using Jellyfin.Api.Attributes; -using Jellyfin.Api.Constants; -using Jellyfin.Api.Helpers; -using Jellyfin.Api.Models.PlaybackDtos; -using Jellyfin.Api.Models.StreamingDtos; -using MediaBrowser.Common.Configuration; -using MediaBrowser.Controller.Configuration; -using MediaBrowser.Controller.Devices; -using MediaBrowser.Controller.Dlna; -using MediaBrowser.Controller.Library; -using MediaBrowser.Controller.MediaEncoding; -using MediaBrowser.Controller.Net; -using MediaBrowser.Model.Configuration; -using MediaBrowser.Model.Dlna; -using MediaBrowser.Model.Net; -using Microsoft.AspNetCore.Authorization; -using Microsoft.AspNetCore.Http; -using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Logging; - -namespace Jellyfin.Api.Controllers -{ - /// - /// The video hls controller. - /// - [Route("")] - [Authorize(Policy = Policies.DefaultAuthorization)] - public class VideoHlsController : BaseJellyfinApiController - { - private const string DefaultEncoderPreset = "superfast"; - private const TranscodingJobType TranscodingJobType = MediaBrowser.Controller.MediaEncoding.TranscodingJobType.Hls; - - private readonly EncodingHelper _encodingHelper; - private readonly IDlnaManager _dlnaManager; - private readonly IAuthorizationContext _authContext; - private readonly IUserManager _userManager; - private readonly ILibraryManager _libraryManager; - private readonly IMediaSourceManager _mediaSourceManager; - private readonly IServerConfigurationManager _serverConfigurationManager; - private readonly IMediaEncoder _mediaEncoder; - private readonly IDeviceManager _deviceManager; - private readonly TranscodingJobHelper _transcodingJobHelper; - private readonly ILogger _logger; - private readonly EncodingOptions _encodingOptions; - - /// - /// Initializes a new instance of the class. - /// - /// Instance of the interface. - /// Instance of the interface. - /// Instance of the interface. - /// Instance of the interface. - /// Instance of the interface. - /// Instance of the interface. - /// Instance of the interface. - /// Instance of the interface. - /// The singleton. - /// Instance of the . - /// Instance of . - public VideoHlsController( - IMediaEncoder mediaEncoder, - IDlnaManager dlnaManager, - IUserManager userManger, - IAuthorizationContext authorizationContext, - ILibraryManager libraryManager, - IMediaSourceManager mediaSourceManager, - IServerConfigurationManager serverConfigurationManager, - IDeviceManager deviceManager, - TranscodingJobHelper transcodingJobHelper, - ILogger logger, - EncodingHelper encodingHelper) - { - _dlnaManager = dlnaManager; - _authContext = authorizationContext; - _userManager = userManger; - _libraryManager = libraryManager; - _mediaSourceManager = mediaSourceManager; - _serverConfigurationManager = serverConfigurationManager; - _mediaEncoder = mediaEncoder; - _deviceManager = deviceManager; - _transcodingJobHelper = transcodingJobHelper; - _logger = logger; - _encodingHelper = encodingHelper; - - _encodingOptions = serverConfigurationManager.GetEncodingOptions(); - } - - /// - /// Gets a hls live stream. - /// - /// The item id. - /// The audio container. - /// Optional. If true, the original file will be streamed statically without any encoding. Use either no url extension or the original file extension. true/false. - /// The streaming parameters. - /// The tag. - /// Optional. The dlna device profile id to utilize. - /// The play session id. - /// The segment container. - /// The segment lenght. - /// The minimum number of segments. - /// The media version id, if playing an alternate version. - /// The device id of the client requesting. Used to stop encoding processes when needed. - /// Optional. Specify a audio codec to encode to, e.g. mp3. If omitted the server will auto-select using the url's extension. Options: aac, mp3, vorbis, wma. - /// Whether or not to allow automatic stream copy if requested values match the original source. Defaults to true. - /// Whether or not to allow copying of the video stream url. - /// Whether or not to allow copying of the audio stream url. - /// Optional. Whether to break on non key frames. - /// Optional. Specify a specific audio sample rate, e.g. 44100. - /// Optional. The maximum audio bit depth. - /// Optional. Specify an audio bitrate to encode to, e.g. 128000. If omitted this will be left to encoder defaults. - /// Optional. Specify a specific number of audio channels to encode to, e.g. 2. - /// Optional. Specify a maximum number of audio channels to encode to, e.g. 2. - /// Optional. Specify a specific an encoder profile (varies by encoder), e.g. main, baseline, high. - /// Optional. Specify a level for the encoder profile (varies by encoder), e.g. 3, 3.1. - /// Optional. A specific video framerate to encode to, e.g. 23.976. Generally this should be omitted unless the device has specific requirements. - /// Optional. A specific maximum video framerate to encode to, e.g. 23.976. Generally this should be omitted unless the device has specific requirements. - /// Whether or not to copy timestamps when transcoding with an offset. Defaults to false. - /// Optional. Specify a starting offset, in ticks. 1 tick = 10000 ms. - /// Optional. The fixed horizontal resolution of the encoded video. - /// Optional. The fixed vertical resolution of the encoded video. - /// Optional. Specify a video bitrate to encode to, e.g. 500000. If omitted this will be left to encoder defaults. - /// Optional. The index of the subtitle stream to use. If omitted no subtitles will be used. - /// Optional. Specify the subtitle delivery method. - /// Optional. - /// Optional. The maximum video bit depth. - /// Optional. Whether to require avc. - /// Optional. Whether to deinterlace the video. - /// Optional. Whether to require a non anamorphic stream. - /// Optional. The maximum number of audio channels to transcode. - /// Optional. The limit of how many cpu cores to use. - /// The live stream id. - /// Optional. Whether to enable the MpegtsM2Ts mode. - /// Optional. Specify a video codec to encode to, e.g. h264. If omitted the server will auto-select using the url's extension. Options: h265, h264, mpeg4, theora, vp8, vp9, vpx (deprecated), wmv. - /// Optional. Specify a subtitle codec to encode to. - /// Optional. The transcoding reason. - /// Optional. The index of the audio stream to use. If omitted the first audio stream will be used. - /// Optional. The index of the video stream to use. If omitted the first video stream will be used. - /// Optional. The . - /// Optional. The streaming options. - /// Optional. The max width. - /// Optional. The max height. - /// Optional. Whether to enable subtitles in the manifest. - /// Hls live stream retrieved. - /// A containing the hls file. - [HttpGet("Videos/{itemId}/live.m3u8")] - [ProducesResponseType(StatusCodes.Status200OK)] - [ProducesPlaylistFile] - public async Task GetLiveHlsStream( - [FromRoute, Required] Guid itemId, - [FromQuery] string? container, - [FromQuery] bool? @static, - [FromQuery] string? @params, - [FromQuery] string? tag, - [FromQuery] string? deviceProfileId, - [FromQuery] string? playSessionId, - [FromQuery] string? segmentContainer, - [FromQuery] int? segmentLength, - [FromQuery] int? minSegments, - [FromQuery] string? mediaSourceId, - [FromQuery] string? deviceId, - [FromQuery] string? audioCodec, - [FromQuery] bool? enableAutoStreamCopy, - [FromQuery] bool? allowVideoStreamCopy, - [FromQuery] bool? allowAudioStreamCopy, - [FromQuery] bool? breakOnNonKeyFrames, - [FromQuery] int? audioSampleRate, - [FromQuery] int? maxAudioBitDepth, - [FromQuery] int? audioBitRate, - [FromQuery] int? audioChannels, - [FromQuery] int? maxAudioChannels, - [FromQuery] string? profile, - [FromQuery] string? level, - [FromQuery] float? framerate, - [FromQuery] float? maxFramerate, - [FromQuery] bool? copyTimestamps, - [FromQuery] long? startTimeTicks, - [FromQuery] int? width, - [FromQuery] int? height, - [FromQuery] int? videoBitRate, - [FromQuery] int? subtitleStreamIndex, - [FromQuery] SubtitleDeliveryMethod? subtitleMethod, - [FromQuery] int? maxRefFrames, - [FromQuery] int? maxVideoBitDepth, - [FromQuery] bool? requireAvc, - [FromQuery] bool? deInterlace, - [FromQuery] bool? requireNonAnamorphic, - [FromQuery] int? transcodingMaxAudioChannels, - [FromQuery] int? cpuCoreLimit, - [FromQuery] string? liveStreamId, - [FromQuery] bool? enableMpegtsM2TsMode, - [FromQuery] string? videoCodec, - [FromQuery] string? subtitleCodec, - [FromQuery] string? transcodeReasons, - [FromQuery] int? audioStreamIndex, - [FromQuery] int? videoStreamIndex, - [FromQuery] EncodingContext? context, - [FromQuery] Dictionary streamOptions, - [FromQuery] int? maxWidth, - [FromQuery] int? maxHeight, - [FromQuery] bool? enableSubtitlesInManifest) - { - VideoRequestDto streamingRequest = new VideoRequestDto - { - Id = itemId, - Container = container, - Static = @static ?? false, - Params = @params, - Tag = tag, - DeviceProfileId = deviceProfileId, - PlaySessionId = playSessionId, - SegmentContainer = segmentContainer, - SegmentLength = segmentLength, - MinSegments = minSegments, - MediaSourceId = mediaSourceId, - DeviceId = deviceId, - AudioCodec = audioCodec, - EnableAutoStreamCopy = enableAutoStreamCopy ?? true, - AllowAudioStreamCopy = allowAudioStreamCopy ?? true, - AllowVideoStreamCopy = allowVideoStreamCopy ?? true, - BreakOnNonKeyFrames = breakOnNonKeyFrames ?? false, - AudioSampleRate = audioSampleRate, - MaxAudioChannels = maxAudioChannels, - AudioBitRate = audioBitRate, - MaxAudioBitDepth = maxAudioBitDepth, - AudioChannels = audioChannels, - Profile = profile, - Level = level, - Framerate = framerate, - MaxFramerate = maxFramerate, - CopyTimestamps = copyTimestamps ?? false, - StartTimeTicks = startTimeTicks, - Width = width, - Height = height, - VideoBitRate = videoBitRate, - SubtitleStreamIndex = subtitleStreamIndex, - SubtitleMethod = subtitleMethod ?? SubtitleDeliveryMethod.Encode, - MaxRefFrames = maxRefFrames, - MaxVideoBitDepth = maxVideoBitDepth, - RequireAvc = requireAvc ?? false, - DeInterlace = deInterlace ?? false, - RequireNonAnamorphic = requireNonAnamorphic ?? false, - TranscodingMaxAudioChannels = transcodingMaxAudioChannels, - CpuCoreLimit = cpuCoreLimit, - LiveStreamId = liveStreamId, - EnableMpegtsM2TsMode = enableMpegtsM2TsMode ?? false, - VideoCodec = videoCodec, - SubtitleCodec = subtitleCodec, - TranscodeReasons = transcodeReasons, - AudioStreamIndex = audioStreamIndex, - VideoStreamIndex = videoStreamIndex, - Context = context ?? EncodingContext.Streaming, - StreamOptions = streamOptions, - MaxHeight = maxHeight, - MaxWidth = maxWidth, - EnableSubtitlesInManifest = enableSubtitlesInManifest ?? true - }; - - // CTS lifecycle is managed internally. - var cancellationTokenSource = new CancellationTokenSource(); - // Due to CTS.Token calling ThrowIfDisposed (https://github.com/dotnet/runtime/issues/29970) we have to "cache" the token - // since it gets disposed when ffmpeg exits - var cancellationToken = cancellationTokenSource.Token; - using var state = await StreamingHelpers.GetStreamingState( - streamingRequest, - Request, - _authContext, - _mediaSourceManager, - _userManager, - _libraryManager, - _serverConfigurationManager, - _mediaEncoder, - _encodingHelper, - _dlnaManager, - _deviceManager, - _transcodingJobHelper, - TranscodingJobType, - cancellationToken) - .ConfigureAwait(false); - - TranscodingJobDto? job = null; - var playlistPath = Path.ChangeExtension(state.OutputFilePath, ".m3u8"); - - if (!System.IO.File.Exists(playlistPath)) - { - var transcodingLock = _transcodingJobHelper.GetTranscodingLock(playlistPath); - await transcodingLock.WaitAsync(cancellationToken).ConfigureAwait(false); - try - { - if (!System.IO.File.Exists(playlistPath)) - { - // If the playlist doesn't already exist, startup ffmpeg - try - { - job = await _transcodingJobHelper.StartFfMpeg( - state, - playlistPath, - GetCommandLineArguments(playlistPath, state), - Request, - TranscodingJobType, - cancellationTokenSource) - .ConfigureAwait(false); - job.IsLiveOutput = true; - } - catch - { - state.Dispose(); - throw; - } - - minSegments = state.MinSegments; - if (minSegments > 0) - { - await HlsHelpers.WaitForMinimumSegmentCount(playlistPath, minSegments, _logger, cancellationToken).ConfigureAwait(false); - } - } - } - finally - { - transcodingLock.Release(); - } - } - - job ??= _transcodingJobHelper.OnTranscodeBeginRequest(playlistPath, TranscodingJobType); - - if (job != null) - { - _transcodingJobHelper.OnTranscodeEndRequest(job); - } - - var playlistText = HlsHelpers.GetLivePlaylistText(playlistPath, state); - - return Content(playlistText, MimeTypes.GetMimeType("playlist.m3u8")); - } - - /// - /// Gets the command line arguments for ffmpeg. - /// - /// The output path of the file. - /// The . - /// The command line arguments as a string. - private string GetCommandLineArguments(string outputPath, StreamState state) - { - var videoCodec = _encodingHelper.GetVideoEncoder(state, _encodingOptions); - var threads = EncodingHelper.GetNumberOfThreads(state, _encodingOptions, videoCodec); // GetNumberOfThreads is static. - var inputModifier = _encodingHelper.GetInputModifier(state, _encodingOptions); - var mapArgs = state.IsOutputVideo ? _encodingHelper.GetMapArgs(state) : string.Empty; - - var directory = Path.GetDirectoryName(outputPath) ?? throw new ArgumentException($"Provided path ({outputPath}) is not valid.", nameof(outputPath)); - var outputFileNameWithoutExtension = Path.GetFileNameWithoutExtension(outputPath); - var outputPrefix = Path.Combine(directory, outputFileNameWithoutExtension); - var outputExtension = EncodingHelper.GetSegmentFileExtension(state.Request.SegmentContainer); - var outputTsArg = outputPrefix + "%d" + outputExtension; - - var segmentFormat = outputExtension.TrimStart('.'); - if (string.Equals(segmentFormat, "ts", StringComparison.OrdinalIgnoreCase)) - { - segmentFormat = "mpegts"; - } - else if (string.Equals(segmentFormat, "mp4", StringComparison.OrdinalIgnoreCase)) - { - string outputFmp4HeaderArg; - if (OperatingSystem.IsWindows()) - { - // on Windows, the path of fmp4 header file needs to be configured - outputFmp4HeaderArg = " -hls_fmp4_init_filename \"" + outputPrefix + "-1" + outputExtension + "\""; - } - else - { - // on Linux/Unix, ffmpeg generate fmp4 header file to m3u8 output folder - outputFmp4HeaderArg = " -hls_fmp4_init_filename \"" + outputFileNameWithoutExtension + "-1" + outputExtension + "\""; - } - - segmentFormat = "fmp4" + outputFmp4HeaderArg; - } - else - { - _logger.LogError("Invalid HLS segment container: {SegmentFormat}", segmentFormat); - } - - var maxMuxingQueueSize = _encodingOptions.MaxMuxingQueueSize > 128 - ? _encodingOptions.MaxMuxingQueueSize.ToString(CultureInfo.InvariantCulture) - : "128"; - - var baseUrlParam = string.Format( - CultureInfo.InvariantCulture, - "\"hls/{0}/\"", - Path.GetFileNameWithoutExtension(outputPath)); - - return string.Format( - CultureInfo.InvariantCulture, - "{0} {1} -map_metadata -1 -map_chapters -1 -threads {2} {3} {4} {5} -copyts -avoid_negative_ts disabled -max_muxing_queue_size {6} -f hls -max_delay 5000000 -hls_time {7} -hls_segment_type {8} -start_number 0 -hls_base_url {9} -hls_playlist_type event -hls_segment_filename \"{10}\" -y \"{11}\"", - inputModifier, - _encodingHelper.GetInputArgument(state, _encodingOptions), - threads, - mapArgs, - GetVideoArguments(state), - GetAudioArguments(state), - maxMuxingQueueSize, - state.SegmentLength.ToString(CultureInfo.InvariantCulture), - segmentFormat, - baseUrlParam, - outputTsArg, - outputPath).Trim(); - } - - /// - /// Gets the audio arguments for transcoding. - /// - /// The . - /// The command line arguments for audio transcoding. - private string GetAudioArguments(StreamState state) - { - if (state.AudioStream == null) - { - return string.Empty; - } - - var audioCodec = _encodingHelper.GetAudioEncoder(state); - - if (!state.IsOutputVideo) - { - if (EncodingHelper.IsCopyCodec(audioCodec)) - { - var bitStreamArgs = EncodingHelper.GetAudioBitStreamArguments(state, state.Request.SegmentContainer, state.MediaSource.Container); - - return "-acodec copy -strict -2" + bitStreamArgs; - } - - var audioTranscodeParams = string.Empty; - - audioTranscodeParams += "-acodec " + audioCodec; - - if (state.OutputAudioBitrate.HasValue) - { - audioTranscodeParams += " -ab " + state.OutputAudioBitrate.Value.ToString(CultureInfo.InvariantCulture); - } - - if (state.OutputAudioChannels.HasValue) - { - audioTranscodeParams += " -ac " + state.OutputAudioChannels.Value.ToString(CultureInfo.InvariantCulture); - } - - if (state.OutputAudioSampleRate.HasValue) - { - audioTranscodeParams += " -ar " + state.OutputAudioSampleRate.Value.ToString(CultureInfo.InvariantCulture); - } - - audioTranscodeParams += " -vn"; - return audioTranscodeParams; - } - - if (EncodingHelper.IsCopyCodec(audioCodec)) - { - var bitStreamArgs = EncodingHelper.GetAudioBitStreamArguments(state, state.Request.SegmentContainer, state.MediaSource.Container); - - return "-acodec copy -strict -2" + bitStreamArgs; - } - - var args = "-codec:a:0 " + audioCodec; - - var channels = state.OutputAudioChannels; - - if (channels.HasValue) - { - args += " -ac " + channels.Value; - } - - var bitrate = state.OutputAudioBitrate; - - if (bitrate.HasValue) - { - args += " -ab " + bitrate.Value.ToString(CultureInfo.InvariantCulture); - } - - if (state.OutputAudioSampleRate.HasValue) - { - args += " -ar " + state.OutputAudioSampleRate.Value.ToString(CultureInfo.InvariantCulture); - } - - args += _encodingHelper.GetAudioFilterParam(state, _encodingOptions); - - return args; - } - - /// - /// Gets the video arguments for transcoding. - /// - /// The . - /// The command line arguments for video transcoding. - private string GetVideoArguments(StreamState state) - { - if (state.VideoStream == null) - { - return string.Empty; - } - - if (!state.IsOutputVideo) - { - return string.Empty; - } - - var codec = _encodingHelper.GetVideoEncoder(state, _encodingOptions); - - var args = "-codec:v:0 " + codec; - - // Prefer hvc1 to hev1. - if (string.Equals(state.ActualOutputVideoCodec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(state.ActualOutputVideoCodec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase)) - { - args += " -tag:v:0 hvc1"; - } - - // if (state.EnableMpegtsM2TsMode) - // { - // args += " -mpegts_m2ts_mode 1"; - // } - - // See if we can save come cpu cycles by avoiding encoding. - if (EncodingHelper.IsCopyCodec(codec)) - { - // If h264_mp4toannexb is ever added, do not use it for live tv. - if (state.VideoStream != null && !string.Equals(state.VideoStream.NalLengthSize, "0", StringComparison.OrdinalIgnoreCase)) - { - string bitStreamArgs = EncodingHelper.GetBitStreamArgs(state.VideoStream); - if (!string.IsNullOrEmpty(bitStreamArgs)) - { - args += " " + bitStreamArgs; - } - } - - args += " -start_at_zero"; - } - else - { - args += _encodingHelper.GetVideoQualityParam(state, codec, _encodingOptions, DefaultEncoderPreset); - - // Set the key frame params for video encoding to match the hls segment time. - args += _encodingHelper.GetHlsVideoKeyFrameArguments(state, codec, state.SegmentLength, true, null); - - // Currenly b-frames in libx265 breaks the FMP4-HLS playback on iOS, disable it for now. - if (string.Equals(codec, "libx265", StringComparison.OrdinalIgnoreCase)) - { - args += " -bf 0"; - } - - var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; - - if (hasGraphicalSubs) - { - // Graphical subs overlay and resolution params. - args += _encodingHelper.GetGraphicalSubtitleParam(state, _encodingOptions, codec); - } - else - { - // Resolution params. - args += _encodingHelper.GetOutputSizeParam(state, _encodingOptions, codec); - } - - if (state.SubtitleStream == null || !state.SubtitleStream.IsExternal || state.SubtitleStream.IsTextSubtitleStream) - { - args += " -start_at_zero"; - } - } - - args += " -flags -global_header"; - - if (!string.IsNullOrEmpty(state.OutputVideoSync)) - { - args += " -vsync " + state.OutputVideoSync; - } - - args += _encodingHelper.GetOutputFFlags(state); - - return args; - } - } -} diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 92b345f126..c139d9d40f 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -24,7 +24,7 @@ namespace MediaBrowser.Controller.MediaEncoding private readonly IMediaEncoder _mediaEncoder; private readonly ISubtitleEncoder _subtitleEncoder; - private static readonly string[] _videoProfiles = new[] + private static readonly string[] _videoProfilesH264 = new[] { "ConstrainedBaseline", "Baseline", @@ -32,10 +32,22 @@ namespace MediaBrowser.Controller.MediaEncoding "Main", "High", "ProgressiveHigh", - "ConstrainedHigh" + "ConstrainedHigh", + "High10" }; - private static readonly Version _minVersionForCudaOverlay = new Version(4, 4); + private static readonly string[] _videoProfilesH265 = new[] + { + "Main", + "Main10" + }; + + private const string QsvAlias = "qs"; + private const string VaapiAlias = "va"; + private const string D3d11vaAlias = "dx11"; + private const string VideotoolboxAlias = "vt"; + private const string OpenclAlias = "ocl"; + private const string CudaAlias = "cu"; public EncodingHelper( IMediaEncoder mediaEncoder, @@ -62,15 +74,13 @@ namespace MediaBrowser.Controller.MediaEncoding var codecMap = new Dictionary(StringComparer.OrdinalIgnoreCase) { - { "qsv", hwEncoder + "_qsv" }, - { hwEncoder + "_qsv", hwEncoder + "_qsv" }, - { "nvenc", hwEncoder + "_nvenc" }, { "amf", hwEncoder + "_amf" }, - { "omx", hwEncoder + "_omx" }, - { hwEncoder + "_v4l2m2m", hwEncoder + "_v4l2m2m" }, - { "mediacodec", hwEncoder + "_mediacodec" }, + { "nvenc", hwEncoder + "_nvenc" }, + { "qsv", hwEncoder + "_qsv" }, { "vaapi", hwEncoder + "_vaapi" }, - { "videotoolbox", hwEncoder + "_videotoolbox" } + { "videotoolbox", hwEncoder + "_videotoolbox" }, + { "v4l2m2m", hwEncoder + "_v4l2m2m" }, + { "omx", hwEncoder + "_omx" }, }; if (!string.IsNullOrEmpty(hwType) @@ -91,11 +101,9 @@ namespace MediaBrowser.Controller.MediaEncoding private bool IsVaapiSupported(EncodingJobInfo state) { - var videoStream = state.VideoStream; - // vaapi will throw an error with this input // [vaapi @ 0x7faed8000960] No VAAPI support for codec mpeg4 profile -99. - if (string.Equals(videoStream?.Codec, "mpeg4", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(state.VideoStream?.Codec, "mpeg4", StringComparison.OrdinalIgnoreCase)) { return false; } @@ -103,82 +111,59 @@ namespace MediaBrowser.Controller.MediaEncoding return _mediaEncoder.SupportsHwaccel("vaapi"); } - private bool IsCudaSupported() + private bool IsVaapiFullSupported() { - return _mediaEncoder.SupportsHwaccel("cuda") - && _mediaEncoder.SupportsFilter("scale_cuda") - && _mediaEncoder.SupportsFilter("yadif_cuda") - && _mediaEncoder.SupportsFilter("hwupload_cuda"); + return _mediaEncoder.SupportsHwaccel("vaapi") + && _mediaEncoder.SupportsFilter("scale_vaapi") + && _mediaEncoder.SupportsFilter("deinterlace_vaapi") + && _mediaEncoder.SupportsFilter("tonemap_vaapi") + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVaapiFrameSync) + && _mediaEncoder.SupportsFilter("hwupload_vaapi"); } - private bool IsOpenclTonemappingSupported(EncodingJobInfo state, EncodingOptions options) + private bool IsOpenclFullSupported() { - var videoStream = state.VideoStream; - if (videoStream == null) - { - return false; - } + return _mediaEncoder.SupportsHwaccel("opencl") + && _mediaEncoder.SupportsFilter("scale_opencl") + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapOpenclBt2390) + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayOpenclFrameSync); + } - return options.EnableTonemapping - && (string.Equals(videoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoStream.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase)) - && IsColorDepth10(state) - && _mediaEncoder.SupportsHwaccel("opencl") - && _mediaEncoder.SupportsFilter("tonemap_opencl"); + private bool IsCudaFullSupported() + { + return _mediaEncoder.SupportsHwaccel("cuda") + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.ScaleCudaFormat) + && _mediaEncoder.SupportsFilter("yadif_cuda") + && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapCudaName) + && _mediaEncoder.SupportsFilter("overlay_cuda") + && _mediaEncoder.SupportsFilter("hwupload_cuda"); } - private bool IsCudaTonemappingSupported(EncodingJobInfo state, EncodingOptions options) + private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) { - var videoStream = state.VideoStream; - if (videoStream == null) + if (state.VideoStream == null) { return false; } return options.EnableTonemapping - && (string.Equals(videoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoStream.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase)) - && IsColorDepth10(state) - && _mediaEncoder.SupportsHwaccel("cuda") - && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapCudaName); + && (string.Equals(state.VideoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.VideoStream.ColorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase)) + && GetVideoColorBitDepth(state) == 10; } - private bool IsVppTonemappingSupported(EncodingJobInfo state, EncodingOptions options) + private bool IsVaapiVppTonemapAvailable(EncodingJobInfo state, EncodingOptions options) { - var videoStream = state.VideoStream; - if (videoStream == null) + if (state.VideoStream == null) { - // Remote stream doesn't have media info, disable vpp tonemapping. return false; } - var codec = videoStream.Codec; - if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) - { - // Limited to HEVC for now since the filter doesn't accept master data from VP9. - return options.EnableVppTonemapping - && string.Equals(videoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase) - && IsColorDepth10(state) - && string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - && _mediaEncoder.SupportsHwaccel("vaapi") - && _mediaEncoder.SupportsFilter("tonemap_vaapi"); - } - - // Hybrid VPP tonemapping for QSV with VAAPI - if (OperatingSystem.IsLinux() && string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) - { - // Limited to HEVC for now since the filter doesn't accept master data from VP9. - return options.EnableVppTonemapping - && string.Equals(videoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase) - && IsColorDepth10(state) - && string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - && _mediaEncoder.SupportsHwaccel("vaapi") - && _mediaEncoder.SupportsFilter("tonemap_vaapi") - && _mediaEncoder.SupportsHwaccel("qsv"); - } - // Native VPP tonemapping may come to QSV in the future. - return false; + + return options.EnableVppTonemapping + && string.Equals(state.VideoStream.ColorTransfer, "smpte2084", StringComparison.OrdinalIgnoreCase) + && GetVideoColorBitDepth(state) == 10; } /// @@ -463,11 +448,20 @@ namespace MediaBrowser.Controller.MediaEncoding return "copy"; } - public int GetVideoProfileScore(string profile) + public int GetVideoProfileScore(string videoCodec, string videoProfile) { // strip spaces because they may be stripped out on the query string - profile = profile.Replace(" ", string.Empty, StringComparison.Ordinal); - return Array.FindIndex(_videoProfiles, x => string.Equals(x, profile, StringComparison.OrdinalIgnoreCase)); + string profile = videoProfile.Replace(" ", string.Empty, StringComparison.Ordinal); + if (string.Equals("h264", videoCodec, StringComparison.OrdinalIgnoreCase)) + { + return Array.FindIndex(_videoProfilesH264, x => string.Equals(x, profile, StringComparison.OrdinalIgnoreCase)); + } + else if (string.Equals("hevc", videoCodec, StringComparison.OrdinalIgnoreCase)) + { + return Array.FindIndex(_videoProfilesH265, x => string.Equals(x, profile, StringComparison.OrdinalIgnoreCase)); + } + + return -1; } public string GetInputPathArgument(EncodingJobInfo state) @@ -526,161 +520,342 @@ namespace MediaBrowser.Controller.MediaEncoding return codec.ToLowerInvariant(); } + private string GetVideoToolboxDeviceArgs(string alias) + { + alias ??= VideotoolboxAlias; + + // device selection in vt is not supported. + return " -init_hw_device videotoolbox=" + alias; + } + + private string GetCudaDeviceArgs(int deviceIndex, string alias) + { + alias ??= CudaAlias; + deviceIndex = deviceIndex >= 0 + ? deviceIndex + : 0; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device cuda={0}:{1}", + alias, + deviceIndex); + } + + private string GetOpenclDeviceArgs(int deviceIndex, string deviceVendorName, string srcDeviceAlias, string alias) + { + alias ??= OpenclAlias; + deviceIndex = deviceIndex >= 0 + ? deviceIndex + : 0; + var vendorOpts = string.IsNullOrEmpty(deviceVendorName) + ? ":0.0" + : ":." + deviceIndex + ",device_vendor=\"" + deviceVendorName + "\""; + var options = string.IsNullOrEmpty(srcDeviceAlias) + ? vendorOpts + : "@" + srcDeviceAlias; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device opencl={0}{1}", + alias, + options); + } + + private string GetD3d11vaDeviceArgs(int deviceIndex, string deviceVendorId, string alias) + { + alias ??= D3d11vaAlias; + deviceIndex = deviceIndex >= 0 ? deviceIndex : 0; + var options = string.IsNullOrEmpty(deviceVendorId) + ? deviceIndex.ToString(CultureInfo.InvariantCulture) + : ",vendor=" + deviceVendorId; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device d3d11va={0}:{1}", + alias, + options); + } + + private string GetVaapiDeviceArgs(string renderNodePath, string kernelDriver, string driver, string alias) + { + alias ??= VaapiAlias; + renderNodePath = renderNodePath ?? "/dev/dri/renderD128"; + var options = string.IsNullOrEmpty(kernelDriver) || string.IsNullOrEmpty(driver) + ? renderNodePath + : ",kernel_driver=" + kernelDriver + ",driver=" + driver; + + return string.Format( + CultureInfo.InvariantCulture, + " -init_hw_device vaapi={0}:{1}", + alias, + options); + } + + private string GetQsvDeviceArgs(string alias) + { + var arg = " -init_hw_device qsv=" + (alias ?? QsvAlias); + if (OperatingSystem.IsLinux()) + { + // derive qsv from vaapi device + return GetVaapiDeviceArgs(null, "i915", "iHD", VaapiAlias) + arg + "@" + VaapiAlias; + } + + if (OperatingSystem.IsWindows()) + { + // derive qsv from d3d11va device + return GetD3d11vaDeviceArgs(0, "0x8086", D3d11vaAlias) + arg + "@" + D3d11vaAlias; + } + + return null; + } + + private string GetFilterHwDeviceArgs(string alias) + { + return string.IsNullOrEmpty(alias) + ? string.Empty + : " -filter_hw_device " + alias; + } + + public string GetGraphicalSubCanvasSize(EncodingJobInfo state) + { + if (state.SubtitleStream != null + && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode + && !state.SubtitleStream.IsTextSubtitleStream) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + + // setup a relative small canvas_size for overlay_qsv/vaapi to reduce transfer overhead + var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, 1080); + + if (overlayW.HasValue && overlayH.HasValue) + { + return string.Format( + CultureInfo.InvariantCulture, + " -canvas_size {0}x{1}", + overlayW.Value, + overlayH.Value); + } + } + + return string.Empty; + } + /// - /// Gets the input argument. + /// Gets the input video hwaccel argument. /// /// Encoding state. /// Encoding options. - /// Input arguments. - public string GetInputArgument(EncodingJobInfo state, EncodingOptions options) + /// Input video hwaccel arguments. + public string GetInputVideoHwaccelArgs(EncodingJobInfo state, EncodingOptions options) { - var arg = new StringBuilder(); - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options) ?? string.Empty; - var outputVideoCodec = GetVideoEncoder(state, options) ?? string.Empty; + if (!state.IsVideoRequest) + { + return string.Empty; + } + + var vidEncoder = GetVideoEncoder(state, options) ?? string.Empty; + if (IsCopyCodec(vidEncoder)) + { + return string.Empty; + } + + var args = new StringBuilder(); var isWindows = OperatingSystem.IsWindows(); var isLinux = OperatingSystem.IsLinux(); var isMacOS = OperatingSystem.IsMacOS(); -#pragma warning disable CA1508 // Defaults to string.Empty - var isSwDecoder = string.IsNullOrEmpty(videoDecoder); -#pragma warning restore CA1508 - var isD3d11vaDecoder = videoDecoder.IndexOf("d3d11va", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiDecoder = videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiEncoder = outputVideoCodec.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvDecoder = videoDecoder.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvEncoder = outputVideoCodec.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1; - var isNvdecDecoder = videoDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); - var isCuvidHevcDecoder = videoDecoder.Contains("hevc_cuvid", StringComparison.OrdinalIgnoreCase); - var isCuvidVp9Decoder = videoDecoder.Contains("vp9_cuvid", StringComparison.OrdinalIgnoreCase); - var isOpenclTonemappingSupported = IsOpenclTonemappingSupported(state, options); - var isVppTonemappingSupported = IsVppTonemappingSupported(state, options); - var isCudaTonemappingSupported = IsCudaTonemappingSupported(state, options); - - if (!IsCopyCodec(outputVideoCodec)) - { - if (state.IsVideoRequest - && _mediaEncoder.SupportsHwaccel("vaapi") - && string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) - { - if (isVaapiDecoder) + var optHwaccelType = options.HardwareAccelerationType; + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isHwTonemapAvailable = IsHwTonemapAvailable(state, options); + + if (string.Equals(optHwaccelType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + if (!isLinux || !_mediaEncoder.SupportsHwaccel("vaapi")) + { + return string.Empty; + } + + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + if (!isVaapiDecoder && !isVaapiEncoder) + { + return string.Empty; + } + + args.Append(GetVaapiDeviceArgs(options.VaapiDevice, null, null, VaapiAlias)); + var filterDevArgs = GetFilterHwDeviceArgs(VaapiAlias); + + if (isHwTonemapAvailable && IsOpenclFullSupported()) + { + if (_mediaEncoder.IsVaapiDeviceInteliHD || _mediaEncoder.IsVaapiDeviceInteli965) { - if (isOpenclTonemappingSupported && !isVppTonemappingSupported) - { - arg.Append("-init_hw_device vaapi=va:") - .Append(options.VaapiDevice) - .Append(" -init_hw_device opencl=ocl@va ") - .Append("-hwaccel_device va ") - .Append("-hwaccel_output_format vaapi ") - .Append("-filter_hw_device ocl "); - } - else + if (!isVaapiDecoder) { - arg.Append("-hwaccel_output_format vaapi ") - .Append("-vaapi_device ") - .Append(options.VaapiDevice) - .Append(' '); + args.Append(GetOpenclDeviceArgs(0, null, VaapiAlias, OpenclAlias)); + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); } } - else if (!isVaapiDecoder && isVaapiEncoder) + else if (_mediaEncoder.IsVaapiDeviceAmd) + { + args.Append(GetOpenclDeviceArgs(0, "Advanced Micro Devices", null, OpenclAlias)); + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); + } + else { - arg.Append("-vaapi_device ") - .Append(options.VaapiDevice) - .Append(' '); + args.Append(GetOpenclDeviceArgs(0, null, null, OpenclAlias)); + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); } + } - arg.Append("-autorotate 0 "); + args.Append(filterDevArgs); + } + else if (string.Equals(optHwaccelType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + if ((!isLinux && !isWindows) || !_mediaEncoder.SupportsHwaccel("qsv")) + { + return string.Empty; } - if (state.IsVideoRequest - && string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isQsvDecoder = vidDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isQsvEncoder = vidEncoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isHwDecoder = isQsvDecoder || isVaapiDecoder || isD3d11vaDecoder; + if (!isHwDecoder && !isQsvEncoder) { - var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + return string.Empty; + } - if (isQsvEncoder) + args.Append(GetQsvDeviceArgs(QsvAlias)); + var filterDevArgs = GetFilterHwDeviceArgs(QsvAlias); + // child device used by qsv. + if (_mediaEncoder.SupportsHwaccel("vaapi") || _mediaEncoder.SupportsHwaccel("d3d11va")) + { + if (isHwTonemapAvailable && IsOpenclFullSupported()) { - if (isQsvDecoder) + var srcAlias = isLinux ? VaapiAlias : D3d11vaAlias; + args.Append(GetOpenclDeviceArgs(0, null, srcAlias, OpenclAlias)); + if (!isHwDecoder) { - if (isLinux) - { - if (hasGraphicalSubs) - { - arg.Append("-init_hw_device qsv=hw -filter_hw_device hw "); - } - else - { - arg.Append("-hwaccel qsv "); - } - } - - if (isWindows) - { - arg.Append("-hwaccel qsv "); - } + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); } + } + } - // While using SW decoder - else if (isSwDecoder) - { - arg.Append("-init_hw_device qsv=hw -filter_hw_device hw "); - } + args.Append(filterDevArgs); + } + else if (string.Equals(optHwaccelType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + if ((!isLinux && !isWindows) || !IsCudaFullSupported()) + { + return string.Empty; + } - // Hybrid VPP tonemapping with VAAPI - else if (isVaapiDecoder && isVppTonemappingSupported) - { - arg.Append("-init_hw_device vaapi=va:") - .Append(options.VaapiDevice) - .Append(" -init_hw_device qsv@va ") - .Append("-hwaccel_output_format vaapi "); - } + var isCuvidDecoder = vidDecoder.Contains("cuvid", StringComparison.OrdinalIgnoreCase); + var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); + var isNvencEncoder = vidEncoder.Contains("nvenc", StringComparison.OrdinalIgnoreCase); + var isHwDecoder = isNvdecDecoder || isCuvidDecoder; + if (!isHwDecoder && !isNvencEncoder) + { + return string.Empty; + } - arg.Append("-autorotate 0 "); - } + args.Append(GetCudaDeviceArgs(0, CudaAlias)) + .Append(GetFilterHwDeviceArgs(CudaAlias)); + + // workaround for "No decoder surfaces left" error, + // but will increase vram usage. https://trac.ffmpeg.org/ticket/7562 + args.Append(" -extra_hw_frames 3"); + } + else if (string.Equals(optHwaccelType, "amf", StringComparison.OrdinalIgnoreCase)) + { + if (!isWindows || !_mediaEncoder.SupportsHwaccel("d3d11va")) + { + return string.Empty; } - if (state.IsVideoRequest - && string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) - && isNvdecDecoder) + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + var isAmfEncoder = vidEncoder.Contains("amf", StringComparison.OrdinalIgnoreCase); + if (!isD3d11vaDecoder && !isAmfEncoder) { - // Fix for 'No decoder surfaces left' error. https://trac.ffmpeg.org/ticket/7562 - arg.Append("-hwaccel_output_format cuda -extra_hw_frames 3 -autorotate 0 "); + return string.Empty; } - if (state.IsVideoRequest - && string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) - && (isNvdecDecoder || isCuvidHevcDecoder || isCuvidVp9Decoder || isSwDecoder)) + // no dxva video processor hw filter. + args.Append(GetD3d11vaDeviceArgs(0, "0x1002", D3d11vaAlias)); + var filterDevArgs = string.Empty; + if (IsOpenclFullSupported()) { - if (!isCudaTonemappingSupported && isOpenclTonemappingSupported) - { - arg.Append("-init_hw_device opencl=ocl:") - .Append(options.OpenclDevice) - .Append(" -filter_hw_device ocl "); - } + args.Append(GetOpenclDeviceArgs(0, null, D3d11vaAlias, OpenclAlias)); + filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias); } - if (state.IsVideoRequest - && string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase) - && (isD3d11vaDecoder || isSwDecoder)) + args.Append(filterDevArgs); + } + else if (string.Equals(optHwaccelType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + { + if (!isMacOS || !_mediaEncoder.SupportsHwaccel("videotoolbox")) { - if (isOpenclTonemappingSupported) - { - arg.Append("-init_hw_device opencl=ocl:") - .Append(options.OpenclDevice) - .Append(" -filter_hw_device ocl "); - } + return string.Empty; } - if (state.IsVideoRequest - && string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + var isVideotoolboxDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + var isVideotoolboxEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + if (!isVideotoolboxDecoder && !isVideotoolboxEncoder) { - arg.Append("-hwaccel videotoolbox "); + return string.Empty; } + + // no videotoolbox hw filter. + args.Append(GetVideoToolboxDeviceArgs(VideotoolboxAlias)); + } + + if (!string.IsNullOrEmpty(vidDecoder)) + { + args.Append(vidDecoder); + } + + // hw transpose filters should be added manually. + args.Append(" -autorotate 0"); + + return args.ToString().Trim(); + } + + /// + /// Gets the input argument. + /// + /// Encoding state. + /// Encoding options. + /// Input arguments. + public string GetInputArgument(EncodingJobInfo state, EncodingOptions options) + { + var arg = new StringBuilder(); + var inputVidHwaccelArgs = GetInputVideoHwaccelArgs(state, options); + + if (!string.IsNullOrEmpty(inputVidHwaccelArgs)) + { + arg.Append(inputVidHwaccelArgs); + } + + var canvasArgs = GetGraphicalSubCanvasSize(state); + if (!string.IsNullOrEmpty(canvasArgs)) + { + arg.Append(canvasArgs); } - arg.Append("-i ") + arg.Append(" -i ") .Append(GetInputPathArgument(state)); + // sub2video for external graphical subtitles if (state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode - && state.SubtitleStream.IsExternal && !state.SubtitleStream.IsTextSubtitleStream) + && !state.SubtitleStream.IsTextSubtitleStream + && state.SubtitleStream.IsExternal) { var subtitlePath = state.SubtitleStream.Path; @@ -693,7 +868,19 @@ namespace MediaBrowser.Controller.MediaEncoding } } - arg.Append(" -i \"").Append(subtitlePath).Append('\"'); + // Also seek the external subtitles stream. + var seekSubParam = GetFastSeekCommandLineParameter(state, options); + if (!string.IsNullOrEmpty(seekSubParam)) + { + arg.Append(' ').Append(seekSubParam); + } + + if (!string.IsNullOrEmpty(canvasArgs)) + { + arg.Append(canvasArgs); + } + + arg.Append(" -i file:\"").Append(subtitlePath).Append('\"'); } if (state.AudioStream != null && state.AudioStream.IsExternal) @@ -814,6 +1001,26 @@ namespace MediaBrowser.Controller.MediaEncoding return FormattableString.Invariant($" -maxrate {bitrate} -bufsize {bufsize}"); } + if (string.Equals(videoCodec, "h264_amf", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "hevc_amf", StringComparison.OrdinalIgnoreCase)) + { + return FormattableString.Invariant($" -qmin 18 -qmax 32 -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); + } + + if (string.Equals(videoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoCodec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase)) + { + // VBR in i965 driver may result in pixelated output. + if (_mediaEncoder.IsVaapiDeviceInteli965) + { + return FormattableString.Invariant($" -rc_mode CBR -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); + } + else + { + return FormattableString.Invariant($" -rc_mode VBR -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); + } + } + return FormattableString.Invariant($" -b:v {bitrate} -maxrate {bitrate} -bufsize {bufsize}"); } @@ -850,8 +1057,10 @@ namespace MediaBrowser.Controller.MediaEncoding /// Gets the text subtitle param. /// /// The state. + /// Enable alpha processing. + /// Enable sub2video mode. /// System.String. - public string GetTextSubtitleParam(EncodingJobInfo state) + public string GetTextSubtitlesFilter(EncodingJobInfo state, bool enableAlpha, bool enableSub2video) { var seconds = Math.Round(TimeSpan.FromTicks(state.StartTimeTicks ?? 0).TotalSeconds); @@ -860,6 +1069,9 @@ namespace MediaBrowser.Controller.MediaEncoding ? string.Empty : string.Format(CultureInfo.InvariantCulture, ",setpts=PTS -{0}/TB", seconds); + var alphaParam = enableAlpha ? ":alpha=1" : string.Empty; + var sub2videoParam = enableSub2video ? ":sub2video=1" : string.Empty; + // TODO // var fallbackFontPath = Path.Combine(_appPaths.ProgramDataPath, "fonts", "DroidSansFallback.ttf"); // string fallbackFontParam = string.Empty; @@ -881,7 +1093,6 @@ namespace MediaBrowser.Controller.MediaEncoding if (state.SubtitleStream.IsExternal) { var subtitlePath = state.SubtitleStream.Path; - var charsetParam = string.Empty; if (!string.IsNullOrEmpty(state.SubtitleStream.Language)) @@ -901,9 +1112,11 @@ namespace MediaBrowser.Controller.MediaEncoding // TODO: Perhaps also use original_size=1920x800 ?? return string.Format( CultureInfo.InvariantCulture, - "subtitles=filename='{0}'{1}{2}", + "subtitles=f='{0}'{1}{2}{3}{4}", _mediaEncoder.EscapeSubtitleFilterPath(subtitlePath), charsetParam, + alphaParam, + sub2videoParam, // fallbackFontParam, setPtsParam); } @@ -912,9 +1125,11 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Format( CultureInfo.InvariantCulture, - "subtitles='{0}:si={1}'{2}", + "subtitles='{0}:si={1}{2}{3}'{4}", _mediaEncoder.EscapeSubtitleFilterPath(mediaPath), state.InternalSubtitleStreamOffset.ToString(CultureInfo.InvariantCulture), + alphaParam, + sub2videoParam, // fallbackFontParam, setPtsParam); } @@ -999,11 +1214,11 @@ namespace MediaBrowser.Controller.MediaEncoding || string.Equals(codec, "h264_vaapi", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase)) { - args += " " + keyFrameArg; + args += keyFrameArg; } else { - args += " " + keyFrameArg + gopArg; + args += keyFrameArg + gopArg; } return args; @@ -1021,54 +1236,49 @@ namespace MediaBrowser.Controller.MediaEncoding { var param = string.Empty; - if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) - && !string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) - { - param += " -pix_fmt yuv420p"; - } - - if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "h264_amf", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) - { - var videoStream = state.VideoStream; - var isColorDepth10 = IsColorDepth10(state); - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions) ?? string.Empty; - var isNvdecDecoder = videoDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); - - if (!isNvdecDecoder) - { - if (isColorDepth10 - && _mediaEncoder.SupportsHwaccel("opencl") - && encodingOptions.EnableTonemapping - && !string.IsNullOrEmpty(videoStream.VideoRange) - && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase)) - { - param += " -pix_fmt nv12"; - } - else - { - param += " -pix_fmt yuv420p"; - } + // Tutorials: Enable Intel GuC / HuC firmware loading for Low Power Encoding. + // https://01.org/linuxgraphics/downloads/firmware + // https://wiki.archlinux.org/title/intel_graphics#Enable_GuC_/_HuC_firmware_loading + // Intel Low Power Encoding can save unnecessary CPU-GPU synchronization, + // which will reduce overhead in performance intensive tasks such as 4k transcoding and tonemapping. + var intelLowPowerHwEncoding = false; + + if (string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + var isIntelVaapiDriver = _mediaEncoder.IsVaapiDeviceInteliHD || _mediaEncoder.IsVaapiDeviceInteli965; + + if (string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase)) + { + intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerH264HwEncoder && isIntelVaapiDriver; + } + else if (string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase)) + { + intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerHevcHwEncoder && isIntelVaapiDriver; + } + } + else if (string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase)) + { + intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerH264HwEncoder; + } + else if (string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) + { + intelLowPowerHwEncoding = encodingOptions.EnableIntelLowPowerHevcHwEncoder; } } + if (intelLowPowerHwEncoding) + { + param += " -low_power 1"; + } + if (string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase)) { param += " -pix_fmt nv21"; } - var isVc1 = state.VideoStream != null && - string.Equals(state.VideoStream.Codec, "vc1", StringComparison.OrdinalIgnoreCase); + var isVc1 = string.Equals(state.VideoStream.Codec, "vc1", StringComparison.OrdinalIgnoreCase); var isLibX265 = string.Equals(videoEncoder, "libx265", StringComparison.OrdinalIgnoreCase); if (string.Equals(videoEncoder, "libx264", StringComparison.OrdinalIgnoreCase) || isLibX265) @@ -1179,19 +1389,6 @@ namespace MediaBrowser.Controller.MediaEncoding break; } - var videoStream = state.VideoStream; - var isColorDepth10 = IsColorDepth10(state); - - if (isColorDepth10 - && _mediaEncoder.SupportsHwaccel("opencl") - && encodingOptions.EnableTonemapping - && !string.IsNullOrEmpty(videoStream.VideoRange) - && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase)) - { - // Enhance workload when tone mapping with AMF on some APUs - param += " -preanalysis true"; - } - if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase)) { param += " -header_insertion_mode gop -gops_per_idr 1"; @@ -1364,13 +1561,6 @@ namespace MediaBrowser.Controller.MediaEncoding profile = "constrained_high"; } - // Currently hevc_amf only support encoding HEVC Main Profile, otherwise force Main Profile. - if (string.Equals(videoEncoder, "hevc_amf", StringComparison.OrdinalIgnoreCase) - && profile.Contains("main10", StringComparison.OrdinalIgnoreCase)) - { - profile = "main"; - } - if (!string.IsNullOrEmpty(profile)) { if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase) @@ -1387,7 +1577,7 @@ namespace MediaBrowser.Controller.MediaEncoding { level = NormalizeTranscodingLevel(state, level); - // libx264, QSV, AMF, VAAPI can adjust the given level to match the output. + // libx264, QSV, AMF can adjust the given level to match the output. if (string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) || string.Equals(videoEncoder, "libx264", StringComparison.OrdinalIgnoreCase)) { @@ -1407,10 +1597,13 @@ namespace MediaBrowser.Controller.MediaEncoding param += " -level " + level; } else if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase)) + || string.Equals(videoEncoder, "hevc_nvenc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase)) { // level option may cause NVENC to fail. // NVENC cannot adjust the given level, just throw an error. + // level option may cause corrupted frames on AMD VAAPI. } else if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase) || !string.Equals(videoEncoder, "libx265", StringComparison.OrdinalIgnoreCase)) @@ -1496,8 +1689,8 @@ namespace MediaBrowser.Controller.MediaEncoding if (!string.IsNullOrEmpty(videoStream.Profile) && !requestedProfiles.Contains(videoStream.Profile.Replace(" ", string.Empty, StringComparison.Ordinal), StringComparer.OrdinalIgnoreCase)) { - var currentScore = GetVideoProfileScore(videoStream.Profile); - var requestedScore = GetVideoProfileScore(requestedProfile); + var currentScore = GetVideoProfileScore(videoStream.Codec, videoStream.Profile); + var requestedScore = GetVideoProfileScore(videoStream.Codec, requestedProfile); if (currentScore == -1 || currentScore > requestedScore) { @@ -1708,7 +1901,8 @@ namespace MediaBrowser.Controller.MediaEncoding { if (string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) || string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase)) + || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase) + || string.Equals(codec, "av1", StringComparison.OrdinalIgnoreCase)) { return .6; } @@ -1945,19 +2139,35 @@ namespace MediaBrowser.Controller.MediaEncoding /// /// Gets the fast seek command line parameter. /// - /// The request. + /// The state. + /// The options. /// System.String. /// The fast seek command line parameter. - public string GetFastSeekCommandLineParameter(BaseEncodingJobOptions request) + public string GetFastSeekCommandLineParameter(EncodingJobInfo state, EncodingOptions options) { - var time = request.StartTimeTicks ?? 0; + var time = state.BaseRequest.StartTimeTicks ?? 0; + var seekParam = string.Empty; if (time > 0) { - return string.Format(CultureInfo.InvariantCulture, "-ss {0}", _mediaEncoder.GetTimeParameter(time)); + seekParam += string.Format(CultureInfo.InvariantCulture, "-ss {0}", _mediaEncoder.GetTimeParameter(time)); + + if (state.IsVideoRequest) + { + var outputVideoCodec = GetVideoEncoder(state, options); + + // Important: If this is ever re-enabled, make sure not to use it with wtv because it breaks seeking + if (!string.Equals(state.InputContainer, "wtv", StringComparison.OrdinalIgnoreCase) + && state.TranscodingType != TranscodingJobType.Progressive + && !state.EnableBreakOnNonKeyFrames(outputVideoCodec) + && (state.BaseRequest.StartTimeTicks ?? 0) > 0) + { + seekParam += " -noaccurate_seek"; + } + } } - return string.Empty; + return seekParam; } /// @@ -2080,180 +2290,93 @@ namespace MediaBrowser.Controller.MediaEncoding return returnFirstIfNoIndex ? streams.FirstOrDefault() : null; } - /// - /// Gets the graphical subtitle parameter. - /// - /// Encoding state. - /// Encoding options. - /// Video codec to use. - /// Graphical subtitle parameter. - public string GetGraphicalSubtitleParam( - EncodingJobInfo state, - EncodingOptions options, - string outputVideoCodec) + public static (int? width, int? height) GetFixedOutputSize( + int? videoWidth, + int? videoHeight, + int? requestedWidth, + int? requestedHeight, + int? requestedMaxWidth, + int? requestedMaxHeight) { - outputVideoCodec ??= string.Empty; - - var outputSizeParam = ReadOnlySpan.Empty; - var request = state.BaseRequest; + if (!videoWidth.HasValue && !requestedWidth.HasValue) + { + return (null, null); + } - outputSizeParam = GetOutputSizeParamInternal(state, options, outputVideoCodec); - - var videoSizeParam = string.Empty; - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options) ?? string.Empty; - var isLinux = OperatingSystem.IsLinux(); - - var isVaapiDecoder = videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiH264Encoder = outputVideoCodec.IndexOf("h264_vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiHevcEncoder = outputVideoCodec.IndexOf("hevc_vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvH264Encoder = outputVideoCodec.Contains("h264_qsv", StringComparison.OrdinalIgnoreCase); - var isQsvHevcEncoder = outputVideoCodec.Contains("hevc_qsv", StringComparison.OrdinalIgnoreCase); - var isNvdecDecoder = videoDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); - var isNvencEncoder = outputVideoCodec.Contains("nvenc", StringComparison.OrdinalIgnoreCase); - var isTonemappingSupportedOnVaapi = string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isVaapiH264Encoder || isVaapiHevcEncoder); - var isTonemappingSupportedOnQsv = string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isQsvH264Encoder || isQsvHevcEncoder); - var isOpenclTonemappingSupported = IsOpenclTonemappingSupported(state, options); - var isVppTonemappingSupported = IsVppTonemappingSupported(state, options); - - var mediaEncoderVersion = _mediaEncoder.GetMediaEncoderVersion(); - var isCudaOverlaySupported = _mediaEncoder.SupportsFilter("overlay_cuda") && mediaEncoderVersion != null && mediaEncoderVersion >= _minVersionForCudaOverlay; - var isCudaFormatConversionSupported = _mediaEncoder.SupportsFilterWithOption(FilterOptionType.ScaleCudaFormat); - - // Tonemapping and burn-in graphical subtitles requires overlay_vaapi. - // But it's still in ffmpeg mailing list. Disable it for now. - if (isTonemappingSupportedOnVaapi && isOpenclTonemappingSupported && !isVppTonemappingSupported) + if (!videoHeight.HasValue && !requestedHeight.HasValue) { - return GetOutputSizeParam(state, options, outputVideoCodec); + return (null, null); } - // Setup subtitle scaling - if (state.VideoStream != null && state.VideoStream.Width.HasValue && state.VideoStream.Height.HasValue) - { - // Adjust the size of graphical subtitles to fit the video stream. - var videoStream = state.VideoStream; - var inputWidth = videoStream.Width; - var inputHeight = videoStream.Height; - var (width, height) = GetFixedOutputSize(inputWidth, inputHeight, request.Width, request.Height, request.MaxWidth, request.MaxHeight); - - if (width.HasValue && height.HasValue) - { - videoSizeParam = string.Format( - CultureInfo.InvariantCulture, - "scale={0}x{1}", - width.Value, - height.Value); - } + int inputWidth = Convert.ToInt32(videoWidth ?? requestedWidth, CultureInfo.InvariantCulture); + int inputHeight = Convert.ToInt32(videoHeight ?? requestedHeight, CultureInfo.InvariantCulture); + int outputWidth = requestedWidth ?? inputWidth; + int outputHeight = requestedHeight ?? inputHeight; - if (!string.IsNullOrEmpty(videoSizeParam) - && !(isTonemappingSupportedOnQsv && isVppTonemappingSupported)) - { - // upload graphical subtitle to QSV - if (isLinux && (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "hevc_qsv", StringComparison.OrdinalIgnoreCase))) - { - videoSizeParam += ",hwupload=extra_hw_frames=64"; - } - } + // Don't transcode video to bigger than 4k when using HW. + int maximumWidth = Math.Min(requestedMaxWidth ?? outputWidth, 4096); + int maximumHeight = Math.Min(requestedMaxHeight ?? outputHeight, 4096); - if (!string.IsNullOrEmpty(videoSizeParam)) - { - // upload graphical subtitle to cuda - if (isNvdecDecoder && isNvencEncoder && isCudaOverlaySupported && isCudaFormatConversionSupported) - { - videoSizeParam += ",hwupload_cuda"; - } - } + if (outputWidth > maximumWidth || outputHeight > maximumHeight) + { + var scaleW = (double)maximumWidth / (double)outputWidth; + var scaleH = (double)maximumHeight / (double)outputHeight; + var scale = Math.Min(scaleW, scaleH); + outputWidth = Math.Min(maximumWidth, (int)(outputWidth * scale)); + outputHeight = Math.Min(maximumHeight, (int)(outputHeight * scale)); } - var mapPrefix = state.SubtitleStream.IsExternal ? - 1 : - 0; + outputWidth = 2 * (outputWidth / 2); + outputHeight = 2 * (outputHeight / 2); + + return (outputWidth, outputHeight); + } - var subtitleStreamIndex = state.SubtitleStream.IsExternal - ? 0 - : state.SubtitleStream.Index; + public static string GetHwScaleFilter( + string hwScaleSuffix, + string videoFormat, + int? videoWidth, + int? videoHeight, + int? requestedWidth, + int? requestedHeight, + int? requestedMaxWidth, + int? requestedMaxHeight) + { + var (outWidth, outHeight) = GetFixedOutputSize( + videoWidth, + videoHeight, + requestedWidth, + requestedHeight, + requestedMaxWidth, + requestedMaxHeight); - // Setup default filtergraph utilizing FFMpeg overlay() and FFMpeg scale() (see the return of this function for index reference) - // Always put the scaler before the overlay for better performance - var retStr = outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay\""; + var isFormatFixed = !string.IsNullOrEmpty(videoFormat); + var isSizeFixed = !videoWidth.HasValue + || outWidth.Value != videoWidth.Value + || !videoHeight.HasValue + || outHeight.Value != videoHeight.Value; - // When the input may or may not be hardware VAAPI decodable - if (string.Equals(outputVideoCodec, "h264_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "hevc_vaapi", StringComparison.OrdinalIgnoreCase)) + var arg1 = isSizeFixed ? ("=w=" + outWidth.Value + ":h=" + outHeight.Value) : string.Empty; + var arg2 = isFormatFixed ? ("format=" + videoFormat) : string.Empty; + if (isFormatFixed) { - /* - [base]: HW scaling video to OutputSize - [sub]: SW scaling subtitle to FixedOutputSize - [base][sub]: SW overlay - */ - retStr = outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]hwdownload[base];[base][sub]overlay,format=nv12,hwupload\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3},hwdownload[base];[base][sub]overlay,format=nv12,hwupload\""; + arg2 = (isSizeFixed ? ':' : '=') + arg2; } - // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first - else if (_mediaEncoder.SupportsHwaccel("vaapi") && videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1 - && (string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "libx265", StringComparison.OrdinalIgnoreCase))) - { - /* - [base]: SW scaling video to OutputSize - [sub]: SW scaling subtitle to FixedOutputSize - [base][sub]: SW overlay - */ - retStr = outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay\""; - } - else if (string.Equals(outputVideoCodec, "h264_qsv", StringComparison.OrdinalIgnoreCase) - || string.Equals(outputVideoCodec, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) - { - /* - QSV in FFMpeg can now setup hardware overlay for transcodes. - For software decoding and hardware encoding option, frames must be hwuploaded into hardware - with fixed frame size. - Currently only supports linux. - */ - if (isTonemappingSupportedOnQsv && isVppTonemappingSupported) - { - retStr = " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3},hwdownload,format=nv12[base];[base][sub]overlay\""; - } - else if (isLinux) - { - retStr = outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay_qsv\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay_qsv\""; - } - } - else if (isNvdecDecoder && isNvencEncoder) + if (!string.IsNullOrEmpty(hwScaleSuffix) && (isSizeFixed || isFormatFixed)) { - if (isCudaOverlaySupported && isCudaFormatConversionSupported) - { - retStr = outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]scale_cuda=format=yuv420p[base];[base][sub]overlay_cuda\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay_cuda\""; - } - else - { - retStr = outputSizeParam.IsEmpty - ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay,format=nv12|yuv420p,hwupload_cuda\"" - : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay,format=nv12|yuv420p,hwupload_cuda\""; - } + return string.Format( + CultureInfo.InvariantCulture, + "scale_{0}{1}{2}", + hwScaleSuffix, + arg1, + arg2); } - return string.Format( - CultureInfo.InvariantCulture, - retStr, - mapPrefix, - subtitleStreamIndex, - state.VideoStream.Index, - outputSizeParam.ToString(), - videoSizeParam); + return string.Empty; } - public static (int? width, int? height) GetFixedOutputSize( + public static string GetCustomSwScaleFilter( int? videoWidth, int? videoHeight, int? requestedWidth, @@ -2261,51 +2384,39 @@ namespace MediaBrowser.Controller.MediaEncoding int? requestedMaxWidth, int? requestedMaxHeight) { - if (!videoWidth.HasValue && !requestedWidth.HasValue) - { - return (null, null); - } - - if (!videoHeight.HasValue && !requestedHeight.HasValue) - { - return (null, null); - } - - decimal inputWidth = Convert.ToDecimal(videoWidth ?? requestedWidth, CultureInfo.InvariantCulture); - decimal inputHeight = Convert.ToDecimal(videoHeight ?? requestedHeight, CultureInfo.InvariantCulture); - decimal outputWidth = requestedWidth.HasValue ? Convert.ToDecimal(requestedWidth.Value) : inputWidth; - decimal outputHeight = requestedHeight.HasValue ? Convert.ToDecimal(requestedHeight.Value) : inputHeight; - decimal maximumWidth = requestedMaxWidth.HasValue ? Convert.ToDecimal(requestedMaxWidth.Value) : outputWidth; - decimal maximumHeight = requestedMaxHeight.HasValue ? Convert.ToDecimal(requestedMaxHeight.Value) : outputHeight; + var (outWidth, outHeight) = GetFixedOutputSize( + videoWidth, + videoHeight, + requestedWidth, + requestedHeight, + requestedMaxWidth, + requestedMaxHeight); - if (outputWidth > maximumWidth || outputHeight > maximumHeight) + if (outWidth.HasValue && outHeight.HasValue) { - var scale = Math.Min(maximumWidth / outputWidth, maximumHeight / outputHeight); - outputWidth = Math.Min(maximumWidth, Math.Truncate(outputWidth * scale)); - outputHeight = Math.Min(maximumHeight, Math.Truncate(outputHeight * scale)); + return string.Format( + CultureInfo.InvariantCulture, + "scale=s={0}x{1}:flags=fast_bilinear", + outWidth.Value, + outHeight.Value); } - outputWidth = 2 * Math.Truncate(outputWidth / 2); - outputHeight = 2 * Math.Truncate(outputHeight / 2); - - return (Convert.ToInt32(outputWidth), Convert.ToInt32(outputHeight)); + return string.Empty; } - public List GetScalingFilters( + public static string GetAlphaSrcFilter( EncodingJobInfo state, - EncodingOptions options, int? videoWidth, int? videoHeight, - Video3DFormat? threedFormat, - string videoDecoder, - string videoEncoder, int? requestedWidth, int? requestedHeight, int? requestedMaxWidth, - int? requestedMaxHeight) + int? requestedMaxHeight, + int? framerate) { - var filters = new List(); - var (width, height) = GetFixedOutputSize( + var reqTicks = state.BaseRequest.StartTimeTicks ?? 0; + var startTime = TimeSpan.FromTicks(reqTicks).ToString(@"hh\\\:mm\\\:ss\\\.fff", CultureInfo.InvariantCulture); + var (outWidth, outHeight) = GetFixedOutputSize( videoWidth, videoHeight, requestedWidth, @@ -2313,280 +2424,128 @@ namespace MediaBrowser.Controller.MediaEncoding requestedMaxWidth, requestedMaxHeight); - if ((string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_vaapi", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase)) - && width.HasValue - && height.HasValue) - { - // Given the input dimensions (inputWidth, inputHeight), determine the output dimensions - // (outputWidth, outputHeight). The user may request precise output dimensions or maximum - // output dimensions. Output dimensions are guaranteed to be even. - var outputWidth = width.Value; - var outputHeight = height.Value; - var qsv_or_vaapi = string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoEncoder, "hevc_qsv", StringComparison.OrdinalIgnoreCase); - var isDeintEnabled = state.DeInterlace("h264", true) - || state.DeInterlace("avc", true) - || state.DeInterlace("h265", true) - || state.DeInterlace("hevc", true); - - var isVaapiDecoder = videoDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); - var isVaapiH264Encoder = videoEncoder.Contains("h264_vaapi", StringComparison.OrdinalIgnoreCase); - var isVaapiHevcEncoder = videoEncoder.Contains("hevc_vaapi", StringComparison.OrdinalIgnoreCase); - var isQsvH264Encoder = videoEncoder.Contains("h264_qsv", StringComparison.OrdinalIgnoreCase); - var isQsvHevcEncoder = videoEncoder.Contains("hevc_qsv", StringComparison.OrdinalIgnoreCase); - var isOpenclTonemappingSupported = IsOpenclTonemappingSupported(state, options); - var isVppTonemappingSupported = IsVppTonemappingSupported(state, options); - var isTonemappingSupportedOnVaapi = string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isVaapiH264Encoder || isVaapiHevcEncoder); - var isTonemappingSupportedOnQsv = string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isQsvH264Encoder || isQsvHevcEncoder); - var isP010PixFmtRequired = (isTonemappingSupportedOnVaapi && (isOpenclTonemappingSupported || isVppTonemappingSupported)) - || (isTonemappingSupportedOnQsv && isVppTonemappingSupported); - - var outputPixFmt = "format=nv12"; - if (isP010PixFmtRequired) - { - outputPixFmt = "format=p010"; - } - - if (isTonemappingSupportedOnQsv && isVppTonemappingSupported) - { - qsv_or_vaapi = false; - } - - if (!videoWidth.HasValue - || outputWidth != videoWidth.Value - || !videoHeight.HasValue - || outputHeight != videoHeight.Value) - { - // Force nv12 pixel format to enable 10-bit to 8-bit colour conversion. - // use vpp_qsv filter to avoid green bar when the fixed output size is requested. - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "{0}=w={1}:h={2}{3}{4}", - qsv_or_vaapi ? "vpp_qsv" : "scale_vaapi", - outputWidth, - outputHeight, - ":" + outputPixFmt, - (qsv_or_vaapi && isDeintEnabled) ? ":deinterlace=1" : string.Empty)); - } + if (outWidth.HasValue && outHeight.HasValue) + { + return string.Format( + CultureInfo.InvariantCulture, + "alphasrc=s={0}x{1}:r={2}:start='{3}'", + outWidth.Value, + outHeight.Value, + framerate ?? 10, + reqTicks > 0 ? startTime : 0); + } - // Assert 10-bit is P010 so as we can avoid the extra scaler to get a bit more fps on high res HDR videos. - else if (!isP010PixFmtRequired) - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "{0}={1}{2}", - qsv_or_vaapi ? "vpp_qsv" : "scale_vaapi", - outputPixFmt, - (qsv_or_vaapi && isDeintEnabled) ? ":deinterlace=1" : string.Empty)); - } - } - else if ((videoDecoder ?? string.Empty).Contains("cuda", StringComparison.OrdinalIgnoreCase) - && width.HasValue - && height.HasValue) - { - var outputWidth = width.Value; - var outputHeight = height.Value; - - var isNvencEncoder = videoEncoder.Contains("nvenc", StringComparison.OrdinalIgnoreCase); - var isOpenclTonemappingSupported = IsOpenclTonemappingSupported(state, options); - var isCudaTonemappingSupported = IsCudaTonemappingSupported(state, options); - var isTonemappingSupportedOnNvenc = string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase); - var mediaEncoderVersion = _mediaEncoder.GetMediaEncoderVersion(); - var isCudaOverlaySupported = _mediaEncoder.SupportsFilter("overlay_cuda") && mediaEncoderVersion != null && mediaEncoderVersion >= _minVersionForCudaOverlay; - var isCudaFormatConversionSupported = _mediaEncoder.SupportsFilterWithOption(FilterOptionType.ScaleCudaFormat); - var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + return string.Empty; + } - var outputPixFmt = string.Empty; - if (isCudaFormatConversionSupported) - { - outputPixFmt = (hasGraphicalSubs && isCudaOverlaySupported && isNvencEncoder) - ? "format=yuv420p" - : "format=nv12"; - if ((isOpenclTonemappingSupported || isCudaTonemappingSupported) - && isTonemappingSupportedOnNvenc) - { - outputPixFmt = "format=p010"; - } - } + public static string GetSwScaleFilter( + EncodingJobInfo state, + EncodingOptions options, + string videoEncoder, + int? videoWidth, + int? videoHeight, + Video3DFormat? threedFormat, + int? requestedWidth, + int? requestedHeight, + int? requestedMaxWidth, + int? requestedMaxHeight) + { + var isV4l2 = string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase); + var scaleVal = isV4l2 ? 64 : 2; - if (!videoWidth.HasValue - || outputWidth != videoWidth.Value - || !videoHeight.HasValue - || outputHeight != videoHeight.Value) + // If fixed dimensions were supplied + if (requestedWidth.HasValue && requestedHeight.HasValue) + { + if (isV4l2) { - filters.Add( - string.Format( + var widthParam = requestedWidth.Value.ToString(CultureInfo.InvariantCulture); + var heightParam = requestedHeight.Value.ToString(CultureInfo.InvariantCulture); + + return string.Format( CultureInfo.InvariantCulture, - "scale_cuda=w={0}:h={1}{2}", - outputWidth, - outputHeight, - isCudaFormatConversionSupported ? (":" + outputPixFmt) : string.Empty)); + "scale=trunc({0}/64)*64:trunc({1}/2)*2", + widthParam, + heightParam); } - else if (isCudaFormatConversionSupported) + else { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale_cuda={0}", - outputPixFmt)); + return GetFixedSwScaleFilter(threedFormat, requestedWidth.Value, requestedHeight.Value); } } - else if ((videoDecoder ?? string.Empty).IndexOf("cuvid", StringComparison.OrdinalIgnoreCase) != -1 - && width.HasValue - && height.HasValue) - { - // Nothing to do, it's handled as an input resize filter - } - else + + // If Max dimensions were supplied, for width selects lowest even number between input width and width req size and selects lowest even number from in width*display aspect and requested size + else if (requestedMaxWidth.HasValue && requestedMaxHeight.HasValue) { - var isExynosV4L2 = string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase); + var maxWidthParam = requestedMaxWidth.Value.ToString(CultureInfo.InvariantCulture); + var maxHeightParam = requestedMaxHeight.Value.ToString(CultureInfo.InvariantCulture); - // If fixed dimensions were supplied - if (requestedWidth.HasValue && requestedHeight.HasValue) - { - if (isExynosV4L2) - { - var widthParam = requestedWidth.Value.ToString(CultureInfo.InvariantCulture); - var heightParam = requestedHeight.Value.ToString(CultureInfo.InvariantCulture); - - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc({0}/64)*64:trunc({1}/2)*2", - widthParam, - heightParam)); - } - else - { - filters.Add(GetFixedSizeScalingFilter(threedFormat, requestedWidth.Value, requestedHeight.Value)); - } - } + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc(min(max(iw\\,ih*dar)\\,min({0}\\,{1}*dar))/{2})*{2}:trunc(min(max(iw/dar\\,ih)\\,min({0}/dar\\,{1}))/2)*2", + maxWidthParam, + maxHeightParam, + scaleVal); + } - // If Max dimensions were supplied, for width selects lowest even number between input width and width req size and selects lowest even number from in width*display aspect and requested size - else if (requestedMaxWidth.HasValue && requestedMaxHeight.HasValue) + // If a fixed width was requested + else if (requestedWidth.HasValue) + { + if (threedFormat.HasValue) { - var maxWidthParam = requestedMaxWidth.Value.ToString(CultureInfo.InvariantCulture); - var maxHeightParam = requestedMaxHeight.Value.ToString(CultureInfo.InvariantCulture); - - if (isExynosV4L2) - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(min(max(iw\\,ih*dar)\\,min({0}\\,{1}*dar))/64)*64:trunc(min(max(iw/dar\\,ih)\\,min({0}/dar\\,{1}))/2)*2", - maxWidthParam, - maxHeightParam)); - } - else - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(min(max(iw\\,ih*dar)\\,min({0}\\,{1}*dar))/2)*2:trunc(min(max(iw/dar\\,ih)\\,min({0}/dar\\,{1}))/2)*2", - maxWidthParam, - maxHeightParam)); - } + // This method can handle 0 being passed in for the requested height + return GetFixedSwScaleFilter(threedFormat, requestedWidth.Value, 0); } - - // If a fixed width was requested - else if (requestedWidth.HasValue) + else { - if (threedFormat.HasValue) - { - // This method can handle 0 being passed in for the requested height - filters.Add(GetFixedSizeScalingFilter(threedFormat, requestedWidth.Value, 0)); - } - else - { - var widthParam = requestedWidth.Value.ToString(CultureInfo.InvariantCulture); + var widthParam = requestedWidth.Value.ToString(CultureInfo.InvariantCulture); - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale={0}:trunc(ow/a/2)*2", - widthParam)); - } + return string.Format( + CultureInfo.InvariantCulture, + "scale={0}:trunc(ow/a/2)*2", + widthParam); } + } - // If a fixed height was requested - else if (requestedHeight.HasValue) - { - var heightParam = requestedHeight.Value.ToString(CultureInfo.InvariantCulture); + // If a fixed height was requested + else if (requestedHeight.HasValue) + { + var heightParam = requestedHeight.Value.ToString(CultureInfo.InvariantCulture); - if (isExynosV4L2) - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(oh*a/64)*64:{0}", - heightParam)); - } - else - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(oh*a/2)*2:{0}", - heightParam)); - } - } + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc(oh*a/{1})*{1}:{0}", + heightParam, + scaleVal); + } - // If a max width was requested - else if (requestedMaxWidth.HasValue) - { - var maxWidthParam = requestedMaxWidth.Value.ToString(CultureInfo.InvariantCulture); + // If a max width was requested + else if (requestedMaxWidth.HasValue) + { + var maxWidthParam = requestedMaxWidth.Value.ToString(CultureInfo.InvariantCulture); - if (isExynosV4L2) - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(min(max(iw\\,ih*dar)\\,{0})/64)*64:trunc(ow/dar/2)*2", - maxWidthParam)); - } - else - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(min(max(iw\\,ih*dar)\\,{0})/2)*2:trunc(ow/dar/2)*2", - maxWidthParam)); - } - } + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc(min(max(iw\\,ih*dar)\\,{0})/{1})*{1}:trunc(ow/dar/2)*2", + maxWidthParam, + scaleVal); + } - // If a max height was requested - else if (requestedMaxHeight.HasValue) - { - var maxHeightParam = requestedMaxHeight.Value.ToString(CultureInfo.InvariantCulture); + // If a max height was requested + else if (requestedMaxHeight.HasValue) + { + var maxHeightParam = requestedMaxHeight.Value.ToString(CultureInfo.InvariantCulture); - if (isExynosV4L2) - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(oh*a/64)*64:min(max(iw/dar\\,ih)\\,{0})", - maxHeightParam)); - } - else - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "scale=trunc(oh*a/2)*2:min(max(iw/dar\\,ih)\\,{0})", - maxHeightParam)); - } - } + return string.Format( + CultureInfo.InvariantCulture, + "scale=trunc(oh*a/{1})*{1}:min(max(iw/dar\\,ih)\\,{0})", + maxHeightParam, + scaleVal); } - return filters; + return string.Empty; } - private string GetFixedSizeScalingFilter(Video3DFormat? threedFormat, int requestedWidth, int requestedHeight) + private static string GetFixedSwScaleFilter(Video3DFormat? threedFormat, int requestedWidth, int requestedHeight) { var widthParam = requestedWidth.ToString(CultureInfo.InvariantCulture); var heightParam = requestedHeight.ToString(CultureInfo.InvariantCulture); @@ -2634,572 +2593,2171 @@ namespace MediaBrowser.Controller.MediaEncoding return string.Format(CultureInfo.InvariantCulture, filter, widthParam, heightParam); } - /// - /// Gets the output size parameter. - /// - /// Encoding state. - /// Encoding options. - /// Video codec to use. - /// The output size parameter. - public string GetOutputSizeParam( - EncodingJobInfo state, - EncodingOptions options, - string outputVideoCodec) + public static string GetSwDeinterlaceFilter(EncodingJobInfo state, EncodingOptions options) + { + var doubleRateDeint = options.DeinterlaceDoubleRate && state.VideoStream?.AverageFrameRate <= 30; + return string.Format( + CultureInfo.InvariantCulture, + "{0}={1}:-1:0", + string.Equals(options.DeinterlaceMethod, "bwdif", StringComparison.OrdinalIgnoreCase) ? "bwdif" : "yadif", + doubleRateDeint ? "1" : "0"); + } + + public static string GetHwDeinterlaceFilter(EncodingJobInfo state, EncodingOptions options, string hwDeintSuffix) + { + var doubleRateDeint = options.DeinterlaceDoubleRate && (state.VideoStream?.AverageFrameRate ?? 60) <= 30; + if (hwDeintSuffix.Contains("cuda", StringComparison.OrdinalIgnoreCase)) + { + return string.Format( + CultureInfo.InvariantCulture, + "yadif_cuda={0}:-1:0", + doubleRateDeint ? "1" : "0"); + } + else if (hwDeintSuffix.Contains("vaapi", StringComparison.OrdinalIgnoreCase)) + { + return string.Format( + CultureInfo.InvariantCulture, + "deinterlace_vaapi=rate={0}", + doubleRateDeint ? "field" : "frame"); + } + else if (hwDeintSuffix.Contains("qsv", StringComparison.OrdinalIgnoreCase)) + { + return "deinterlace_qsv=mode=2"; + } + + return string.Empty; + } + + public static string GetHwTonemapFilter(EncodingOptions options, string hwTonemapSuffix, string videoFormat) { - string filters = GetOutputSizeParamInternal(state, options, outputVideoCodec); - return string.IsNullOrEmpty(filters) ? string.Empty : " -vf \"" + filters + "\""; + if (string.IsNullOrEmpty(hwTonemapSuffix)) + { + return string.Empty; + } + + var args = "tonemap_{0}=format={1}:p=bt709:t=bt709:m=bt709"; + + if (!hwTonemapSuffix.Contains("vaapi", StringComparison.OrdinalIgnoreCase)) + { + args += ":tonemap={2}:peak={3}:desat={4}"; + + if (options.TonemappingParam != 0) + { + args += ":param={5}"; + } + + if (!string.Equals(options.TonemappingRange, "auto", StringComparison.OrdinalIgnoreCase)) + { + args += ":range={6}"; + } + } + + return string.Format( + CultureInfo.InvariantCulture, + args, + hwTonemapSuffix, + videoFormat ?? "nv12", + options.TonemappingAlgorithm, + options.TonemappingPeak, + options.TonemappingDesat, + options.TonemappingParam, + options.TonemappingRange); } /// - /// Gets the output size parameter. - /// If we're going to put a fixed size on the command line, this will calculate it. + /// Gets the parameter of software filter chain. /// /// Encoding state. /// Encoding options. - /// Video codec to use. - /// The output size parameter. - public string GetOutputSizeParamInternal( + /// Video encoder to use. + /// The tuple contains three lists: main, sub and overlay filters + public (List, List, List) GetSwVidFilterChain( EncodingJobInfo state, EncodingOptions options, - string outputVideoCodec) + string vidEncoder) { - // http://sonnati.wordpress.com/2012/10/19/ffmpeg-the-swiss-army-knife-of-internet-streaming-part-vi/ - - var request = state.BaseRequest; - var videoStream = state.VideoStream; - var filters = new List(); - - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, options) ?? string.Empty; - var inputWidth = videoStream?.Width; - var inputHeight = videoStream?.Height; + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; var threeDFormat = state.MediaSource.Video3DFormat; - var isSwDecoder = string.IsNullOrEmpty(videoDecoder); - var isD3d11vaDecoder = videoDecoder.IndexOf("d3d11va", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiDecoder = videoDecoder.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiEncoder = outputVideoCodec.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiH264Encoder = outputVideoCodec.IndexOf("h264_vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isVaapiHevcEncoder = outputVideoCodec.IndexOf("hevc_vaapi", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvH264Encoder = outputVideoCodec.IndexOf("h264_qsv", StringComparison.OrdinalIgnoreCase) != -1; - var isQsvHevcEncoder = outputVideoCodec.IndexOf("hevc_qsv", StringComparison.OrdinalIgnoreCase) != -1; - var isNvdecDecoder = videoDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); - var isNvencEncoder = outputVideoCodec.Contains("nvenc", StringComparison.OrdinalIgnoreCase); - var isCuvidH264Decoder = videoDecoder.Contains("h264_cuvid", StringComparison.OrdinalIgnoreCase); - var isCuvidHevcDecoder = videoDecoder.Contains("hevc_cuvid", StringComparison.OrdinalIgnoreCase); - var isCuvidVp9Decoder = videoDecoder.Contains("vp9_cuvid", StringComparison.OrdinalIgnoreCase); - var isLibX264Encoder = outputVideoCodec.IndexOf("libx264", StringComparison.OrdinalIgnoreCase) != -1; - var isLibX265Encoder = outputVideoCodec.IndexOf("libx265", StringComparison.OrdinalIgnoreCase) != -1; - var isLinux = OperatingSystem.IsLinux(); - var isColorDepth10 = IsColorDepth10(state); - - var isTonemappingSupportedOnNvenc = string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase) && (isNvdecDecoder || isCuvidHevcDecoder || isCuvidVp9Decoder || isSwDecoder); - var isTonemappingSupportedOnAmf = string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase) && (isD3d11vaDecoder || isSwDecoder); - var isTonemappingSupportedOnVaapi = string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isVaapiH264Encoder || isVaapiHevcEncoder); - var isTonemappingSupportedOnQsv = string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) && isVaapiDecoder && (isQsvH264Encoder || isQsvHevcEncoder); - var isOpenclTonemappingSupported = IsOpenclTonemappingSupported(state, options); - var isVppTonemappingSupported = IsVppTonemappingSupported(state, options); - var isCudaTonemappingSupported = IsCudaTonemappingSupported(state, options); - var mediaEncoderVersion = _mediaEncoder.GetMediaEncoderVersion(); - var isCudaOverlaySupported = _mediaEncoder.SupportsFilter("overlay_cuda") && mediaEncoderVersion != null && mediaEncoderVersion >= _minVersionForCudaOverlay; + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); - var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; - var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; - var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; - - // If double rate deinterlacing is enabled and the input framerate is 30fps or below, otherwise the output framerate will be too high for many devices - var doubleRateDeinterlace = options.DeinterlaceDoubleRate && (videoStream?.AverageFrameRate ?? 60) <= 30; - - var isScalingInAdvance = false; - var isCudaDeintInAdvance = false; - var isHwuploadCudaRequired = false; - var isNoTonemapFilterApplied = true; - var isDeinterlaceH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); - var isDeinterlaceHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); - - // Add OpenCL tonemapping filter for NVENC/AMF/VAAPI. - if ((isTonemappingSupportedOnNvenc && !isCudaTonemappingSupported) || isTonemappingSupportedOnAmf || (isTonemappingSupportedOnVaapi && !isVppTonemappingSupported)) - { - // NVIDIA Pascal and Turing or higher are recommended. - // AMD Polaris and Vega or higher are recommended. - // Intel Kaby Lake or newer is required. - if (isOpenclTonemappingSupported) - { - isNoTonemapFilterApplied = false; - var inputHdrParams = GetInputHdrParams(videoStream.ColorTransfer); - if (!string.IsNullOrEmpty(inputHdrParams)) - { - filters.Add(inputHdrParams); - } + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; - var parameters = "tonemap_opencl=format=nv12:primaries=bt709:transfer=bt709:matrix=bt709:tonemap={0}:desat={1}:threshold={2}:peak={3}"; + var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; - if (options.TonemappingParam != 0) - { - parameters += ":param={4}"; - } + /* Make main filters for video stream */ + var mainFilters = new List(); - if (!string.Equals(options.TonemappingRange, "auto", StringComparison.OrdinalIgnoreCase)) - { - parameters += ":range={5}"; - } + mainFilters.Add(GetOverwriteColorPropertiesParam(state, false)); - if (isSwDecoder || isD3d11vaDecoder) - { - isScalingInAdvance = true; - // Add zscale filter before tone mapping filter for performance. - var (width, height) = GetFixedOutputSize(inputWidth, inputHeight, request.Width, request.Height, request.MaxWidth, request.MaxHeight); - if (width.HasValue && height.HasValue) - { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "zscale=s={0}x{1}", - width.Value, - height.Value)); - } + // INPUT sw surface(memory/copy-back from vram) + // sw deint + if (doDeintH2645) + { + var deintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(deintFilter); + } - // Convert to hardware pixel format p010 when using SW decoder. - filters.Add("format=p010"); - } + var outFormat = isSwDecoder ? "yuv420p" : "nv12"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + if (isVaapiEncoder) + { + outFormat = "nv12"; + } - if ((isDeinterlaceH264 || isDeinterlaceHevc) && isNvdecDecoder) - { - isCudaDeintInAdvance = true; - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "yadif_cuda={0}:-1:0", - doubleRateDeinterlace ? "1" : "0")); - } + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); - if (isVaapiDecoder || isNvdecDecoder) - { - isScalingInAdvance = true; - filters.AddRange( - GetScalingFilters( - state, - options, - inputWidth, - inputHeight, - threeDFormat, - videoDecoder, - outputVideoCodec, - request.Width, - request.Height, - request.MaxWidth, - request.MaxHeight)); - } + // sw tonemap <= TODO: finsh the fast tonemap filter - // hwmap the HDR data to opencl device by cl-va p010 interop. - if (isVaapiDecoder) - { - filters.Add("hwmap"); - } + // OUTPUT yuv420p/nv12 surface(memory) - // convert cuda device data to p010 host data. - if (isNvdecDecoder) - { - filters.Add("hwdownload,format=p010"); - } + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List(); + var overlayFilters = new List(); + if (hasTextSubs) + { + // subtitles=f='*.ass':alpha=0 + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + else if (hasGraphicalSubs) + { + // [0:s]scale=s=1280x720 + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=endall:shortest=1:repeatlast=0"); + } + + return (mainFilters, subFilters, overlayFilters); + } + + /// + /// Gets the parameter of Nvidia NVENC filter chain. + /// + /// Encoding state. + /// Encoding options. + /// Video encoder to use. + /// The tuple contains three lists: main, sub and overlay filters + public (List, List, List) GetNvidiaVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + if (!string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + return (null, null, null); + } + + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !vidEncoder.Contains("nvenc", StringComparison.OrdinalIgnoreCase); + + // legacy cuvid(resize/deint/sw) pipeline(copy-back) + if ((isSwDecoder && isSwEncoder) + || !IsCudaFullSupported() + || !options.EnableEnhancedNvdecDecoder + || !_mediaEncoder.SupportsFilter("alphasrc")) + { + return GetSwVidFilterChain(state, options, vidEncoder); + } + + // prefered nvdec + cuda filters + nvenc pipeline + return GetNvidiaVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + public (List, List, List) GetNvidiaVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); + var isNvencEncoder = vidEncoder.Contains("nvenc", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isNvencEncoder; + var isCuInCuOut = isNvdecDecoder && isNvencEncoder; + + var doubleRateDeint = options.DeinterlaceDoubleRate && (state.VideoStream?.AverageFrameRate ?? 60) <= 30; + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doCuTonemap = IsHwTonemapAvailable(state, options); + + var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doCuTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doCuTonemap ? "yuv420p10le" : "yuv420p"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // sw => hw + if (doCuTonemap) + { + mainFilters.Add("hwupload"); + } + } + if (isNvdecDecoder) + { + // INPUT cuda surface(vram) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "cuda"); + mainFilters.Add(deintFilter); + } + + var outFormat = doCuTonemap ? string.Empty : "yuv420p"; + var hwScaleFilter = GetHwScaleFilter("cuda", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw scale + mainFilters.Add(hwScaleFilter); + } + + // hw tonemap + if (doCuTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "cuda", "yuv420p"); + mainFilters.Add(tonemapFilter); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doCuTonemap; + if ((isNvdecDecoder && isSwEncoder) || isUploadForOclTonemap) + { + memoryOutput = true; + + // OUTPUT yuv420p surface(memory) + mainFilters.Add("hwdownload"); + mainFilters.Add("format=yuv420p"); + } + + // OUTPUT yuv420p surface(memory) + if (isSwDecoder && isNvencEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + // OUTPUT cuda(yuv420p) surface(vram) + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List(); + var overlayFilters = new List(); + if (isCuInCuOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) + { + // scale=s=1280x720,format=yuva420p,hwupload + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + subFilters.Add("format=yuva420p"); + } + else if (hasTextSubs) + { + // alphasrc=s=1280x720:r=10:start=0,format=yuva420p,subtitles,hwupload + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=yuva420p"); + subFilters.Add(subTextSubtitlesFilter); + } + + subFilters.Add("hwupload"); + overlayFilters.Add("overlay_cuda=eof_action=endall:shortest=1:repeatlast=0"); + } + } + else + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=endall:shortest=1:repeatlast=0"); + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + /// + /// Gets the parameter of AMD AMF filter chain. + /// + /// Encoding state. + /// Encoding options. + /// Video encoder to use. + /// The tuple contains three lists: main, sub and overlay filters + public (List, List, List) GetAmdVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + if (!string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + return (null, null, null); + } + + var isWindows = OperatingSystem.IsWindows(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !vidEncoder.Contains("amf", StringComparison.OrdinalIgnoreCase); + var isAmfDx11OclSupported = isWindows && _mediaEncoder.SupportsHwaccel("d3d11va") && IsOpenclFullSupported(); + + // legacy d3d11va pipeline(copy-back) + if ((isSwDecoder && isSwEncoder) + || !isAmfDx11OclSupported + || !_mediaEncoder.SupportsFilter("alphasrc")) + { + return GetSwVidFilterChain(state, options, vidEncoder); + } + + // prefered d3d11va + opencl filters + amf pipeline + return GetAmdDx11VidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + public (List, List, List) GetAmdDx11VidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + var isAmfEncoder = vidEncoder.Contains("amf", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isAmfEncoder; + var isDxInDxOut = isD3d11vaDecoder && isAmfEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doOclTonemap = IsHwTonemapAvailable(state, options); + + var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doOclTonemap ? "yuv420p10le" : "yuv420p"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload"); + } + } + + if (isD3d11vaDecoder) + { + // INPUT d3d11 surface(vram) + // map from d3d11va to opencl via d3d11-opencl interop. + mainFilters.Add("hwmap=derive_device=opencl"); + + // hw deint <= TODO: finsh the 'yadif_opencl' filter + + var outFormat = doOclTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter("opencl", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw scale + mainFilters.Add(hwScaleFilter); + } + + // hw tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doOclTonemap; + if ((isD3d11vaDecoder && isSwEncoder) || isUploadForOclTonemap) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl. + var hwTransferFilter = hasGraphicalSubs ? "hwdownload" : "hwmap"; + mainFilters.Add(hwTransferFilter); + mainFilters.Add("format=nv12"); + } + + // OUTPUT yuv420p surface + if (isSwDecoder && isAmfEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if (isDxInDxOut && !hasSubs) + { + // OUTPUT d3d11(nv12) surface(vram) + // reverse-mapping via d3d11-opencl interop. + mainFilters.Add("hwmap=derive_device=d3d11va:reverse=1"); + mainFilters.Add("format=d3d11"); + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List(); + var overlayFilters = new List(); + if (isDxInDxOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) + { + // scale=s=1280x720,format=yuva420p,hwupload + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + subFilters.Add("format=yuva420p"); + } + else if (hasTextSubs) + { + // alphasrc=s=1280x720:r=10:start=0,format=yuva420p,subtitles,hwupload + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=yuva420p"); + subFilters.Add(subTextSubtitlesFilter); + } + + subFilters.Add("hwupload"); + overlayFilters.Add("overlay_opencl=eof_action=endall:shortest=1:repeatlast=0"); + overlayFilters.Add("hwmap=derive_device=d3d11va:reverse=1"); + overlayFilters.Add("format=d3d11"); + } + } + else if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=endall:shortest=1:repeatlast=0"); + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + /// + /// Gets the parameter of Intel QSV filter chain. + /// + /// Encoding state. + /// Encoding options. + /// Video encoder to use. + /// The tuple contains three lists: main, sub and overlay filters + public (List, List, List) GetIntelVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + if (!string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + return (null, null, null); + } + + var isWindows = OperatingSystem.IsWindows(); + var isLinux = OperatingSystem.IsLinux(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !vidEncoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isQsvOclSupported = _mediaEncoder.SupportsHwaccel("qsv") && IsOpenclFullSupported(); + var isIntelDx11OclSupported = isWindows + && _mediaEncoder.SupportsHwaccel("d3d11va") + && isQsvOclSupported; + var isIntelVaapiOclSupported = isLinux + && IsVaapiSupported(state) + && isQsvOclSupported; + + // legacy qsv pipeline(copy-back) + if ((isSwDecoder && isSwEncoder) + || (!isIntelVaapiOclSupported && !isIntelDx11OclSupported) + || !_mediaEncoder.SupportsFilter("alphasrc")) + { + return GetSwVidFilterChain(state, options, vidEncoder); + } + + // prefered qsv(vaapi) + opencl filters pipeline + if (isIntelVaapiOclSupported) + { + return GetIntelQsvVaapiVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + // prefered qsv(d3d11) + opencl filters pipeline + if (isIntelDx11OclSupported) + { + return GetIntelQsvDx11VidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + return (null, null, null); + } + + public (List, List, List) GetIntelQsvDx11VidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); + var isQsvDecoder = vidDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isQsvEncoder = vidEncoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isHwDecoder = isD3d11vaDecoder || isQsvDecoder; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isQsvEncoder; + var isQsvInQsvOut = isHwDecoder && isQsvEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doOclTonemap = IsHwTonemapAvailable(state, options); + + var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doOclTonemap ? "yuv420p10le" : "yuv420p"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload"); + } + } + else if (isD3d11vaDecoder || isQsvDecoder) + { + var outFormat = doOclTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter("qsv", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + + if (isD3d11vaDecoder) + { + if (!string.IsNullOrEmpty(hwScaleFilter) || doDeintH2645) + { + // INPUT d3d11 surface(vram) + // map from d3d11va to qsv. + mainFilters.Add("hwmap=derive_device=qsv"); + } + } + + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "qsv"); + mainFilters.Add(deintFilter); + } + + // hw scale + mainFilters.Add(hwScaleFilter); + } + + if (doOclTonemap && isHwDecoder) + { + // map from qsv to opencl via qsv(d3d11)-opencl interop. + mainFilters.Add("hwmap=derive_device=opencl"); + } + + // hw tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doOclTonemap; + var isHwmapUsable = isSwEncoder && doOclTonemap; + if ((isHwDecoder && isSwEncoder) || isUploadForOclTonemap) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl. + // qsv hwmap is not fully implemented for the time being. + mainFilters.Add(isHwmapUsable ? "hwmap" : "hwdownload"); + mainFilters.Add("format=nv12"); + } + + // OUTPUT nv12 surface(memory) + if (isSwDecoder && isQsvEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if (isQsvInQsvOut && doOclTonemap) + { + // OUTPUT qsv(nv12) surface(vram) + // reverse-mapping via qsv(d3d11)-opencl interop. + mainFilters.Add("hwmap=derive_device=qsv:reverse=1"); + mainFilters.Add("format=qsv"); + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List(); + var overlayFilters = new List(); + if (isQsvInQsvOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) + { + // scale,format=bgra,hwupload + // overlay_qsv can handle overlay scaling, + // add a dummy scale filter to pair with -canvas_size. + subFilters.Add("scale=flags=fast_bilinear"); + subFilters.Add("format=bgra"); + } + else if (hasTextSubs) + { + // alphasrc=s=1280x720:r=10:start=0,format=bgra,subtitles,hwupload + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); + } + + // qsv requires a fixed pool size. + subFilters.Add("hwupload=extra_hw_frames=32"); + + var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var overlaySize = (overlayW.HasValue && overlayH.HasValue) + ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + : string.Empty; + var overlayQsvFilter = string.Format( + CultureInfo.InvariantCulture, + "overlay_qsv=eof_action=endall:shortest=1:repeatlast=0{0}", + overlaySize); + overlayFilters.Add(overlayQsvFilter); + } + } + else if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=endall:shortest=1:repeatlast=0"); + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + public (List, List, List) GetIntelQsvVaapiVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isQsvDecoder = vidDecoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isQsvEncoder = vidEncoder.Contains("qsv", StringComparison.OrdinalIgnoreCase); + var isHwDecoder = isVaapiDecoder || isQsvDecoder; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isQsvEncoder; + var isQsvInQsvOut = isHwDecoder && isQsvEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doVaVppTonemap = IsVaapiVppTonemapAvailable(state, options); + var doOclTonemap = !doVaVppTonemap && IsHwTonemapAvailable(state, options); + var doTonemap = doVaVppTonemap || doOclTonemap; + var doDeintH2645 = doDeintH264 || doDeintHevc; + + var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doOclTonemap ? "yuv420p10le" : "yuv420p"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload"); + } + } + else if (isVaapiDecoder || isQsvDecoder) + { + // INPUT vaapi/qsv surface(vram) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, isVaapiDecoder ? "vaapi" : "qsv"); + mainFilters.Add(deintFilter); + } + + var outFormat = doTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter(isVaapiDecoder ? "vaapi" : "qsv", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw scale + mainFilters.Add(hwScaleFilter); + } + + // vaapi vpp tonemap + if (doVaVppTonemap && isHwDecoder) + { + if (isQsvDecoder) + { + // map from qsv to vaapi. + mainFilters.Add("hwmap=derive_device=vaapi"); + } + + var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12"); + mainFilters.Add(tonemapFilter); + + if (isQsvDecoder) + { + // map from vaapi to qsv. + mainFilters.Add("hwmap=derive_device=qsv"); + } + } + + if (doOclTonemap && isHwDecoder) + { + // map from qsv to opencl via qsv(vaapi)-opencl interop. + mainFilters.Add("hwmap=derive_device=opencl"); + } + + // ocl tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doOclTonemap; + var isHwmapUsable = isSwEncoder && (doOclTonemap || isVaapiDecoder); + if ((isHwDecoder && isSwEncoder) || isUploadForOclTonemap) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl/vaapi. + // qsv hwmap is not fully implemented for the time being. + mainFilters.Add(isHwmapUsable ? "hwmap" : "hwdownload"); + mainFilters.Add("format=nv12"); + } + + // OUTPUT nv12 surface(memory) + if (isSwDecoder && isQsvEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if (isQsvInQsvOut) + { + if (doOclTonemap) + { + // OUTPUT qsv(nv12) surface(vram) + // reverse-mapping via qsv(vaapi)-opencl interop. + // add extra pool size to avoid the 'cannot allocate memory' error on hevc_qsv. + mainFilters.Add("hwmap=derive_device=qsv:reverse=1:extra_hw_frames=16"); + mainFilters.Add("format=qsv"); + } + else if (isVaapiDecoder) + { + mainFilters.Add("hwmap=derive_device=qsv"); + mainFilters.Add("format=qsv"); + } + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List(); + var overlayFilters = new List(); + if (isQsvInQsvOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) + { + subFilters.Add("scale=flags=fast_bilinear"); + subFilters.Add("format=bgra"); + } + else if (hasTextSubs) + { + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); + } + + // qsv requires a fixed pool size. + subFilters.Add("hwupload=extra_hw_frames=32"); + + var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var overlaySize = (overlayW.HasValue && overlayH.HasValue) + ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + : string.Empty; + var overlayQsvFilter = string.Format( + CultureInfo.InvariantCulture, + "overlay_qsv=eof_action=endall:shortest=1:repeatlast=0{0}", + overlaySize); + overlayFilters.Add(overlayQsvFilter); + } + } + else if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=pass:shortest=1:repeatlast=0"); + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + /// + /// Gets the parameter of Intel/AMD VAAPI filter chain. + /// + /// Encoding state. + /// Encoding options. + /// Video encoder to use. + /// The tuple contains three lists: main, sub and overlay filters + public (List, List, List) GetVaapiVidFilterChain( + EncodingJobInfo state, + EncodingOptions options, + string vidEncoder) + { + if (!string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + return (null, null, null); + } + + var isLinux = OperatingSystem.IsLinux(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiOclSupported = isLinux && IsVaapiSupported(state) && IsVaapiFullSupported() && IsOpenclFullSupported(); + + // legacy vaapi pipeline(copy-back) + if ((isSwDecoder && isSwEncoder) + || !isVaapiOclSupported + || !_mediaEncoder.SupportsFilter("alphasrc")) + { + var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder); + + if (!isSwEncoder) + { + var newfilters = new List(); + var noOverlay = swFilterChain.Item3.Count == 0; + newfilters.AddRange(noOverlay ? swFilterChain.Item1 : swFilterChain.Item3); + newfilters.Add("hwupload"); + + var mainFilters = noOverlay ? newfilters : swFilterChain.Item1; + var overlayFilters = noOverlay ? swFilterChain.Item3 : newfilters; + return (mainFilters, swFilterChain.Item2, overlayFilters); + } + + return swFilterChain; + } + + // prefered vaapi + opencl filters pipeline + if (_mediaEncoder.IsVaapiDeviceInteliHD) + { + // Intel iHD path, with extra vpp tonemap and overlay support. + return GetVaapiFullVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + // Intel i965 and Amd radeonsi/r600 path, only featuring scale and deinterlace support. + return GetVaapiLimitedVidFiltersPrefered(state, options, vidDecoder, vidEncoder); + } + + public (List, List, List) GetVaapiFullVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isVaapiEncoder; + var isVaInVaOut = isVaapiDecoder && isVaapiEncoder; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doVaVppTonemap = isVaapiDecoder && IsVaapiVppTonemapAvailable(state, options); + var doOclTonemap = !doVaVppTonemap && IsHwTonemapAvailable(state, options); + var doTonemap = doVaVppTonemap || doOclTonemap; + var doDeintH2645 = doDeintH264 || doDeintHevc; + + var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + /* Make main filters for video stream */ + var mainFilters = new List(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doTonemap)); + + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + var outFormat = doOclTonemap ? "yuv420p10le" : "nv12"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload"); + } + } + else if (isVaapiDecoder) + { + // INPUT vaapi surface(vram) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "vaapi"); + mainFilters.Add(deintFilter); + } + + var outFormat = doTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw scale + mainFilters.Add(hwScaleFilter); + } + + // vaapi vpp tonemap + if (doVaVppTonemap && isVaapiDecoder) + { + var tonemapFilter = GetHwTonemapFilter(options, "vaapi", "nv12"); + mainFilters.Add(tonemapFilter); + } + + if (doOclTonemap && isVaapiDecoder) + { + // map from vaapi to opencl via vaapi-opencl interop(Intel only). + mainFilters.Add("hwmap=derive_device=opencl"); + } + + // ocl tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + if (doOclTonemap && isVaInVaOut) + { + // OUTPUT vaapi(nv12) surface(vram) + // reverse-mapping via vaapi-opencl interop. + mainFilters.Add("hwmap=derive_device=vaapi:reverse=1"); + mainFilters.Add("format=vaapi"); + } + + var memoryOutput = false; + var isUploadForOclTonemap = isSwDecoder && doOclTonemap; + var isHwmapNotUsable = isUploadForOclTonemap && isVaapiEncoder; + if ((isVaapiDecoder && isSwEncoder) || isUploadForOclTonemap) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl/vaapi. + mainFilters.Add(isHwmapNotUsable ? "hwdownload" : "hwmap"); + mainFilters.Add("format=nv12"); + } + + // OUTPUT nv12 surface(memory) + if (isSwDecoder && isVaapiEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if (memoryOutput && isVaapiEncoder) + { + if (!hasGraphicalSubs) + { + mainFilters.Add("hwupload_vaapi"); + } + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List(); + var overlayFilters = new List(); + if (isVaInVaOut) + { + if (hasSubs) + { + if (hasGraphicalSubs) + { + subFilters.Add("scale=flags=fast_bilinear"); + subFilters.Add("format=bgra"); + } + else if (hasTextSubs) + { + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, 1080, hasAssSubs ? 10 : 5); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); + } + + subFilters.Add("hwupload"); + + var (overlayW, overlayH) = GetFixedOutputSize(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + var overlaySize = (overlayW.HasValue && overlayH.HasValue) + ? (":w=" + overlayW.Value + ":h=" + overlayH.Value) + : string.Empty; + var overlayVaapiFilter = string.Format( + CultureInfo.InvariantCulture, + "overlay_vaapi=eof_action=endall:shortest=1:repeatlast=0{0}", + overlaySize); + overlayFilters.Add(overlayVaapiFilter); + } + } + else if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=pass:shortest=1:repeatlast=0"); + + if (isVaapiEncoder) + { + overlayFilters.Add("hwupload_vaapi"); + } + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + public (List, List, List) GetVaapiLimitedVidFiltersPrefered( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { + var inW = state.VideoStream?.Width; + var inH = state.VideoStream?.Height; + var reqW = state.BaseRequest.Width; + var reqH = state.BaseRequest.Height; + var reqMaxW = state.BaseRequest.MaxWidth; + var reqMaxH = state.BaseRequest.MaxHeight; + var threeDFormat = state.MediaSource.Video3DFormat; + + var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); + var isSwDecoder = string.IsNullOrEmpty(vidDecoder); + var isSwEncoder = !isVaapiEncoder; + var isVaInVaOut = isVaapiDecoder && isVaapiEncoder; + var isi965Driver = _mediaEncoder.IsVaapiDeviceInteli965; + var isAmdDriver = _mediaEncoder.IsVaapiDeviceAmd; + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doOclTonemap = IsHwTonemapAvailable(state, options); + + var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + + /* Make main filters for video stream */ + var mainFilters = new List(); + + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); + + var outFormat = string.Empty; + if (isSwDecoder) + { + // INPUT sw surface(memory) + // sw deint + if (doDeintH2645) + { + var swDeintFilter = GetSwDeinterlaceFilter(state, options); + mainFilters.Add(swDeintFilter); + } + + outFormat = doOclTonemap ? "yuv420p10le" : "nv12"; + var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); + // sw scale + mainFilters.Add(swScaleFilter); + mainFilters.Add("format=" + outFormat); + + // keep video at memory except ocl tonemap, + // since the overhead caused by hwupload >>> using sw filter. + // sw => hw + if (doOclTonemap) + { + mainFilters.Add("hwupload"); + } + } + else if (isVaapiDecoder) + { + // INPUT vaapi surface(vram) + // hw deint + if (doDeintH2645) + { + var deintFilter = GetHwDeinterlaceFilter(state, options, "vaapi"); + mainFilters.Add(deintFilter); + } + + outFormat = doOclTonemap ? string.Empty : "nv12"; + var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); + // hw scale + mainFilters.Add(hwScaleFilter); + } + + if (doOclTonemap && isVaapiDecoder) + { + if (isi965Driver) + { + // map from vaapi to opencl via vaapi-opencl interop(Intel only). + mainFilters.Add("hwmap=derive_device=opencl"); + } + else + { + mainFilters.Add("hwdownload"); + mainFilters.Add("format=p010le"); + mainFilters.Add("hwupload"); + } + } + + // ocl tonemap + if (doOclTonemap) + { + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + } + + if (doOclTonemap && isVaInVaOut) + { + if (isi965Driver) + { + // OUTPUT vaapi(nv12) surface(vram) + // reverse-mapping via vaapi-opencl interop. + mainFilters.Add("hwmap=derive_device=vaapi:reverse=1"); + mainFilters.Add("format=vaapi"); + } + } + + var memoryOutput = false; + var isUploadForOclTonemap = doOclTonemap && (isSwDecoder || (isVaapiDecoder && !isi965Driver)); + var isHwmapNotUsable = hasGraphicalSubs || isUploadForOclTonemap; + var isHwmapForSubs = hasSubs && isVaapiDecoder; + var isHwUnmapForTextSubs = hasTextSubs && isVaInVaOut && !isUploadForOclTonemap; + if ((isVaapiDecoder && isSwEncoder) || isUploadForOclTonemap || isHwmapForSubs) + { + memoryOutput = true; + + // OUTPUT nv12 surface(memory) + // prefer hwmap to hwdownload on opencl/vaapi. + mainFilters.Add(isHwmapNotUsable ? "hwdownload" : "hwmap"); + mainFilters.Add("format=nv12"); + } + + // OUTPUT nv12 surface(memory) + if (isSwDecoder && isVaapiEncoder) + { + memoryOutput = true; + } + + if (memoryOutput) + { + // text subtitles + if (hasTextSubs) + { + var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false); + mainFilters.Add(textSubtitlesFilter); + } + } + + if (isHwUnmapForTextSubs) + { + mainFilters.Add("hwmap"); + mainFilters.Add("format=vaapi"); + } + else if (memoryOutput && isVaapiEncoder) + { + if (!hasGraphicalSubs) + { + mainFilters.Add("hwupload_vaapi"); + } + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List(); + var overlayFilters = new List(); + if (memoryOutput) + { + if (hasGraphicalSubs) + { + var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subSwScaleFilter); + overlayFilters.Add("overlay=eof_action=pass:shortest=1:repeatlast=0"); + + if (isVaapiEncoder) + { + overlayFilters.Add("hwupload_vaapi"); + } + } + } + + return (mainFilters, subFilters, overlayFilters); + } + + /// + /// Gets the parameter of video processing filters. + /// + /// Encoding state. + /// Encoding options. + /// Video codec to use. + /// The video processing filters parameter. + public string GetVideoProcessingFilterParam( + EncodingJobInfo state, + EncodingOptions options, + string outputVideoCodec) + { + var videoStream = state.VideoStream; + if (videoStream == null) + { + return string.Empty; + } + + var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + + List mainFilters; + List subFilters; + List overlayFilters; + + if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetVaapiVidFilterChain(state, options, outputVideoCodec); + } + else if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetIntelVidFilterChain(state, options, outputVideoCodec); + } + else if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetNvidiaVidFilterChain(state, options, outputVideoCodec); + } + else if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + (mainFilters, subFilters, overlayFilters) = GetAmdVidFilterChain(state, options, outputVideoCodec); + } + else + { + (mainFilters, subFilters, overlayFilters) = GetSwVidFilterChain(state, options, outputVideoCodec); + } + + mainFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); + subFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); + overlayFilters?.RemoveAll(filter => string.IsNullOrEmpty(filter)); + + var mainStr = string.Empty; + if (mainFilters?.Count > 0) + { + mainStr = string.Format( + CultureInfo.InvariantCulture, + "{0}", + string.Join(',', mainFilters)); + } + + if (overlayFilters?.Count == 0) + { + // -vf "scale..." + return string.IsNullOrEmpty(mainStr) ? string.Empty : " -vf \"" + mainStr + "\""; + } + + if (overlayFilters?.Count > 0 + && subFilters?.Count > 0 + && state.SubtitleStream != null) + { + // overlay graphical/text subtitles + var subStr = string.Format( + CultureInfo.InvariantCulture, + "{0}", + string.Join(',', subFilters)); + + var overlayStr = string.Format( + CultureInfo.InvariantCulture, + "{0}", + string.Join(',', overlayFilters)); + + + var mapPrefix = Convert.ToInt32(state.SubtitleStream.IsExternal); + var subtitleStreamIndex = state.SubtitleStream.IsExternal + ? 0 + : state.SubtitleStream.Index; + + if (hasSubs) + { + // -filter_complex "[0:s]scale=s[sub]..." + var filterStr = string.IsNullOrEmpty(mainStr) + ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]{5}\"" + : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[main];[main][sub]{5}\""; + + if (hasTextSubs) + { + filterStr = string.IsNullOrEmpty(mainStr) + ? " -filter_complex \"{4}[sub];[0:{2}][sub]{5}\"" + : " -filter_complex \"{4}[sub];[0:{2}]{3}[main];[main][sub]{5}\""; + } + + return string.Format( + CultureInfo.InvariantCulture, + filterStr, + mapPrefix, + subtitleStreamIndex, + state.VideoStream.Index, + mainStr, + subStr, + overlayStr); + } + } + + return string.Empty; + } + + public string GetOverwriteColorPropertiesParam(EncodingJobInfo state, bool isTonemapAvailable) + { + if (isTonemapAvailable) + { + return GetInputHdrParam(state.VideoStream?.ColorTransfer); + } + + return GetOutputSdrParam(null); + } + + public string GetInputHdrParam(string colorTransfer) + { + if (string.Equals(colorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase)) + { + // HLG + return "setparams=color_primaries=bt2020:color_trc=arib-std-b67:colorspace=bt2020nc"; + } + + // HDR10 + return "setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc"; + } + + public string GetOutputSdrParam(string tonemappingRange) + { + // SDR + if (string.Equals(tonemappingRange, "tv", StringComparison.OrdinalIgnoreCase)) + { + return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv"; + } + + if (string.Equals(tonemappingRange, "pc", StringComparison.OrdinalIgnoreCase)) + { + return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=pc"; + } - if (isNvdecDecoder - || isCuvidHevcDecoder - || isCuvidVp9Decoder - || isSwDecoder - || isD3d11vaDecoder) - { - // Upload the HDR10 or HLG data to the OpenCL device, - // use tonemap_opencl filter for tone mapping, - // and then download the SDR data to memory. - filters.Add("hwupload"); - } + return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709"; + } - // Fallback to hable if bt2390 is chosen but not supported in tonemap_opencl. - var isBt2390SupportedInOpenclTonemap = _mediaEncoder.SupportsFilterWithOption(FilterOptionType.TonemapOpenclBt2390); - if (string.Equals(options.TonemappingAlgorithm, "bt2390", StringComparison.OrdinalIgnoreCase) - && !isBt2390SupportedInOpenclTonemap) - { - options.TonemappingAlgorithm = "hable"; - } + public static int GetVideoColorBitDepth(EncodingJobInfo state) + { + var videoStream = state.VideoStream; + if (videoStream != null) + { + if (videoStream.BitDepth.HasValue) + { + return videoStream.BitDepth.Value; + } + else if (string.Equals(videoStream.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p", StringComparison.OrdinalIgnoreCase)) + { + return 8; + } + else if (string.Equals(videoStream.PixelFormat, "yuv420p10le", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p10le", StringComparison.OrdinalIgnoreCase)) + { + return 10; + } + else if (string.Equals(videoStream.PixelFormat, "yuv420p12le", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.PixelFormat, "yuv444p12le", StringComparison.OrdinalIgnoreCase)) + { + return 12; + } + else + { + return 8; + } + } - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - parameters, - options.TonemappingAlgorithm, - options.TonemappingDesat, - options.TonemappingThreshold, - options.TonemappingPeak, - options.TonemappingParam, - options.TonemappingRange)); - - if (isNvdecDecoder - || isCuvidHevcDecoder - || isCuvidVp9Decoder - || isSwDecoder - || isD3d11vaDecoder) - { - filters.Add("hwdownload"); - filters.Add("format=nv12"); - } + return 0; + } + + /// + /// Gets the ffmpeg option string for the hardware accelerated video decoder. + /// + /// The encoding job info. + /// The encoding options. + /// The option string or null if none available. + protected string GetHardwareVideoDecoder(EncodingJobInfo state, EncodingOptions options) + { + var videoStream = state.VideoStream; + if (videoStream == null) + { + return null; + } + + // Only use alternative encoders for video files. + var videoType = state.MediaSource.VideoType ?? VideoType.VideoFile; + if (videoType != VideoType.VideoFile) + { + return null; + } + + if (IsCopyCodec(state.OutputVideoCodec)) + { + return null; + } + + if (!string.IsNullOrEmpty(videoStream.Codec) && !string.IsNullOrEmpty(options.HardwareAccelerationType)) + { + var bitDepth = GetVideoColorBitDepth(state); + + // Only HEVC, VP9 and AV1 formats have 10-bit hardware decoder support now. + if (bitDepth == 10 + && !(string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "av1", StringComparison.OrdinalIgnoreCase))) + { + return null; + } + + if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + return GetQsvHwVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + return GetNvdecVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + return GetAmfVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + return GetVaapiVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) + { + return GetVideotoolboxVidDecoder(state, options, videoStream, bitDepth); + } + + if (string.Equals(options.HardwareAccelerationType, "omx", StringComparison.OrdinalIgnoreCase)) + { + return GetOmxVidDecoder(state, options, videoStream, bitDepth); + } + } + + var whichCodec = videoStream.Codec; + if (string.Equals(whichCodec, "avc", StringComparison.OrdinalIgnoreCase)) + { + whichCodec = "h264"; + } + else if (string.Equals(whichCodec, "h265", StringComparison.OrdinalIgnoreCase)) + { + whichCodec = "hevc"; + } + + // Avoid a second attempt if no hardware acceleration is being used + options.HardwareDecodingCodecs = Array.FindAll(options.HardwareDecodingCodecs, val => !string.Equals(val, whichCodec, StringComparison.OrdinalIgnoreCase)); + + // leave blank so ffmpeg will decide + return null; + } + + /// + /// Gets a hw decoder name. + /// + /// Encoding options. + /// Decoder prefix. + /// Decoder suffix. + /// Video codec to use. + /// Video color bit depth. + /// Hardware decoder name. + public string GetHwDecoderName(EncodingOptions options, string decoderPrefix, string decoderSuffix, string videoCodec, int bitDepth) + { + if (string.IsNullOrEmpty(decoderPrefix) || string.IsNullOrEmpty(decoderSuffix)) + { + return null; + } + + var decoderName = decoderPrefix + '_' + decoderSuffix; + + var isCodecAvailable = _mediaEncoder.SupportsDecoder(decoderName) && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase); + if (bitDepth == 10 && isCodecAvailable) + { + if ((options.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Hevc) + || (options.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Vp9)) + { + return null; + } + } + + if (string.Equals(decoderSuffix, "cuvid", StringComparison.OrdinalIgnoreCase) && options.EnableEnhancedNvdecDecoder) + { + return null; + } + + if (string.Equals(decoderSuffix, "qsv", StringComparison.OrdinalIgnoreCase) && options.PreferSystemNativeHwDecoder) + { + return null; + } + + return isCodecAvailable ? (" -c:v " + decoderName) : null; + } + + /// + /// Gets a hwaccel type to use as a hardware decoder depending on the system. + /// + /// Encoding state. + /// Encoding options. + /// Video codec to use. + /// Video color bit depth. + /// Specifies if output hw surface. + /// Hardware accelerator type. + public string GetHwaccelType(EncodingJobInfo state, EncodingOptions options, string videoCodec, int bitDepth, bool outputHwSurface) + { + var isWindows = OperatingSystem.IsWindows(); + var isLinux = OperatingSystem.IsLinux(); + var isMacOS = OperatingSystem.IsMacOS(); + var isD3d11Supported = isWindows && _mediaEncoder.SupportsHwaccel("d3d11va"); + var isVaapiSupported = isLinux && IsVaapiSupported(state); + var isCudaSupported = (isLinux || isWindows) && IsCudaFullSupported(); + var isQsvSupported = (isLinux || isWindows) && _mediaEncoder.SupportsHwaccel("qsv"); + var isVideotoolboxSupported = isMacOS && _mediaEncoder.SupportsHwaccel("videotoolbox"); + var isCodecAvailable = options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase); + + // Set the av1 codec explicitly to trigger hw accelerator, otherwise libdav1d will be used. + var isAv1 = string.Equals(videoCodec, "av1", StringComparison.OrdinalIgnoreCase); + + if (bitDepth == 10 && isCodecAvailable) + { + if ((options.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Hevc) + || (options.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Vp9)) + { + return null; + } + } - if (isNvdecDecoder && isNvencEncoder) + // Intel qsv/d3d11va/vaapi + if (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) + { + if (options.PreferSystemNativeHwDecoder) + { + if (isVaapiSupported && isCodecAvailable) { - isHwuploadCudaRequired = true; + return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); } - if (isVaapiDecoder) + if (isD3d11Supported && isCodecAvailable) { - // Reverse the data route from opencl to vaapi. - filters.Add("hwmap=derive_device=vaapi:reverse=1"); + return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); } - - var outputSdrParams = GetOutputSdrParams(options.TonemappingRange); - if (!string.IsNullOrEmpty(outputSdrParams)) + } + else + { + if (isQsvSupported && isCodecAvailable) { - filters.Add(outputSdrParams); + return " -hwaccel qsv" + (outputHwSurface ? " -hwaccel_output_format qsv" : string.Empty); } } } - // When the input may or may not be hardware VAAPI decodable. - if ((isVaapiH264Encoder || isVaapiHevcEncoder) - && !(isTonemappingSupportedOnVaapi && (isOpenclTonemappingSupported || isVppTonemappingSupported))) + // Nvidia cuda + if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + if (options.EnableEnhancedNvdecDecoder && isCudaSupported && isCodecAvailable) + { + return " -hwaccel cuda" + (outputHwSurface ? " -hwaccel_output_format cuda" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); + } + } + + // Amd d3d11va + if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) + { + if (isD3d11Supported && isCodecAvailable) + { + return " -hwaccel d3d11va" + (outputHwSurface ? " -hwaccel_output_format d3d11" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); + } + } + + // Vaapi + if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) + && isVaapiSupported + && isCodecAvailable) { - filters.Add("format=nv12|vaapi"); - filters.Add("hwupload"); + return " -hwaccel vaapi" + (outputHwSurface ? " -hwaccel_output_format vaapi" : string.Empty) + (isAv1 ? " -c:v av1" : string.Empty); } - // When burning in graphical subtitles using overlay_qsv, upload videostream to the same qsv context. - else if (isLinux && hasGraphicalSubs && (isQsvH264Encoder || isQsvHevcEncoder) - && !(isTonemappingSupportedOnQsv && isVppTonemappingSupported)) + if (string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase) + && isVideotoolboxSupported + && isCodecAvailable) { - filters.Add("hwupload=extra_hw_frames=64"); + return " -hwaccel videotoolbox" + (outputHwSurface ? " -hwaccel_output_format videotoolbox_vld" : string.Empty); } - // If we're hardware VAAPI decoding and software encoding, download frames from the decoder first. - else if ((IsVaapiSupported(state) && isVaapiDecoder) && (isLibX264Encoder || isLibX265Encoder) - && !(isTonemappingSupportedOnQsv && isVppTonemappingSupported)) + return null; + } + + public string GetQsvHwVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + var isWindows = OperatingSystem.IsWindows(); + var isLinux = OperatingSystem.IsLinux(); + + if ((!isWindows && !isLinux) + || !string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) { - var codec = videoStream.Codec; + return null; + } + + var isQsvOclSupported = _mediaEncoder.SupportsHwaccel("qsv") && IsOpenclFullSupported(); + var isIntelDx11OclSupported = isWindows + && _mediaEncoder.SupportsHwaccel("d3d11va") + && isQsvOclSupported; + var isIntelVaapiOclSupported = isLinux + && IsVaapiSupported(state) + && isQsvOclSupported; + var hwSurface = (isIntelDx11OclSupported || isIntelVaapiOclSupported) + && _mediaEncoder.SupportsFilter("alphasrc"); - // Assert 10-bit hardware VAAPI decodable - if (isColorDepth10 && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase))) + var is8bitSwFormatsQsv = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsQsv = is8bitSwFormatsQsv || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + // TODO: add more 8/10bit and 4:4:4 formats for Qsv after finishing the ffcheck tool + + if (is8bitSwFormatsQsv) + { + if (string.Equals(videoStream.Codec, "avc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "h264", StringComparison.OrdinalIgnoreCase)) { - /* - Download data from GPU to CPU as p010le format. - Colorspace conversion is unnecessary here as libx264 will handle it. - If this step is missing, it will fail on AMD but not on intel. - */ - filters.Add("hwdownload"); - filters.Add("format=p010le"); + return GetHwaccelType(state, options, "h264", bitDepth, hwSurface) + GetHwDecoderName(options, "h264", "qsv", "h264", bitDepth); } - // Assert 8-bit hardware VAAPI decodable - else if (!isColorDepth10) + if (string.Equals(videoStream.Codec, "vc1", StringComparison.OrdinalIgnoreCase)) { - filters.Add("hwdownload"); - filters.Add("format=nv12"); + return GetHwaccelType(state, options, "vc1", bitDepth, hwSurface) + GetHwDecoderName(options, "vc1", "qsv", "vc1", bitDepth); } - } - // Add hardware deinterlace filter before scaling filter. - if (isDeinterlaceH264 || isDeinterlaceHevc) - { - if (isVaapiEncoder - || (isTonemappingSupportedOnQsv && isVppTonemappingSupported)) + if (string.Equals(videoStream.Codec, "vp8", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "deinterlace_vaapi=rate={0}", - doubleRateDeinterlace ? "field" : "frame")); + return GetHwaccelType(state, options, "vp8", bitDepth, hwSurface) + GetHwDecoderName(options, "vp8", "qsv", "vp8", bitDepth); } - else if (isNvdecDecoder && !isCudaDeintInAdvance) + + if (string.Equals(videoStream.Codec, "mpeg2video", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "yadif_cuda={0}:-1:0", - doubleRateDeinterlace ? "1" : "0")); + return GetHwaccelType(state, options, "mpeg2video", bitDepth, hwSurface) + GetHwDecoderName(options, "mpeg2", "qsv", "mpeg2video", bitDepth); } } - // Add software deinterlace filter before scaling filter. - if ((isDeinterlaceH264 || isDeinterlaceHevc) - && !isVaapiH264Encoder - && !isVaapiHevcEncoder - && !isQsvH264Encoder - && !isQsvHevcEncoder - && !isNvdecDecoder - && !isCuvidH264Decoder) + if (is8_10bitSwFormatsQsv) { - if (string.Equals(options.DeinterlaceMethod, "bwdif", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) + || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "bwdif={0}:-1:0", - doubleRateDeinterlace ? "1" : "0")); + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface) + GetHwDecoderName(options, "hevc", "qsv", "hevc", bitDepth); } - else + + if (string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase)) { - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - "yadif={0}:-1:0", - doubleRateDeinterlace ? "1" : "0")); + return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface) + GetHwDecoderName(options, "vp9", "qsv", "vp9", bitDepth); + } + + if (string.Equals(videoStream.Codec, "av1", StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "av1", bitDepth, hwSurface) + GetHwDecoderName(options, "av1", "qsv", "av1", bitDepth); } } - // Add scaling filter: scale_*=format=nv12 or scale_*=w=*:h=*:format=nv12 or scale=expr - if (!isScalingInAdvance) + return null; + } + + public string GetNvdecVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + if ((!OperatingSystem.IsWindows() && !OperatingSystem.IsLinux()) + || !string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) { - filters.AddRange( - GetScalingFilters( - state, - options, - inputWidth, - inputHeight, - threeDFormat, - videoDecoder, - outputVideoCodec, - request.Width, - request.Height, - request.MaxWidth, - request.MaxHeight)); + return null; } - // Add Cuda tonemapping filter. - if (isNvdecDecoder && isCudaTonemappingSupported) + var hwSurface = IsCudaFullSupported() + && options.EnableEnhancedNvdecDecoder + && _mediaEncoder.SupportsFilter("alphasrc"); + var is8bitSwFormatsNvdec = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsNvdec = is8bitSwFormatsNvdec || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + // TODO: add more 8/10/12bit and 4:4:4 formats for Nvdec after finishing the ffcheck tool + + if (is8bitSwFormatsNvdec) { - isNoTonemapFilterApplied = false; - var inputHdrParams = GetInputHdrParams(videoStream.ColorTransfer); - if (!string.IsNullOrEmpty(inputHdrParams)) + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "h264", bitDepth, hwSurface) + GetHwDecoderName(options, "h264", "cuvid", "h264", bitDepth); + } + + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - filters.Add(inputHdrParams); + return GetHwaccelType(state, options, "mpeg2video", bitDepth, hwSurface) + GetHwDecoderName(options, "mpeg2", "cuvid", "mpeg2video", bitDepth); } - var parameters = (hasGraphicalSubs && isCudaOverlaySupported && isNvencEncoder) - ? "tonemap_cuda=format=yuv420p:primaries=bt709:transfer=bt709:matrix=bt709:tonemap={0}:peak={1}:desat={2}" - : "tonemap_cuda=format=nv12:primaries=bt709:transfer=bt709:matrix=bt709:tonemap={0}:peak={1}:desat={2}"; + if (string.Equals("vc1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vc1", bitDepth, hwSurface) + GetHwDecoderName(options, "vc1", "cuvid", "vc1", bitDepth); + } - if (options.TonemappingParam != 0) + if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - parameters += ":param={3}"; + return GetHwaccelType(state, options, "mpeg4", bitDepth, hwSurface) + GetHwDecoderName(options, "mpeg4", "cuvid", "mpeg4", bitDepth); } - if (!string.Equals(options.TonemappingRange, "auto", StringComparison.OrdinalIgnoreCase)) + if (string.Equals("vp8", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - parameters += ":range={4}"; + return GetHwaccelType(state, options, "vp8", bitDepth, hwSurface) + GetHwDecoderName(options, "vp8", "cuvid", "vp8", bitDepth); } + } - filters.Add( - string.Format( - CultureInfo.InvariantCulture, - parameters, - options.TonemappingAlgorithm, - options.TonemappingPeak, - options.TonemappingDesat, - options.TonemappingParam, - options.TonemappingRange)); - - if (isLibX264Encoder - || isLibX265Encoder - || hasTextSubs - || (hasGraphicalSubs && !isCudaOverlaySupported && isNvencEncoder)) - { - if (isNvencEncoder) - { - isHwuploadCudaRequired = true; - } + if (is8_10bitSwFormatsNvdec) + { + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface) + GetHwDecoderName(options, "hevc", "cuvid", "hevc", bitDepth); + } - filters.Add("hwdownload"); - filters.Add("format=nv12"); + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface) + GetHwDecoderName(options, "vp9", "cuvid", "vp9", bitDepth); } - var outputSdrParams = GetOutputSdrParams(options.TonemappingRange); - if (!string.IsNullOrEmpty(outputSdrParams)) + if (string.Equals("av1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - filters.Add(outputSdrParams); + return GetHwaccelType(state, options, "av1", bitDepth, hwSurface) + GetHwDecoderName(options, "av1", "cuvid", "av1", bitDepth); } } - // Add VPP tonemapping filter for VAAPI. - // Full hardware based video post processing, faster than OpenCL but lacks fine tuning options. - if ((isTonemappingSupportedOnVaapi || isTonemappingSupportedOnQsv) - && isVppTonemappingSupported) + return null; + } + + public string GetAmfVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + if (!OperatingSystem.IsWindows() + || !string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) { - filters.Add("tonemap_vaapi=format=nv12:transfer=bt709:matrix=bt709:primaries=bt709"); + return null; } - // Another case is when using Nvenc decoder. - if (isNvdecDecoder && !isOpenclTonemappingSupported && !isCudaTonemappingSupported) + var hwSurface = _mediaEncoder.SupportsHwaccel("d3d11va") + && IsOpenclFullSupported() + && _mediaEncoder.SupportsFilter("alphasrc"); + var is8bitSwFormatsAmf = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsAmf = is8bitSwFormatsAmf || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + + if (is8bitSwFormatsAmf) { - var codec = videoStream.Codec; - var isCudaFormatConversionSupported = _mediaEncoder.SupportsFilterWithOption(FilterOptionType.ScaleCudaFormat); + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "h264", bitDepth, hwSurface); + } - // Assert 10-bit hardware decodable - if (isColorDepth10 && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(codec, "vp9", StringComparison.OrdinalIgnoreCase))) + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - if (isCudaFormatConversionSupported) - { - if (isLibX264Encoder - || isLibX265Encoder - || hasTextSubs - || (hasGraphicalSubs && !isCudaOverlaySupported && isNvencEncoder)) - { - if (isNvencEncoder) - { - isHwuploadCudaRequired = true; - } + return GetHwaccelType(state, options, "mpeg2video", bitDepth, hwSurface); + } - filters.Add("hwdownload"); - filters.Add("format=nv12"); - } - } - else - { - // Download data from GPU to CPU as p010 format. - filters.Add("hwdownload"); - filters.Add("format=p010"); + if (string.Equals("vc1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vc1", bitDepth, hwSurface); + } - // Cuda lacks of a pixel format converter. - if (isNvencEncoder) - { - isHwuploadCudaRequired = true; - filters.Add("format=yuv420p"); - } - } + if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "mpeg4", bitDepth, hwSurface); + } + } + + if (is8_10bitSwFormatsAmf) + { + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface); + } + + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface); + } + + if (string.Equals("av1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "av1", bitDepth, hwSurface); + } + } + + return null; + } + + public string GetVaapiVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + if (!OperatingSystem.IsLinux() + || !string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + var hwSurface = IsVaapiSupported(state) + && IsVaapiFullSupported() + && IsOpenclFullSupported() + && _mediaEncoder.SupportsFilter("alphasrc"); + var is8bitSwFormatsVaapi = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsVaapi = is8bitSwFormatsVaapi || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + + if (is8bitSwFormatsVaapi) + { + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "h264", bitDepth, hwSurface); + } + + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "mpeg2video", bitDepth, hwSurface); } - // Assert 8-bit hardware decodable - else if (!isColorDepth10 - && (isLibX264Encoder - || isLibX265Encoder - || hasTextSubs - || (hasGraphicalSubs && !isCudaOverlaySupported && isNvencEncoder))) + if (string.Equals("vc1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - if (isNvencEncoder) - { - isHwuploadCudaRequired = true; - } - - filters.Add("hwdownload"); - filters.Add("format=nv12"); + return GetHwaccelType(state, options, "vc1", bitDepth, hwSurface); } - } - // Add parameters to use VAAPI with burn-in text subtitles (GH issue #642) - if (isVaapiH264Encoder - || isVaapiHevcEncoder - || (isTonemappingSupportedOnQsv && isVppTonemappingSupported)) - { - if (hasTextSubs) + if (string.Equals("vp8", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - // Convert hw context from ocl to va. - // For tonemapping and text subs burn-in. - if (isTonemappingSupportedOnVaapi && isOpenclTonemappingSupported && !isVppTonemappingSupported) - { - filters.Add("scale_vaapi"); - } - - // Test passed on Intel and AMD gfx - filters.Add("hwmap=mode=read+write"); - filters.Add("format=nv12"); + return GetHwaccelType(state, options, "vp8", bitDepth, hwSurface); } } - if (hasTextSubs) + if (is8_10bitSwFormatsVaapi) { - var subParam = GetTextSubtitleParam(state); - - filters.Add(subParam); - - // Ensure proper filters are passed to ffmpeg in case of hardware acceleration via VA-API - // Reference: https://trac.ffmpeg.org/wiki/Hardware/VAAPI - if (isVaapiH264Encoder || isVaapiHevcEncoder) + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - filters.Add("hwmap"); + return GetHwaccelType(state, options, "hevc", bitDepth, hwSurface); } - if (isTonemappingSupportedOnQsv && isVppTonemappingSupported) + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - filters.Add("hwmap,format=vaapi"); + return GetHwaccelType(state, options, "vp9", bitDepth, hwSurface); } - if (isNvdecDecoder && isNvencEncoder) + if (string.Equals("av1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - isHwuploadCudaRequired = true; + return GetHwaccelType(state, options, "av1", bitDepth, hwSurface); } } - // Interop the VAAPI data to QSV for hybrid tonemapping - if (isTonemappingSupportedOnQsv && isVppTonemappingSupported && !hasGraphicalSubs) - { - filters.Add("hwmap=derive_device=qsv,scale_qsv"); - } + return null; + } - if (isHwuploadCudaRequired && !hasGraphicalSubs) + public string GetVideotoolboxVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) + { + if (!OperatingSystem.IsMacOS() + || !string.Equals(options.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) { - filters.Add("hwupload_cuda"); + return null; } - // If no tonemap filter is applied, - // tag the video range as SDR to prevent the encoder from encoding HDR video. - if (isNoTonemapFilterApplied) + var is8bitSwFormatsVt = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + + if (is8bitSwFormatsVt) { - var outputSdrParams = GetOutputSdrParams(null); - if (!string.IsNullOrEmpty(outputSdrParams)) + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "h264", bitDepth, false); + } + + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "mpeg2video", bitDepth, false); + } + + if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - filters.Add(outputSdrParams); + return GetHwaccelType(state, options, "mpeg4", bitDepth, false); } } - var output = string.Empty; - if (filters.Count > 0) + if (is8_10bitSwFormatsVt) { - output += string.Format( - CultureInfo.InvariantCulture, - "{0}", - string.Join(',', filters)); + if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "hevc", bitDepth, false); + } + + if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwaccelType(state, options, "vp9", bitDepth, false); + } } - return output; + return null; } - public static string GetInputHdrParams(string colorTransfer) + public string GetOmxVidDecoder(EncodingJobInfo state, EncodingOptions options, MediaStream videoStream, int bitDepth) { - if (string.Equals(colorTransfer, "arib-std-b67", StringComparison.OrdinalIgnoreCase)) - { - // HLG - return "setparams=color_primaries=bt2020:color_trc=arib-std-b67:colorspace=bt2020nc"; - } - else + if (!OperatingSystem.IsLinux() + || !string.Equals(options.HardwareAccelerationType, "omx", StringComparison.OrdinalIgnoreCase)) { - // HDR10 - return "setparams=color_primaries=bt2020:color_trc=smpte2084:colorspace=bt2020nc"; + return null; } - } - public static string GetOutputSdrParams(string tonemappingRange) - { - // SDR - if (string.Equals(tonemappingRange, "tv", StringComparison.OrdinalIgnoreCase)) - { - return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=tv"; - } + var is8bitSwFormatsOmx = string.Equals("yuv420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); - if (string.Equals(tonemappingRange, "pc", StringComparison.OrdinalIgnoreCase)) + if (is8bitSwFormatsOmx) { - return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709:range=pc"; + if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) + || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwDecoderName(options, "h264", "mmal", "h264", bitDepth); + } + + if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwDecoderName(options, "mpeg2", "mmal", "mpeg2video", bitDepth); + } + + if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwDecoderName(options, "mpeg4", "mmal", "mpeg4", bitDepth); + } + + if (string.Equals("vc1", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) + { + return GetHwDecoderName(options, "vc1", "mmal", "vc1", bitDepth); + } } - return "setparams=color_primaries=bt709:color_trc=bt709:colorspace=bt709"; + return null; } /// @@ -3302,7 +4860,7 @@ namespace MediaBrowser.Controller.MediaEncoding inputModifier = inputModifier.Trim(); - inputModifier += " " + GetFastSeekCommandLineParameter(state.BaseRequest); + inputModifier += " " + GetFastSeekCommandLineParameter(state, encodingOptions); inputModifier = inputModifier.Trim(); if (state.InputProtocol == MediaProtocol.Rtsp) @@ -3356,61 +4914,8 @@ namespace MediaBrowser.Controller.MediaEncoding inputModifier += " -fflags " + string.Join(string.Empty, flags); } - var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions); - - if (!string.IsNullOrEmpty(videoDecoder)) - { - inputModifier += " " + videoDecoder; - - if (!IsCopyCodec(state.OutputVideoCodec) - && videoDecoder.Contains("cuvid", StringComparison.OrdinalIgnoreCase)) - { - var videoStream = state.VideoStream; - var inputWidth = videoStream?.Width; - var inputHeight = videoStream?.Height; - var request = state.BaseRequest; - - var (width, height) = GetFixedOutputSize(inputWidth, inputHeight, request.Width, request.Height, request.MaxWidth, request.MaxHeight); - - if (videoDecoder.Contains("cuvid", StringComparison.OrdinalIgnoreCase) - && width.HasValue - && height.HasValue) - { - if (width.HasValue && height.HasValue) - { - inputModifier += string.Format( - CultureInfo.InvariantCulture, - " -resize {0}x{1}", - width.Value, - height.Value); - } - - if (state.DeInterlace("h264", true)) - { - inputModifier += " -deint 1"; - - if (!encodingOptions.DeinterlaceDoubleRate || (videoStream?.AverageFrameRate ?? 60) > 30) - { - inputModifier += " -drop_second_field 1"; - } - } - } - } - } - if (state.IsVideoRequest) { - var outputVideoCodec = GetVideoEncoder(state, encodingOptions); - - // Important: If this is ever re-enabled, make sure not to use it with wtv because it breaks seeking - if (!string.Equals(state.InputContainer, "wtv", StringComparison.OrdinalIgnoreCase) - && state.TranscodingType != TranscodingJobType.Progressive - && !state.EnableBreakOnNonKeyFrames(outputVideoCodec) - && (state.BaseRequest.StartTimeTicks ?? 0) > 0) - { - inputModifier += " -noaccurate_seek"; - } - if (!string.IsNullOrEmpty(state.InputContainer) && state.VideoType == VideoType.VideoFile && string.IsNullOrEmpty(encodingOptions.HardwareAccelerationType)) { var inputFormat = GetInputFormat(state.InputContainer); @@ -3623,322 +5128,6 @@ namespace MediaBrowser.Controller.MediaEncoding } } - /// - /// Gets the ffmpeg option string for the hardware accelerated video decoder. - /// - /// The encoding job info. - /// The encoding options. - /// The option string or null if none available. - protected string GetHardwareAcceleratedVideoDecoder(EncodingJobInfo state, EncodingOptions encodingOptions) - { - var videoStream = state.VideoStream; - - if (videoStream == null) - { - return null; - } - - var videoType = state.MediaSource.VideoType ?? VideoType.VideoFile; - // Only use alternative encoders for video files. - // When using concat with folder rips, if the mfx session fails to initialize, ffmpeg will be stuck retrying and will not exit gracefully - // Since transcoding of folder rips is experimental anyway, it's not worth adding additional variables such as this. - if (videoType != VideoType.VideoFile) - { - return null; - } - - if (IsCopyCodec(state.OutputVideoCodec)) - { - return null; - } - - if (!string.IsNullOrEmpty(videoStream.Codec) && !string.IsNullOrEmpty(encodingOptions.HardwareAccelerationType)) - { - var isColorDepth10 = IsColorDepth10(state); - - // Only hevc and vp9 formats have 10-bit hardware decoder support now. - if (isColorDepth10 && !(string.Equals(videoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoStream.Codec, "h265", StringComparison.OrdinalIgnoreCase) - || string.Equals(videoStream.Codec, "vp9", StringComparison.OrdinalIgnoreCase))) - { - return null; - } - - // Hybrid VPP tonemapping with VAAPI - if (string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) - && IsVppTonemappingSupported(state, encodingOptions)) - { - var outputVideoCodec = GetVideoEncoder(state, encodingOptions) ?? string.Empty; - var isQsvEncoder = outputVideoCodec.Contains("qsv", StringComparison.OrdinalIgnoreCase); - if (isQsvEncoder) - { - // Since tonemap_vaapi only support HEVC for now, no need to check the codec again. - return GetHwaccelType(state, encodingOptions, "hevc", isColorDepth10); - } - } - - if (string.Equals(encodingOptions.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwDecoderName(encodingOptions, "h264_qsv", "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwDecoderName(encodingOptions, "hevc_qsv", "hevc", isColorDepth10); - case "mpeg2video": - return GetHwDecoderName(encodingOptions, "mpeg2_qsv", "mpeg2video", isColorDepth10); - case "vc1": - return GetHwDecoderName(encodingOptions, "vc1_qsv", "vc1", isColorDepth10); - case "vp8": - return GetHwDecoderName(encodingOptions, "vp8_qsv", "vp8", isColorDepth10); - case "vp9": - return GetHwDecoderName(encodingOptions, "vp9_qsv", "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "h264", isColorDepth10) - : GetHwDecoderName(encodingOptions, "h264_cuvid", "h264", isColorDepth10); - case "hevc": - case "h265": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "hevc", isColorDepth10) - : GetHwDecoderName(encodingOptions, "hevc_cuvid", "hevc", isColorDepth10); - case "mpeg2video": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "mpeg2video", isColorDepth10) - : GetHwDecoderName(encodingOptions, "mpeg2_cuvid", "mpeg2video", isColorDepth10); - case "vc1": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "vc1", isColorDepth10) - : GetHwDecoderName(encodingOptions, "vc1_cuvid", "vc1", isColorDepth10); - case "mpeg4": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "mpeg4", isColorDepth10) - : GetHwDecoderName(encodingOptions, "mpeg4_cuvid", "mpeg4", isColorDepth10); - case "vp8": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "vp8", isColorDepth10) - : GetHwDecoderName(encodingOptions, "vp8_cuvid", "vp8", isColorDepth10); - case "vp9": - return encodingOptions.EnableEnhancedNvdecDecoder && IsCudaSupported() - ? GetHwaccelType(state, encodingOptions, "vp9", isColorDepth10) - : GetHwDecoderName(encodingOptions, "vp9_cuvid", "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "mediacodec", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwDecoderName(encodingOptions, "h264_mediacodec", "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwDecoderName(encodingOptions, "hevc_mediacodec", "hevc", isColorDepth10); - case "mpeg2video": - return GetHwDecoderName(encodingOptions, "mpeg2_mediacodec", "mpeg2video", isColorDepth10); - case "mpeg4": - return GetHwDecoderName(encodingOptions, "mpeg4_mediacodec", "mpeg4", isColorDepth10); - case "vp8": - return GetHwDecoderName(encodingOptions, "vp8_mediacodec", "vp8", isColorDepth10); - case "vp9": - return GetHwDecoderName(encodingOptions, "vp9_mediacodec", "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "omx", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwDecoderName(encodingOptions, "h264_mmal", "h264", isColorDepth10); - case "mpeg2video": - return GetHwDecoderName(encodingOptions, "mpeg2_mmal", "mpeg2video", isColorDepth10); - case "mpeg4": - return GetHwDecoderName(encodingOptions, "mpeg4_mmal", "mpeg4", isColorDepth10); - case "vc1": - return GetHwDecoderName(encodingOptions, "vc1_mmal", "vc1", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwaccelType(state, encodingOptions, "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwaccelType(state, encodingOptions, "hevc", isColorDepth10); - case "mpeg2video": - return GetHwaccelType(state, encodingOptions, "mpeg2video", isColorDepth10); - case "vc1": - return GetHwaccelType(state, encodingOptions, "vc1", isColorDepth10); - case "mpeg4": - return GetHwaccelType(state, encodingOptions, "mpeg4", isColorDepth10); - case "vp9": - return GetHwaccelType(state, encodingOptions, "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwaccelType(state, encodingOptions, "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwaccelType(state, encodingOptions, "hevc", isColorDepth10); - case "mpeg2video": - return GetHwaccelType(state, encodingOptions, "mpeg2video", isColorDepth10); - case "vc1": - return GetHwaccelType(state, encodingOptions, "vc1", isColorDepth10); - case "vp8": - return GetHwaccelType(state, encodingOptions, "vp8", isColorDepth10); - case "vp9": - return GetHwaccelType(state, encodingOptions, "vp9", isColorDepth10); - } - } - else if (string.Equals(encodingOptions.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) - { - switch (videoStream.Codec.ToLowerInvariant()) - { - case "avc": - case "h264": - return GetHwDecoderName(encodingOptions, "h264_opencl", "h264", isColorDepth10); - case "hevc": - case "h265": - return GetHwDecoderName(encodingOptions, "hevc_opencl", "hevc", isColorDepth10); - case "mpeg2video": - return GetHwDecoderName(encodingOptions, "mpeg2_opencl", "mpeg2video", isColorDepth10); - case "mpeg4": - return GetHwDecoderName(encodingOptions, "mpeg4_opencl", "mpeg4", isColorDepth10); - case "vc1": - return GetHwDecoderName(encodingOptions, "vc1_opencl", "vc1", isColorDepth10); - case "vp8": - return GetHwDecoderName(encodingOptions, "vp8_opencl", "vp8", isColorDepth10); - case "vp9": - return GetHwDecoderName(encodingOptions, "vp9_opencl", "vp9", isColorDepth10); - } - } - } - - var whichCodec = videoStream.Codec?.ToLowerInvariant(); - switch (whichCodec) - { - case "avc": - whichCodec = "h264"; - break; - case "h265": - whichCodec = "hevc"; - break; - } - - // Avoid a second attempt if no hardware acceleration is being used - encodingOptions.HardwareDecodingCodecs = encodingOptions.HardwareDecodingCodecs.Where(val => val != whichCodec).ToArray(); - - // leave blank so ffmpeg will decide - return null; - } - - /// - /// Gets a hw decoder name. - /// - /// Encoding options. - /// Decoder to use. - /// Video codec to use. - /// Specifies if color depth 10. - /// Hardware decoder name. - public string GetHwDecoderName(EncodingOptions options, string decoder, string videoCodec, bool isColorDepth10) - { - var isCodecAvailable = _mediaEncoder.SupportsDecoder(decoder) && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase); - if (isColorDepth10 && isCodecAvailable) - { - if ((options.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Hevc) - || (options.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Vp9)) - { - return null; - } - } - - return isCodecAvailable ? ("-c:v " + decoder) : null; - } - - /// - /// Gets a hwaccel type to use as a hardware decoder(dxva/vaapi) depending on the system. - /// - /// Encoding state. - /// Encoding options. - /// Video codec to use. - /// Specifies if color depth 10. - /// Hardware accelerator type. - public string GetHwaccelType(EncodingJobInfo state, EncodingOptions options, string videoCodec, bool isColorDepth10) - { - var isWindows = OperatingSystem.IsWindows(); - var isLinux = OperatingSystem.IsLinux(); - var isWindows8orLater = Environment.OSVersion.Version.Major > 6 || (Environment.OSVersion.Version.Major == 6 && Environment.OSVersion.Version.Minor > 1); - var isDxvaSupported = _mediaEncoder.SupportsHwaccel("dxva2") || _mediaEncoder.SupportsHwaccel("d3d11va"); - var isCodecAvailable = options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase); - - if (isColorDepth10 && isCodecAvailable) - { - if ((options.HardwareDecodingCodecs.Contains("hevc", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Hevc) - || (options.HardwareDecodingCodecs.Contains("vp9", StringComparer.OrdinalIgnoreCase) && !options.EnableDecodingColorDepth10Vp9)) - { - return null; - } - } - - if (string.Equals(options.HardwareAccelerationType, "amf", StringComparison.OrdinalIgnoreCase)) - { - // Currently there is no AMF decoder on Linux, only have h264 encoder. - if (isDxvaSupported && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase)) - { - if (isWindows && isWindows8orLater) - { - return "-hwaccel d3d11va"; - } - - if (isWindows && !isWindows8orLater) - { - return "-hwaccel dxva2"; - } - } - } - - if (string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase) - || (string.Equals(options.HardwareAccelerationType, "qsv", StringComparison.OrdinalIgnoreCase) - && IsVppTonemappingSupported(state, options))) - { - if (IsVaapiSupported(state) && options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase)) - { - if (isLinux) - { - return "-hwaccel vaapi"; - } - } - } - - if (string.Equals(options.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) - { - if (options.HardwareDecodingCodecs.Contains(videoCodec, StringComparer.OrdinalIgnoreCase)) - { - return "-hwaccel cuda"; - } - } - - return null; - } - public string GetSubtitleEmbedArguments(EncodingJobInfo state) { if (state.SubtitleStream == null || state.SubtitleDeliveryMethod != SubtitleDeliveryMethod.Embed) @@ -4056,25 +5245,12 @@ namespace MediaBrowser.Controller.MediaEncoding var hasCopyTs = false; - // Add resolution params, if specified - if (!hasGraphicalSubs) - { - var outputSizeParam = GetOutputSizeParam(state, encodingOptions, videoCodec); - - args += outputSizeParam; - - hasCopyTs = outputSizeParam.IndexOf("copyts", StringComparison.OrdinalIgnoreCase) != -1; - } - - // This is for graphical subs - if (hasGraphicalSubs) - { - var graphicalSubtitleParam = GetGraphicalSubtitleParam(state, encodingOptions, videoCodec); + // video processing filters. + var videoProcessParam = GetVideoProcessingFilterParam(state, encodingOptions, videoCodec); - args += graphicalSubtitleParam; + args += videoProcessParam; - hasCopyTs = graphicalSubtitleParam.IndexOf("copyts", StringComparison.OrdinalIgnoreCase) != -1; - } + hasCopyTs = videoProcessParam.Contains("copyts", StringComparison.OrdinalIgnoreCase); if (state.RunTimeTicks.HasValue && state.BaseRequest.CopyTimestamps) { @@ -4199,41 +5375,5 @@ namespace MediaBrowser.Controller.MediaEncoding { return string.Equals(codec, "copy", StringComparison.OrdinalIgnoreCase); } - - public static bool IsColorDepth10(EncodingJobInfo state) - { - var result = false; - var videoStream = state.VideoStream; - - if (videoStream != null) - { - if (videoStream.BitDepth.HasValue) - { - return videoStream.BitDepth.Value == 10; - } - - if (!string.IsNullOrEmpty(videoStream.PixelFormat)) - { - result = videoStream.PixelFormat.Contains("p10", StringComparison.OrdinalIgnoreCase); - if (result) - { - return true; - } - } - - if (!string.IsNullOrEmpty(videoStream.Profile)) - { - result = videoStream.Profile.Contains("Main 10", StringComparison.OrdinalIgnoreCase) - || videoStream.Profile.Contains("High 10", StringComparison.OrdinalIgnoreCase) - || videoStream.Profile.Contains("Profile 2", StringComparison.OrdinalIgnoreCase); - if (result) - { - return true; - } - } - } - - return result; - } } } diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingJobInfo.cs b/MediaBrowser.Controller/MediaEncoding/EncodingJobInfo.cs index e92c4a08a6..c4affa5678 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingJobInfo.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingJobInfo.cs @@ -110,23 +110,7 @@ namespace MediaBrowser.Controller.MediaEncoding public string OutputContainer { get; set; } - public string OutputVideoSync - { - get - { - // For live tv + in progress recordings - if (string.Equals(InputContainer, "mpegts", StringComparison.OrdinalIgnoreCase) - || string.Equals(InputContainer, "ts", StringComparison.OrdinalIgnoreCase)) - { - if (!MediaSource.RunTimeTicks.HasValue) - { - return "cfr"; - } - } - - return "-1"; - } - } + public string OutputVideoSync { get; set; } public string AlbumCoverPath { get; set; } diff --git a/MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs b/MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs index 7ce707b19e..a4869cb670 100644 --- a/MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs +++ b/MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs @@ -18,6 +18,16 @@ namespace MediaBrowser.Controller.MediaEncoding /// /// The tonemap_opencl_bt2390. /// - TonemapOpenclBt2390 = 2 + TonemapOpenclBt2390 = 2, + + /// + /// The overlay_opencl_framesync. + /// + OverlayOpenclFrameSync = 3, + + /// + /// The overlay_vaapi_framesync. + /// + OverlayVaapiFrameSync = 4 } } diff --git a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs index 1418e583e7..27d618a3f7 100644 --- a/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs +++ b/MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs @@ -25,6 +25,30 @@ namespace MediaBrowser.Controller.MediaEncoding /// The encoder path. string EncoderPath { get; } + /// + /// Gets the version of encoder. + /// + /// The version of encoder. + Version EncoderVersion { get; } + + /// + /// Whether the configured Vaapi device is from AMD(radeonsi/r600 Mesa driver). + /// + /// true if the Vaapi device is an AMD(radeonsi/r600 Mesa driver) GPU, false otherwise. + bool IsVaapiDeviceAmd { get; } + + /// + /// Whether the configured Vaapi device is from Intel(iHD driver). + /// + /// true if the Vaapi device is an Intel(iHD driver) GPU, false otherwise. + bool IsVaapiDeviceInteliHD { get; } + + /// + /// Whether the configured Vaapi device is from Intel(legacy i965 driver). + /// + /// true if the Vaapi device is an Intel(legacy i965 driver) GPU, false otherwise. + bool IsVaapiDeviceInteli965 { get; } + /// /// Whether given encoder codec is supported. /// @@ -60,12 +84,6 @@ namespace MediaBrowser.Controller.MediaEncoding /// true if the filter is supported, false otherwise. bool SupportsFilterWithOption(FilterOptionType option); - /// - /// Get the version of media encoder. - /// - /// The version of media encoder. - Version GetMediaEncoderVersion(); - /// /// Extracts the audio image. /// diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs index 60a2d39e5a..fe30699340 100644 --- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs +++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs @@ -16,6 +16,12 @@ namespace MediaBrowser.MediaEncoding.Encoder { "h264", "hevc", + "vp8", + "libvpx", + "vp9", + "libvpx-vp9", + "av1", + "libdav1d", "mpeg2video", "mpeg4", "msmpeg4", @@ -30,6 +36,7 @@ namespace MediaBrowser.MediaEncoding.Encoder "vc1_qsv", "vp8_qsv", "vp9_qsv", + "av1_qsv", "h264_cuvid", "hevc_cuvid", "mpeg2_cuvid", @@ -37,16 +44,11 @@ namespace MediaBrowser.MediaEncoding.Encoder "mpeg4_cuvid", "vp8_cuvid", "vp9_cuvid", + "av1_cuvid", "h264_mmal", "mpeg2_mmal", "mpeg4_mmal", "vc1_mmal", - "h264_mediacodec", - "hevc_mediacodec", - "mpeg2_mediacodec", - "mpeg4_mediacodec", - "vp8_mediacodec", - "vp9_mediacodec", "h264_opencl", "hevc_opencl", "mpeg2_opencl", @@ -89,20 +91,39 @@ namespace MediaBrowser.MediaEncoding.Encoder private static readonly string[] _requiredFilters = new[] { + // sw + "alphasrc", + "zscale", + // qsv + "scale_qsv", + "vpp_qsv", + "deinterlace_qsv", + "overlay_qsv", + // cuda "scale_cuda", "yadif_cuda", - "hwupload_cuda", - "overlay_cuda", "tonemap_cuda", + "overlay_cuda", + "hwupload_cuda", + // opencl + "scale_opencl", "tonemap_opencl", + "overlay_opencl", + // vaapi + "scale_vaapi", + "deinterlace_vaapi", "tonemap_vaapi", + "overlay_vaapi", + "hwupload_vaapi" }; private static readonly IReadOnlyDictionary _filterOptionsDict = new Dictionary { { 0, new string[] { "scale_cuda", "Output format (default \"same\")" } }, { 1, new string[] { "tonemap_cuda", "GPU accelerated HDR to SDR tonemapping" } }, - { 2, new string[] { "tonemap_opencl", "bt2390" } } + { 2, new string[] { "tonemap_opencl", "bt2390" } }, + { 3, new string[] { "overlay_opencl", "Action to take when encountering EOF from secondary input" } }, + { 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } } }; // These are the library versions that corresponds to our minimum ffmpeg version 4.x according to the version table below @@ -144,7 +165,7 @@ namespace MediaBrowser.MediaEncoding.Encoder string output; try { - output = GetProcessOutput(_encoderPath, "-version"); + output = GetProcessOutput(_encoderPath, "-version", false); } catch (Exception ex) { @@ -225,7 +246,7 @@ namespace MediaBrowser.MediaEncoding.Encoder string output; try { - output = GetProcessOutput(_encoderPath, "-version"); + output = GetProcessOutput(_encoderPath, "-version", false); } catch (Exception ex) { @@ -318,12 +339,36 @@ namespace MediaBrowser.MediaEncoding.Encoder return map; } + public bool CheckVaapiDeviceByDriverName(string driverName, string renderNodePath) + { + if (!OperatingSystem.IsLinux()) + { + return false; + } + + if (string.IsNullOrEmpty(driverName) || string.IsNullOrEmpty(renderNodePath)) + { + return false; + } + + try + { + var output = GetProcessOutput(_encoderPath, "-v verbose -hide_banner -init_hw_device vaapi=va:" + renderNodePath, true); + return output.Contains(driverName, StringComparison.Ordinal); + } + catch (Exception ex) + { + _logger.LogError(ex, "Error detecting the given vaapi render node path"); + return false; + } + } + private IEnumerable GetHwaccelTypes() { string? output = null; try { - output = GetProcessOutput(_encoderPath, "-hwaccels"); + output = GetProcessOutput(_encoderPath, "-hwaccels", false); } catch (Exception ex) { @@ -351,7 +396,7 @@ namespace MediaBrowser.MediaEncoding.Encoder string output; try { - output = GetProcessOutput(_encoderPath, "-h filter=" + filter); + output = GetProcessOutput(_encoderPath, "-h filter=" + filter, false); } catch (Exception ex) { @@ -375,7 +420,7 @@ namespace MediaBrowser.MediaEncoding.Encoder string output; try { - output = GetProcessOutput(_encoderPath, "-" + codecstr); + output = GetProcessOutput(_encoderPath, "-" + codecstr, false); } catch (Exception ex) { @@ -406,7 +451,7 @@ namespace MediaBrowser.MediaEncoding.Encoder string output; try { - output = GetProcessOutput(_encoderPath, "-filters"); + output = GetProcessOutput(_encoderPath, "-filters", false); } catch (Exception ex) { @@ -444,7 +489,7 @@ namespace MediaBrowser.MediaEncoding.Encoder return dict; } - private string GetProcessOutput(string path, string arguments) + private string GetProcessOutput(string path, string arguments, bool readStdErr) { using (var process = new Process() { @@ -455,7 +500,6 @@ namespace MediaBrowser.MediaEncoding.Encoder WindowStyle = ProcessWindowStyle.Hidden, ErrorDialog = false, RedirectStandardOutput = true, - // ffmpeg uses stderr to log info, don't show this RedirectStandardError = true } }) @@ -464,7 +508,7 @@ namespace MediaBrowser.MediaEncoding.Encoder process.Start(); - return process.StandardOutput.ReadToEnd(); + return readStdErr ? process.StandardError.ReadToEnd() : process.StandardOutput.ReadToEnd(); } } } diff --git a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs index 1c97a19828..fce71bf1a0 100644 --- a/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs +++ b/MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs @@ -65,6 +65,10 @@ namespace MediaBrowser.MediaEncoding.Encoder private List _filters = new List(); private IDictionary _filtersWithOption = new Dictionary(); + private bool _isVaapiDeviceAmd = false; + private bool _isVaapiDeviceInteliHD = false; + private bool _isVaapiDeviceInteli965 = false; + private Version _ffmpegVersion = null; private string _ffmpegPath = string.Empty; private string _ffprobePath; @@ -87,6 +91,10 @@ namespace MediaBrowser.MediaEncoding.Encoder /// public string EncoderPath => _ffmpegPath; + public Version EncoderVersion => _ffmpegVersion; + public bool IsVaapiDeviceAmd => _isVaapiDeviceAmd; + public bool IsVaapiDeviceInteliHD => _isVaapiDeviceInteliHD; + public bool IsVaapiDeviceInteli965 => _isVaapiDeviceInteli965; /// /// Run at startup or if the user removes a Custom path from transcode page. @@ -114,9 +122,9 @@ namespace MediaBrowser.MediaEncoding.Encoder } // Write the FFmpeg path to the config/encoding.xml file as so it appears in UI - var config = _configurationManager.GetEncodingOptions(); - config.EncoderAppPathDisplay = _ffmpegPath ?? string.Empty; - _configurationManager.SaveConfiguration("encoding", config); + var options = _configurationManager.GetEncodingOptions(); + options.EncoderAppPathDisplay = _ffmpegPath ?? string.Empty; + _configurationManager.SaveConfiguration("encoding", options); // Only if mpeg path is set, try and set path to probe if (_ffmpegPath != null) @@ -134,7 +142,30 @@ namespace MediaBrowser.MediaEncoding.Encoder SetAvailableHwaccels(validator.GetHwaccels()); SetMediaEncoderVersion(validator); - _threads = EncodingHelper.GetNumberOfThreads(null, _configurationManager.GetEncodingOptions(), null); + _threads = EncodingHelper.GetNumberOfThreads(null, options, null); + + // Check the Vaapi device vendor + if (OperatingSystem.IsLinux() + && SupportsHwaccel("vaapi") + && !string.IsNullOrEmpty(options.VaapiDevice) + && string.Equals(options.HardwareAccelerationType, "vaapi", StringComparison.OrdinalIgnoreCase)) + { + _isVaapiDeviceAmd = validator.CheckVaapiDeviceByDriverName("Mesa Gallium driver", options.VaapiDevice); + _isVaapiDeviceInteliHD = validator.CheckVaapiDeviceByDriverName("Intel iHD driver", options.VaapiDevice); + _isVaapiDeviceInteli965 = validator.CheckVaapiDeviceByDriverName("Intel i965 driver", options.VaapiDevice); + if (_isVaapiDeviceAmd) + { + _logger.LogInformation("VAAPI device {RenderNodePath} is AMD GPU", options.VaapiDevice); + } + else if (_isVaapiDeviceInteliHD) + { + _logger.LogInformation("VAAPI device {RenderNodePath} is Intel GPU (iHD)", options.VaapiDevice); + } + else if (_isVaapiDeviceInteli965) + { + _logger.LogInformation("VAAPI device {RenderNodePath} is Intel GPU (i965)", options.VaapiDevice); + } + } } _logger.LogInformation("FFmpeg: {FfmpegPath}", _ffmpegPath ?? string.Empty); @@ -301,11 +332,6 @@ namespace MediaBrowser.MediaEncoding.Encoder return false; } - public Version GetMediaEncoderVersion() - { - return _ffmpegVersion; - } - public bool CanEncodeToAudioCodec(string codec) { if (string.Equals(codec, "opus", StringComparison.OrdinalIgnoreCase)) @@ -508,36 +534,9 @@ namespace MediaBrowser.MediaEncoding.Encoder if (!isAudio) { - // The failure of HDR extraction usually occurs when using custom ffmpeg that does not contain the zscale filter. try { - return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, true, true, targetFormat, cancellationToken).ConfigureAwait(false); - } - catch (ArgumentException) - { - throw; - } - catch (Exception ex) - { - _logger.LogError(ex, "I-frame or HDR image extraction failed, will attempt with I-frame extraction disabled. Input: {Arguments}", inputArgument); - } - - try - { - return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, false, true, targetFormat, cancellationToken).ConfigureAwait(false); - } - catch (ArgumentException) - { - throw; - } - catch (Exception ex) - { - _logger.LogError(ex, "HDR image extraction failed, will fallback to SDR image extraction. Input: {Arguments}", inputArgument); - } - - try - { - return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, true, false, targetFormat, cancellationToken).ConfigureAwait(false); + return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, true, targetFormat, cancellationToken).ConfigureAwait(false); } catch (ArgumentException) { @@ -549,10 +548,10 @@ namespace MediaBrowser.MediaEncoding.Encoder } } - return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, false, false, targetFormat, cancellationToken).ConfigureAwait(false); + return await ExtractImageInternal(inputArgument, container, videoStream, imageStreamIndex, threedFormat, offset, false, targetFormat, cancellationToken).ConfigureAwait(false); } - private async Task ExtractImageInternal(string inputPath, string container, MediaStream videoStream, int? imageStreamIndex, Video3DFormat? threedFormat, TimeSpan? offset, bool useIFrame, bool allowTonemap, ImageFormat? targetFormat, CancellationToken cancellationToken) + private async Task ExtractImageInternal(string inputPath, string container, MediaStream videoStream, int? imageStreamIndex, Video3DFormat? threedFormat, TimeSpan? offset, bool useIFrame, ImageFormat? targetFormat, CancellationToken cancellationToken) { if (string.IsNullOrEmpty(inputPath)) { @@ -572,36 +571,32 @@ namespace MediaBrowser.MediaEncoding.Encoder var tempExtractPath = Path.Combine(_configurationManager.ApplicationPaths.TempDirectory, Guid.NewGuid() + outputExtension); Directory.CreateDirectory(Path.GetDirectoryName(tempExtractPath)); + // deint -> scale -> thumbnail -> tonemap. + // put the SW tonemap right after the thumbnail to do it only once to reduce cpu usage. + var filters = new List(); + + // deinterlace using bwdif algorithm for video stream. + if (videoStream != null && videoStream.IsInterlaced) + { + filters.Add("bwdif=0:-1:0"); + } + // apply some filters to thumbnail extracted below (below) crop any black lines that we made and get the correct ar. // This filter chain may have adverse effects on recorded tv thumbnails if ar changes during presentation ex. commercials @ diff ar - var vf = threedFormat switch + var scaler = threedFormat switch { // hsbs crop width in half,scale to correct size, set the display aspect,crop out any black bars we may have made. Work out the correct height based on the display aspect it will maintain the aspect where -1 in this case (3d) may not. - Video3DFormat.HalfSideBySide => "-vf crop=iw/2:ih:0:0,scale=(iw*2):ih,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1", + Video3DFormat.HalfSideBySide => "crop=iw/2:ih:0:0,scale=(iw*2):ih,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1", // fsbs crop width in half,set the display aspect,crop out any black bars we may have made - Video3DFormat.FullSideBySide => "-vf crop=iw/2:ih:0:0,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1", + Video3DFormat.FullSideBySide => "crop=iw/2:ih:0:0,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1", // htab crop heigh in half,scale to correct size, set the display aspect,crop out any black bars we may have made - Video3DFormat.HalfTopAndBottom => "-vf crop=iw:ih/2:0:0,scale=(iw*2):ih),setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1", + Video3DFormat.HalfTopAndBottom => "crop=iw:ih/2:0:0,scale=(iw*2):ih),setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1", // ftab crop heigt in half, set the display aspect,crop out any black bars we may have made - Video3DFormat.FullTopAndBottom => "-vf crop=iw:ih/2:0:0,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1", - _ => string.Empty + Video3DFormat.FullTopAndBottom => "crop=iw:ih/2:0:0,setdar=dar=a,crop=min(iw\\,ih*dar):min(ih\\,iw/dar):(iw-min(iw\\,iw*sar))/2:(ih - min (ih\\,ih/sar))/2,setsar=sar=1", + _ => "scale=trunc(iw*sar):ih" }; - var mapArg = imageStreamIndex.HasValue ? (" -map 0:" + imageStreamIndex.Value.ToString(CultureInfo.InvariantCulture)) : string.Empty; - - var enableHdrExtraction = allowTonemap && string.Equals(videoStream?.VideoRange, "HDR", StringComparison.OrdinalIgnoreCase); - if (enableHdrExtraction) - { - string tonemapFilters = "zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0:peak=100,zscale=t=bt709:m=bt709,format=yuv420p"; - if (vf.Length == 0) - { - vf = "-vf " + tonemapFilters; - } - else - { - vf += "," + tonemapFilters; - } - } + filters.Add(scaler); // Use ffmpeg to sample 100 (we can drop this if required using thumbnail=50 for 50 frames) frames and pick the best thumbnail. Have a fall back just in case. // mpegts need larger batch size otherwise the corrupted thumbnail will be created. Larger batch size will lower the processing speed. @@ -609,18 +604,19 @@ namespace MediaBrowser.MediaEncoding.Encoder if (enableThumbnail) { var useLargerBatchSize = string.Equals("mpegts", container, StringComparison.OrdinalIgnoreCase); - var batchSize = useLargerBatchSize ? "50" : "24"; - if (string.IsNullOrEmpty(vf)) - { - vf = "-vf thumbnail=" + batchSize; - } - else - { - vf += ",thumbnail=" + batchSize; - } + filters.Add("thumbnail=n=" + (useLargerBatchSize ? "50" : "24")); } - var args = string.Format(CultureInfo.InvariantCulture, "-i {0}{3} -threads {4} -v quiet -vframes 1 {2} -f image2 \"{1}\"", inputPath, tempExtractPath, vf, mapArg, _threads); + // Use SW tonemap on HDR video stream only when the zscale filter is available. + var enableHdrExtraction = string.Equals(videoStream?.VideoRange, "HDR", StringComparison.OrdinalIgnoreCase) && SupportsFilter("zscale"); + if (enableHdrExtraction) + { + filters.Add("zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0:peak=100,zscale=t=bt709:m=bt709,format=yuv420p"); + } + + var vf = string.Join(',', filters); + var mapArg = imageStreamIndex.HasValue ? (" -map 0:" + imageStreamIndex.Value.ToString(CultureInfo.InvariantCulture)) : string.Empty; + var args = string.Format(CultureInfo.InvariantCulture, "-i {0}{3} -threads {4} -v quiet -vframes 1 -vf {2} -f image2 \"{1}\"", inputPath, tempExtractPath, vf, mapArg, _threads); if (offset.HasValue) { diff --git a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs index 4e4957ef71..750fd44ebb 100644 --- a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs +++ b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs @@ -777,18 +777,23 @@ namespace MediaBrowser.MediaEncoding.Probing if (!stream.BitDepth.HasValue) { - if (!string.IsNullOrEmpty(streamInfo.PixelFormat) - && streamInfo.PixelFormat.Contains("p10", StringComparison.OrdinalIgnoreCase)) + if (!string.IsNullOrEmpty(streamInfo.PixelFormat)) { - stream.BitDepth = 10; - } - - if (!string.IsNullOrEmpty(streamInfo.Profile) - && (streamInfo.Profile.Contains("Main 10", StringComparison.OrdinalIgnoreCase) - || streamInfo.Profile.Contains("High 10", StringComparison.OrdinalIgnoreCase) - || streamInfo.Profile.Contains("Profile 2", StringComparison.OrdinalIgnoreCase))) - { - stream.BitDepth = 10; + if (string.Equals(streamInfo.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase) + || string.Equals(streamInfo.PixelFormat, "yuv444p", StringComparison.OrdinalIgnoreCase)) + { + stream.BitDepth = 8; + } + else if (string.Equals(streamInfo.PixelFormat, "yuv420p10le", StringComparison.OrdinalIgnoreCase) + || string.Equals(streamInfo.PixelFormat, "yuv444p10le", StringComparison.OrdinalIgnoreCase)) + { + stream.BitDepth = 10; + } + else if (string.Equals(streamInfo.PixelFormat, "yuv420p12le", StringComparison.OrdinalIgnoreCase) + || string.Equals(streamInfo.PixelFormat, "yuv444p12le", StringComparison.OrdinalIgnoreCase)) + { + stream.BitDepth = 12; + } } } diff --git a/MediaBrowser.Model/Configuration/EncodingOptions.cs b/MediaBrowser.Model/Configuration/EncodingOptions.cs index 365bbeef66..d0ded99ea4 100644 --- a/MediaBrowser.Model/Configuration/EncodingOptions.cs +++ b/MediaBrowser.Model/Configuration/EncodingOptions.cs @@ -16,12 +16,9 @@ namespace MediaBrowser.Model.Configuration // This is a DRM device that is almost guaranteed to be there on every intel platform, // plus it's the default one in ffmpeg if you don't specify anything VaapiDevice = "/dev/dri/renderD128"; - // This is the OpenCL device that is used for tonemapping. - // The left side of the dot is the platform number, and the right side is the device number on the platform. - OpenclDevice = "0.0"; EnableTonemapping = false; EnableVppTonemapping = false; - TonemappingAlgorithm = "hable"; + TonemappingAlgorithm = "bt2390"; TonemappingRange = "auto"; TonemappingDesat = 0; TonemappingThreshold = 0.8; @@ -34,6 +31,9 @@ namespace MediaBrowser.Model.Configuration EnableDecodingColorDepth10Hevc = true; EnableDecodingColorDepth10Vp9 = true; EnableEnhancedNvdecDecoder = true; + PreferSystemNativeHwDecoder = true; + EnableIntelLowPowerH264HwEncoder = false; + EnableIntelLowPowerHevcHwEncoder = false; EnableHardwareEncoding = true; AllowHevcEncoding = false; EnableSubtitleExtraction = true; @@ -70,8 +70,6 @@ namespace MediaBrowser.Model.Configuration public string VaapiDevice { get; set; } - public string OpenclDevice { get; set; } - public bool EnableTonemapping { get; set; } public bool EnableVppTonemapping { get; set; } @@ -104,6 +102,12 @@ namespace MediaBrowser.Model.Configuration public bool EnableEnhancedNvdecDecoder { get; set; } + public bool PreferSystemNativeHwDecoder { get; set; } + + public bool EnableIntelLowPowerH264HwEncoder { get; set; } + + public bool EnableIntelLowPowerHevcHwEncoder { get; set; } + public bool EnableHardwareEncoding { get; set; } public bool AllowHevcEncoding { get; set; }