diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index b6738e7cc1..946f7266c3 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -253,6 +253,14 @@ namespace MediaBrowser.Controller.MediaEncoding && _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync); } + private bool IsVideoToolboxFullSupported() + { + return _mediaEncoder.SupportsHwaccel("videotoolbox") + && _mediaEncoder.SupportsFilter("yadif_videotoolbox") + && _mediaEncoder.SupportsFilter("overlay_videotoolbox") + && _mediaEncoder.SupportsFilter("scale_vt"); + } + private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options) { if (state.VideoStream is null @@ -272,7 +280,8 @@ namespace MediaBrowser.Controller.MediaEncoding var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase); var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase); var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase); - return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder; + var isVideoToolBoxDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder || isVideoToolBoxDecoder; } return state.VideoStream.VideoRange == VideoRange.HDR @@ -308,6 +317,21 @@ namespace MediaBrowser.Controller.MediaEncoding && state.VideoStream.VideoRangeType == VideoRangeType.HDR10; } + private bool IsVideoToolboxTonemapAvailable(EncodingJobInfo state, EncodingOptions options) + { + if (state.VideoStream is null + || !options.EnableVideoToolboxTonemapping + || GetVideoColorBitDepth(state) != 10) + { + return false; + } + + // Certain DV profile 5 video works in Safari with direct playing, but the VideoToolBox does not produce correct mapping results with transcoding. + // All other HDR formats working. + return state.VideoStream.VideoRange == VideoRange.HDR + && state.VideoStream.VideoRangeType is VideoRangeType.HDR10 or VideoRangeType.HLG or VideoRangeType.HDR10Plus; + } + /// /// Gets the name of the output video codec. /// @@ -4954,22 +4978,30 @@ namespace MediaBrowser.Controller.MediaEncoding return (null, null, null); } - var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder); + var isMacOS = OperatingSystem.IsMacOS(); + var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty; + var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + var isVtFullSupported = isMacOS && IsVideoToolboxFullSupported(); + var isVtOclSupported = isVtFullSupported && IsOpenclFullSupported(); - if (!options.EnableHardwareEncoding) + // legacy videotoolbox pipeline (disable hw filters) + if (!isVtEncoder + || !isVtOclSupported + || !_mediaEncoder.SupportsFilter("alphasrc")) { - return swFilterChain; + return GetSwVidFilterChain(state, options, vidEncoder); } - if (_mediaEncoder.EncoderVersion.CompareTo(new Version("5.0.0")) < 0) - { - // All features used here requires ffmpeg 5.0 or later, fallback to software filters if using an old ffmpeg - return swFilterChain; - } + // preferred videotoolbox + vt/ocl filters pipeline + return GetAppleVidFiltersPreferred(state, options, vidDecoder, vidEncoder); + } - var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); - var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); - var doDeintH2645 = doDeintH264 || doDeintHevc; + public (List MainFilters, List SubFilters, List OverlayFilters) GetAppleVidFiltersPreferred( + EncodingJobInfo state, + EncodingOptions options, + string vidDecoder, + string vidEncoder) + { var inW = state.VideoStream?.Width; var inH = state.VideoStream?.Height; var reqW = state.BaseRequest.Width; @@ -4977,33 +5009,114 @@ namespace MediaBrowser.Controller.MediaEncoding var reqMaxW = state.BaseRequest.MaxWidth; var reqMaxH = state.BaseRequest.MaxHeight; var threeDFormat = state.MediaSource.Video3DFormat; - var newfilters = new List(); - var noOverlay = swFilterChain.OverlayFilters.Count == 0; - var supportsHwDeint = _mediaEncoder.SupportsFilter("yadif_videotoolbox"); - // fallback to software filters if we are using filters not supported by hardware yet. - var useHardwareFilters = noOverlay && (!doDeintH2645 || supportsHwDeint); - if (!useHardwareFilters) + var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase); + + var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true); + var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true); + var doDeintH2645 = doDeintH264 || doDeintHevc; + var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options); + var doOclTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options); + + var scaleFormat = string.Empty; + if (!string.Equals(state.VideoStream.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase)) { - return swFilterChain; + // Use P010 for OpenCL tone mapping, otherwise force an 8bit output. + scaleFormat = doOclTonemap ? "p010le" : "nv12"; } - // ffmpeg cannot use videotoolbox to scale - var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH); - newfilters.Add(swScaleFilter); + var hwScaleFilter = GetHwScaleFilter("vt", scaleFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH); - // hwupload on videotoolbox encoders can automatically convert AVFrame into its CVPixelBuffer equivalent - // videotoolbox will automatically convert the CVPixelBuffer to a pixel format the encoder supports, so we don't have to set a pixel format explicitly here - // This will reduce CPU usage significantly on UHD videos with 10 bit colors because we bypassed the ffmpeg pixel format conversion - newfilters.Add("hwupload"); + var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream; + var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream; + var hasAssSubs = hasSubs + && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase) + || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase)); + + if (!isVtEncoder) + { + // should not happen. + return (null, null, null); + } + + /* Make main filters for video stream */ + var mainFilters = new List(); + // Color override is only required for OpenCL where hardware surface is in use + if (doOclTonemap) + { + mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap)); + } + + // INPUT videotoolbox/memory surface(vram/uma) + // this will pass-through automatically if in/out format matches. + mainFilters.Add("format=nv12|p010le|videotoolbox_vld"); + mainFilters.Add("hwupload=derive_device=videotoolbox"); + + // hw deint if (doDeintH2645) { var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox"); - newfilters.Add(deintFilter); + mainFilters.Add(deintFilter); } - return (newfilters, swFilterChain.SubFilters, swFilterChain.OverlayFilters); + if (doVtTonemap) + { + const string VtTonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709"; + + // scale_vt can handle scaling & tonemapping in one shot, just like vpp_qsv. + hwScaleFilter = string.IsNullOrEmpty(hwScaleFilter) + ? "scale_vt=" + VtTonemapArgs + : hwScaleFilter + ":" + VtTonemapArgs; + } + + // hw scale & vt tonemap + mainFilters.Add(hwScaleFilter); + + // ocl tonemap + if (doOclTonemap) + { + // map from videotoolbox to opencl via videotoolbox-opencl interop. + mainFilters.Add("hwmap=derive_device=opencl:mode=read"); + + var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12"); + mainFilters.Add(tonemapFilter); + + // OUTPUT videotoolbox(nv12) surface(vram/uma) + // reverse-mapping via videotoolbox-opencl interop. + mainFilters.Add("hwmap=derive_device=videotoolbox:mode=write:reverse=1"); + } + + /* Make sub and overlay filters for subtitle stream */ + var subFilters = new List(); + var overlayFilters = new List(); + + if (hasSubs) + { + if (hasGraphicalSubs) + { + var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH); + subFilters.Add(subPreProcFilters); + subFilters.Add("format=bgra"); + } + else if (hasTextSubs) + { + var framerate = state.VideoStream?.RealFrameRate; + var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10; + + var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate); + var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true); + subFilters.Add(alphaSrcFilter); + subFilters.Add("format=bgra"); + subFilters.Add(subTextSubtitlesFilter); + } + + subFilters.Add("hwupload=derive_device=videotoolbox"); + overlayFilters.Add("overlay_videotoolbox=eof_action=pass:repeatlast=0"); + } + + return (mainFilters, subFilters, overlayFilters); } /// @@ -5995,22 +6108,37 @@ namespace MediaBrowser.Controller.MediaEncoding || string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase); + // Hardware surface only make sense when interop with OpenCL + // VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases. + // For example: https://trac.ffmpeg.org/ticket/10884 + var useOclToneMapping = !IsVideoToolboxTonemapAvailable(state, options) + && options.EnableTonemapping + && state.VideoStream is not null + && GetVideoColorBitDepth(state) == 10 + && state.VideoStream.VideoRange == VideoRange.HDR + && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10 + || state.VideoStream.VideoRangeType == VideoRangeType.HLG + || (state.VideoStream.VideoRangeType == VideoRangeType.DOVI + && string.Equals(state.VideoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase))); + + var useHwSurface = useOclToneMapping && IsVideoToolboxFullSupported() && _mediaEncoder.SupportsFilter("alphasrc"); + if (is8bitSwFormatsVt) { if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) || string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "h264", bitDepth, false); + return GetHwaccelType(state, options, "h264", bitDepth, useHwSurface); } if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "mpeg2video", bitDepth, false); + return GetHwaccelType(state, options, "mpeg2video", bitDepth, useHwSurface); } if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "mpeg4", bitDepth, false); + return GetHwaccelType(state, options, "mpeg4", bitDepth, useHwSurface); } } @@ -6019,12 +6147,12 @@ namespace MediaBrowser.Controller.MediaEncoding if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase) || string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "hevc", bitDepth, false); + return GetHwaccelType(state, options, "hevc", bitDepth, useHwSurface); } if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase)) { - return GetHwaccelType(state, options, "vp9", bitDepth, false); + return GetHwaccelType(state, options, "vp9", bitDepth, useHwSurface); } } diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs index fdca283908..6549125d36 100644 --- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs +++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs @@ -128,6 +128,8 @@ namespace MediaBrowser.MediaEncoding.Encoder "overlay_vulkan", // videotoolbox "yadif_videotoolbox", + "scale_vt", + "overlay_videotoolbox", // rkrga "scale_rkrga", "vpp_rkrga", diff --git a/MediaBrowser.Model/Configuration/EncodingOptions.cs b/MediaBrowser.Model/Configuration/EncodingOptions.cs index 84c735f9ca..ab6f0d867c 100644 --- a/MediaBrowser.Model/Configuration/EncodingOptions.cs +++ b/MediaBrowser.Model/Configuration/EncodingOptions.cs @@ -28,6 +28,7 @@ public class EncodingOptions VaapiDevice = "/dev/dri/renderD128"; EnableTonemapping = false; EnableVppTonemapping = false; + EnableVideoToolboxTonemapping = false; TonemappingAlgorithm = "bt2390"; TonemappingMode = "auto"; TonemappingRange = "auto"; @@ -146,6 +147,11 @@ public class EncodingOptions /// public bool EnableVppTonemapping { get; set; } + /// + /// Gets or sets a value indicating whether videotoolbox tonemapping is enabled. + /// + public bool EnableVideoToolboxTonemapping { get; set; } + /// /// Gets or sets the tone-mapping algorithm. ///