diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
index b6738e7cc1..946f7266c3 100644
--- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
+++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
@@ -253,6 +253,14 @@ namespace MediaBrowser.Controller.MediaEncoding
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync);
}
+ private bool IsVideoToolboxFullSupported()
+ {
+ return _mediaEncoder.SupportsHwaccel("videotoolbox")
+ && _mediaEncoder.SupportsFilter("yadif_videotoolbox")
+ && _mediaEncoder.SupportsFilter("overlay_videotoolbox")
+ && _mediaEncoder.SupportsFilter("scale_vt");
+ }
+
private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
{
if (state.VideoStream is null
@@ -272,7 +280,8 @@ namespace MediaBrowser.Controller.MediaEncoding
var isNvdecDecoder = vidDecoder.Contains("cuda", StringComparison.OrdinalIgnoreCase);
var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase);
var isD3d11vaDecoder = vidDecoder.Contains("d3d11va", StringComparison.OrdinalIgnoreCase);
- return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder;
+ var isVideoToolBoxDecoder = vidDecoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
+ return isSwDecoder || isNvdecDecoder || isVaapiDecoder || isD3d11vaDecoder || isVideoToolBoxDecoder;
}
return state.VideoStream.VideoRange == VideoRange.HDR
@@ -308,6 +317,21 @@ namespace MediaBrowser.Controller.MediaEncoding
&& state.VideoStream.VideoRangeType == VideoRangeType.HDR10;
}
+ private bool IsVideoToolboxTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
+ {
+ if (state.VideoStream is null
+ || !options.EnableVideoToolboxTonemapping
+ || GetVideoColorBitDepth(state) != 10)
+ {
+ return false;
+ }
+
+ // Certain DV profile 5 video works in Safari with direct playing, but the VideoToolBox does not produce correct mapping results with transcoding.
+ // All other HDR formats working.
+ return state.VideoStream.VideoRange == VideoRange.HDR
+ && state.VideoStream.VideoRangeType is VideoRangeType.HDR10 or VideoRangeType.HLG or VideoRangeType.HDR10Plus;
+ }
+
///
/// Gets the name of the output video codec.
///
@@ -4954,22 +4978,30 @@ namespace MediaBrowser.Controller.MediaEncoding
return (null, null, null);
}
- var swFilterChain = GetSwVidFilterChain(state, options, vidEncoder);
+ var isMacOS = OperatingSystem.IsMacOS();
+ var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty;
+ var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
+ var isVtFullSupported = isMacOS && IsVideoToolboxFullSupported();
+ var isVtOclSupported = isVtFullSupported && IsOpenclFullSupported();
- if (!options.EnableHardwareEncoding)
+ // legacy videotoolbox pipeline (disable hw filters)
+ if (!isVtEncoder
+ || !isVtOclSupported
+ || !_mediaEncoder.SupportsFilter("alphasrc"))
{
- return swFilterChain;
+ return GetSwVidFilterChain(state, options, vidEncoder);
}
- if (_mediaEncoder.EncoderVersion.CompareTo(new Version("5.0.0")) < 0)
- {
- // All features used here requires ffmpeg 5.0 or later, fallback to software filters if using an old ffmpeg
- return swFilterChain;
- }
+ // preferred videotoolbox + vt/ocl filters pipeline
+ return GetAppleVidFiltersPreferred(state, options, vidDecoder, vidEncoder);
+ }
- var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
- var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
- var doDeintH2645 = doDeintH264 || doDeintHevc;
+ public (List MainFilters, List SubFilters, List OverlayFilters) GetAppleVidFiltersPreferred(
+ EncodingJobInfo state,
+ EncodingOptions options,
+ string vidDecoder,
+ string vidEncoder)
+ {
var inW = state.VideoStream?.Width;
var inH = state.VideoStream?.Height;
var reqW = state.BaseRequest.Width;
@@ -4977,33 +5009,114 @@ namespace MediaBrowser.Controller.MediaEncoding
var reqMaxW = state.BaseRequest.MaxWidth;
var reqMaxH = state.BaseRequest.MaxHeight;
var threeDFormat = state.MediaSource.Video3DFormat;
- var newfilters = new List();
- var noOverlay = swFilterChain.OverlayFilters.Count == 0;
- var supportsHwDeint = _mediaEncoder.SupportsFilter("yadif_videotoolbox");
- // fallback to software filters if we are using filters not supported by hardware yet.
- var useHardwareFilters = noOverlay && (!doDeintH2645 || supportsHwDeint);
- if (!useHardwareFilters)
+ var isVtEncoder = vidEncoder.Contains("videotoolbox", StringComparison.OrdinalIgnoreCase);
+
+ var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
+ var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
+ var doDeintH2645 = doDeintH264 || doDeintHevc;
+ var doVtTonemap = IsVideoToolboxTonemapAvailable(state, options);
+ var doOclTonemap = !doVtTonemap && IsHwTonemapAvailable(state, options);
+
+ var scaleFormat = string.Empty;
+ if (!string.Equals(state.VideoStream.PixelFormat, "yuv420p", StringComparison.OrdinalIgnoreCase))
{
- return swFilterChain;
+ // Use P010 for OpenCL tone mapping, otherwise force an 8bit output.
+ scaleFormat = doOclTonemap ? "p010le" : "nv12";
}
- // ffmpeg cannot use videotoolbox to scale
- var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
- newfilters.Add(swScaleFilter);
+ var hwScaleFilter = GetHwScaleFilter("vt", scaleFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
- // hwupload on videotoolbox encoders can automatically convert AVFrame into its CVPixelBuffer equivalent
- // videotoolbox will automatically convert the CVPixelBuffer to a pixel format the encoder supports, so we don't have to set a pixel format explicitly here
- // This will reduce CPU usage significantly on UHD videos with 10 bit colors because we bypassed the ffmpeg pixel format conversion
- newfilters.Add("hwupload");
+ var hasSubs = state.SubtitleStream is not null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
+ var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream;
+ var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream;
+ var hasAssSubs = hasSubs
+ && (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
+ || string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
+
+ if (!isVtEncoder)
+ {
+ // should not happen.
+ return (null, null, null);
+ }
+
+ /* Make main filters for video stream */
+ var mainFilters = new List();
+ // Color override is only required for OpenCL where hardware surface is in use
+ if (doOclTonemap)
+ {
+ mainFilters.Add(GetOverwriteColorPropertiesParam(state, doOclTonemap));
+ }
+
+ // INPUT videotoolbox/memory surface(vram/uma)
+ // this will pass-through automatically if in/out format matches.
+ mainFilters.Add("format=nv12|p010le|videotoolbox_vld");
+ mainFilters.Add("hwupload=derive_device=videotoolbox");
+
+ // hw deint
if (doDeintH2645)
{
var deintFilter = GetHwDeinterlaceFilter(state, options, "videotoolbox");
- newfilters.Add(deintFilter);
+ mainFilters.Add(deintFilter);
}
- return (newfilters, swFilterChain.SubFilters, swFilterChain.OverlayFilters);
+ if (doVtTonemap)
+ {
+ const string VtTonemapArgs = "color_matrix=bt709:color_primaries=bt709:color_transfer=bt709";
+
+ // scale_vt can handle scaling & tonemapping in one shot, just like vpp_qsv.
+ hwScaleFilter = string.IsNullOrEmpty(hwScaleFilter)
+ ? "scale_vt=" + VtTonemapArgs
+ : hwScaleFilter + ":" + VtTonemapArgs;
+ }
+
+ // hw scale & vt tonemap
+ mainFilters.Add(hwScaleFilter);
+
+ // ocl tonemap
+ if (doOclTonemap)
+ {
+ // map from videotoolbox to opencl via videotoolbox-opencl interop.
+ mainFilters.Add("hwmap=derive_device=opencl:mode=read");
+
+ var tonemapFilter = GetHwTonemapFilter(options, "opencl", "nv12");
+ mainFilters.Add(tonemapFilter);
+
+ // OUTPUT videotoolbox(nv12) surface(vram/uma)
+ // reverse-mapping via videotoolbox-opencl interop.
+ mainFilters.Add("hwmap=derive_device=videotoolbox:mode=write:reverse=1");
+ }
+
+ /* Make sub and overlay filters for subtitle stream */
+ var subFilters = new List();
+ var overlayFilters = new List();
+
+ if (hasSubs)
+ {
+ if (hasGraphicalSubs)
+ {
+ var subPreProcFilters = GetGraphicalSubPreProcessFilters(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
+ subFilters.Add(subPreProcFilters);
+ subFilters.Add("format=bgra");
+ }
+ else if (hasTextSubs)
+ {
+ var framerate = state.VideoStream?.RealFrameRate;
+ var subFramerate = hasAssSubs ? Math.Min(framerate ?? 25, 60) : 10;
+
+ var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, subFramerate);
+ var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
+ subFilters.Add(alphaSrcFilter);
+ subFilters.Add("format=bgra");
+ subFilters.Add(subTextSubtitlesFilter);
+ }
+
+ subFilters.Add("hwupload=derive_device=videotoolbox");
+ overlayFilters.Add("overlay_videotoolbox=eof_action=pass:repeatlast=0");
+ }
+
+ return (mainFilters, subFilters, overlayFilters);
}
///
@@ -5995,22 +6108,37 @@ namespace MediaBrowser.Controller.MediaEncoding
|| string.Equals("yuvj420p", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
var is8_10bitSwFormatsVt = is8bitSwFormatsVt || string.Equals("yuv420p10le", videoStream.PixelFormat, StringComparison.OrdinalIgnoreCase);
+ // Hardware surface only make sense when interop with OpenCL
+ // VideoToolbox's Hardware surface in ffmpeg is not only slower than hwupload, but also breaks HDR in many cases.
+ // For example: https://trac.ffmpeg.org/ticket/10884
+ var useOclToneMapping = !IsVideoToolboxTonemapAvailable(state, options)
+ && options.EnableTonemapping
+ && state.VideoStream is not null
+ && GetVideoColorBitDepth(state) == 10
+ && state.VideoStream.VideoRange == VideoRange.HDR
+ && (state.VideoStream.VideoRangeType == VideoRangeType.HDR10
+ || state.VideoStream.VideoRangeType == VideoRangeType.HLG
+ || (state.VideoStream.VideoRangeType == VideoRangeType.DOVI
+ && string.Equals(state.VideoStream.Codec, "hevc", StringComparison.OrdinalIgnoreCase)));
+
+ var useHwSurface = useOclToneMapping && IsVideoToolboxFullSupported() && _mediaEncoder.SupportsFilter("alphasrc");
+
if (is8bitSwFormatsVt)
{
if (string.Equals("avc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
|| string.Equals("h264", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
{
- return GetHwaccelType(state, options, "h264", bitDepth, false);
+ return GetHwaccelType(state, options, "h264", bitDepth, useHwSurface);
}
if (string.Equals("mpeg2video", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
{
- return GetHwaccelType(state, options, "mpeg2video", bitDepth, false);
+ return GetHwaccelType(state, options, "mpeg2video", bitDepth, useHwSurface);
}
if (string.Equals("mpeg4", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
{
- return GetHwaccelType(state, options, "mpeg4", bitDepth, false);
+ return GetHwaccelType(state, options, "mpeg4", bitDepth, useHwSurface);
}
}
@@ -6019,12 +6147,12 @@ namespace MediaBrowser.Controller.MediaEncoding
if (string.Equals("hevc", videoStream.Codec, StringComparison.OrdinalIgnoreCase)
|| string.Equals("h265", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
{
- return GetHwaccelType(state, options, "hevc", bitDepth, false);
+ return GetHwaccelType(state, options, "hevc", bitDepth, useHwSurface);
}
if (string.Equals("vp9", videoStream.Codec, StringComparison.OrdinalIgnoreCase))
{
- return GetHwaccelType(state, options, "vp9", bitDepth, false);
+ return GetHwaccelType(state, options, "vp9", bitDepth, useHwSurface);
}
}
diff --git a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
index fdca283908..6549125d36 100644
--- a/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
+++ b/MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
@@ -128,6 +128,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
"overlay_vulkan",
// videotoolbox
"yadif_videotoolbox",
+ "scale_vt",
+ "overlay_videotoolbox",
// rkrga
"scale_rkrga",
"vpp_rkrga",
diff --git a/MediaBrowser.Model/Configuration/EncodingOptions.cs b/MediaBrowser.Model/Configuration/EncodingOptions.cs
index 84c735f9ca..ab6f0d867c 100644
--- a/MediaBrowser.Model/Configuration/EncodingOptions.cs
+++ b/MediaBrowser.Model/Configuration/EncodingOptions.cs
@@ -28,6 +28,7 @@ public class EncodingOptions
VaapiDevice = "/dev/dri/renderD128";
EnableTonemapping = false;
EnableVppTonemapping = false;
+ EnableVideoToolboxTonemapping = false;
TonemappingAlgorithm = "bt2390";
TonemappingMode = "auto";
TonemappingRange = "auto";
@@ -146,6 +147,11 @@ public class EncodingOptions
///
public bool EnableVppTonemapping { get; set; }
+ ///
+ /// Gets or sets a value indicating whether videotoolbox tonemapping is enabled.
+ ///
+ public bool EnableVideoToolboxTonemapping { get; set; }
+
///
/// Gets or sets the tone-mapping algorithm.
///