open-source-mirrors
/
jellyfin
mirror of https://github.com/jellyfin/jellyfin
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Vulkan filtering support for AMD VAAPI (Vega/gfx9+)

Browse Source 
This requires:
- VK_EXT_image_drm_format_modifier extension
- Linux kernel version >= 5.15
- jellyfin-ffmpeg5 >= 5.0.1-2

Signed-off-by: nyanmisaka <nst799610810@gmail.com>
pull/7708/head
nyanmisaka 2 years ago
parent 976dd012e7
commit 560d0838c7
5 changed files with 340 additions and 15 deletions
Show Stats Download Patch File Download Diff File

281
MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs
Unescape View File

@ -31,10 +31,13 @@ namespace MediaBrowser.Controller.MediaEncoding
private const string VideotoolboxAlias = "vt";
private const string OpenclAlias = "ocl";
private const string CudaAlias = "cu";
private const string DrmAlias = "dr";
private const string VulkanAlias = "vk";
private readonly IApplicationPaths _appPaths;
private readonly IMediaEncoder _mediaEncoder;
private readonly ISubtitleEncoder _subtitleEncoder;
private readonly IConfiguration _config;
private readonly Version _minKernelVersionAmdVkFmtModifier = new Version(5, 15);
private readonly Version _minKernelVersioni915Hang = new Version(5, 18);
private static readonly string[] _videoProfilesH264 = new[]
@ -149,6 +152,14 @@ namespace MediaBrowser.Controller.MediaEncoding
&& _mediaEncoder.SupportsFilter("hwupload_cuda");
}
private bool IsVulkanFullSupported()
{
return _mediaEncoder.SupportsHwaccel("vulkan")
&& _mediaEncoder.SupportsFilter("libplacebo")
&& _mediaEncoder.SupportsFilter("scale_vulkan")
&& _mediaEncoder.SupportsFilterWithOption(FilterOptionType.OverlayVulkanFrameSync);
}
private bool IsHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
{
if (state.VideoStream == null
@ -176,6 +187,19 @@ namespace MediaBrowser.Controller.MediaEncoding
|| string.Equals(state.VideoStream.VideoRangeType, "HLG", StringComparison.OrdinalIgnoreCase));
}
private bool IsVulkanHwTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
{
if (state.VideoStream == null)
{
return false;
}
// libplacebo has partial Dolby Vision to SDR tonemapping support.
return options.EnableTonemapping
&& string.Equals(state.VideoStream.VideoRange, "HDR", StringComparison.OrdinalIgnoreCase)
&& GetVideoColorBitDepth(state) == 10;
}
private bool IsVaapiVppTonemapAvailable(EncodingJobInfo state, EncodingOptions options)
{
if (state.VideoStream == null
@ -756,8 +780,13 @@ namespace MediaBrowser.Controller.MediaEncoding
}
else if (_mediaEncoder.IsVaapiDeviceAmd)
{
args.Append(GetOpenclDeviceArgs(0, "Advanced Micro Devices", null, OpenclAlias));
filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias);
if (!IsVulkanFullSupported()
|| !_mediaEncoder.IsVaapiDeviceSupportVulkanFmtModifier
|| Environment.OSVersion.Version < _minKernelVersionAmdVkFmtModifier)
{
args.Append(GetOpenclDeviceArgs(0, "Advanced Micro Devices", null, OpenclAlias));
filterDevArgs = GetFilterHwDeviceArgs(OpenclAlias);
}
}
else
{
@ -2774,22 +2803,41 @@ namespace MediaBrowser.Controller.MediaEncoding
return string.Empty;
}
var args = "tonemap_{0}=format={1}:p=bt709:t=bt709:m=bt709";
var args = string.Empty;
var algorithm = options.TonemappingAlgorithm;
if (hwTonemapSuffix.Contains("vaapi", StringComparison.OrdinalIgnoreCase))
if (string.Equals(hwTonemapSuffix, "vaapi", StringComparison.OrdinalIgnoreCase))
{
args += ",procamp_vaapi=b={2}:c={3}:extra_hw_frames=16";
args = "tonemap_vaapi=format={0}:p=bt709:t=bt709:m=bt709,procamp_vaapi=b={1}:c={2}:extra_hw_frames=16";
return string.Format(
CultureInfo.InvariantCulture,
args,
hwTonemapSuffix,
videoFormat ?? "nv12",
options.VppTonemappingBrightness,
options.VppTonemappingContrast);
}
else if (string.Equals(hwTonemapSuffix, "vulkan", StringComparison.OrdinalIgnoreCase))
{
args = "libplacebo=format={1}:tonemapping={2}:color_primaries=bt709:color_trc=bt709:colorspace=bt709:peak_detect=0:upscaler=none:downscaler=none";
if (!string.Equals(options.TonemappingRange, "auto", StringComparison.OrdinalIgnoreCase))
{
args += ":range={6}";
}
if (string.Equals(options.TonemappingAlgorithm, "bt2390", StringComparison.OrdinalIgnoreCase))
{
algorithm = "bt.2390";
}
else if (string.Equals(options.TonemappingAlgorithm, "none", StringComparison.OrdinalIgnoreCase))
{
algorithm = "clip";
}
}
else
{
args += ":tonemap={2}:peak={3}:desat={4}";
args = "tonemap_{0}=format={1}:p=bt709:t=bt709:m=bt709:tonemap={2}:peak={3}:desat={4}";
if (options.TonemappingParam != 0)
{
@ -2807,7 +2855,7 @@ namespace MediaBrowser.Controller.MediaEncoding
args,
hwTonemapSuffix,
videoFormat ?? "nv12",
options.TonemappingAlgorithm,
algorithm,
options.TonemappingPeak,
options.TonemappingDesat,
options.TonemappingParam,
@ -3770,7 +3818,9 @@ namespace MediaBrowser.Controller.MediaEncoding
var vidDecoder = GetHardwareVideoDecoder(state, options) ?? string.Empty;
var isSwDecoder = string.IsNullOrEmpty(vidDecoder);
var isSwEncoder = !vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase);
var isVaapiOclSupported = isLinux && IsVaapiSupported(state) && IsVaapiFullSupported() && IsOpenclFullSupported();
var isVaapiFullSupported = isLinux && IsVaapiSupported(state) && IsVaapiFullSupported();
var isVaapiOclSupported = isVaapiFullSupported && IsOpenclFullSupported();
var isVaapiVkSupported = isVaapiFullSupported && IsVulkanFullSupported();
// legacy vaapi pipeline(copy-back)
if ((isSwDecoder && isSwEncoder)
@ -3798,14 +3848,24 @@ namespace MediaBrowser.Controller.MediaEncoding
if (_mediaEncoder.IsVaapiDeviceInteliHD)
{
// Intel iHD path, with extra vpp tonemap and overlay support.
return GetVaapiFullVidFiltersPrefered(state, options, vidDecoder, vidEncoder);
return GetIntelVaapiFullVidFiltersPrefered(state, options, vidDecoder, vidEncoder);
}
// prefered vaapi + vulkan filters pipeline
if (_mediaEncoder.IsVaapiDeviceAmd
&& isVaapiVkSupported
&& _mediaEncoder.IsVaapiDeviceSupportVulkanFmtModifier
&& Environment.OSVersion.Version >= _minKernelVersionAmdVkFmtModifier)
{
// AMD radeonsi path(Vega/gfx9+, kernel>=5.15), with extra vulkan tonemap and overlay support.
return GetAmdVaapiFullVidFiltersPrefered(state, options, vidDecoder, vidEncoder);
}
// Intel i965 and Amd radeonsi/r600 path, only featuring scale and deinterlace support.
// Intel i965 and Amd radeonsi/r600 path(Polaris/gfx8-), only featuring scale and deinterlace support.
return GetVaapiLimitedVidFiltersPrefered(state, options, vidDecoder, vidEncoder);
}
public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetVaapiFullVidFiltersPrefered(
public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetIntelVaapiFullVidFiltersPrefered(
EncodingJobInfo state,
EncodingOptions options,
string vidDecoder,
@ -4003,6 +4063,203 @@ namespace MediaBrowser.Controller.MediaEncoding
return (mainFilters, subFilters, overlayFilters);
}
public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetAmdVaapiFullVidFiltersPrefered(
EncodingJobInfo state,
EncodingOptions options,
string vidDecoder,
string vidEncoder)
{
var inW = state.VideoStream?.Width;
var inH = state.VideoStream?.Height;
var reqW = state.BaseRequest.Width;
var reqH = state.BaseRequest.Height;
var reqMaxW = state.BaseRequest.MaxWidth;
var reqMaxH = state.BaseRequest.MaxHeight;
var threeDFormat = state.MediaSource.Video3DFormat;
var isVaapiDecoder = vidDecoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase);
var isVaapiEncoder = vidEncoder.Contains("vaapi", StringComparison.OrdinalIgnoreCase);
var isSwDecoder = string.IsNullOrEmpty(vidDecoder);
var isSwEncoder = !isVaapiEncoder;
var isVaInVaOut = isVaapiDecoder && isVaapiEncoder;
var doDeintH264 = state.DeInterlace("h264", true) || state.DeInterlace("avc", true);
var doDeintHevc = state.DeInterlace("h265", true) || state.DeInterlace("hevc", true);
var doVkTonemap = IsVulkanHwTonemapAvailable(state, options);
var doDeintH2645 = doDeintH264 || doDeintHevc;
var hasSubs = state.SubtitleStream != null && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode;
var hasTextSubs = hasSubs && state.SubtitleStream.IsTextSubtitleStream;
var hasGraphicalSubs = hasSubs && !state.SubtitleStream.IsTextSubtitleStream;
var hasAssSubs = hasSubs
&& (string.Equals(state.SubtitleStream.Codec, "ass", StringComparison.OrdinalIgnoreCase)
|| string.Equals(state.SubtitleStream.Codec, "ssa", StringComparison.OrdinalIgnoreCase));
/* Make main filters for video stream */
var mainFilters = new List<string>();
mainFilters.Add(GetOverwriteColorPropertiesParam(state, doVkTonemap));
if (isSwDecoder)
{
// INPUT sw surface(memory)
// sw deint
if (doDeintH2645)
{
var swDeintFilter = GetSwDeinterlaceFilter(state, options);
mainFilters.Add(swDeintFilter);
}
var outFormat = doVkTonemap ? "yuv420p10le" : "nv12";
var swScaleFilter = GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH);
// sw scale
mainFilters.Add(swScaleFilter);
mainFilters.Add("format=" + outFormat);
// keep video at memory except vk tonemap,
// since the overhead caused by hwupload >>> using sw filter.
// sw => hw
if (doVkTonemap)
{
mainFilters.Add("hwupload=derive_device=vulkan:extra_hw_frames=16");
}
}
else if (isVaapiDecoder)
{
// INPUT vaapi surface(vram)
// hw deint
if (doDeintH2645)
{
var deintFilter = GetHwDeinterlaceFilter(state, options, "vaapi");
mainFilters.Add(deintFilter);
}
var outFormat = doVkTonemap ? string.Empty : (hasSubs && isVaInVaOut ? "bgra" : "nv12");
var hwScaleFilter = GetHwScaleFilter("vaapi", outFormat, inW, inH, reqW, reqH, reqMaxW, reqMaxH);
// allocate extra pool sizes for overlay_vulkan
if (!string.IsNullOrEmpty(hwScaleFilter) && isVaInVaOut && hasSubs)
{
hwScaleFilter += ":extra_hw_frames=32";
}
// hw scale
mainFilters.Add(hwScaleFilter);
}
if ((isVaapiDecoder && doVkTonemap) || (isVaInVaOut && (doVkTonemap || hasSubs)))
{
// map from vaapi to vulkan via vaapi-vulkan interop (Vega/gfx9+).
mainFilters.Add("hwmap=derive_device=vulkan");
}
// vk tonemap
if (doVkTonemap)
{
var outFormat = isVaInVaOut && hasSubs ? "bgra" : "nv12";
var tonemapFilter = GetHwTonemapFilter(options, "vulkan", outFormat);
mainFilters.Add(tonemapFilter);
}
if (doVkTonemap && isVaInVaOut && !hasSubs)
{
// OUTPUT vaapi(nv12/bgra) surface(vram)
// reverse-mapping via vaapi-vulkan interop.
mainFilters.Add("hwmap=derive_device=vaapi:reverse=1");
mainFilters.Add("format=vaapi");
}
var memoryOutput = false;
var isUploadForVkTonemap = isSwDecoder && doVkTonemap;
if ((isVaapiDecoder && isSwEncoder) || isUploadForVkTonemap)
{
memoryOutput = true;
// OUTPUT nv12 surface(memory)
mainFilters.Add("hwdownload");
mainFilters.Add("format=nv12");
}
// OUTPUT nv12 surface(memory)
if (isSwDecoder && isVaapiEncoder)
{
memoryOutput = true;
}
if (memoryOutput)
{
// text subtitles
if (hasTextSubs)
{
var textSubtitlesFilter = GetTextSubtitlesFilter(state, false, false);
mainFilters.Add(textSubtitlesFilter);
}
}
if (memoryOutput && isVaapiEncoder)
{
if (!hasGraphicalSubs)
{
mainFilters.Add("hwupload_vaapi");
}
}
/* Make sub and overlay filters for subtitle stream */
var subFilters = new List<string>();
var overlayFilters = new List<string>();
if (isVaInVaOut)
{
if (hasSubs)
{
if (hasGraphicalSubs)
{
// scale=s=1280x720,format=bgra,hwupload
var subSwScaleFilter = GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subSwScaleFilter);
subFilters.Add("format=bgra");
}
else if (hasTextSubs)
{
var alphaSrcFilter = GetAlphaSrcFilter(state, inW, inH, reqW, reqH, reqMaxW, reqMaxH, hasAssSubs ? 10 : 5);
var subTextSubtitlesFilter = GetTextSubtitlesFilter(state, true, true);
subFilters.Add(alphaSrcFilter);
subFilters.Add("format=bgra");
subFilters.Add(subTextSubtitlesFilter);
}
subFilters.Add("hwupload=derive_device=vulkan:extra_hw_frames=16");
overlayFilters.Add("overlay_vulkan=eof_action=endall:shortest=1:repeatlast=0");
// explicitly sync using libplacebo.
overlayFilters.Add("libplacebo=format=nv12:upscaler=none:downscaler=none");
// OUTPUT vaapi(nv12/bgra) surface(vram)
// reverse-mapping via vaapi-vulkan interop.
overlayFilters.Add("hwmap=derive_device=vaapi:reverse=1");
overlayFilters.Add("format=vaapi");
}
}
else if (memoryOutput)
{
if (hasGraphicalSubs)
{
var subSwScaleFilter = isSwDecoder
? GetSwScaleFilter(state, options, vidEncoder, inW, inH, threeDFormat, reqW, reqH, reqMaxW, reqMaxH)
: GetCustomSwScaleFilter(inW, inH, reqW, reqH, reqMaxW, reqMaxH);
subFilters.Add(subSwScaleFilter);
overlayFilters.Add("overlay=eof_action=pass:shortest=1:repeatlast=0");
if (isVaapiEncoder)
{
overlayFilters.Add("hwupload_vaapi");
}
}
}
return (mainFilters, subFilters, overlayFilters);
}
public (List<string> MainFilters, List<string> SubFilters, List<string> OverlayFilters) GetVaapiLimitedVidFiltersPrefered(
EncodingJobInfo state,
EncodingOptions options,

7
MediaBrowser.Controller/MediaEncoding/FilterOptionType.cs
Unescape View File

@ -28,6 +28,11 @@ namespace MediaBrowser.Controller.MediaEncoding
/// <summary>
/// The overlay_vaapi_framesync.
/// </summary>
OverlayVaapiFrameSync = 4
OverlayVaapiFrameSync = 4,
/// <summary>
/// The overlay_vulkan_framesync.
/// </summary>
OverlayVulkanFrameSync = 5
}
}

6
MediaBrowser.Controller/MediaEncoding/IMediaEncoder.cs
Unescape View File

@ -61,6 +61,12 @@ namespace MediaBrowser.Controller.MediaEncoding
/// <value><c>true</c> if the Vaapi device is an Intel(legacy i965 driver) GPU, <c>false</c> otherwise.</value>
bool IsVaapiDeviceInteli965 { get; }
/// <summary>
/// Gets a value indicating whether the configured Vaapi device supports vulkan drm format modifier.
/// </summary>
/// <value><c>true</c> if the Vaapi device supports vulkan drm format modifier, <c>false</c> otherwise.</value>
bool IsVaapiDeviceSupportVulkanFmtModifier { get; }
/// <summary>
/// Whether given encoder codec is supported.
/// </summary>

42
MediaBrowser.MediaEncoding/Encoder/EncoderValidator.cs
Unescape View File

@ -102,7 +102,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
"tonemap_vaapi",
"procamp_vaapi",
"overlay_vaapi",
"hwupload_vaapi"
"hwupload_vaapi",
// vulkan
"libplacebo",
"scale_vulkan",
"overlay_vulkan"
};
private static readonly IReadOnlyDictionary<int, string[]> _filterOptionsDict = new Dictionary<int, string[]>
@ -111,7 +115,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
{ 1, new string[] { "tonemap_cuda", "GPU accelerated HDR to SDR tonemapping" } },
{ 2, new string[] { "tonemap_opencl", "bt2390" } },
{ 3, new string[] { "overlay_opencl", "Action to take when encountering EOF from secondary input" } },
{ 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } }
{ 4, new string[] { "overlay_vaapi", "Action to take when encountering EOF from secondary input" } },
{ 5, new string[] { "overlay_vulkan", "Action to take when encountering EOF from secondary input" } }
};
// These are the library versions that corresponds to our minimum ffmpeg version 4.x according to the version table below
@ -351,6 +356,39 @@ namespace MediaBrowser.MediaEncoding.Encoder
}
}
public bool CheckVulkanDrmDeviceByExtensionName(string renderNodePath, string[] vulkanExtensions)
{
if (!OperatingSystem.IsLinux())
{
return false;
}
if (string.IsNullOrEmpty(renderNodePath))
{
return false;
}
try
{
var command = "-v verbose -hide_banner -init_hw_device drm=dr:" + renderNodePath + " -init_hw_device vulkan=vk@dr";
var output = GetProcessOutput(_encoderPath, command, true, null);
foreach (string ext in vulkanExtensions)
{
if (!output.Contains(ext, StringComparison.Ordinal))
{
return false;
}
}
return true;
}
catch (Exception ex)
{
_logger.LogError(ex, "Error detecting the given drm render node path");
return false;
}
}
private IEnumerable<string> GetHwaccelTypes()
{
string? output = null;

19
MediaBrowser.MediaEncoding/Encoder/MediaEncoder.cs
Unescape View File

@ -72,6 +72,16 @@ namespace MediaBrowser.MediaEncoding.Encoder
private bool _isVaapiDeviceAmd = false;
private bool _isVaapiDeviceInteliHD = false;
private bool _isVaapiDeviceInteli965 = false;
private bool _isVaapiDeviceSupportVulkanFmtModifier = false;
private static string[] _vulkanFmtModifierExts = {
"VK_KHR_sampler_ycbcr_conversion",
"VK_EXT_image_drm_format_modifier",
"VK_KHR_external_memory_fd",
"VK_EXT_external_memory_dma_buf",
"VK_KHR_external_semaphore_fd",
"VK_EXT_external_memory_host"
};
private Version _ffmpegVersion = null;
private string _ffmpegPath = string.Empty;
@ -110,6 +120,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
public bool IsVaapiDeviceInteli965 => _isVaapiDeviceInteli965;
public bool IsVaapiDeviceSupportVulkanFmtModifier => _isVaapiDeviceSupportVulkanFmtModifier;
/// <summary>
/// Run at startup or if the user removes a Custom path from transcode page.
/// Sets global variables FFmpegPath.
@ -169,6 +181,8 @@ namespace MediaBrowser.MediaEncoding.Encoder
_isVaapiDeviceAmd = validator.CheckVaapiDeviceByDriverName("Mesa Gallium driver", options.VaapiDevice);
_isVaapiDeviceInteliHD = validator.CheckVaapiDeviceByDriverName("Intel iHD driver", options.VaapiDevice);
_isVaapiDeviceInteli965 = validator.CheckVaapiDeviceByDriverName("Intel i965 driver", options.VaapiDevice);
_isVaapiDeviceSupportVulkanFmtModifier = validator.CheckVulkanDrmDeviceByExtensionName(options.VaapiDevice, _vulkanFmtModifierExts);
if (_isVaapiDeviceAmd)
{
_logger.LogInformation("VAAPI device {RenderNodePath} is AMD GPU", options.VaapiDevice);
@ -181,6 +195,11 @@ namespace MediaBrowser.MediaEncoding.Encoder
{
_logger.LogInformation("VAAPI device {RenderNodePath} is Intel GPU (i965)", options.VaapiDevice);
}
if (_isVaapiDeviceSupportVulkanFmtModifier)
{
_logger.LogInformation("VAAPI device {RenderNodePath} supports Vulkan DRM format modifier", options.VaapiDevice);
}
}
}

Write Preview
Loading…
Cancel
Save