From 7b862bba5aad345f0bc8d76bfb950590471ae232 Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Sat, 25 Jul 2020 00:57:34 +0800 Subject: [PATCH 1/6] add Tonemapping relaying on nvdec and ocl --- .../MediaEncoding/EncodingHelper.cs | 109 ++++++++++++++++-- .../Probing/MediaStreamInfo.cs | 14 +++ .../Probing/ProbeResultNormalizer.cs | 10 ++ .../Configuration/EncodingOptions.cs | 29 ++++- MediaBrowser.Model/Entities/MediaStream.cs | 18 ++- 5 files changed, 166 insertions(+), 14 deletions(-) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 0242338b70..4a33f22e63 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -454,6 +454,7 @@ namespace MediaBrowser.Controller.MediaEncoding var isVaapiEncoder = outputVideoCodec.IndexOf("vaapi", StringComparison.OrdinalIgnoreCase) != -1; var isQsvDecoder = videoDecoder.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1; var isQsvEncoder = outputVideoCodec.IndexOf("qsv", StringComparison.OrdinalIgnoreCase) != -1; + var isNvencHevcDecoder = videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1; var isWindows = RuntimeInformation.IsOSPlatform(OSPlatform.Windows); var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); @@ -511,6 +512,25 @@ namespace MediaBrowser.Controller.MediaEncoding } } } + + if (state.IsVideoRequest + && string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) + { + var codec = state.VideoStream.Codec.ToLowerInvariant(); + var isColorDepth10 = IsColorDepth10(state); + + if (isNvencHevcDecoder && isColorDepth10 + && _mediaEncoder.SupportsHwaccel("opencl") + && encodingOptions.EnableTonemapping + && !string.IsNullOrEmpty(state.VideoStream.VideoRange) + && state.VideoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase)) + { + arg.Append("-init_hw_device opencl=ocl:") + .Append(encodingOptions.OpenclDevice) + .Append(' ') + .Append("-filter_hw_device ocl "); + } + } } arg.Append("-i ") @@ -994,11 +1014,33 @@ namespace MediaBrowser.Controller.MediaEncoding if (!string.Equals(videoEncoder, "h264_omx", StringComparison.OrdinalIgnoreCase) && !string.Equals(videoEncoder, "h264_qsv", StringComparison.OrdinalIgnoreCase) && !string.Equals(videoEncoder, "h264_vaapi", StringComparison.OrdinalIgnoreCase) + && !string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase) && !string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase)) { param = "-pix_fmt yuv420p " + param; } + if (string.Equals(videoEncoder, "h264_nvenc", StringComparison.OrdinalIgnoreCase)) + { + var videoDecoder = GetHardwareAcceleratedVideoDecoder(state, encodingOptions) ?? string.Empty; + var videoStream = state.VideoStream; + var isColorDepth10 = IsColorDepth10(state); + + if (videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1 + && isColorDepth10 + && _mediaEncoder.SupportsHwaccel("opencl") + && encodingOptions.EnableTonemapping + && !string.IsNullOrEmpty(videoStream.VideoRange) + && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase)) + { + param = "-pix_fmt nv12 " + param; + } + else + { + param = "-pix_fmt yuv420p " + param; + } + } + if (string.Equals(videoEncoder, "h264_v4l2m2m", StringComparison.OrdinalIgnoreCase)) { param = "-pix_fmt nv21 " + param; @@ -1599,46 +1641,54 @@ namespace MediaBrowser.Controller.MediaEncoding { outputSizeParam = GetOutputSizeParam(state, options, outputVideoCodec).TrimEnd('"'); - var index = outputSizeParam.IndexOf("hwdownload", StringComparison.OrdinalIgnoreCase); + var index = outputSizeParam.IndexOf("hwupload,tonemap_opencl", StringComparison.OrdinalIgnoreCase); if (index != -1) { outputSizeParam = outputSizeParam.Substring(index); } else { - index = outputSizeParam.IndexOf("hwupload=extra_hw_frames", StringComparison.OrdinalIgnoreCase); + index = outputSizeParam.IndexOf("hwdownload", StringComparison.OrdinalIgnoreCase); if (index != -1) { outputSizeParam = outputSizeParam.Substring(index); } else { - index = outputSizeParam.IndexOf("format", StringComparison.OrdinalIgnoreCase); + index = outputSizeParam.IndexOf("hwupload=extra_hw_frames", StringComparison.OrdinalIgnoreCase); if (index != -1) { outputSizeParam = outputSizeParam.Substring(index); } else { - index = outputSizeParam.IndexOf("yadif", StringComparison.OrdinalIgnoreCase); + index = outputSizeParam.IndexOf("format", StringComparison.OrdinalIgnoreCase); if (index != -1) { outputSizeParam = outputSizeParam.Substring(index); } else { - index = outputSizeParam.IndexOf("scale", StringComparison.OrdinalIgnoreCase); + index = outputSizeParam.IndexOf("yadif", StringComparison.OrdinalIgnoreCase); if (index != -1) { outputSizeParam = outputSizeParam.Substring(index); } else { - index = outputSizeParam.IndexOf("vpp", StringComparison.OrdinalIgnoreCase); + index = outputSizeParam.IndexOf("scale", StringComparison.OrdinalIgnoreCase); if (index != -1) { outputSizeParam = outputSizeParam.Substring(index); } + else + { + index = outputSizeParam.IndexOf("vpp", StringComparison.OrdinalIgnoreCase); + if (index != -1) + { + outputSizeParam = outputSizeParam.Substring(index); + } + } } } } @@ -2058,12 +2108,58 @@ namespace MediaBrowser.Controller.MediaEncoding var isVaapiH264Encoder = outputVideoCodec.IndexOf("h264_vaapi", StringComparison.OrdinalIgnoreCase) != -1; var isQsvH264Encoder = outputVideoCodec.IndexOf("h264_qsv", StringComparison.OrdinalIgnoreCase) != -1; var isNvdecH264Decoder = videoDecoder.IndexOf("h264_cuvid", StringComparison.OrdinalIgnoreCase) != -1; + var isNvdecHevcDecoder = videoDecoder.IndexOf("hevc_cuvid", StringComparison.OrdinalIgnoreCase) != -1; var isLibX264Encoder = outputVideoCodec.IndexOf("libx264", StringComparison.OrdinalIgnoreCase) != -1; var isLinux = RuntimeInformation.IsOSPlatform(OSPlatform.Linux); + var isColorDepth10 = IsColorDepth10(state); var hasTextSubs = state.SubtitleStream != null && state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; var hasGraphicalSubs = state.SubtitleStream != null && !state.SubtitleStream.IsTextSubtitleStream && state.SubtitleDeliveryMethod == SubtitleDeliveryMethod.Encode; + // Currently only with the use of NVENC decoder can we get a decent performance. + // Currently only the HEVC/H265 format is supported. + // NVIDIA Pascal and Turing or higher are recommended. + if (isNvdecHevcDecoder && isColorDepth10 + && _mediaEncoder.SupportsHwaccel("opencl") + && options.EnableTonemapping + && !string.IsNullOrEmpty(videoStream.VideoRange) + && videoStream.VideoRange.Contains("HDR", StringComparison.OrdinalIgnoreCase)) + { + var parameters = "tonemap_opencl=format=nv12:primaries=bt709:transfer=bt709:matrix=bt709:tonemap={0}:desat={1}:threshold={2}:peak={3}"; + + if (options.TonemappingParam != 0) + { + parameters += ":param={4}"; + } + + if (!string.Equals(options.TonemappingRange, "auto", StringComparison.OrdinalIgnoreCase)) + { + parameters += ":range={5}"; + } + + // Upload the HDR10 or HLG data to the OpenCL device, + // use tonemap_opencl filter for tone mapping, + // and then download the SDR data to memory. + filters.Add("hwupload"); + filters.Add( + string.Format( + CultureInfo.InvariantCulture, + parameters, + options.TonemappingAlgorithm, + options.TonemappingDesat, + options.TonemappingThreshold, + options.TonemappingPeak, + options.TonemappingParam, + options.TonemappingRange)); + filters.Add("hwdownload"); + + if (hasGraphicalSubs || state.DeInterlace("h265", true) || state.DeInterlace("hevc", true) + || string.Equals(outputVideoCodec, "libx264", StringComparison.OrdinalIgnoreCase)) + { + filters.Add("format=nv12"); + } + } + // When the input may or may not be hardware VAAPI decodable if (isVaapiH264Encoder) { @@ -2081,7 +2177,6 @@ namespace MediaBrowser.Controller.MediaEncoding else if (IsVaapiSupported(state) && isVaapiDecoder && isLibX264Encoder) { var codec = videoStream.Codec.ToLowerInvariant(); - var isColorDepth10 = IsColorDepth10(state); // Assert 10-bit hardware VAAPI decodable if (isColorDepth10 && (string.Equals(codec, "hevc", StringComparison.OrdinalIgnoreCase) diff --git a/MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs b/MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs index a2ea0766a8..d9658cba24 100644 --- a/MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs +++ b/MediaBrowser.MediaEncoding/Probing/MediaStreamInfo.cs @@ -279,6 +279,20 @@ namespace MediaBrowser.MediaEncoding.Probing [JsonPropertyName("disposition")] public IReadOnlyDictionary Disposition { get; set; } + /// + /// Gets or sets the color range. + /// + /// The color range. + [JsonPropertyName("color_range")] + public string ColorRange { get; set; } + + /// + /// Gets or sets the color space. + /// + /// The color space. + [JsonPropertyName("color_space")] + public string ColorSpace { get; set; } + /// /// Gets or sets the color transfer. /// diff --git a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs index c85d963ed1..3815e00d47 100644 --- a/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs +++ b/MediaBrowser.MediaEncoding/Probing/ProbeResultNormalizer.cs @@ -703,6 +703,16 @@ namespace MediaBrowser.MediaEncoding.Probing stream.RefFrames = streamInfo.Refs; } + if (!string.IsNullOrEmpty(streamInfo.ColorRange)) + { + stream.ColorRange = streamInfo.ColorRange; + } + + if (!string.IsNullOrEmpty(streamInfo.ColorSpace)) + { + stream.ColorSpace = streamInfo.ColorSpace; + } + if (!string.IsNullOrEmpty(streamInfo.ColorTransfer)) { stream.ColorTransfer = streamInfo.ColorTransfer; diff --git a/MediaBrowser.Model/Configuration/EncodingOptions.cs b/MediaBrowser.Model/Configuration/EncodingOptions.cs index 9a30f7e9f0..843ff3ff9c 100644 --- a/MediaBrowser.Model/Configuration/EncodingOptions.cs +++ b/MediaBrowser.Model/Configuration/EncodingOptions.cs @@ -29,6 +29,22 @@ namespace MediaBrowser.Model.Configuration public string VaapiDevice { get; set; } + public string OpenclDevice { get; set; } + + public bool EnableTonemapping { get; set; } + + public string TonemappingAlgorithm { get; set; } + + public string TonemappingRange { get; set; } + + public double TonemappingDesat { get; set; } + + public double TonemappingThreshold { get; set; } + + public double TonemappingPeak { get; set; } + + public double TonemappingParam { get; set; } + public int H264Crf { get; set; } public int H265Crf { get; set; } @@ -53,8 +69,19 @@ namespace MediaBrowser.Model.Configuration EnableThrottling = false; ThrottleDelaySeconds = 180; EncodingThreadCount = -1; - // This is a DRM device that is almost guaranteed to be there on every intel platform, plus it's the default one in ffmpeg if you don't specify anything + // This is a DRM device that is almost guaranteed to be there on every intel platform, + // plus it's the default one in ffmpeg if you don't specify anything VaapiDevice = "/dev/dri/renderD128"; + // This is the OpenCL device that is used for tonemapping. + // The left side of the dot is the platform number, and the right side is the device number on the platform. + OpenclDevice = "0.0"; + EnableTonemapping = false; + TonemappingAlgorithm = "reinhard"; + TonemappingRange = "auto"; + TonemappingDesat = 0; + TonemappingThreshold = 0.8; + TonemappingPeak = 0; + TonemappingParam = 0; H264Crf = 23; H265Crf = 28; DeinterlaceMethod = "yadif"; diff --git a/MediaBrowser.Model/Entities/MediaStream.cs b/MediaBrowser.Model/Entities/MediaStream.cs index 1b37cfc939..19eb79acc4 100644 --- a/MediaBrowser.Model/Entities/MediaStream.cs +++ b/MediaBrowser.Model/Entities/MediaStream.cs @@ -35,6 +35,18 @@ namespace MediaBrowser.Model.Entities /// The language. public string Language { get; set; } + /// + /// Gets or sets the color range. + /// + /// The color range. + public string ColorRange { get; set; } + + /// + /// Gets or sets the color space. + /// + /// The color space. + public string ColorSpace { get; set; } + /// /// Gets or sets the color transfer. /// @@ -47,12 +59,6 @@ namespace MediaBrowser.Model.Entities /// The color primaries. public string ColorPrimaries { get; set; } - /// - /// Gets or sets the color space. - /// - /// The color space. - public string ColorSpace { get; set; } - /// /// Gets or sets the comment. /// From 5716453542172443124db43790d4fde0f0abff10 Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Sat, 25 Jul 2020 01:03:34 +0800 Subject: [PATCH 2/6] minor changes --- MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 668411342b..42b89be023 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -1686,7 +1686,7 @@ namespace MediaBrowser.Controller.MediaEncoding index = outputSizeParam.IndexOf("vpp", StringComparison.OrdinalIgnoreCase); if (index != -1) { - outputSizeParam = outputSizeParam.Substring(index); + outputSizeParam = outputSizeParam.Slice(index); } } } @@ -1771,9 +1771,9 @@ namespace MediaBrowser.Controller.MediaEncoding */ if (isLinux) { - retStr = !outputSizeParam.IsEmpty ? - " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay_qsv\"" : - " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay_qsv\""; + retStr = !outputSizeParam.IsEmpty + ? " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}]{3}[base];[base][sub]overlay_qsv\"" + : " -filter_complex \"[{0}:{1}]{4}[sub];[0:{2}][sub]overlay_qsv\""; } } From 95b961c8647b9db98d52631561cfc98224b905f7 Mon Sep 17 00:00:00 2001 From: Nyanmisaka Date: Sun, 26 Jul 2020 12:38:57 +0800 Subject: [PATCH 3/6] remove unused line --- MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 42b89be023..1cdc8b736d 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -516,7 +516,6 @@ namespace MediaBrowser.Controller.MediaEncoding if (state.IsVideoRequest && string.Equals(encodingOptions.HardwareAccelerationType, "nvenc", StringComparison.OrdinalIgnoreCase)) { - var codec = state.VideoStream.Codec.ToLowerInvariant(); var isColorDepth10 = IsColorDepth10(state); if (isNvencHevcDecoder && isColorDepth10 From 17527d68dfe2fa6fb7755f819f03ada5165cb23d Mon Sep 17 00:00:00 2001 From: Nyanmisaka Date: Mon, 27 Jul 2020 14:11:05 +0800 Subject: [PATCH 4/6] fix build --- MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 323fc5f75d..a04ddbaa0f 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -530,6 +530,7 @@ namespace MediaBrowser.Controller.MediaEncoding .Append(' ') .Append("-filter_hw_device ocl "); } + } if (state.IsVideoRequest && string.Equals(encodingOptions.HardwareAccelerationType, "videotoolbox", StringComparison.OrdinalIgnoreCase)) From 7b3f76b81fd92d30ac8783508fb7b6793d32ec95 Mon Sep 17 00:00:00 2001 From: Nyanmisaka Date: Mon, 3 Aug 2020 14:55:28 +0800 Subject: [PATCH 5/6] resolve conflicts --- .../MediaEncoding/EncodingHelper.cs | 63 +++++-------------- 1 file changed, 16 insertions(+), 47 deletions(-) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index a04ddbaa0f..9433d33b4a 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -1647,57 +1647,26 @@ namespace MediaBrowser.Controller.MediaEncoding { outputSizeParam = GetOutputSizeParam(state, options, outputVideoCodec).TrimEnd('"'); - var index = outputSizeParam.IndexOf("hwupload,tonemap_opencl", StringComparison.OrdinalIgnoreCase); - if (index != -1) - { - outputSizeParam = outputSizeParam.Slice(index); - } - else + // All possible beginning of video filters + // Don't break the order + string[] beginOfParam = new[] + { + "hwupload,tonemap_opencl", + "hwupload=extra_hw_frames", + "vpp", + "hwdownload", + "format", + "yadif", + "scale" + }; + + for (int i = 0, index = -1; i < beginOfParam.Length; i++) { - index = outputSizeParam.IndexOf("hwdownload", StringComparison.OrdinalIgnoreCase); + index = outputSizeParam.IndexOf(beginOfParam[i], StringComparison.OrdinalIgnoreCase); if (index != -1) { outputSizeParam = outputSizeParam.Slice(index); - } - else - { - index = outputSizeParam.IndexOf("hwupload=extra_hw_frames", StringComparison.OrdinalIgnoreCase); - if (index != -1) - { - outputSizeParam = outputSizeParam.Slice(index); - } - else - { - index = outputSizeParam.IndexOf("format", StringComparison.OrdinalIgnoreCase); - if (index != -1) - { - outputSizeParam = outputSizeParam.Slice(index); - } - else - { - index = outputSizeParam.IndexOf("yadif", StringComparison.OrdinalIgnoreCase); - if (index != -1) - { - outputSizeParam = outputSizeParam.Slice(index); - } - else - { - index = outputSizeParam.IndexOf("scale", StringComparison.OrdinalIgnoreCase); - if (index != -1) - { - outputSizeParam = outputSizeParam.Slice(index); - } - else - { - index = outputSizeParam.IndexOf("vpp", StringComparison.OrdinalIgnoreCase); - if (index != -1) - { - outputSizeParam = outputSizeParam.Slice(index); - } - } - } - } - } + break; } } } From 54349fc94597824714f623b8c31583fc044274aa Mon Sep 17 00:00:00 2001 From: nyanmisaka Date: Thu, 20 Aug 2020 19:08:36 +0800 Subject: [PATCH 6/6] fix outputSize --- .../MediaEncoding/EncodingHelper.cs | 64 +++++++++++-------- 1 file changed, 37 insertions(+), 27 deletions(-) diff --git a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs index 9433d33b4a..7ceb25bade 100644 --- a/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs +++ b/MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs @@ -1639,35 +1639,45 @@ namespace MediaBrowser.Controller.MediaEncoding var outputSizeParam = ReadOnlySpan.Empty; var request = state.BaseRequest; - // Add resolution params, if specified - if (request.Width.HasValue - || request.Height.HasValue - || request.MaxHeight.HasValue - || request.MaxWidth.HasValue) - { - outputSizeParam = GetOutputSizeParam(state, options, outputVideoCodec).TrimEnd('"'); - - // All possible beginning of video filters - // Don't break the order - string[] beginOfParam = new[] - { - "hwupload,tonemap_opencl", - "hwupload=extra_hw_frames", - "vpp", - "hwdownload", - "format", - "yadif", - "scale" - }; + outputSizeParam = GetOutputSizeParam(state, options, outputVideoCodec).TrimEnd('"'); + + // All possible beginning of video filters + // Don't break the order + string[] beginOfOutputSizeParam = new[] + { + // for tonemap_opencl + "hwupload,tonemap_opencl", + + // hwupload=extra_hw_frames=64,vpp_qsv (for overlay_qsv on linux) + "hwupload=extra_hw_frames", + + // vpp_qsv + "vpp", + + // hwdownload,format=p010le (hardware decode + software encode for vaapi) + "hwdownload", + + // format=nv12|vaapi,hwupload,scale_vaapi + "format", + + // bwdif,scale=expr + "bwdif", - for (int i = 0, index = -1; i < beginOfParam.Length; i++) + // yadif,scale=expr + "yadif", + + // scale=expr + "scale" + }; + + var index = -1; + foreach (var param in beginOfOutputSizeParam) + { + index = outputSizeParam.IndexOf(param, StringComparison.OrdinalIgnoreCase); + if (index != -1) { - index = outputSizeParam.IndexOf(beginOfParam[i], StringComparison.OrdinalIgnoreCase); - if (index != -1) - { - outputSizeParam = outputSizeParam.Slice(index); - break; - } + outputSizeParam = outputSizeParam.Slice(index); + break; } }