Backport pull request #12973 from jellyfin/release-10.10.z

Fix pixel format in HEVC RExt SDR transcoding

Original-merge: aa08d3f2bf

Merged-by: crobibero <cody@robibe.ro>

Backported-by: Joshua M. Boniface <joshua@boniface.me>

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -4131,7 +4131,7 @@ namespace MediaBrowser.Controller.MediaEncoding
             else if (isD3d11vaDecoder || isQsvDecoder)
             {
                 var isRext = IsVideoStreamHevcRext(state);
-                var twoPassVppTonemap = isRext;
+                var twoPassVppTonemap = false;
                 var doVppFullRangeOut = isMjpegEncoder
                     && _mediaEncoder.EncoderVersion >= _minFFmpegQsvVppOutRangeOption;
                 var doVppScaleModeHq = isMjpegEncoder
@@ -4140,6 +4140,12 @@ namespace MediaBrowser.Controller.MediaEncoding
                 var procampParams = string.Empty;
                 if (doVppTonemap)
                 {
+                    if (isRext)
+                    {
+                        // VPP tonemap requires p010 input
+                        twoPassVppTonemap = true;
+                    }
+
                     if (options.VppTonemappingBrightness != 0
                         && options.VppTonemappingBrightness >= -100
                         && options.VppTonemappingBrightness <= 100)