fix: only apply custom downmix to 5.1 audios

The current implementation of custom downmixing algorithm is flawed and caused multiple issues:

- The algorithm is designed for 5.1 channel audios but will also apply to 7.1 audio, resulting in very wrong outputs.
- The channel check only applies ffmpeg's native downmixing when channels >5, causing regressions on output channel limit control for inputs with fewer channels.
- The channel count is not set before calculating bitrate, resulting in the wrong bitrate being applied to the output.

Signed-off-by: gnattu <gnattuoc@me.com>

Jellyfin.Api/Controllers/DynamicHlsController.cs

@@ -1712,12 +1712,11 @@ public class DynamicHlsController : BaseJellyfinApiController
 
         var channels = state.OutputAudioChannels;
 
+        var useDownMixAlgorithm = state.AudioStream.Channels is 6 && _encodingOptions.DownMixStereoAlgorithm != DownMixStereoAlgorithms.None;
+
         if (channels.HasValue
             && (channels.Value != 2
-                || (state.AudioStream is not null
-                    && state.AudioStream.Channels.HasValue
-                    && state.AudioStream.Channels.Value > 5
-                    && _encodingOptions.DownMixStereoAlgorithm == DownMixStereoAlgorithms.None)))
+                || (state.AudioStream?.Channels != null && !useDownMixAlgorithm)))
         {
             args += " -ac " + channels.Value;
         }

Jellyfin.Api/Helpers/StreamingHelpers.cs

@@ -166,6 +166,9 @@ public static class StreamingHelpers
         }
 
         var outputAudioCodec = streamingRequest.AudioCodec;
+        state.OutputAudioCodec = outputAudioCodec;
+        state.OutputContainer = (containerInternal ?? string.Empty).TrimStart('.');
+        state.OutputAudioChannels = encodingHelper.GetNumAudioChannelsParam(state, state.AudioStream, state.OutputAudioCodec);
         if (EncodingHelper.LosslessAudioCodecs.Contains(outputAudioCodec))
         {
             state.OutputAudioBitrate = state.AudioStream.BitRate ?? 0;
@@ -180,10 +183,6 @@ public static class StreamingHelpers
             containerInternal = ".pcm";
         }
 
-        state.OutputAudioCodec = outputAudioCodec;
-        state.OutputContainer = (containerInternal ?? string.Empty).TrimStart('.');
-        state.OutputAudioChannels = encodingHelper.GetNumAudioChannelsParam(state, state.AudioStream, state.OutputAudioCodec);
-
         if (state.VideoRequest is not null)
         {
             state.OutputVideoCodec = state.Request.VideoCodec;

MediaBrowser.Controller/MediaEncoding/EncodingHelper.cs

@@ -2627,7 +2627,7 @@ namespace MediaBrowser.Controller.MediaEncoding
                 && channels.Value == 2
                 && state.AudioStream is not null
                 && state.AudioStream.Channels.HasValue
-                && state.AudioStream.Channels.Value > 5)
+                && state.AudioStream.Channels.Value == 6)
             {
                 switch (encodingOptions.DownMixStereoAlgorithm)
                 {
@@ -6903,7 +6903,7 @@ namespace MediaBrowser.Controller.MediaEncoding
 
             var channels = state.OutputAudioChannels;
 
-            if (channels.HasValue && ((channels.Value != 2 && state.AudioStream.Channels <= 5) || encodingOptions.DownMixStereoAlgorithm == DownMixStereoAlgorithms.None))
+            if (channels.HasValue && ((channels.Value != 2 && state.AudioStream?.Channels != 6) || encodingOptions.DownMixStereoAlgorithm == DownMixStereoAlgorithms.None))
             {
                 args += " -ac " + channels.Value;
             }