From a8677bcc8f2f9561be070cb9412c5a2be9e5be7f Mon Sep 17 00:00:00 2001 From: Geoffrey McRae Date: Sun, 26 Nov 2023 15:33:22 +1100 Subject: [PATCH] libavcodec/dcadec: adjust the `ch_layout` when downmix is active Applications making use of this codec with the `downmix` option are segfaulting unless the `ch_layout` is overridden after `avcodec_open2` as can be seen in projects like MythTV[1] This patch fixes this by overriding the ch_layout as done in other decoders such as AC3. 1: https://github.com/MythTV/mythtv/blob/af6f362a140cd59b9ed784a8c639fd456b5f6967/mythtv/libs/libmythtv/decoders/avformatdecoder.cpp#L4607 Signed-off-by: Geoffrey McRae Signed-off-by: James Almer --- libavcodec/dcadec.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/libavcodec/dcadec.c b/libavcodec/dcadec.c index 1fee49cf4d..cc23731b90 100644 --- a/libavcodec/dcadec.c +++ b/libavcodec/dcadec.c @@ -366,12 +366,19 @@ FF_ENABLE_DEPRECATION_WARNINGS if (s->downmix_layout.nb_channels) { if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO) || - !av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO_DOWNMIX)) + !av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO_DOWNMIX)) { s->request_channel_layout = DCA_SPEAKER_LAYOUT_STEREO; - else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0)) + av_channel_layout_uninit(&avctx->ch_layout); + avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_STEREO; + } else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0)) { s->request_channel_layout = DCA_SPEAKER_LAYOUT_5POINT0; - else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1)) + av_channel_layout_uninit(&avctx->ch_layout); + avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT0; + } else if (!av_channel_layout_compare(&s->downmix_layout, &(AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1)) { s->request_channel_layout = DCA_SPEAKER_LAYOUT_5POINT1; + av_channel_layout_uninit(&avctx->ch_layout); + avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1; + } else av_log(avctx, AV_LOG_WARNING, "Invalid downmix layout\n"); }