From a06a2d89434abd8feb760a943924f3cfacedfda6 Mon Sep 17 00:00:00 2001 From: Andreas Rheinhardt Date: Fri, 16 Sep 2022 02:27:25 +0200 Subject: [PATCH] avcodec/avcodec: Check for more invalid channel layouts In particular, check the provided channel layout for encoders without AVCodec.ch_layouts set. This fixes an infinite loop in the WavPack encoder (and maybe other issues in other encoders as well) in case the channel count is zero. Reviewed-by: James Almer Signed-off-by: Andreas Rheinhardt --- libavcodec/avcodec.c | 14 ++++++++++++++ libavcodec/decode.c | 5 ----- libavcodec/encode.c | 5 ----- 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/libavcodec/avcodec.c b/libavcodec/avcodec.c index 81f30a4a58..737c4bcd04 100644 --- a/libavcodec/avcodec.c +++ b/libavcodec/avcodec.c @@ -246,6 +246,20 @@ FF_DISABLE_DEPRECATION_WARNINGS FF_ENABLE_DEPRECATION_WARNINGS #endif + /* AV_CODEC_CAP_CHANNEL_CONF is a decoder-only flag; so the code below + * in particular checks that nb_channels is set for all audio encoders. */ + if (avctx->codec_type == AVMEDIA_TYPE_AUDIO && !avctx->ch_layout.nb_channels + && !(codec->capabilities & AV_CODEC_CAP_CHANNEL_CONF)) { + av_log(avctx, AV_LOG_ERROR, "%s requires channel layout to be set\n", + av_codec_is_decoder(codec) ? "Decoder" : "Encoder"); + ret = AVERROR(EINVAL); + goto free_and_end; + } + if (avctx->ch_layout.nb_channels && !av_channel_layout_check(&avctx->ch_layout)) { + av_log(avctx, AV_LOG_ERROR, "Invalid channel layout\n"); + ret = AVERROR(EINVAL); + goto free_and_end; + } if (avctx->ch_layout.nb_channels > FF_SANE_NB_CHANNELS) { av_log(avctx, AV_LOG_ERROR, "Too many channels: %d\n", avctx->ch_layout.nb_channels); ret = AVERROR(EINVAL); diff --git a/libavcodec/decode.c b/libavcodec/decode.c index 2961705c9d..6be2d3d6ed 100644 --- a/libavcodec/decode.c +++ b/libavcodec/decode.c @@ -1595,11 +1595,6 @@ FF_DISABLE_DEPRECATION_WARNINGS FF_ENABLE_DEPRECATION_WARNINGS #endif - if (avctx->codec_type == AVMEDIA_TYPE_AUDIO && avctx->ch_layout.nb_channels == 0 && - !(avctx->codec->capabilities & AV_CODEC_CAP_CHANNEL_CONF)) { - av_log(avctx, AV_LOG_ERROR, "Decoder requires channel count but channels not set\n"); - return AVERROR(EINVAL); - } if (avctx->codec->max_lowres < avctx->lowres || avctx->lowres < 0) { av_log(avctx, AV_LOG_WARNING, "The maximum value for lowres supported by the decoder is %d\n", avctx->codec->max_lowres); diff --git a/libavcodec/encode.c b/libavcodec/encode.c index ffac2c1ff3..9f4804cd69 100644 --- a/libavcodec/encode.c +++ b/libavcodec/encode.c @@ -602,11 +602,6 @@ static int encode_preinit_audio(AVCodecContext *avctx) return AVERROR(EINVAL); } if (avctx->codec->ch_layouts) { - if (!av_channel_layout_check(&avctx->ch_layout)) { - av_log(avctx, AV_LOG_WARNING, "Channel layout not specified correctly\n"); - return AVERROR(EINVAL); - } - for (i = 0; avctx->codec->ch_layouts[i].nb_channels; i++) { if (!av_channel_layout_compare(&avctx->ch_layout, &avctx->codec->ch_layouts[i])) break;