swsresample/swresample: error out on invalid layouts

If it's unsupported or invalid, then there's no point trying to rebuild it
using a value that may have been derived from the same layout to begin with.

Move the checks before the attempts at copying the layout while at it.

Fixes ticket #9908.

Signed-off-by: James Almer <jamrial@gmail.com>
This commit is contained in:
James Almer 2022-09-08 19:43:03 -03:00
parent e402bd65b1
commit 9903ba28c2
1 changed files with 32 additions and 16 deletions

View File

@ -227,7 +227,7 @@ av_cold int swr_init(struct SwrContext *s){
s->in_ch_layout.order = AV_CHANNEL_ORDER_UNSPEC;
s->in_ch_layout.nb_channels = s->user_in_ch_count;
}
} else
} else if (av_channel_layout_check(&s->user_in_chlayout))
av_channel_layout_copy(&s->in_ch_layout, &s->user_in_chlayout);
if ((s->user_out_ch_count && s->user_out_ch_count != s->user_out_chlayout.nb_channels) ||
@ -240,17 +240,45 @@ av_cold int swr_init(struct SwrContext *s){
s->out_ch_layout.order = AV_CHANNEL_ORDER_UNSPEC;
s->out_ch_layout.nb_channels = s->user_out_ch_count;
}
} else
} else if (av_channel_layout_check(&s->user_out_chlayout))
av_channel_layout_copy(&s->out_ch_layout, &s->user_out_chlayout);
if (!s->out.ch_count && !s->user_out_ch_layout)
s->out.ch_count = s->out_ch_layout.nb_channels;
if (!s-> in.ch_count && !s-> user_in_ch_layout)
s-> in.ch_count = s->in_ch_layout.nb_channels;
if (!(ret = av_channel_layout_check(&s->in_ch_layout)) || s->in_ch_layout.nb_channels > SWR_CH_MAX) {
if (ret)
av_channel_layout_describe(&s->in_ch_layout, l1, sizeof(l1));
av_log(s, AV_LOG_WARNING, "Input channel layout \"%s\" is invalid or unsupported.\n", ret ? l1 : "");
return AVERROR(EINVAL);
}
if (!(ret = av_channel_layout_check(&s->out_ch_layout)) || s->out_ch_layout.nb_channels > SWR_CH_MAX) {
if (ret)
av_channel_layout_describe(&s->out_ch_layout, l2, sizeof(l2));
av_log(s, AV_LOG_WARNING, "Output channel layout \"%s\" is invalid or unsupported.\n", ret ? l2 : "");
return AVERROR(EINVAL);
}
#else
s->out.ch_count = s-> user_out_chlayout.nb_channels;
s-> in.ch_count = s-> user_in_chlayout.nb_channels;
if (!(ret = av_channel_layout_check(&s->user_in_chlayout)) || s->user_in_chlayout.nb_channels > SWR_CH_MAX) {
if (ret)
av_channel_layout_describe(&s->user_in_chlayout, l1, sizeof(l1));
av_log(s, AV_LOG_WARNING, "Input channel layout \"%s\" is invalid or unsupported.\n", ret ? l1 : "");
return AVERROR(EINVAL);
}
if (!(ret = av_channel_layout_check(&s->user_out_chlayout)) || s->user_out_chlayout.nb_channels > SWR_CH_MAX) {
if (ret)
av_channel_layout_describe(&s->user_out_chlayout, l2, sizeof(l2));
av_log(s, AV_LOG_WARNING, "Output channel layout \"%s\" is invalid or unsupported.\n", ret ? l2 : "");
return AVERROR(EINVAL);
}
ret = av_channel_layout_copy(&s->in_ch_layout, &s->user_in_chlayout);
ret |= av_channel_layout_copy(&s->out_ch_layout, &s->user_out_chlayout);
if (ret < 0)
@ -261,18 +289,6 @@ av_cold int swr_init(struct SwrContext *s){
s->dither.method = s->user_dither_method;
if (!av_channel_layout_check(&s->in_ch_layout) || s->in_ch_layout.nb_channels > SWR_CH_MAX) {
av_channel_layout_describe(&s->in_ch_layout, l1, sizeof(l1));
av_log(s, AV_LOG_WARNING, "Input channel layout \"%s\" is invalid or unsupported.\n", l1);
av_channel_layout_uninit(&s->in_ch_layout);
}
if (!av_channel_layout_check(&s->out_ch_layout) || s->out_ch_layout.nb_channels > SWR_CH_MAX) {
av_channel_layout_describe(&s->out_ch_layout, l2, sizeof(l2));
av_log(s, AV_LOG_WARNING, "Output channel layout \"%s\" is invalid or unsupported.\n", l2);
av_channel_layout_uninit(&s->out_ch_layout);
}
switch(s->engine){
#if CONFIG_LIBSOXR
case SWR_ENGINE_SOXR: s->resampler = &swri_soxr_resampler; break;
@ -291,9 +307,9 @@ av_cold int swr_init(struct SwrContext *s){
av_channel_layout_uninit(&s->in_ch_layout);
}
if (!s->in_ch_layout.nb_channels || s->in_ch_layout.order == AV_CHANNEL_ORDER_UNSPEC)
if (s->in_ch_layout.order == AV_CHANNEL_ORDER_UNSPEC)
av_channel_layout_default(&s->in_ch_layout, s->used_ch_count);
if (!s->out_ch_layout.nb_channels || s->out_ch_layout.order == AV_CHANNEL_ORDER_UNSPEC)
if (s->out_ch_layout.order == AV_CHANNEL_ORDER_UNSPEC)
av_channel_layout_default(&s->out_ch_layout, s->out.ch_count);
s->rematrix = av_channel_layout_compare(&s->out_ch_layout, &s->in_ch_layout) ||