avcodec/mpeg4videodec: avoid invalid values and reinitialize in format changes for studio profile

Fixes: out of array access
Fixes: 23327/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_MPEG4_fuzzer-5134822992510976

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2020-06-13 11:56:01 +02:00
parent bd0f81526d
commit e53235f06c
1 changed files with 14 additions and 5 deletions

View File

@ -3134,6 +3134,7 @@ static int decode_studio_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
MpegEncContext *s = &ctx->m;
int width, height;
int bits_per_raw_sample;
int rgb, chroma_format;
// random_accessible_vol and video_object_type_indication have already
// been read by the caller decode_vol_header()
@ -3141,28 +3142,36 @@ static int decode_studio_vol_header(Mpeg4DecContext *ctx, GetBitContext *gb)
ctx->shape = get_bits(gb, 2); /* video_object_layer_shape */
skip_bits(gb, 4); /* video_object_layer_shape_extension */
skip_bits1(gb); /* progressive_sequence */
if (ctx->shape != RECT_SHAPE) {
avpriv_request_sample(s->avctx, "MPEG-4 Studio profile non rectangular shape");
return AVERROR_PATCHWELCOME;
}
if (ctx->shape != BIN_ONLY_SHAPE) {
ctx->rgb = get_bits1(gb); /* rgb_components */
s->chroma_format = get_bits(gb, 2); /* chroma_format */
if (!s->chroma_format) {
rgb = get_bits1(gb); /* rgb_components */
chroma_format = get_bits(gb, 2); /* chroma_format */
if (!chroma_format || chroma_format == CHROMA_420 || (rgb && chroma_format == CHROMA_422)) {
av_log(s->avctx, AV_LOG_ERROR, "illegal chroma format\n");
return AVERROR_INVALIDDATA;
}
bits_per_raw_sample = get_bits(gb, 4); /* bit_depth */
if (bits_per_raw_sample == 10) {
if (ctx->rgb) {
if (rgb) {
s->avctx->pix_fmt = AV_PIX_FMT_GBRP10;
}
else {
s->avctx->pix_fmt = s->chroma_format == CHROMA_422 ? AV_PIX_FMT_YUV422P10 : AV_PIX_FMT_YUV444P10;
s->avctx->pix_fmt = chroma_format == CHROMA_422 ? AV_PIX_FMT_YUV422P10 : AV_PIX_FMT_YUV444P10;
}
}
else {
avpriv_request_sample(s->avctx, "MPEG-4 Studio profile bit-depth %u", bits_per_raw_sample);
return AVERROR_PATCHWELCOME;
}
if (rgb != ctx->rgb || s->chroma_format != chroma_format)
s->context_reinit = 1;
s->avctx->bits_per_raw_sample = bits_per_raw_sample;
ctx->rgb = rgb;
s->chroma_format = chroma_format;
}
if (ctx->shape == RECT_SHAPE) {
check_marker(s->avctx, gb, "before video_object_layer_width");