avcodec/mpeg4videodec: Remove use of FF_PROFILE_MPEG4_SIMPLE_STUDIO as indicator of studio profile

The profile field is changed by code inside and outside the decoder,
its not a reliable indicator of the internal codec state.
Maintaining it consistency with studio_profile is messy.
Its easier to just avoid it and use only studio_profile

Fixes: assertion failure
Fixes: ffmpeg_crash_9.avi

Found-by: Thuan Pham, Marcel Böhme, Andrew Santosa and Alexandru Razvan Caciulescu with AFLSmart
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2018-07-03 00:27:04 +02:00
parent 3b10bb8772
commit bd27a9364c
3 changed files with 5 additions and 5 deletions

View File

@ -814,8 +814,7 @@ static int er_supported(ERContext *s)
{
if(s->avctx->hwaccel && s->avctx->hwaccel->decode_slice ||
!s->cur_pic.f ||
s->cur_pic.field_picture ||
s->avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO
s->cur_pic.field_picture
)
return 0;
return 1;

View File

@ -47,9 +47,10 @@
static enum AVPixelFormat h263_get_format(AVCodecContext *avctx)
{
MpegEncContext *s = avctx->priv_data;
/* MPEG-4 Studio Profile only, not supported by hardware */
if (avctx->bits_per_raw_sample > 8) {
av_assert1(avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO);
av_assert1(s->studio_profile);
return avctx->pix_fmt;
}
@ -670,7 +671,8 @@ retry:
av_assert1(s->bitstream_buffer_size == 0);
frame_end:
ff_er_frame_end(&s->er);
if (!s->studio_profile)
ff_er_frame_end(&s->er);
if (avctx->hwaccel) {
ret = avctx->hwaccel->end_frame(avctx);

View File

@ -3244,7 +3244,6 @@ end:
s->avctx->has_b_frames = !s->low_delay;
if (s->studio_profile) {
av_assert0(s->avctx->profile == FF_PROFILE_MPEG4_SIMPLE_STUDIO);
if (!s->avctx->bits_per_raw_sample) {
av_log(s->avctx, AV_LOG_ERROR, "Missing VOL header\n");
return AVERROR_INVALIDDATA;