mp3: properly forward mp_decode_frame errors

The function can return either a parsing error or a memory management
error.

Fixes: CVE-2012-2797

(cherry picked from commit 9ab0874ea8)

Conflicts:

	libavcodec/mpegaudiodec.c

Signed-off-by: Reinhard Tartler <siretart@tauware.de>
This commit is contained in:
Luca Barbato 2012-10-22 19:11:05 +02:00 committed by Reinhard Tartler
parent 56c1e18a52
commit a5290800f5

View File

@ -1632,7 +1632,7 @@ static int decode_frame(AVCodecContext * avctx, void *data, int *got_frame_ptr,
int buf_size = avpkt->size; int buf_size = avpkt->size;
MPADecodeContext *s = avctx->priv_data; MPADecodeContext *s = avctx->priv_data;
uint32_t header; uint32_t header;
int out_size; int ret;
if (buf_size < HEADER_SIZE) if (buf_size < HEADER_SIZE)
return AVERROR_INVALIDDATA; return AVERROR_INVALIDDATA;
@ -1663,21 +1663,22 @@ static int decode_frame(AVCodecContext * avctx, void *data, int *got_frame_ptr,
buf_size= s->frame_size; buf_size= s->frame_size;
} }
out_size = mp_decode_frame(s, NULL, buf, buf_size); ret = mp_decode_frame(s, NULL, buf, buf_size);
if (out_size >= 0) { if (ret >= 0) {
*got_frame_ptr = 1; *got_frame_ptr = 1;
*(AVFrame *)data = s->frame; *(AVFrame *)data = s->frame;
avctx->sample_rate = s->sample_rate; avctx->sample_rate = s->sample_rate;
//FIXME maybe move the other codec info stuff from above here too //FIXME maybe move the other codec info stuff from above here too
} else { } else {
av_log(avctx, AV_LOG_ERROR, "Error while decoding MPEG audio frame.\n"); av_log(avctx, AV_LOG_ERROR, "Error while decoding MPEG audio frame.\n");
/* Only return an error if the bad frame makes up the whole packet. /* Only return an error if the bad frame makes up the whole packet or
If there is more data in the packet, just consume the bad frame * the error is related to buffer management.
instead of returning an error, which would discard the whole * If there is more data in the packet, just consume the bad frame
packet. */ * instead of returning an error, which would discard the whole
* packet. */
*got_frame_ptr = 0; *got_frame_ptr = 0;
if (buf_size == avpkt->size) if (buf_size == avpkt->size || ret != AVERROR_INVALIDDATA)
return out_size; return ret;
} }
s->frame_size = 0; s->frame_size = 0;
return buf_size; return buf_size;
@ -1698,7 +1699,7 @@ static int decode_frame_adu(AVCodecContext *avctx, void *data,
int buf_size = avpkt->size; int buf_size = avpkt->size;
MPADecodeContext *s = avctx->priv_data; MPADecodeContext *s = avctx->priv_data;
uint32_t header; uint32_t header;
int len, out_size; int len, out_size, ret = 0;
len = buf_size; len = buf_size;
@ -1735,7 +1736,11 @@ static int decode_frame_adu(AVCodecContext *avctx, void *data,
out_size = buf_size; out_size = buf_size;
else else
#endif #endif
out_size = mp_decode_frame(s, NULL, buf, buf_size); ret = mp_decode_frame(s, NULL, buf, buf_size);
if (ret < 0) {
av_log(avctx, AV_LOG_ERROR, "Error while decoding MPEG audio frame.\n");
return ret;
}
*got_frame_ptr = 1; *got_frame_ptr = 1;
*(AVFrame *)data = s->frame; *(AVFrame *)data = s->frame;
@ -1942,7 +1947,10 @@ static int decode_frame_mp3on4(AVCodecContext *avctx, void *data,
} }
ch += m->nb_channels; ch += m->nb_channels;
out_size += mp_decode_frame(m, outptr, buf, fsize); if ((ret = mp_decode_frame(m, outptr, buf, fsize)) < 0)
return ret;
out_size += ret;
buf += fsize; buf += fsize;
len -= fsize; len -= fsize;