avcodec/wmaenc: Check input for finiteness

Fixes out of array read
Fixes: 1b79b985cdf860ffa228c00ee5497051/signal_sigsegv_1f99d24_315_f0f1fdb451264b1138e9c24eb7d49181.wv

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2016-01-16 21:40:49 +01:00
parent 057549a9cc
commit aa6aa2ef09
1 changed files with 11 additions and 2 deletions

View File

@ -98,7 +98,7 @@ static av_cold int encode_init(AVCodecContext *avctx)
return 0;
}
static void apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame)
static int apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame)
{
WMACodecContext *s = avctx->priv_data;
float **audio = (float **) frame->extended_data;
@ -117,7 +117,13 @@ static void apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame)
win, len);
s->fdsp->vector_fmul(s->frame_out[ch], s->frame_out[ch], win, len);
mdct->mdct_calc(mdct, s->coefs[ch], s->output);
if (!isfinite(s->coefs[ch][0])) {
av_log(avctx, AV_LOG_ERROR, "Input contains NaN/+-Inf\n");
return AVERROR(EINVAL);
}
}
return 0;
}
// FIXME use for decoding too
@ -364,7 +370,10 @@ static int encode_superframe(AVCodecContext *avctx, AVPacket *avpkt,
s->block_len_bits = s->frame_len_bits; // required by non variable block len
s->block_len = 1 << s->block_len_bits;
apply_window_and_mdct(avctx, frame);
ret = apply_window_and_mdct(avctx, frame);
if (ret < 0)
return ret;
if (s->ms_stereo) {
float a, b;