From aa6aa2ef091818c2669c48051286ce361401f31b Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 16 Jan 2016 21:40:49 +0100 Subject: [PATCH] avcodec/wmaenc: Check input for finiteness Fixes out of array read Fixes: 1b79b985cdf860ffa228c00ee5497051/signal_sigsegv_1f99d24_315_f0f1fdb451264b1138e9c24eb7d49181.wv Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind Signed-off-by: Michael Niedermayer --- libavcodec/wmaenc.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/libavcodec/wmaenc.c b/libavcodec/wmaenc.c index 54e83b66ad..abe8e4b876 100644 --- a/libavcodec/wmaenc.c +++ b/libavcodec/wmaenc.c @@ -98,7 +98,7 @@ static av_cold int encode_init(AVCodecContext *avctx) return 0; } -static void apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame) +static int apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame) { WMACodecContext *s = avctx->priv_data; float **audio = (float **) frame->extended_data; @@ -117,7 +117,13 @@ static void apply_window_and_mdct(AVCodecContext *avctx, const AVFrame *frame) win, len); s->fdsp->vector_fmul(s->frame_out[ch], s->frame_out[ch], win, len); mdct->mdct_calc(mdct, s->coefs[ch], s->output); + if (!isfinite(s->coefs[ch][0])) { + av_log(avctx, AV_LOG_ERROR, "Input contains NaN/+-Inf\n"); + return AVERROR(EINVAL); + } } + + return 0; } // FIXME use for decoding too @@ -364,7 +370,10 @@ static int encode_superframe(AVCodecContext *avctx, AVPacket *avpkt, s->block_len_bits = s->frame_len_bits; // required by non variable block len s->block_len = 1 << s->block_len_bits; - apply_window_and_mdct(avctx, frame); + ret = apply_window_and_mdct(avctx, frame); + + if (ret < 0) + return ret; if (s->ms_stereo) { float a, b;