From 0ef8f03133a0bd83c74200a8cf30982c0f574016 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Fri, 14 Jul 2017 00:45:29 +0200 Subject: [PATCH] avcodec/aacdec_template: Fix undefined integer overflow in apply_tns() Fixes: runtime error: signed integer overflow: -2147483648 - 1202286525 cannot be represented in type 'int' Fixes: 2071/clusterfuzz-testcase-minimized-6036414271586304 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/aac_defines.h | 2 ++ libavcodec/aacdec_template.c | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libavcodec/aac_defines.h b/libavcodec/aac_defines.h index 3c79a8a4a1..438d78a7aa 100644 --- a/libavcodec/aac_defines.h +++ b/libavcodec/aac_defines.h @@ -35,6 +35,7 @@ #define AAC_RENAME(x) x ## _fixed #define AAC_RENAME_32(x) x ## _fixed_32 typedef int INTFLOAT; +typedef unsigned UINTFLOAT; ///< Equivalent to INTFLOAT, Used as temporal cast to avoid undefined sign overflow operations. typedef int64_t INT64FLOAT; typedef int16_t SHORTFLOAT; typedef SoftFloat AAC_FLOAT; @@ -83,6 +84,7 @@ typedef int AAC_SIGNE; #define AAC_RENAME(x) x #define AAC_RENAME_32(x) x typedef float INTFLOAT; +typedef float UINTFLOAT; typedef float INT64FLOAT; typedef float SHORTFLOAT; typedef float AAC_FLOAT; diff --git a/libavcodec/aacdec_template.c b/libavcodec/aacdec_template.c index 4cbbc6023a..c60a6ca096 100644 --- a/libavcodec/aacdec_template.c +++ b/libavcodec/aacdec_template.c @@ -2389,7 +2389,7 @@ static int decode_extension_payload(AACContext *ac, GetBitContext *gb, int cnt, * @param decode 1 if tool is used normally, 0 if tool is used in LTP. * @param coef spectral coefficients */ -static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns, +static void apply_tns(INTFLOAT coef_param[1024], TemporalNoiseShaping *tns, IndividualChannelStream *ics, int decode) { const int mmm = FFMIN(ics->tns_max_bands, ics->max_sfb); @@ -2397,6 +2397,7 @@ static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns, int bottom, top, order, start, end, size, inc; INTFLOAT lpc[TNS_MAX_ORDER]; INTFLOAT tmp[TNS_MAX_ORDER+1]; + UINTFLOAT *coef = coef_param; for (w = 0; w < ics->num_windows; w++) { bottom = ics->num_swb; @@ -2426,7 +2427,7 @@ static void apply_tns(INTFLOAT coef[1024], TemporalNoiseShaping *tns, // ar filter for (m = 0; m < size; m++, start += inc) for (i = 1; i <= FFMIN(m, order); i++) - coef[start] -= AAC_MUL26(coef[start - i * inc], lpc[i - 1]); + coef[start] -= AAC_MUL26((INTFLOAT)coef[start - i * inc], lpc[i - 1]); } else { // ma filter for (m = 0; m < size; m++, start += inc) {