From 511e6f17f493719058229630c7db4d8d7c05aeac Mon Sep 17 00:00:00 2001 From: Andrew D'Addesio Date: Sat, 2 Dec 2017 11:36:25 -0600 Subject: [PATCH] opus_silk: Fix arithmetic overflow (per RFC8251) As per Sec.6 of RFC8251: Integer Wrap-Around in Inverse Gain Computation 32-bit integer overflow in Levinson recursion. Affects silk_is_lpc_stable(). Signed-off-by: Andrew D'Addesio --- libavcodec/opus_silk.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/libavcodec/opus_silk.c b/libavcodec/opus_silk.c index 3c9c849c21..344333cc18 100644 --- a/libavcodec/opus_silk.c +++ b/libavcodec/opus_silk.c @@ -185,8 +185,15 @@ static inline int silk_is_lpc_stable(const int16_t lpc[16], int order) row = lpc32[k & 1]; for (j = 0; j < k; j++) { - int x = prevrow[j] - ROUND_MULL(prevrow[k - j - 1], rc, 31); - row[j] = ROUND_MULL(x, gain, fbits); + int x = av_sat_sub32(prevrow[j], ROUND_MULL(prevrow[k - j - 1], rc, 31)); + int64_t tmp = ROUND_MULL(x, gain, fbits); + + /* per RFC 8251 section 6, if this calculation overflows, the filter + is considered unstable. */ + if (tmp < INT32_MIN || tmp > INT32_MAX) + return 0; + + row[j] = (int32_t)tmp; } } }