RepoMirrors
/
ffmpeg
mirror of https://git.ffmpeg.org/ffmpeg.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

avcodec/mss34dsp: Fix multiple signed integer overflow 

Fixes: 1387/clusterfuzz-testcase-minimized-4802757766676480

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/targets/ffmpeg
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 464c4b86ee)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
This commit is contained in:
Michael Niedermayer 2017-05-07 14:12:04 +02:00
parent 7bf97c4039
commit 5e07278783
1 changed files with 22 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
libavcodec/mss34dsp.c
View File

@ -62,30 +62,30 @@ void ff_mss34_gen_quant_mat(uint16_t *qmat, int quality, int luma)
}
#define DCT_TEMPLATE(blk, step, SOP, shift) \
const int t0 = -39409 * blk[7 * step] - 58980 * blk[1 * step]; \
const int t1 = 39410 * blk[1 * step] - 58980 * blk[7 * step]; \
const int t2 = -33410 * blk[5 * step] - 167963 * blk[3 * step]; \
const int t3 = 33410 * blk[3 * step] - 167963 * blk[5 * step]; \
const int t4 = blk[3 * step] + blk[7 * step]; \
const int t5 = blk[1 * step] + blk[5 * step]; \
const int t6 = 77062 * t4 + 51491 * t5; \
const int t7 = 77062 * t5 - 51491 * t4; \
const int t8 = 35470 * blk[2 * step] - 85623 * blk[6 * step]; \
const int t9 = 35470 * blk[6 * step] + 85623 * blk[2 * step]; \
const int tA = SOP(blk[0 * step] - blk[4 * step]); \
const int tB = SOP(blk[0 * step] + blk[4 * step]); \
const unsigned t0 =-39409U * blk[7 * step] - 58980U * blk[1 * step]; \
const unsigned t1 = 39410U * blk[1 * step] - 58980U * blk[7 * step]; \
const unsigned t2 =-33410U * blk[5 * step] -167963U * blk[3 * step]; \
const unsigned t3 = 33410U * blk[3 * step] -167963U * blk[5 * step]; \
const unsigned t4 = blk[3 * step] + blk[7 * step]; \
const unsigned t5 = blk[1 * step] + blk[5 * step]; \
const unsigned t6 = 77062U * t4 + 51491U * t5; \
const unsigned t7 = 77062U * t5 - 51491U * t4; \
const unsigned t8 = 35470U * blk[2 * step] - 85623U * blk[6 * step]; \
const unsigned t9 = 35470U * blk[6 * step] + 85623U * blk[2 * step]; \
const unsigned tA = SOP(blk[0 * step] - blk[4 * step]); \
const unsigned tB = SOP(blk[0 * step] + blk[4 * step]); \
\
blk[0 * step] = ( t1 + t6 + t9 + tB) >> shift; \
blk[1 * step] = ( t3 + t7 + t8 + tA) >> shift; \
blk[2 * step] = ( t2 + t6 - t8 + tA) >> shift; \
blk[3 * step] = ( t0 + t7 - t9 + tB) >> shift; \
blk[4 * step] = (-(t0 + t7) - t9 + tB) >> shift; \
blk[5 * step] = (-(t2 + t6) - t8 + tA) >> shift; \
blk[6 * step] = (-(t3 + t7) + t8 + tA) >> shift; \
blk[7 * step] = (-(t1 + t6) + t9 + tB) >> shift; \
blk[0 * step] = (int)( t1 + t6 + t9 + tB) >> shift; \
blk[1 * step] = (int)( t3 + t7 + t8 + tA) >> shift; \
blk[2 * step] = (int)( t2 + t6 - t8 + tA) >> shift; \
blk[3 * step] = (int)( t0 + t7 - t9 + tB) >> shift; \
blk[4 * step] = (int)(-(t0 + t7) - t9 + tB) >> shift; \
blk[5 * step] = (int)(-(t2 + t6) - t8 + tA) >> shift; \
blk[6 * step] = (int)(-(t3 + t7) + t8 + tA) >> shift; \
blk[7 * step] = (int)(-(t1 + t6) + t9 + tB) >> shift; \
#define SOP_ROW(a) (((a) << 16) + 0x2000)
#define SOP_COL(a) (((a) + 32) << 16)
#define SOP_ROW(a) (((a) * (1U << 16)) + 0x2000)
#define SOP_COL(a) (((a) + 32) * (1U << 16))
void ff_mss34_dct_put(uint8_t *dst, int stride, int *block)
{