From a84fbd747119841942c67d2f55cc796ab25cd245 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Wed, 19 Jun 2024 20:58:05 +0200 Subject: [PATCH] avcodec/j2kenc: Merge dwt_norm into lambda This moves computations out of a loop This may help with UB in vsynth*-jpeg2000-yuva444p16 Fixes: signed integer overflow: 31665934879948800 * 9998 cannot be represented in type 'long' Fixes: 69024/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_JPEG2000_fuzzer-5949662967169024 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/j2kenc.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index 8cf82f7216..91e66d8104 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -1349,7 +1349,7 @@ static void makelayers(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) } } -static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) +static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda) { int passno, res = 0; for (passno = 0; passno < cblk->npasses; passno++){ @@ -1361,7 +1361,7 @@ static int getcut(Jpeg2000Cblk *cblk, uint64_t lambda, int dwt_norm) dd = cblk->passes[passno].disto - (res ? cblk->passes[res-1].disto : 0); - if (((dd * dwt_norm) >> WMSEDEC_SHIFT) * dwt_norm >= dr * lambda) + if (dd >= dr * lambda) res = passno+1; } return res; @@ -1384,11 +1384,12 @@ static void truncpasses(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile) Jpeg2000Band *band = reslevel->band + bandno; Jpeg2000Prec *prec = band->prec + precno; + int64_t dwt_norm = dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15; + int64_t lambda_prime = av_rescale(s->lambda, 1 << WMSEDEC_SHIFT, dwt_norm * dwt_norm); for (cblkno = 0; cblkno < prec->nb_codeblocks_height * prec->nb_codeblocks_width; cblkno++){ Jpeg2000Cblk *cblk = prec->cblk + cblkno; - cblk->ninclpasses = getcut(cblk, s->lambda, - (int64_t)dwt_norms[codsty->transform == FF_DWT53][bandpos][lev] * (int64_t)band->i_stepsize >> 15); + cblk->ninclpasses = getcut(cblk, lambda_prime); cblk->layers[0].data_start = cblk->data; cblk->layers[0].cum_passes = cblk->ninclpasses; cblk->layers[0].npasses = cblk->ninclpasses;