From b5587fd2c6ce39bad7a5e7ebb3bd86b6469648de Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Thu, 30 Nov 2017 21:51:56 +0100 Subject: [PATCH] avcodec/jpeg2000: Only allocate Jpeg2000Pass for the encoder Reduces memory needed. Fixes: OOM Fixes: 4427/clusterfuzz-testcase-minimized-5106919271301120 Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/j2kenc.c | 4 +++- libavcodec/jpeg2000.c | 1 + libavcodec/jpeg2000.h | 2 +- 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index 78ec88a694..3e542af3c6 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -941,7 +941,9 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno } if (!prec->cblk[cblkno].data) prec->cblk[cblkno].data = av_malloc(1 + 8192); - if (!prec->cblk[cblkno].data) + if (!prec->cblk[cblkno].passes) + prec->cblk[cblkno].passes = av_malloc_array(JPEG2000_MAX_PASSES, sizeof (*prec->cblk[cblkno].passes)); + if (!prec->cblk[cblkno].data || !prec->cblk[cblkno].passes) return AVERROR(ENOMEM); encode_cblk(s, &t1, prec->cblk + cblkno, tile, xx1 - xx0, yy1 - yy0, bandpos, codsty->nreslevels - reslevelno - 1); diff --git a/libavcodec/jpeg2000.c b/libavcodec/jpeg2000.c index 8551cf8d6c..5f3965047f 100644 --- a/libavcodec/jpeg2000.c +++ b/libavcodec/jpeg2000.c @@ -606,6 +606,7 @@ void ff_jpeg2000_cleanup(Jpeg2000Component *comp, Jpeg2000CodingStyle *codsty) for (cblkno = 0; cblkno < nb_code_blocks; cblkno ++) { Jpeg2000Cblk *cblk = &prec->cblk[cblkno]; av_freep(&cblk->data); + av_freep(&cblk->passes); } av_freep(&prec->cblk); } diff --git a/libavcodec/jpeg2000.h b/libavcodec/jpeg2000.h index eaf7faf342..752feae96b 100644 --- a/libavcodec/jpeg2000.h +++ b/libavcodec/jpeg2000.h @@ -173,7 +173,7 @@ typedef struct Jpeg2000Cblk { int nb_terminations; int nb_terminationsinc; int data_start[JPEG2000_MAX_PASSES]; - Jpeg2000Pass passes[JPEG2000_MAX_PASSES]; + Jpeg2000Pass *passes; int coord[2][2]; // border coordinates {{x0, x1}, {y0, y1}} } Jpeg2000Cblk; // code block