From 3d5822d9cf07d08bce82903e4715658f46b01b5c Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sat, 18 Nov 2017 01:33:18 +0100 Subject: [PATCH] avcodec/jpeg2000: Dynamically allocate codeblock data Fixes: OOM Fixes: 3541/clusterfuzz-testcase-minimized-6469958596820992 Adds support for decoding codeblock data larger than 8kb Reduces decoder memory consumption Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg Signed-off-by: Michael Niedermayer --- libavcodec/j2kenc.c | 8 ++++++-- libavcodec/jpeg2000.c | 12 ++++++++++-- libavcodec/jpeg2000.h | 4 ++-- libavcodec/jpeg2000dec.c | 22 +++++++++++++++++++--- 4 files changed, 37 insertions(+), 9 deletions(-) diff --git a/libavcodec/j2kenc.c b/libavcodec/j2kenc.c index 2135ece540..362c7f0f49 100644 --- a/libavcodec/j2kenc.c +++ b/libavcodec/j2kenc.c @@ -663,7 +663,8 @@ static void encode_cblk(Jpeg2000EncoderContext *s, Jpeg2000T1Context *t1, Jpeg20 bpno = cblk->nonzerobits - 1; } - ff_mqc_initenc(&t1->mqc, cblk->data); + cblk->data[0] = 0; + ff_mqc_initenc(&t1->mqc, cblk->data + 1); for (passno = 0; bpno >= 0; passno++){ nmsedec=0; @@ -796,7 +797,7 @@ static int encode_packet(Jpeg2000EncoderContext *s, Jpeg2000ResLevel *rlevel, in if (cblk->ninclpasses){ if (s->buf_end - s->buf < cblk->passes[cblk->ninclpasses-1].rate) return -1; - bytestream_put_buffer(&s->buf, cblk->data, cblk->passes[cblk->ninclpasses-1].rate + bytestream_put_buffer(&s->buf, cblk->data + 1, cblk->passes[cblk->ninclpasses-1].rate - cblk->passes[cblk->ninclpasses-1].flushed_len); bytestream_put_buffer(&s->buf, cblk->passes[cblk->ninclpasses-1].flushed, cblk->passes[cblk->ninclpasses-1].flushed_len); @@ -937,6 +938,9 @@ static int encode_tile(Jpeg2000EncoderContext *s, Jpeg2000Tile *tile, int tileno } } } + prec->cblk[cblkno].data = av_malloc(1 + 8192); + if (!prec->cblk[cblkno].data) + return AVERROR(ENOMEM); encode_cblk(s, &t1, prec->cblk + cblkno, tile, xx1 - xx0, yy1 - yy0, bandpos, codsty->nreslevels - reslevelno - 1); xx0 = xx1; diff --git a/libavcodec/jpeg2000.c b/libavcodec/jpeg2000.c index afeb9df27c..8551cf8d6c 100644 --- a/libavcodec/jpeg2000.c +++ b/libavcodec/jpeg2000.c @@ -357,7 +357,6 @@ static int init_prec(Jpeg2000Band *band, comp->reslevel[reslevelno-1].coord[1][0]; } - cblk->zero = 0; cblk->lblock = 3; cblk->length = 0; memset(cblk->lengthinc, 0, sizeof(cblk->lengthinc)); @@ -598,9 +597,18 @@ void ff_jpeg2000_cleanup(Jpeg2000Component *comp, Jpeg2000CodingStyle *codsty) for (precno = 0; precno < reslevel->num_precincts_x * reslevel->num_precincts_y; precno++) { if (band->prec) { Jpeg2000Prec *prec = band->prec + precno; + int nb_code_blocks = prec->nb_codeblocks_height * prec->nb_codeblocks_width; + av_freep(&prec->zerobits); av_freep(&prec->cblkincl); - av_freep(&prec->cblk); + if (prec->cblk) { + int cblkno; + for (cblkno = 0; cblkno < nb_code_blocks; cblkno ++) { + Jpeg2000Cblk *cblk = &prec->cblk[cblkno]; + av_freep(&cblk->data); + } + av_freep(&prec->cblk); + } } } diff --git a/libavcodec/jpeg2000.h b/libavcodec/jpeg2000.h index 8a022ad918..eaf7faf342 100644 --- a/libavcodec/jpeg2000.h +++ b/libavcodec/jpeg2000.h @@ -168,8 +168,8 @@ typedef struct Jpeg2000Cblk { uint16_t lengthinc[JPEG2000_MAX_PASSES]; uint8_t nb_lengthinc; uint8_t lblock; - uint8_t zero; - uint8_t data[8192]; + uint8_t *data; + size_t data_allocated; int nb_terminations; int nb_terminationsinc; int data_start[JPEG2000_MAX_PASSES]; diff --git a/libavcodec/jpeg2000dec.c b/libavcodec/jpeg2000dec.c index 62b9009a82..0309b1f6fb 100644 --- a/libavcodec/jpeg2000dec.c +++ b/libavcodec/jpeg2000dec.c @@ -1001,10 +1001,18 @@ static int jpeg2000_decode_packet(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile, if ((ret = get_bits(s, av_log2(newpasses1) + cblk->lblock)) < 0) return ret; - if (ret > sizeof(cblk->data)) { + if (ret > cblk->data_allocated) { + size_t new_size = FFMAX(2*cblk->data_allocated, ret); + void *new = av_realloc(cblk->data, new_size); + if (new) { + cblk->data = new; + cblk->data_allocated = new_size; + } + } + if (ret > cblk->data_allocated) { avpriv_request_sample(s->avctx, "Block with lengthinc greater than %"SIZE_SPECIFIER"", - sizeof(cblk->data)); + cblk->data_allocated); return AVERROR_PATCHWELCOME; } cblk->lengthinc[cblk->nb_lengthinc++] = ret; @@ -1030,8 +1038,16 @@ static int jpeg2000_decode_packet(Jpeg2000DecoderContext *s, Jpeg2000Tile *tile, for (cblkno = 0; cblkno < nb_code_blocks; cblkno++) { Jpeg2000Cblk *cblk = prec->cblk + cblkno; for (cwsno = 0; cwsno < cblk->nb_lengthinc; cwsno ++) { + if (cblk->data_allocated < cblk->length + cblk->lengthinc[cwsno] + 4) { + size_t new_size = FFMAX(2*cblk->data_allocated, cblk->length + cblk->lengthinc[cwsno] + 4); + void *new = av_realloc(cblk->data, new_size); + if (new) { + cblk->data = new; + cblk->data_allocated = new_size; + } + } if ( bytestream2_get_bytes_left(&s->g) < cblk->lengthinc[cwsno] - || sizeof(cblk->data) < cblk->length + cblk->lengthinc[cwsno] + 4 + || cblk->data_allocated < cblk->length + cblk->lengthinc[cwsno] + 4 ) { av_log(s->avctx, AV_LOG_ERROR, "Block length %"PRIu16" or lengthinc %d is too large, left %d\n",