From a7aac19933a91e22d77b0b4dd4ecd61edf52d43f Mon Sep 17 00:00:00 2001 From: Rostislav Pehlivanov Date: Wed, 8 Nov 2017 23:50:04 +0000 Subject: [PATCH] vc2enc_dwt: pad the temporary buffer by the slice size Since non-Haar wavelets need to look into pixels outside the frame, we need to pad the buffer. The old factor of two seemed to be a workaround that fact and only padded to the left and bottom. This correctly pads by the slice size and as such reduces memory usage and potential exploits. Reported by Liu Bingchang. Ideally, there should be no temporary buffer but the encoder is designed to deinterleave the coefficients into the classical wavelet structure with the lower frequency values in the top left corner. Signed-off-by: Rostislav Pehlivanov (cherry picked from commit 3228ac730c11eca49d5680d5550128e397061c85) --- libavcodec/vc2enc.c | 3 ++- libavcodec/vc2enc_dwt.c | 12 +++++++++--- libavcodec/vc2enc_dwt.h | 4 +++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/libavcodec/vc2enc.c b/libavcodec/vc2enc.c index eda390163f..745c6e974d 100644 --- a/libavcodec/vc2enc.c +++ b/libavcodec/vc2enc.c @@ -1190,7 +1190,8 @@ static av_cold int vc2_encode_init(AVCodecContext *avctx) /* DWT init */ if (ff_vc2enc_init_transforms(&s->transform_args[i].t, s->plane[i].coef_stride, - s->plane[i].dwt_height)) + s->plane[i].dwt_height, + s->slice_width, s->slice_height)) goto alloc_fail; } diff --git a/libavcodec/vc2enc_dwt.c b/libavcodec/vc2enc_dwt.c index c60b003a31..d22af8a313 100644 --- a/libavcodec/vc2enc_dwt.c +++ b/libavcodec/vc2enc_dwt.c @@ -255,21 +255,27 @@ static void vc2_subband_dwt_haar_shift(VC2TransformContext *t, dwtcoef *data, dwt_haar(t, data, stride, width, height, 1); } -av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_width, int p_height) +av_cold int ff_vc2enc_init_transforms(VC2TransformContext *s, int p_stride, + int p_height, int slice_w, int slice_h) { s->vc2_subband_dwt[VC2_TRANSFORM_9_7] = vc2_subband_dwt_97; s->vc2_subband_dwt[VC2_TRANSFORM_5_3] = vc2_subband_dwt_53; s->vc2_subband_dwt[VC2_TRANSFORM_HAAR] = vc2_subband_dwt_haar; s->vc2_subband_dwt[VC2_TRANSFORM_HAAR_S] = vc2_subband_dwt_haar_shift; - s->buffer = av_malloc(2*p_width*p_height*sizeof(dwtcoef)); + /* Pad by the slice size, only matters for non-Haar wavelets */ + s->buffer = av_calloc((p_stride + slice_w)*(p_height + slice_h), sizeof(dwtcoef)); if (!s->buffer) return 1; + s->padding = (slice_h >> 1)*p_stride + (slice_w >> 1); + s->buffer += s->padding; + return 0; } av_cold void ff_vc2enc_free_transforms(VC2TransformContext *s) { - av_freep(&s->buffer); + av_free(s->buffer - s->padding); + s->buffer = NULL; } diff --git a/libavcodec/vc2enc_dwt.h b/libavcodec/vc2enc_dwt.h index 7fbbfbe0ed..a6932bcdaf 100644 --- a/libavcodec/vc2enc_dwt.h +++ b/libavcodec/vc2enc_dwt.h @@ -41,12 +41,14 @@ enum VC2TransformType { typedef struct VC2TransformContext { dwtcoef *buffer; + int padding; void (*vc2_subband_dwt[VC2_TRANSFORMS_NB])(struct VC2TransformContext *t, dwtcoef *data, ptrdiff_t stride, int width, int height); } VC2TransformContext; -int ff_vc2enc_init_transforms(VC2TransformContext *t, int p_width, int p_height); +int ff_vc2enc_init_transforms(VC2TransformContext *t, int p_stride, int p_height, + int slice_w, int slice_h); void ff_vc2enc_free_transforms(VC2TransformContext *t); #endif /* AVCODEC_VC2ENC_DWT_H */