From eec67f25224a48047da57be18b610b11b0fd0bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fr=C3=A9d=C3=A9ric=20Devernay?= Date: Tue, 12 Sep 2017 15:26:15 +0200 Subject: [PATCH] avcodec/dnxhdenc: fix DNxHR 444 encoding crashes Fixes #6649. --- libavcodec/dnxhdenc.c | 16 ++++++++-------- libavcodec/dnxhdenc.h | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/libavcodec/dnxhdenc.c b/libavcodec/dnxhdenc.c index 5a0e6de6a5..0d80381a2d 100644 --- a/libavcodec/dnxhdenc.c +++ b/libavcodec/dnxhdenc.c @@ -749,14 +749,14 @@ void dnxhd_get_blocks(DNXHDEncContext *ctx, int mb_x, int mb_y) ptr_y = &ctx->edge_buf_y[0]; ptr_u = &ctx->edge_buf_uv[0][0]; ptr_v = &ctx->edge_buf_uv[1][0]; - } else if (ctx->bit_depth == 10 && vdsp->emulated_edge_mc && ((mb_x << 3) + 8 > ctx->m.avctx->width || - (mb_y << 3) + 8 > ctx->m.avctx->height)) { - int y_w = ctx->m.avctx->width - (mb_x << 3); - int y_h = ctx->m.avctx->height - (mb_y << 3); + } else if (ctx->bit_depth == 10 && vdsp->emulated_edge_mc && ((mb_x << 4) + 16 > ctx->m.avctx->width || + (mb_y << 4) + 16 > ctx->m.avctx->height)) { + int y_w = ctx->m.avctx->width - (mb_x << 4); + int y_h = ctx->m.avctx->height - (mb_y << 4); int uv_w = ctx->is_444 ? y_w : (y_w + 1) / 2; int uv_h = y_h; - linesize = 16; - uvlinesize = 8 + 8 * ctx->is_444; + linesize = 32; + uvlinesize = 16 + 16 * ctx->is_444; vdsp->emulated_edge_mc(&ctx->edge_buf_y[0], ptr_y, linesize, ctx->m.linesize, @@ -771,8 +771,8 @@ void dnxhd_get_blocks(DNXHDEncContext *ctx, int mb_x, int mb_y) uvlinesize / 2, 16, 0, 0, uv_w, uv_h); - dct_y_offset = bw * linesize; - dct_uv_offset = bw * uvlinesize; + dct_y_offset = bw * linesize / 2; + dct_uv_offset = bw * uvlinesize / 2; ptr_y = &ctx->edge_buf_y[0]; ptr_u = &ctx->edge_buf_uv[0][0]; ptr_v = &ctx->edge_buf_uv[1][0]; diff --git a/libavcodec/dnxhdenc.h b/libavcodec/dnxhdenc.h index 9b43f6e9c6..26c3eec695 100644 --- a/libavcodec/dnxhdenc.h +++ b/libavcodec/dnxhdenc.h @@ -75,8 +75,8 @@ typedef struct DNXHDEncContext { int intra_quant_bias; DECLARE_ALIGNED(16, int16_t, blocks)[12][64]; - DECLARE_ALIGNED(16, uint8_t, edge_buf_y)[256]; - DECLARE_ALIGNED(16, uint8_t, edge_buf_uv)[2][256]; + DECLARE_ALIGNED(16, uint8_t, edge_buf_y)[512]; // has to hold 16x16 uint16 when depth=10 + DECLARE_ALIGNED(16, uint8_t, edge_buf_uv)[2][512]; // has to hold 16x16 uint16_t when depth=10 int (*qmatrix_c) [64]; int (*qmatrix_l) [64];