avcodec/dnxhdenc: fix DNxHR 444 encoding crashes

Fixes #6649.
This commit is contained in:
Frédéric Devernay 2017-09-12 15:26:15 +02:00 committed by Paul B Mahol
parent 89a2472ec5
commit eec67f2522
2 changed files with 10 additions and 10 deletions

View File

@ -749,14 +749,14 @@ void dnxhd_get_blocks(DNXHDEncContext *ctx, int mb_x, int mb_y)
ptr_y = &ctx->edge_buf_y[0];
ptr_u = &ctx->edge_buf_uv[0][0];
ptr_v = &ctx->edge_buf_uv[1][0];
} else if (ctx->bit_depth == 10 && vdsp->emulated_edge_mc && ((mb_x << 3) + 8 > ctx->m.avctx->width ||
(mb_y << 3) + 8 > ctx->m.avctx->height)) {
int y_w = ctx->m.avctx->width - (mb_x << 3);
int y_h = ctx->m.avctx->height - (mb_y << 3);
} else if (ctx->bit_depth == 10 && vdsp->emulated_edge_mc && ((mb_x << 4) + 16 > ctx->m.avctx->width ||
(mb_y << 4) + 16 > ctx->m.avctx->height)) {
int y_w = ctx->m.avctx->width - (mb_x << 4);
int y_h = ctx->m.avctx->height - (mb_y << 4);
int uv_w = ctx->is_444 ? y_w : (y_w + 1) / 2;
int uv_h = y_h;
linesize = 16;
uvlinesize = 8 + 8 * ctx->is_444;
linesize = 32;
uvlinesize = 16 + 16 * ctx->is_444;
vdsp->emulated_edge_mc(&ctx->edge_buf_y[0], ptr_y,
linesize, ctx->m.linesize,
@ -771,8 +771,8 @@ void dnxhd_get_blocks(DNXHDEncContext *ctx, int mb_x, int mb_y)
uvlinesize / 2, 16,
0, 0, uv_w, uv_h);
dct_y_offset = bw * linesize;
dct_uv_offset = bw * uvlinesize;
dct_y_offset = bw * linesize / 2;
dct_uv_offset = bw * uvlinesize / 2;
ptr_y = &ctx->edge_buf_y[0];
ptr_u = &ctx->edge_buf_uv[0][0];
ptr_v = &ctx->edge_buf_uv[1][0];

View File

@ -75,8 +75,8 @@ typedef struct DNXHDEncContext {
int intra_quant_bias;
DECLARE_ALIGNED(16, int16_t, blocks)[12][64];
DECLARE_ALIGNED(16, uint8_t, edge_buf_y)[256];
DECLARE_ALIGNED(16, uint8_t, edge_buf_uv)[2][256];
DECLARE_ALIGNED(16, uint8_t, edge_buf_y)[512]; // has to hold 16x16 uint16 when depth=10
DECLARE_ALIGNED(16, uint8_t, edge_buf_uv)[2][512]; // has to hold 16x16 uint16_t when depth=10
int (*qmatrix_c) [64];
int (*qmatrix_l) [64];