From 72e691314027b08955679319394dd0d8477973b7 Mon Sep 17 00:00:00 2001 From: Michael Niedermayer Date: Sun, 23 Feb 2014 18:20:05 +0100 Subject: [PATCH] avcodec/h264: clear chroma planes when flags gray is used Fixes Ticket3397 Signed-off-by: Michael Niedermayer --- libavcodec/h264.c | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index a01febf1b9..f9d10a2f79 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -454,6 +454,18 @@ static int alloc_picture(H264Context *h, Picture *pic) pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data; } } + if (!h->avctx->hwaccel && CONFIG_GRAY && h->flags & CODEC_FLAG_GRAY && pic->f.data[2]) { + int h_chroma_shift, v_chroma_shift; + av_pix_fmt_get_chroma_sub_sample(pic->f.format, + &h_chroma_shift, &v_chroma_shift); + + for(i=0; iavctx->height, v_chroma_shift); i++) { + memset(pic->f.data[1] + pic->f.linesize[1]*i, + 0x80, FF_CEIL_RSHIFT(h->avctx->width, h_chroma_shift)); + memset(pic->f.data[2] + pic->f.linesize[2]*i, + 0x80, FF_CEIL_RSHIFT(h->avctx->width, h_chroma_shift)); + } + } if (!h->qscale_table_pool) { ret = init_table_pools(h);