From 2e8f3cbcda536970aee03f5543185d7fc01232a1 Mon Sep 17 00:00:00 2001 From: Uoti Urpala Date: Tue, 7 Aug 2012 14:48:06 +0300 Subject: [PATCH] h264: vdpau: fix crash with unsupported colorspace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The h264_vdpau decoder crashed if output colorspace was not 8-bit 420. Add a check to error out instead (current hardware does not support other colorspaces, so successful decoding is not possible). Signed-off-by: Martin Storsjö --- libavcodec/h264.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 2019de26ba..a191bc74de 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -3876,6 +3876,15 @@ again: if (avctx->bits_per_raw_sample != h->sps.bit_depth_luma || h->cur_chroma_format_idc != h->sps.chroma_format_idc) { + if (s->avctx->codec->capabilities & CODEC_CAP_HWACCEL_VDPAU + && (h->sps.bit_depth_luma != 8 || + h->sps.chroma_format_idc > 1)) { + av_log(avctx, AV_LOG_ERROR, + "VDPAU decoding does not support video " + "colorspace\n"); + buf_index = -1; + goto end; + } if (h->sps.bit_depth_luma >= 8 && h->sps.bit_depth_luma <= 10) { avctx->bits_per_raw_sample = h->sps.bit_depth_luma; h->cur_chroma_format_idc = h->sps.chroma_format_idc;