From 9c9ede44f37e291677c0db1c5fabf1c3f6106008 Mon Sep 17 00:00:00 2001 From: Hendrik Leppkes Date: Mon, 18 Mar 2013 18:54:43 +0100 Subject: [PATCH 1/2] h264: allocate hwaccel privdata after the frame buffer This ensures the hwaccel privdata does not leak when a frame buffer could not be allocated (and toggle the assert when the frame is re-used). Having no frame buffer available is quite common when using the DXVA2 hwaccel in situations where the DXVA2 renderer is being re-allocated, for example when moving between displays. Signed-off-by: Anton Khirnov --- libavcodec/h264.c | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 9b196c711d..98ace67425 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -318,6 +318,15 @@ static int alloc_picture(H264Context *h, Picture *pic) av_assert0(!pic->f.data[0]); + pic->tf.f = &pic->f; + ret = ff_thread_get_buffer(h->avctx, &pic->tf, pic->reference ? + AV_GET_BUFFER_FLAG_REF : 0); + if (ret < 0) + goto fail; + + h->linesize = pic->f.linesize[0]; + h->uvlinesize = pic->f.linesize[1]; + if (h->avctx->hwaccel) { const AVHWAccel *hwaccel = h->avctx->hwaccel; av_assert0(!pic->hwaccel_picture_private); @@ -328,14 +337,6 @@ static int alloc_picture(H264Context *h, Picture *pic) pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data; } } - pic->tf.f = &pic->f; - ret = ff_thread_get_buffer(h->avctx, &pic->tf, pic->reference ? - AV_GET_BUFFER_FLAG_REF : 0); - if (ret < 0) - goto fail; - - h->linesize = pic->f.linesize[0]; - h->uvlinesize = pic->f.linesize[1]; if (!h->qscale_table_pool) { ret = init_table_pools(h); From c3ebfcd6e1327ca7bbcaee822e593c2da6cfd352 Mon Sep 17 00:00:00 2001 From: Hendrik Leppkes Date: Mon, 18 Mar 2013 18:54:44 +0100 Subject: [PATCH 2/2] mpegvideo: allocate hwaccel privdata after the frame buffer This ensures the hwaccel privdata does not leak when a frame buffer could not be allocated (and toggle the assert when the frame is re-used). Having no frame buffer available is quite common when using the DXVA2 hwaccel in situations where the DXVA2 renderer is being re-allocated, for example when moving between displays. Signed-off-by: Anton Khirnov --- libavcodec/mpegvideo.c | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/libavcodec/mpegvideo.c b/libavcodec/mpegvideo.c index ae6cdefe6b..dab1485fcc 100644 --- a/libavcodec/mpegvideo.c +++ b/libavcodec/mpegvideo.c @@ -266,18 +266,6 @@ static int alloc_frame_buffer(MpegEncContext *s, Picture *pic) { int r, ret; - if (s->avctx->hwaccel) { - assert(!pic->hwaccel_picture_private); - if (s->avctx->hwaccel->priv_data_size) { - pic->hwaccel_priv_buf = av_buffer_allocz(s->avctx->hwaccel->priv_data_size); - if (!pic->hwaccel_priv_buf) { - av_log(s->avctx, AV_LOG_ERROR, "alloc_frame_buffer() failed (hwaccel private data allocation)\n"); - return -1; - } - pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data; - } - } - pic->tf.f = &pic->f; if (s->codec_id != AV_CODEC_ID_WMV3IMAGE && s->codec_id != AV_CODEC_ID_VC1IMAGE && @@ -294,11 +282,21 @@ static int alloc_frame_buffer(MpegEncContext *s, Picture *pic) if (r < 0 || !pic->f.data[0]) { av_log(s->avctx, AV_LOG_ERROR, "get_buffer() failed (%d %p)\n", r, pic->f.data[0]); - av_buffer_unref(&pic->hwaccel_priv_buf); - pic->hwaccel_picture_private = NULL; return -1; } + if (s->avctx->hwaccel) { + assert(!pic->hwaccel_picture_private); + if (s->avctx->hwaccel->priv_data_size) { + pic->hwaccel_priv_buf = av_buffer_allocz(s->avctx->hwaccel->priv_data_size); + if (!pic->hwaccel_priv_buf) { + av_log(s->avctx, AV_LOG_ERROR, "alloc_frame_buffer() failed (hwaccel private data allocation)\n"); + return -1; + } + pic->hwaccel_picture_private = pic->hwaccel_priv_buf->data; + } + } + if (s->linesize && (s->linesize != pic->f.linesize[0] || s->uvlinesize != pic->f.linesize[1])) { av_log(s->avctx, AV_LOG_ERROR,