From f701f088f4eedf86a51920cdbdddcd9895615d40 Mon Sep 17 00:00:00 2001 From: Lynne Date: Sat, 20 May 2023 15:49:01 +0200 Subject: [PATCH] vulkan_h264: reject end_frame being called without valid session parameters When seeking through MBAFF-coded H264, this can happen. Decoding calls end_frame without calling start_frame. We are unable to decode this, as no frame state has been set. Happens for both VAAPI and Vulkan. Could be an issue elsewhere, hence the individual commit. --- libavcodec/vulkan_h264.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/libavcodec/vulkan_h264.c b/libavcodec/vulkan_h264.c index 3d86255966..7df2e99617 100644 --- a/libavcodec/vulkan_h264.c +++ b/libavcodec/vulkan_h264.c @@ -508,10 +508,14 @@ static int vk_h264_end_frame(AVCodecContext *avctx) const H264Context *h = avctx->priv_data; H264Picture *pic = h->cur_pic_ptr; H264VulkanDecodePicture *hp = pic->hwaccel_picture_private; + FFVulkanDecodeContext *dec = avctx->internal->hwaccel_priv_data; FFVulkanDecodePicture *vp = &hp->vp; FFVulkanDecodePicture *rvp[H264_MAX_PICTURE_COUNT] = { 0 }; AVFrame *rav[H264_MAX_PICTURE_COUNT] = { 0 }; + if (!dec->session_params) + return AVERROR(EINVAL); + for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) { H264Picture *rp = hp->ref_src[i]; H264VulkanDecodePicture *rhp = rp->hwaccel_picture_private;