From 2b3206891649f317c20993411efef4bee39ae784 Mon Sep 17 00:00:00 2001 From: Linjie Fu Date: Tue, 31 Mar 2020 23:34:00 +0800 Subject: [PATCH] lavc/vaapi_encode: add FF_CODEC_CAP_INIT_CLEANUP caps for encoders ff_vaapi_encode_close() is not enough to free the resources like cbs if initialization failure happens after codec->configure (except for vp8/vp9). We need to call avctx->codec->close() to deallocate, otherwise memory leak happens. Add FF_CODEC_CAP_INIT_CLEANUP for vaapi encoders and deallocate the resources at free_and_end inside avcodec_open2(). Reviewed-by: Timo Rothenpieler Signed-off-by: Linjie Fu --- libavcodec/vaapi_encode.c | 1 - libavcodec/vaapi_encode_h264.c | 1 + libavcodec/vaapi_encode_h265.c | 1 + libavcodec/vaapi_encode_mjpeg.c | 1 + libavcodec/vaapi_encode_mpeg2.c | 1 + libavcodec/vaapi_encode_vp8.c | 1 + libavcodec/vaapi_encode_vp9.c | 1 + 7 files changed, 6 insertions(+), 1 deletion(-) diff --git a/libavcodec/vaapi_encode.c b/libavcodec/vaapi_encode.c index f551967ad6..cb05ebd774 100644 --- a/libavcodec/vaapi_encode.c +++ b/libavcodec/vaapi_encode.c @@ -2366,7 +2366,6 @@ av_cold int ff_vaapi_encode_init(AVCodecContext *avctx) return 0; fail: - ff_vaapi_encode_close(avctx); return err; } diff --git a/libavcodec/vaapi_encode_h264.c b/libavcodec/vaapi_encode_h264.c index 67b1ecae1b..e195650ef7 100644 --- a/libavcodec/vaapi_encode_h264.c +++ b/libavcodec/vaapi_encode_h264.c @@ -1356,6 +1356,7 @@ AVCodec ff_h264_vaapi_encoder = { .close = &vaapi_encode_h264_close, .priv_class = &vaapi_encode_h264_class, .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, .defaults = vaapi_encode_h264_defaults, .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_VAAPI, diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c index e20e8cbdd0..92e0510911 100644 --- a/libavcodec/vaapi_encode_h265.c +++ b/libavcodec/vaapi_encode_h265.c @@ -1292,6 +1292,7 @@ AVCodec ff_hevc_vaapi_encoder = { .close = &vaapi_encode_h265_close, .priv_class = &vaapi_encode_h265_class, .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, .defaults = vaapi_encode_h265_defaults, .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_VAAPI, diff --git a/libavcodec/vaapi_encode_mjpeg.c b/libavcodec/vaapi_encode_mjpeg.c index 2b5d3bf4f9..8b441d36c3 100644 --- a/libavcodec/vaapi_encode_mjpeg.c +++ b/libavcodec/vaapi_encode_mjpeg.c @@ -565,6 +565,7 @@ AVCodec ff_mjpeg_vaapi_encoder = { .priv_class = &vaapi_encode_mjpeg_class, .capabilities = AV_CODEC_CAP_HARDWARE | AV_CODEC_CAP_INTRA_ONLY, + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, .defaults = vaapi_encode_mjpeg_defaults, .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_VAAPI, diff --git a/libavcodec/vaapi_encode_mpeg2.c b/libavcodec/vaapi_encode_mpeg2.c index 0398c4cd8c..02c76552ef 100644 --- a/libavcodec/vaapi_encode_mpeg2.c +++ b/libavcodec/vaapi_encode_mpeg2.c @@ -702,6 +702,7 @@ AVCodec ff_mpeg2_vaapi_encoder = { .close = &vaapi_encode_mpeg2_close, .priv_class = &vaapi_encode_mpeg2_class, .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, .defaults = vaapi_encode_mpeg2_defaults, .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_VAAPI, diff --git a/libavcodec/vaapi_encode_vp8.c b/libavcodec/vaapi_encode_vp8.c index 16cbcbd81c..cff926baae 100644 --- a/libavcodec/vaapi_encode_vp8.c +++ b/libavcodec/vaapi_encode_vp8.c @@ -257,6 +257,7 @@ AVCodec ff_vp8_vaapi_encoder = { .close = &ff_vaapi_encode_close, .priv_class = &vaapi_encode_vp8_class, .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, .defaults = vaapi_encode_vp8_defaults, .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_VAAPI, diff --git a/libavcodec/vaapi_encode_vp9.c b/libavcodec/vaapi_encode_vp9.c index d05319ef39..8514b85991 100644 --- a/libavcodec/vaapi_encode_vp9.c +++ b/libavcodec/vaapi_encode_vp9.c @@ -291,6 +291,7 @@ AVCodec ff_vp9_vaapi_encoder = { .close = &ff_vaapi_encode_close, .priv_class = &vaapi_encode_vp9_class, .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, + .caps_internal = FF_CODEC_CAP_INIT_CLEANUP, .defaults = vaapi_encode_vp9_defaults, .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_VAAPI,