From 6966548c1bd8a07548cab72309c4e792166c9603 Mon Sep 17 00:00:00 2001 From: sharpbai Date: Tue, 13 Aug 2019 14:18:21 +0800 Subject: [PATCH] avcodec/videotoolboxenc: fix encoding frame crash on iOS 11 On iOS 11, encoding a frame may return error with log "Error encoding frame 0", which means vtenc_output_callback is called with status=0 and sample_buffer=NULL. Then the encoding session will be crashed on next callback wether or not closing the codec context. Let us look through the link below introducing VTCompressionOutputCallback, https://developer.apple.com/documentation/videotoolbox/vtcompressionoutputcallback?language=objc "status=0" (noErr) means compression was successful. "sampleBuffer=NULL" means the frame was dropped when compression was successful (status=0) or compression was not successful (status!=0). So we should not set AVERROR_EXTERNAL on "status=0" and "sample_buffer=NULL" as it is not a error. The fix is that we only set AVERROR_EXTERNAL with status value non zero. When sample_buffer is NULL and status value is zero, we simply return with no other operation. This crash often occurs on iOS 11 for example encoding 720p@25fps. Signed-off-by: sharpbai Signed-off-by: Rick Kern --- libavcodec/videotoolboxenc.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libavcodec/videotoolboxenc.c b/libavcodec/videotoolboxenc.c index d76bb7f646..8afdd125d2 100644 --- a/libavcodec/videotoolboxenc.c +++ b/libavcodec/videotoolboxenc.c @@ -569,12 +569,16 @@ static void vtenc_output_callback( return; } - if (status || !sample_buffer) { + if (status) { av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", (int)status); set_async_error(vtctx, AVERROR_EXTERNAL); return; } + if (!sample_buffer) { + return; + } + if (!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) { int set_status = set_extradata(avctx, sample_buffer); if (set_status) {