From f03f78bc1c99b1e29624418e2f7315b8a47981e9 Mon Sep 17 00:00:00 2001 From: Anton Khirnov Date: Wed, 21 Sep 2016 20:21:58 +0200 Subject: [PATCH] mpegvideo_enc: handle encoding errors with b_strategy=2 --- libavcodec/mpegvideo_enc.c | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/libavcodec/mpegvideo_enc.c b/libavcodec/mpegvideo_enc.c index 46e2e2c3fc..6ab9ba57f5 100644 --- a/libavcodec/mpegvideo_enc.c +++ b/libavcodec/mpegvideo_enc.c @@ -1265,6 +1265,7 @@ static int estimate_best_b_count(MpegEncContext *s) int i, j, out_size, p_lambda, b_lambda, lambda2; int64_t best_rd = INT64_MAX; int best_b_count = -1; + int ret = 0; if (!c) return AVERROR(ENOMEM); @@ -1338,6 +1339,10 @@ static int estimate_best_b_count(MpegEncContext *s) s->tmp_frames[0]->quality = 1 * FF_QP2LAMBDA; out_size = encode_frame(c, s->tmp_frames[0]); + if (out_size < 0) { + ret = out_size; + goto fail; + } //rd += (out_size * lambda2) >> FF_LAMBDA_SHIFT; @@ -1349,6 +1354,10 @@ static int estimate_best_b_count(MpegEncContext *s) s->tmp_frames[i + 1]->quality = is_p ? p_lambda : b_lambda; out_size = encode_frame(c, s->tmp_frames[i + 1]); + if (out_size < 0) { + ret = out_size; + goto fail; + } rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3); } @@ -1356,6 +1365,10 @@ static int estimate_best_b_count(MpegEncContext *s) /* get the delayed frames */ while (out_size) { out_size = encode_frame(c, NULL); + if (out_size < 0) { + ret = out_size; + goto fail; + } rd += (out_size * lambda2) >> (FF_LAMBDA_SHIFT - 3); } @@ -1370,6 +1383,9 @@ static int estimate_best_b_count(MpegEncContext *s) avcodec_free_context(&c); return best_b_count; +fail: + avcodec_free_context(&c); + return ret; } static int select_input_picture(MpegEncContext *s) @@ -1450,6 +1466,8 @@ static int select_input_picture(MpegEncContext *s) } } else if (s->b_frame_strategy == 2) { b_frames = estimate_best_b_count(s); + if (b_frames < 0) + return b_frames; } emms_c();