From b3dc260e7fa6f3f852dd5cb7d86763c4b5736714 Mon Sep 17 00:00:00 2001 From: Vittorio Giovara Date: Fri, 2 Aug 2013 18:36:27 +0200 Subject: [PATCH] h264: return meaningful values Signed-off-by: Anton Khirnov --- libavcodec/h264.c | 171 +++++++++++++++++++++++++--------------------- 1 file changed, 94 insertions(+), 77 deletions(-) diff --git a/libavcodec/h264.c b/libavcodec/h264.c index 89f3e2cf9f..f68b189294 100644 --- a/libavcodec/h264.c +++ b/libavcodec/h264.c @@ -414,7 +414,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h) av_log(h->avctx, AV_LOG_ERROR, "top block unavailable for requested intra4x4 mode %d at %d %d\n", status, h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } else if (status) { h->intra4x4_pred_mode_cache[scan8[0] + i] = status; } @@ -430,7 +430,7 @@ int ff_h264_check_intra4x4_pred_mode(H264Context *h) av_log(h->avctx, AV_LOG_ERROR, "left block unavailable for requested intra4x4 mode %d at %d %d\n", status, h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } else if (status) { h->intra4x4_pred_mode_cache[scan8[0] + 8 * i] = status; } @@ -453,7 +453,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma) av_log(h->avctx, AV_LOG_ERROR, "out of range intra chroma pred mode at %d %d\n", h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } if (!(h->top_samples_available & 0x8000)) { @@ -462,7 +462,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma) av_log(h->avctx, AV_LOG_ERROR, "top block unavailable for requested intra mode at %d %d\n", h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } } @@ -478,7 +478,7 @@ int ff_h264_check_intra_pred_mode(H264Context *h, int mode, int is_chroma) av_log(h->avctx, AV_LOG_ERROR, "left block unavailable for requested intra mode at %d %d\n", h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } } @@ -1259,7 +1259,7 @@ int ff_h264_alloc_tables(H264Context *h) fail: free_tables(h, 1); - return -1; + return AVERROR(ENOMEM); } /** @@ -1358,7 +1358,7 @@ static int context_init(H264Context *h) return 0; fail: - return -1; // free_tables will clean up for us + return AVERROR(ENOMEM); // free_tables will clean up for us } static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size, @@ -1367,6 +1367,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size, int ff_h264_decode_extradata(H264Context *h) { AVCodecContext *avctx = h->avctx; + int ret; if (avctx->extradata[0] == 1) { int i, cnt, nalsize; @@ -1376,7 +1377,7 @@ int ff_h264_decode_extradata(H264Context *h) if (avctx->extradata_size < 7) { av_log(avctx, AV_LOG_ERROR, "avcC too short\n"); - return -1; + return AVERROR_INVALIDDATA; } /* sps and pps in the avcC always have length coded with 2 bytes, * so put a fake nal_length_size = 2 while parsing them */ @@ -1387,11 +1388,12 @@ int ff_h264_decode_extradata(H264Context *h) for (i = 0; i < cnt; i++) { nalsize = AV_RB16(p) + 2; if (p - avctx->extradata + nalsize > avctx->extradata_size) - return -1; - if (decode_nal_units(h, p, nalsize, 1) < 0) { + return AVERROR_INVALIDDATA; + ret = decode_nal_units(h, p, nalsize, 1); + if (ret < 0) { av_log(avctx, AV_LOG_ERROR, "Decoding sps %d from avcC failed\n", i); - return -1; + return ret; } p += nalsize; } @@ -1400,11 +1402,12 @@ int ff_h264_decode_extradata(H264Context *h) for (i = 0; i < cnt; i++) { nalsize = AV_RB16(p) + 2; if (p - avctx->extradata + nalsize > avctx->extradata_size) - return -1; - if (decode_nal_units(h, p, nalsize, 1) < 0) { + return AVERROR_INVALIDDATA; + ret = decode_nal_units(h, p, nalsize, 1); + if (ret < 0) { av_log(avctx, AV_LOG_ERROR, "Decoding pps %d from avcC failed\n", i); - return -1; + return ret; } p += nalsize; } @@ -1412,8 +1415,9 @@ int ff_h264_decode_extradata(H264Context *h) h->nal_length_size = (avctx->extradata[4] & 0x03) + 1; } else { h->is_avc = 0; - if (decode_nal_units(h, avctx->extradata, avctx->extradata_size, 1) < 0) - return -1; + ret = decode_nal_units(h, avctx->extradata, avctx->extradata_size, 1); + if (ret < 0) + return ret; } return 0; } @@ -1422,6 +1426,7 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx) { H264Context *h = avctx->priv_data; int i; + int ret; h->avctx = avctx; @@ -1473,9 +1478,11 @@ av_cold int ff_h264_decode_init(AVCodecContext *avctx) avctx->ticks_per_frame = 2; } - if (avctx->extradata_size > 0 && avctx->extradata && - ff_h264_decode_extradata(h)) - return -1; + if (avctx->extradata_size > 0 && avctx->extradata) { + ret = ff_h264_decode_extradata(h); + if (ret < 0) + return ret; + } if (h->sps.bitstream_restriction_flag && h->avctx->has_b_frames < h->sps.num_reorder_frames) { @@ -1625,9 +1632,10 @@ static int decode_update_thread_context(AVCodecContext *dst, h->ref_index_pool = NULL; h->motion_val_pool = NULL; - if (ff_h264_alloc_tables(h) < 0) { + ret = ff_h264_alloc_tables(h); + if (ret < 0) { av_log(dst, AV_LOG_ERROR, "Could not allocate memory for h264\n"); - return AVERROR(ENOMEM); + return ret; } context_init(h); @@ -3002,7 +3010,7 @@ static int h264_slice_header_init(H264Context *h, int reinit) int nb_slices = (HAVE_THREADS && h->avctx->active_thread_type & FF_THREAD_SLICE) ? h->avctx->thread_count : 1; - int i; + int i, ret; h->avctx->sample_aspect_ratio = h->sps.sar; av_assert0(h->avctx->sample_aspect_ratio.den); @@ -3025,10 +3033,11 @@ static int h264_slice_header_init(H264Context *h, int reinit) h->prev_interlaced_frame = 1; init_scan_tables(h); - if (ff_h264_alloc_tables(h) < 0) { + ret = ff_h264_alloc_tables(h); + if (ret < 0) { av_log(h->avctx, AV_LOG_ERROR, "Could not allocate memory for h264\n"); - return AVERROR(ENOMEM); + return ret; } if (nb_slices > MAX_THREADS || (nb_slices > h->mb_height && h->mb_height)) { @@ -3044,9 +3053,10 @@ static int h264_slice_header_init(H264Context *h, int reinit) h->slice_context_count = nb_slices; if (!HAVE_THREADS || !(h->avctx->active_thread_type & FF_THREAD_SLICE)) { - if (context_init(h) < 0) { + ret = context_init(h); + if (ret < 0) { av_log(h->avctx, AV_LOG_ERROR, "context_init() failed.\n"); - return -1; + return ret; } } else { for (i = 1; i < h->slice_context_count; i++) { @@ -3085,9 +3095,9 @@ static int h264_slice_header_init(H264Context *h, int reinit) } for (i = 0; i < h->slice_context_count; i++) - if (context_init(h->thread_context[i]) < 0) { + if ((ret = context_init(h->thread_context[i])) < 0) { av_log(h->avctx, AV_LOG_ERROR, "context_init() failed.\n"); - return -1; + return ret; } } @@ -3141,7 +3151,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) av_log(h->avctx, AV_LOG_ERROR, "slice type too large (%d) at %d %d\n", h->slice_type, h->mb_x, h->mb_y); - return -1; + return AVERROR_INVALIDDATA; } if (slice_type > 4) { slice_type -= 5; @@ -3163,13 +3173,13 @@ static int decode_slice_header(H264Context *h, H264Context *h0) pps_id = get_ue_golomb(&h->gb); if (pps_id >= MAX_PPS_COUNT) { av_log(h->avctx, AV_LOG_ERROR, "pps_id out of range\n"); - return -1; + return AVERROR_INVALIDDATA; } if (!h0->pps_buffers[pps_id]) { av_log(h->avctx, AV_LOG_ERROR, "non-existing PPS %u referenced\n", pps_id); - return -1; + return AVERROR_INVALIDDATA; } h->pps = *h0->pps_buffers[pps_id]; @@ -3177,7 +3187,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) av_log(h->avctx, AV_LOG_ERROR, "non-existing SPS %u referenced\n", h->pps.sps_id); - return -1; + return AVERROR_INVALIDDATA; } if (h->pps.sps_id != h->current_sps_id || @@ -3262,7 +3272,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (h != h0) { av_log(h->avctx, AV_LOG_ERROR, "Cannot (re-)initialize context during parallel decoding.\n"); - return -1; + return AVERROR_PATCHWELCOME; } if ((ret = get_pixel_format(h)) < 0) @@ -3390,19 +3400,20 @@ static int decode_slice_header(H264Context *h, H264Context *h0) Picture *prev = h->short_ref_count ? h->short_ref[0] : NULL; av_log(h->avctx, AV_LOG_DEBUG, "Frame num gap %d %d\n", h->frame_num, h->prev_frame_num); - if (h264_frame_start(h) < 0) - return -1; + ret = h264_frame_start(h); + if (ret < 0) + return ret; h->prev_frame_num++; h->prev_frame_num %= 1 << h->sps.log2_max_frame_num; h->cur_pic_ptr->frame_num = h->prev_frame_num; ff_thread_report_progress(&h->cur_pic_ptr->tf, INT_MAX, 0); ff_thread_report_progress(&h->cur_pic_ptr->tf, INT_MAX, 1); - if ((ret = ff_generate_sliding_window_mmcos(h, 1)) < 0 && - h->avctx->err_recognition & AV_EF_EXPLODE) + ret = ff_generate_sliding_window_mmcos(h, 1); + if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) + return ret; + ret = ff_h264_execute_ref_pic_marking(h, h->mmco, h->mmco_index); + if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) return ret; - if (ff_h264_execute_ref_pic_marking(h, h->mmco, h->mmco_index) < 0 && - (h->avctx->err_recognition & AV_EF_EXPLODE)) - return AVERROR_INVALIDDATA; /* Error concealment: If a ref is missing, copy the previous ref * in its place. * FIXME: Avoiding a memcpy would be nice, but ref handling makes @@ -3459,7 +3470,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (!FIELD_PICTURE(h) || h0->first_field) { if (h264_frame_start(h) < 0) { h0->first_field = 0; - return -1; + return AVERROR_INVALIDDATA; } } else { release_unused_pictures(h, 0); @@ -3474,7 +3485,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (first_mb_in_slice << FIELD_OR_MBAFF_PICTURE(h) >= h->mb_num || first_mb_in_slice >= h->mb_num) { av_log(h->avctx, AV_LOG_ERROR, "first_mb_in_slice overflow\n"); - return -1; + return AVERROR_INVALIDDATA; } h->resync_mb_x = h->mb_x = first_mb_in_slice % h->mb_width; h->resync_mb_y = h->mb_y = (first_mb_in_slice / h->mb_width) << @@ -3553,10 +3564,12 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (!default_ref_list_done) ff_h264_fill_default_ref_list(h); - if (h->slice_type_nos != AV_PICTURE_TYPE_I && - ff_h264_decode_ref_pic_list_reordering(h) < 0) { - h->ref_count[1] = h->ref_count[0] = 0; - return -1; + if (h->slice_type_nos != AV_PICTURE_TYPE_I) { + ret = ff_h264_decode_ref_pic_list_reordering(h); + if (ret < 0) { + h->ref_count[1] = h->ref_count[0] = 0; + return ret; + } } if ((h->pps.weighted_pred && h->slice_type_nos == AV_PICTURE_TYPE_P) || @@ -3579,12 +3592,13 @@ static int decode_slice_header(H264Context *h, H264Context *h0) // or h->mmco, which will cause ref list mix-ups and decoding errors // further down the line. This may break decoding if the first slice is // corrupt, thus we only do this if frame-mt is enabled. - if (h->nal_ref_idc && - ff_h264_decode_ref_pic_marking(h0, &h->gb, - !(h->avctx->active_thread_type & FF_THREAD_FRAME) || - h0->current_slice == 0) < 0 && - (h->avctx->err_recognition & AV_EF_EXPLODE)) - return AVERROR_INVALIDDATA; + if (h->nal_ref_idc) { + ret = ff_h264_decode_ref_pic_marking(h0, &h->gb, + !(h->avctx->active_thread_type & FF_THREAD_FRAME) || + h0->current_slice == 0); + if (ret < 0 && (h->avctx->err_recognition & AV_EF_EXPLODE)) + return AVERROR_INVALIDDATA; + } if (FRAME_MBAFF(h)) { ff_h264_fill_mbaff_ref_list(h); @@ -3603,7 +3617,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) tmp = get_ue_golomb_31(&h->gb); if (tmp > 2) { av_log(h->avctx, AV_LOG_ERROR, "cabac_init_idc overflow\n"); - return -1; + return AVERROR_INVALIDDATA; } h->cabac_init_idc = tmp; } @@ -3612,7 +3626,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) tmp = h->pps.init_qp + get_se_golomb(&h->gb); if (tmp > 51 + 6 * (h->sps.bit_depth_luma - 8)) { av_log(h->avctx, AV_LOG_ERROR, "QP %u out of range\n", tmp); - return -1; + return AVERROR_INVALIDDATA; } h->qscale = tmp; h->chroma_qp[0] = get_chroma_qp(h, 0, h->qscale); @@ -3632,7 +3646,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) if (tmp > 2) { av_log(h->avctx, AV_LOG_ERROR, "deblocking_filter_idc %u out of range\n", tmp); - return -1; + return AVERROR_INVALIDDATA; } h->deblocking_filter = tmp; if (h->deblocking_filter < 2) @@ -3646,7 +3660,7 @@ static int decode_slice_header(H264Context *h, H264Context *h0) av_log(h->avctx, AV_LOG_ERROR, "deblocking filter parameters %d %d out of range\n", h->slice_alpha_c0_offset, h->slice_beta_offset); - return -1; + return AVERROR_INVALIDDATA; } } } @@ -3764,7 +3778,7 @@ int ff_h264_get_slice_type(const H264Context *h) case AV_PICTURE_TYPE_SI: return 4; default: - return -1; + return AVERROR_INVALIDDATA; } } @@ -4180,7 +4194,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) h->cabac.bytestream_end - h->cabac.bytestream); er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x, h->mb_y, ER_MB_ERROR); - return -1; + return AVERROR_INVALIDDATA; } if (++h->mb_x >= h->mb_width) { @@ -4227,7 +4241,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) "error while decoding MB %d %d\n", h->mb_x, h->mb_y); er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x, h->mb_y, ER_MB_ERROR); - return -1; + return ret; } if (++h->mb_x >= h->mb_width) { @@ -4255,7 +4269,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) h->mb_x - 1, h->mb_y, ER_MB_END); - return -1; + return AVERROR_INVALIDDATA; } } } @@ -4263,6 +4277,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) if (get_bits_left(&h->gb) <= 0 && h->mb_skip_run <= 0) { tprintf(h->avctx, "slice end %d %d\n", get_bits_count(&h->gb), h->gb.size_in_bits); + if (get_bits_left(&h->gb) == 0) { er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x - 1, h->mb_y, @@ -4275,7 +4290,7 @@ static int decode_slice(struct AVCodecContext *avctx, void *arg) er_add_slice(h, h->resync_mb_x, h->resync_mb_y, h->mb_x, h->mb_y, ER_MB_ERROR); - return -1; + return AVERROR_INVALIDDATA; } } } @@ -4333,6 +4348,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size, int pass = !(avctx->active_thread_type & FF_THREAD_FRAME); int nals_needed = 0; ///< number of NALs that need decoding before the next frame thread starts int nal_index; + int ret = 0; h->max_contexts = h->slice_context_count; if (!(avctx->flags2 & CODEC_FLAG2_CHUNKS)) { @@ -4391,7 +4407,7 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size, ptr = ff_h264_decode_nal(hx, buf + buf_index, &dst_length, &consumed, next_avc - buf_index); if (ptr == NULL || dst_length < 0) { - buf_index = -1; + ret = -1; goto end; } i = buf_index + consumed; @@ -4462,7 +4478,7 @@ again: if (h->nal_unit_type != NAL_IDR_SLICE) { av_log(h->avctx, AV_LOG_ERROR, "Invalid mix of idr and non-idr slices\n"); - buf_index = -1; + ret = -1; goto end; } idr(h); // FIXME ensure we don't lose some frames if there is reordering @@ -4484,8 +4500,8 @@ again: decode_postinit(h, nal_index >= nals_needed); if (h->avctx->hwaccel && - h->avctx->hwaccel->start_frame(h->avctx, NULL, 0) < 0) - return -1; + (ret = h->avctx->hwaccel->start_frame(h->avctx, NULL, 0)) < 0) + return ret; } if (hx->redundant_pic_count == 0 && @@ -4497,10 +4513,11 @@ again: hx->slice_type_nos == AV_PICTURE_TYPE_I) && avctx->skip_frame < AVDISCARD_ALL) { if (avctx->hwaccel) { - if (avctx->hwaccel->decode_slice(avctx, - &buf[buf_index - consumed], - consumed) < 0) - return -1; + ret = avctx->hwaccel->decode_slice(avctx, + &buf[buf_index - consumed], + consumed); + if (ret < 0) + return ret; } else context_count++; } @@ -4541,8 +4558,8 @@ again: break; case NAL_SPS: init_get_bits(&h->gb, ptr, bit_length); - if (ff_h264_decode_seq_parameter_set(h) < 0 && - h->is_avc && (nalsize != consumed) && nalsize) { + ret = ff_h264_decode_seq_parameter_set(h); + if (ret < 0 && h->is_avc && (nalsize != consumed) && nalsize) { av_log(h->avctx, AV_LOG_DEBUG, "SPS decoding failure, trying again with the complete NAL\n"); init_get_bits(&h->gb, buf + buf_index + 1 - consumed, @@ -4550,10 +4567,10 @@ again: ff_h264_decode_seq_parameter_set(h); } - if (h264_set_parameter_from_sps(h) < 0) { - buf_index = -1; + ret = h264_set_parameter_from_sps(h); + if (ret < 0) goto end; - } + break; case NAL_PPS: init_get_bits(&h->gb, ptr, bit_length); @@ -4602,7 +4619,7 @@ end: h->picture_structure == PICT_BOTTOM_FIELD); } - return buf_index; + return (ret < 0) ? ret : buf_index; } /** @@ -4686,7 +4703,7 @@ out: buf_index = decode_nal_units(h, buf, buf_size, 0); if (buf_index < 0) - return -1; + return AVERROR_INVALIDDATA; if (!h->cur_pic_ptr && h->nal_unit_type == NAL_END_SEQUENCE) { buf_size = 0; @@ -4697,7 +4714,7 @@ out: if (avctx->skip_frame >= AVDISCARD_NONREF) return 0; av_log(avctx, AV_LOG_ERROR, "no frame!\n"); - return -1; + return AVERROR_INVALIDDATA; } if (!(avctx->flags2 & CODEC_FLAG2_CHUNKS) ||