From b5aefb6e70f7a19adbab3dd41633666ba95e1997 Mon Sep 17 00:00:00 2001 From: Nuo Mi Date: Tue, 21 Mar 2023 16:01:16 +0100 Subject: [PATCH] avcodec: add bitstream parser for H266/VVC Add nal parser ff_vvc_parser to parse vvc elementary bitstreams. Co-authored-by: Thomas Siedel Co-authored-by: James Almer Signed-off-by: James Almer --- Changelog | 1 + configure | 1 + libavcodec/Makefile | 1 + libavcodec/parsers.c | 1 + libavcodec/vvc_parser.c | 580 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 584 insertions(+) create mode 100644 libavcodec/vvc_parser.c diff --git a/Changelog b/Changelog index 5e4492fb71..e9cbce7e1c 100644 --- a/Changelog +++ b/Changelog @@ -22,6 +22,7 @@ version : - bwdif_cuda filter - Microsoft RLE video encoder - Raw AC-4 muxer and demuxer +- Raw VVC bitstream parser version 6.0: - Radiance HDR image support diff --git a/configure b/configure index b2665a0fbd..10d12770d6 100755 --- a/configure +++ b/configure @@ -3333,6 +3333,7 @@ hevc_parser_select="hevcparse hevc_sei" mpegaudio_parser_select="mpegaudioheader" mpeg4video_parser_select="h263dsp mpegvideodec qpeldsp" vc1_parser_select="vc1dsp" +vvc_parser_select="cbs_h266" # bitstream_filters aac_adtstoasc_bsf_select="adts_header mpeg4audio" diff --git a/libavcodec/Makefile b/libavcodec/Makefile index a0ebb15a2c..b915464b90 100644 --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -1209,6 +1209,7 @@ OBJS-$(CONFIG_VC1_PARSER) += vc1_parser.o vc1.o vc1data.o \ OBJS-$(CONFIG_VP3_PARSER) += vp3_parser.o OBJS-$(CONFIG_VP8_PARSER) += vp8_parser.o OBJS-$(CONFIG_VP9_PARSER) += vp9_parser.o +OBJS-$(CONFIG_VVC_PARSER) += vvc_parser.o OBJS-$(CONFIG_WEBP_PARSER) += webp_parser.o OBJS-$(CONFIG_XBM_PARSER) += xbm_parser.o OBJS-$(CONFIG_XMA_PARSER) += xma_parser.o diff --git a/libavcodec/parsers.c b/libavcodec/parsers.c index 2c077ec3ae..285f81a901 100644 --- a/libavcodec/parsers.c +++ b/libavcodec/parsers.c @@ -75,6 +75,7 @@ extern const AVCodecParser ff_vorbis_parser; extern const AVCodecParser ff_vp3_parser; extern const AVCodecParser ff_vp8_parser; extern const AVCodecParser ff_vp9_parser; +extern const AVCodecParser ff_vvc_parser; extern const AVCodecParser ff_webp_parser; extern const AVCodecParser ff_xbm_parser; extern const AVCodecParser ff_xma_parser; diff --git a/libavcodec/vvc_parser.c b/libavcodec/vvc_parser.c new file mode 100644 index 0000000000..4f7657f7ab --- /dev/null +++ b/libavcodec/vvc_parser.c @@ -0,0 +1,580 @@ +/* + * H.266 / VVC parser + * + * Copyright (C) 2021 Nuo Mi + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "cbs.h" +#include "cbs_h266.h" +#include "internal.h" +#include "parser.h" +#include "decode.h" + +#define START_CODE 0x000001 ///< start_code_prefix_one_3bytes +#define IS_IDR(nut) (nut == VVC_IDR_W_RADL || nut == VVC_IDR_N_LP) +#define IS_H266_SLICE(nut) (nut <= VVC_RASL_NUT || (nut >= VVC_IDR_W_RADL && nut <= VVC_GDR_NUT)) + +typedef struct PuInfo { + AVBufferRef *sps_ref; + AVBufferRef *pps_ref; + AVBufferRef *slice_ref; + AVBufferRef *ph_ref; + + const H266RawPPS *pps; + const H266RawSPS *sps; + const H266RawPH *ph; + const H266RawSlice *slice; + int pic_type; +} PuInfo; + +typedef struct AuDetector { + uint8_t prev_layer_id; + int prev_tid0_poc; + int prev_poc; +} AuDetector; + +typedef struct VVCParserContext { + ParseContext pc; + CodedBitstreamContext *cbc; + + CodedBitstreamFragment picture_unit; + + PuInfo au_info; + AVPacket au; + AVPacket last_au; + + AuDetector au_detector; + + int parsed_extradata; +} VVCParserContext; + +static const enum AVPixelFormat pix_fmts_8bit[] = { + AV_PIX_FMT_GRAY8, AV_PIX_FMT_YUV420P, + AV_PIX_FMT_YUV422P, AV_PIX_FMT_YUV444P +}; + +static const enum AVPixelFormat pix_fmts_10bit[] = { + AV_PIX_FMT_GRAY10, AV_PIX_FMT_YUV420P10, + AV_PIX_FMT_YUV422P10, AV_PIX_FMT_YUV444P10 +}; + +static int get_format(const H266RawSPS *sps) +{ + switch (sps->sps_bitdepth_minus8) { + case 0: + return pix_fmts_8bit[sps->sps_chroma_format_idc]; + case 2: + return pix_fmts_10bit[sps->sps_chroma_format_idc]; + } + return AV_PIX_FMT_NONE; +} + +/** + * Find the end of the current frame in the bitstream. + * @return the position of the first byte of the next frame, or END_NOT_FOUND + */ +static int find_frame_end(AVCodecParserContext *s, const uint8_t *buf, + int buf_size) +{ + VVCParserContext *ctx = s->priv_data; + ParseContext *pc = &ctx->pc; + int i; + + for (i = 0; i < buf_size; i++) { + int nut, code_len; + + pc->state64 = (pc->state64 << 8) | buf[i]; + + if (((pc->state64 >> 3 * 8) & 0xFFFFFF) != START_CODE) + continue; + + code_len = ((pc->state64 >> 3 * 8) & 0xFFFFFFFF) == 0x01 ? 4 : 3; + + nut = (pc->state64 >> (8 + 3)) & 0x1F; + // 7.4.2.4.3 and 7.4.2.4.4 + if ((nut >= VVC_OPI_NUT && nut <= VVC_PREFIX_APS_NUT && + nut != VVC_PH_NUT) || nut == VVC_AUD_NUT + || (nut == VVC_PREFIX_SEI_NUT && !pc->frame_start_found) + || nut == VVC_RSV_NVCL_26 || nut == VVC_UNSPEC_28 + || nut == VVC_UNSPEC_29) { + if (pc->frame_start_found) { + pc->frame_start_found = 0; + return i - (code_len + 2); + } + } else if (nut == VVC_PH_NUT || IS_H266_SLICE(nut)) { + int sh_picture_header_in_slice_header_flag = buf[i] >> 7; + + if (nut == VVC_PH_NUT || sh_picture_header_in_slice_header_flag) { + if (!pc->frame_start_found) { + pc->frame_start_found = 1; + } else { // First slice of next frame found + pc->frame_start_found = 0; + return i - (code_len + 2); + } + } + } + } + return END_NOT_FOUND; +} + +static int get_pict_type(const CodedBitstreamFragment *pu) +{ + int has_p = 0; + for (int i = 0; i < pu->nb_units; i++) { + CodedBitstreamUnit *unit = &pu->units[i]; + if (IS_H266_SLICE(unit->type)) { + const H266RawSlice *slice = unit->content; + uint8_t type = slice->header.sh_slice_type; + if (type == VVC_SLICE_TYPE_B) { + return AV_PICTURE_TYPE_B; + } + if (type == VVC_SLICE_TYPE_P) { + has_p = 1; + } + } + } + return has_p ? AV_PICTURE_TYPE_P : AV_PICTURE_TYPE_I; +} + +static void pu_info_unref(PuInfo *info) +{ + av_buffer_unref(&info->slice_ref); + av_buffer_unref(&info->ph_ref); + av_buffer_unref(&info->pps_ref); + av_buffer_unref(&info->sps_ref); + info->slice = NULL; + info->ph = NULL; + info->pps = NULL; + info->sps = NULL; + info->pic_type = AV_PICTURE_TYPE_NONE; +} + +static int pu_info_ref(PuInfo *dest, const PuInfo *src) +{ + pu_info_unref(dest); + dest->sps_ref = av_buffer_ref(src->sps_ref); + dest->pps_ref = av_buffer_ref(src->pps_ref); + if (src->ph_ref) + dest->ph_ref = av_buffer_ref(src->ph_ref); + dest->slice_ref = av_buffer_ref(src->slice_ref); + if (!dest->sps_ref || !dest->pps_ref || (src->ph_ref && !dest->ph_ref) + || !dest->slice_ref) { + pu_info_unref(dest); + return AVERROR(ENOMEM); + } + + dest->sps = src->sps; + dest->pps = src->pps; + dest->ph = src->ph; + dest->slice = src->slice; + dest->pic_type = src->pic_type; + return 0; +} + +static void set_parser_ctx(AVCodecParserContext *s, AVCodecContext *avctx, + const PuInfo *pu) +{ + static const uint8_t h266_sub_width_c[] = { + 1, 2, 2, 1 + }; + static const uint8_t h266_sub_height_c[] = { + 1, 2, 1, 1 + }; + const H266RawSPS *sps = pu->sps; + const H266RawPPS *pps = pu->pps; + //const H266RawPH *ph = pu->ph; + const H266RawNALUnitHeader *nal = &pu->slice->header.nal_unit_header; + + s->pict_type = pu->pic_type; + s->format = get_format(sps); + s->picture_structure = AV_PICTURE_STRUCTURE_FRAME; + + s->key_frame = nal->nal_unit_type == VVC_IDR_W_RADL || + nal->nal_unit_type == VVC_IDR_N_LP || + nal->nal_unit_type == VVC_CRA_NUT || + nal->nal_unit_type == VVC_GDR_NUT; + + s->coded_width = pps->pps_pic_width_in_luma_samples; + s->coded_height = pps->pps_pic_height_in_luma_samples; + s->width = pps->pps_pic_width_in_luma_samples - + (pps->pps_conf_win_left_offset + pps->pps_conf_win_right_offset) * + h266_sub_width_c[sps->sps_chroma_format_idc]; + s->height = pps->pps_pic_height_in_luma_samples - + (pps->pps_conf_win_top_offset + pps->pps_conf_win_bottom_offset) * + h266_sub_height_c[sps->sps_chroma_format_idc];; + + avctx->profile = sps->profile_tier_level.general_profile_idc; + avctx->level = sps->profile_tier_level.general_level_idc; + + avctx->colorspace = (enum AVColorSpace) sps->vui.vui_matrix_coeffs; + avctx->color_primaries = (enum AVColorPrimaries) sps->vui.vui_colour_primaries; + avctx->color_trc = (enum AVColorTransferCharacteristic) sps->vui.vui_transfer_characteristics; + avctx->color_range = + sps->vui.vui_full_range_flag ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG; + + avctx->has_b_frames = (sps->sps_max_sublayers_minus1 + 1) > 2 ? 2 : + sps->sps_max_sublayers_minus1; + avctx->max_b_frames = sps->sps_max_sublayers_minus1; + + if (sps->sps_ptl_dpb_hrd_params_present_flag && + sps->sps_timing_hrd_params_present_flag) { + int num = sps->sps_general_timing_hrd_parameters.num_units_in_tick; + int den = sps->sps_general_timing_hrd_parameters.time_scale; + + if (num != 0 && den != 0) + av_reduce(&avctx->framerate.den, &avctx->framerate.num, + num, den, 1 << 30); + } +} + +static int set_ctx(AVCodecParserContext *s, AVCodecContext *avctx, + const PuInfo *next_pu) +{ + VVCParserContext *ctx = s->priv_data; + + int ret = pu_info_ref(&ctx->au_info, next_pu); + if (ret < 0) + return ret; + + set_parser_ctx(s, avctx, &ctx->au_info); + + return 0; +} + +//8.3.1 Decoding process for picture order count. +//VTM did not follow the spec, and it's much simpler than spec. +//We follow the VTM. +static void get_slice_poc(VVCParserContext *s, int *poc, + const H266RawSPS *sps, + const H266RawPH *ph, + const H266RawSliceHeader *slice, void *log_ctx) +{ + int poc_msb, max_poc_lsb, poc_lsb; + AuDetector *d = &s->au_detector; + max_poc_lsb = 1 << (sps->sps_log2_max_pic_order_cnt_lsb_minus4 + 4); + poc_lsb = ph->ph_pic_order_cnt_lsb; + if (IS_IDR(slice->nal_unit_header.nal_unit_type)) { + if (ph->ph_poc_msb_cycle_present_flag) + poc_msb = ph->ph_poc_msb_cycle_val * max_poc_lsb; + else + poc_msb = 0; + } else { + int prev_poc = d->prev_tid0_poc; + int prev_poc_lsb = prev_poc & (max_poc_lsb - 1); + int prev_poc_msb = prev_poc - prev_poc_lsb; + if (ph->ph_poc_msb_cycle_present_flag) { + poc_msb = ph->ph_poc_msb_cycle_val * max_poc_lsb; + } else { + if ((poc_lsb < prev_poc_lsb) && ((prev_poc_lsb - poc_lsb) >= + (max_poc_lsb / 2))) + poc_msb = prev_poc_msb + max_poc_lsb; + else if ((poc_lsb > prev_poc_lsb) && ((poc_lsb - prev_poc_lsb) > + (max_poc_lsb / 2))) + poc_msb = prev_poc_msb - max_poc_lsb; + else + poc_msb = prev_poc_msb; + } + } + + *poc = poc_msb + poc_lsb; +} + +static void au_detector_init(AuDetector *d) +{ + d->prev_layer_id = UINT8_MAX; + d->prev_poc = INT_MAX; + d->prev_tid0_poc = INT_MAX; +} + +static int is_au_start(VVCParserContext *s, const PuInfo *pu, void *log_ctx) +{ + //7.4.2.4.3 + AuDetector *d = &s->au_detector; + const H266RawSPS *sps = pu->sps; + const H266RawNALUnitHeader *nal = &pu->slice->header.nal_unit_header; + const H266RawPH *ph = pu->ph; + const H266RawSlice *slice = pu->slice; + int ret, poc, nut; + + get_slice_poc(s, &poc, sps, ph, &slice->header, log_ctx); + + ret = (nal->nuh_layer_id <= d->prev_layer_id) || (poc != d->prev_poc); + + nut = nal->nal_unit_type; + d->prev_layer_id = nal->nuh_layer_id; + d->prev_poc = poc; + if (nal->nuh_temporal_id_plus1 == 1 && + !ph->ph_non_ref_pic_flag && nut != VVC_RADL_NUT + && nut != VVC_RASL_NUT) { + d->prev_tid0_poc = poc; + } + return ret; +} + +static int get_pu_info(PuInfo *info, const CodedBitstreamH266Context *h266, + const CodedBitstreamFragment *pu, void *logctx) +{ + const H266RawNALUnitHeader *nal; + int ret; + + memset(info, 0, sizeof(*info)); + for (int i = 0; i < pu->nb_units; i++) { + nal = pu->units[i].content; + if (!nal) + continue; + if ( nal->nal_unit_type == VVC_PH_NUT ) { + info->ph = pu->units[i].content; + info->ph_ref = pu->units[i].content_ref; + } else if (IS_H266_SLICE(nal->nal_unit_type)) { + info->slice = pu->units[i].content; + info->slice_ref = pu->units[i].content_ref; + if (info->slice->header.sh_picture_header_in_slice_header_flag) + info->ph = &info->slice->header.sh_picture_header; + if (!info->ph) { + av_log(logctx, AV_LOG_ERROR, + "can't find picture header in picture unit.\n"); + ret = AVERROR_INVALIDDATA; + goto error; + } + break; + } + } + if (!info->slice) { + av_log(logctx, AV_LOG_ERROR, "can't find slice in picture unit.\n"); + ret = AVERROR_INVALIDDATA; + goto error; + } + info->pps = h266->pps[info->ph->ph_pic_parameter_set_id]; + if (!info->pps) { + av_log(logctx, AV_LOG_ERROR, "PPS id %d is not avaliable.\n", + info->ph->ph_pic_parameter_set_id); + ret = AVERROR_INVALIDDATA; + goto error; + } + info->pps_ref = h266->pps_ref[info->ph->ph_pic_parameter_set_id]; + info->sps = h266->sps[info->pps->pps_seq_parameter_set_id]; + if (!info->sps) { + av_log(logctx, AV_LOG_ERROR, "SPS id %d is not avaliable.\n", + info->pps->pps_seq_parameter_set_id); + ret = AVERROR_INVALIDDATA; + goto error; + } + info->sps_ref = h266->sps_ref[info->pps->pps_seq_parameter_set_id]; + info->pic_type = get_pict_type(pu); + return 0; + error: + memset(info, 0, sizeof(*info)); + return ret; +} + +static int append_au(AVPacket *pkt, const uint8_t *buf, int buf_size) +{ + int offset = pkt->size; + int ret; + if ((ret = av_grow_packet(pkt, buf_size)) < 0) + goto end; + memcpy(pkt->data + offset, buf, buf_size); + end: + return ret; +} + +/** + * Parse NAL units of found picture and decode some basic information. + * + * @param s parser context. + * @param avctx codec context. + * @param buf buffer with field/frame data. + * @param buf_size size of the buffer. + * @return < 0 for error, == 0 for a complete au, > 0 is not a completed au. + */ +static int parse_nal_units(AVCodecParserContext *s, const uint8_t *buf, + int buf_size, AVCodecContext *avctx) +{ + VVCParserContext *ctx = s->priv_data; + const CodedBitstreamH266Context *h266 = ctx->cbc->priv_data; + + CodedBitstreamFragment *pu = &ctx->picture_unit; + int ret; + PuInfo info; + + if (!buf_size) { + if (ctx->au.size) { + av_packet_move_ref(&ctx->last_au, &ctx->au); + return 0; + } + return 1; + } + + if ((ret = ff_cbs_read(ctx->cbc, pu, buf, buf_size)) < 0) { + av_log(avctx, AV_LOG_ERROR, "Failed to parse picture unit.\n"); + goto end; + } + if ((ret = get_pu_info(&info, h266, pu, avctx)) < 0) + goto end; + if (append_au(&ctx->au, buf, buf_size) < 0) { + ret = AVERROR(ENOMEM); + goto end; + } + if (is_au_start(ctx, &info, avctx)) { + if ((ret = set_ctx(s, avctx, &info)) < 0) + goto end; + av_packet_move_ref(&ctx->last_au, &ctx->au); + } else { + ret = 1; //not a completed au + } + end: + ff_cbs_fragment_reset(pu); + return ret; +} + +/** + * Combine PU to AU + * + * @param s parser context. + * @param avctx codec context. + * @param buf buffer to a PU. + * @param buf_size size of the buffer. + * @return < 0 for error, == 0 a complete au, > 0 not a completed au. + */ +static int combine_au(AVCodecParserContext *s, AVCodecContext *avctx, + const uint8_t **buf, int *buf_size) +{ + VVCParserContext *ctx = s->priv_data; + int ret; + + ctx->cbc->log_ctx = avctx; + + av_packet_unref(&ctx->last_au); + ret = parse_nal_units(s, *buf, *buf_size, avctx); + if (ret == 0) { + if (ctx->last_au.size) { + *buf = ctx->last_au.data; + *buf_size = ctx->last_au.size; + } else { + ret = 1; //no output + } + } + ctx->cbc->log_ctx = NULL; + return ret; +} + +static int vvc_parser_parse(AVCodecParserContext *s, AVCodecContext *avctx, + const uint8_t **poutbuf, int *poutbuf_size, + const uint8_t *buf, int buf_size) +{ + int next, ret; + VVCParserContext *ctx = s->priv_data; + ParseContext *pc = &ctx->pc; + CodedBitstreamFragment *pu = &ctx->picture_unit; + + int is_dummy_buf = !buf_size; + int flush = !buf_size; + const uint8_t *dummy_buf = buf; + + *poutbuf = NULL; + *poutbuf_size = 0; + + if (avctx->extradata_size && !ctx->parsed_extradata) { + ctx->parsed_extradata = 1; + + ret = ff_cbs_read_extradata_from_codec(ctx->cbc, pu, avctx); + if (ret < 0) + av_log(avctx, AV_LOG_WARNING, "Failed to parse extradata.\n"); + + ff_cbs_fragment_reset(pu); + } + + if (s->flags & PARSER_FLAG_COMPLETE_FRAMES) { + next = buf_size; + } else { + next = find_frame_end(s, buf, buf_size); + if (ff_combine_frame(pc, next, &buf, &buf_size) < 0) + return buf_size; + } + + is_dummy_buf &= (dummy_buf == buf); + + if (!is_dummy_buf) { + ret = combine_au(s, avctx, &buf, &buf_size); + if (ret > 0 && flush) { + buf_size = 0; + ret = combine_au(s, avctx, &buf, &buf_size); + } + if (ret != 0) + return next; + } + + *poutbuf = buf; + *poutbuf_size = buf_size; + + return next; +} + +static const CodedBitstreamUnitType decompose_unit_types[] = { + VVC_TRAIL_NUT, + VVC_STSA_NUT, + VVC_RADL_NUT, + VVC_RASL_NUT, + VVC_IDR_W_RADL, + VVC_IDR_N_LP, + VVC_CRA_NUT, + VVC_GDR_NUT, + VVC_VPS_NUT, + VVC_SPS_NUT, + VVC_PPS_NUT, + VVC_PH_NUT, + VVC_AUD_NUT, +}; + +static av_cold int vvc_parser_init(AVCodecParserContext *s) +{ + VVCParserContext *ctx = s->priv_data; + int ret; + + ret = ff_cbs_init(&ctx->cbc, AV_CODEC_ID_VVC, NULL); + if (ret < 0) + return ret; + au_detector_init(&ctx->au_detector); + + ctx->cbc->decompose_unit_types = decompose_unit_types; + ctx->cbc->nb_decompose_unit_types = FF_ARRAY_ELEMS(decompose_unit_types); + + return ret; +} + +static void vvc_parser_close(AVCodecParserContext *s) +{ + VVCParserContext *ctx = s->priv_data; + + pu_info_unref(&ctx->au_info); + av_packet_unref(&ctx->au); + av_packet_unref(&ctx->last_au); + ff_cbs_fragment_free(&ctx->picture_unit); + + ff_cbs_close(&ctx->cbc); + av_freep(&ctx->pc.buffer); +} + +AVCodecParser ff_vvc_parser = { + .codec_ids = { AV_CODEC_ID_VVC }, + .priv_data_size = sizeof(VVCParserContext), + .parser_init = vvc_parser_init, + .parser_close = vvc_parser_close, + .parser_parse = vvc_parser_parse, +};