From 35ae44c6156216b8c745e4bc65f309840101d21a Mon Sep 17 00:00:00 2001 From: Xinpeng Sun Date: Mon, 18 Mar 2024 16:12:46 +0800 Subject: [PATCH] lavfi/tonemap_vaapi: Add support for HDR to HDR tone mapping Usage example: ffmpeg -y -hwaccel vaapi -hwaccel_output_format vaapi -i hdr.mp4 \ -vf "tonemap_vaapi=display=7500 3000|34000 16000|13250 34500|15635 16450|500 10000000:extra_hw_frames=64" \ -c:v hevc_vaapi output.mp4 Signed-off-by: Xinpeng Sun Signed-off-by: Haihao Xiang --- doc/filters.texi | 42 +++++--- libavfilter/vf_tonemap_vaapi.c | 191 +++++++++++++++++++++++++++++---- 2 files changed, 203 insertions(+), 30 deletions(-) diff --git a/doc/filters.texi b/doc/filters.texi index 09c7349313..fc813f12c1 100644 --- a/doc/filters.texi +++ b/doc/filters.texi @@ -27832,8 +27832,7 @@ The inputs have same memory layout for color channels, the overlay has additiona @section tonemap_vaapi -Perform HDR(High Dynamic Range) to SDR(Standard Dynamic Range) conversion with tone-mapping. -It maps the dynamic range of HDR10 content to the SDR content. +Perform HDR-to-SDR or HDR-to-HDR tone-mapping. It currently only accepts HDR10 as input. It accepts the following parameters: @@ -27842,28 +27841,42 @@ It accepts the following parameters: @item format Specify the output pixel format. -Currently supported formats are: -@table @var -@item p010 -@item nv12 -@end table - -Default is nv12. +Default is nv12 for HDR-to-SDR tone-mapping and p010 for HDR-to-HDR +tone-mapping. @item primaries, p Set the output color primaries. -Default is bt709. +Default is bt709 for HDR-to-SDR tone-mapping and same as input for HDR-to-HDR +tone-mapping. @item transfer, t Set the output transfer characteristics. -Default is bt709. +Default is bt709 for HDR-to-SDR tone-mapping and same as input for HDR-to-HDR +tone-mapping. @item matrix, m Set the output colorspace matrix. -Default is bt709. +Default is bt709 for HDR-to-SDR tone-mapping and same as input for HDR-to-HDR +tone-mapping. + +@item display +Set the output mastering display colour volume. It is given by a '|'-separated +list of two values, two values are space separated. It set display primaries +x & y in G, B, R order, then white point x & y, the nominal minimum & maximum +display luminances. + +HDR-to-HDR tone-mapping will be performed when this option is set. + +@item light +Set the output content light level information. It accepts 2 space-separated +values, the first input is the maximum light level and the second input is +the maximum average light level. + +It is ignored for HDR-to-SDR tone-mapping, and optional for HDR-to-HDR +tone-mapping. @end table @@ -27875,6 +27888,11 @@ Convert HDR(HDR10) video to bt2020-transfer-characteristic p010 format @example tonemap_vaapi=format=p010:t=bt2020-10 @end example +@item +Convert HDR video to HDR video +@example +tonemap_vaapi=display=7500\ 3000|34000\ 16000|13250\ 34500|15635\ 16450|500\ 10000000 +@end example @end itemize @section hstack_vaapi diff --git a/libavfilter/vf_tonemap_vaapi.c b/libavfilter/vf_tonemap_vaapi.c index 5d475e8ff2..7ebcb18f79 100644 --- a/libavfilter/vf_tonemap_vaapi.c +++ b/libavfilter/vf_tonemap_vaapi.c @@ -39,7 +39,11 @@ typedef struct HDRVAAPIContext { enum AVColorTransferCharacteristic color_transfer; enum AVColorSpace color_matrix; + char *mastering_display; + char *content_light; + VAHdrMetaDataHDR10 in_metadata; + VAHdrMetaDataHDR10 out_metadata; AVFrameSideData *src_display; AVFrameSideData *src_light; @@ -146,6 +150,87 @@ static int tonemap_vaapi_save_metadata(AVFilterContext *avctx, AVFrame *input_fr return 0; } +static int tonemap_vaapi_update_sidedata(AVFilterContext *avctx, AVFrame *output_frame) +{ + HDRVAAPIContext *ctx = avctx->priv; + AVFrameSideData *metadata; + AVMasteringDisplayMetadata *hdr_meta; + AVFrameSideData *metadata_lt; + AVContentLightMetadata *hdr_meta_lt; + int i; + const int mapping[3] = {1, 2, 0}; //green, blue, red + const int chroma_den = 50000; + const int luma_den = 10000; + + metadata = av_frame_new_side_data(output_frame, + AV_FRAME_DATA_MASTERING_DISPLAY_METADATA, + sizeof(AVMasteringDisplayMetadata)); + if (!metadata) + return AVERROR(ENOMEM); + + hdr_meta = (AVMasteringDisplayMetadata *)metadata->data; + + for (i = 0; i < 3; i++) { + const int j = mapping[i]; + hdr_meta->display_primaries[j][0].num = ctx->out_metadata.display_primaries_x[i]; + hdr_meta->display_primaries[j][0].den = chroma_den; + + hdr_meta->display_primaries[j][1].num = ctx->out_metadata.display_primaries_y[i]; + hdr_meta->display_primaries[j][1].den = chroma_den; + } + + hdr_meta->white_point[0].num = ctx->out_metadata.white_point_x; + hdr_meta->white_point[0].den = chroma_den; + + hdr_meta->white_point[1].num = ctx->out_metadata.white_point_y; + hdr_meta->white_point[1].den = chroma_den; + hdr_meta->has_primaries = 1; + + hdr_meta->max_luminance.num = ctx->out_metadata.max_display_mastering_luminance; + hdr_meta->max_luminance.den = luma_den; + + hdr_meta->min_luminance.num = ctx->out_metadata.min_display_mastering_luminance; + hdr_meta->min_luminance.den = luma_den; + hdr_meta->has_luminance = 1; + + av_log(avctx, AV_LOG_DEBUG, + "Mastering display colour volume(out):\n"); + av_log(avctx, AV_LOG_DEBUG, + "G(%u,%u) B(%u,%u) R(%u,%u) WP(%u,%u)\n", + ctx->out_metadata.display_primaries_x[0], + ctx->out_metadata.display_primaries_y[0], + ctx->out_metadata.display_primaries_x[1], + ctx->out_metadata.display_primaries_y[1], + ctx->out_metadata.display_primaries_x[2], + ctx->out_metadata.display_primaries_y[2], + ctx->out_metadata.white_point_x, + ctx->out_metadata.white_point_y); + av_log(avctx, AV_LOG_DEBUG, + "max_display_mastering_luminance=%u, min_display_mastering_luminance=%u\n", + ctx->out_metadata.max_display_mastering_luminance, + ctx->out_metadata.min_display_mastering_luminance); + + metadata_lt = av_frame_new_side_data(output_frame, + AV_FRAME_DATA_CONTENT_LIGHT_LEVEL, + sizeof(AVContentLightMetadata)); + if (!metadata_lt) + return AVERROR(ENOMEM); + + hdr_meta_lt = (AVContentLightMetadata *)metadata_lt->data; + + hdr_meta_lt->MaxCLL = FFMIN(ctx->out_metadata.max_content_light_level, 65535); + hdr_meta_lt->MaxFALL = FFMIN(ctx->out_metadata.max_pic_average_light_level, 65535); + + av_log(avctx, AV_LOG_DEBUG, + "Content light level information(out):\n"); + av_log(avctx, AV_LOG_DEBUG, + "MaxCLL(%u) MaxFALL(%u)\n", + ctx->out_metadata.max_content_light_level, + ctx->out_metadata.max_pic_average_light_level); + + return 0; +} + static int tonemap_vaapi_set_filter_params(AVFilterContext *avctx, AVFrame *input_frame) { VAAPIVPPContext *vpp_ctx = avctx->priv; @@ -208,15 +293,26 @@ static int tonemap_vaapi_build_filter_params(AVFilterContext *avctx) return AVERROR(EINVAL); } - for (i = 0; i < num_query_caps; i++) { - if (VA_TONE_MAPPING_HDR_TO_SDR & hdr_cap[i].caps_flag) - break; - } - - if (i >= num_query_caps) { - av_log(avctx, AV_LOG_ERROR, - "VAAPI driver doesn't support HDR to SDR\n"); - return AVERROR(EINVAL); + if (ctx->mastering_display) { + for (i = 0; i < num_query_caps; i++) { + if (VA_TONE_MAPPING_HDR_TO_HDR & hdr_cap[i].caps_flag) + break; + } + if (i >= num_query_caps) { + av_log(avctx, AV_LOG_ERROR, + "VAAPI driver doesn't support HDR to HDR\n"); + return AVERROR(EINVAL); + } + } else { + for (i = 0; i < num_query_caps; i++) { + if (VA_TONE_MAPPING_HDR_TO_SDR & hdr_cap[i].caps_flag) + break; + } + if (i >= num_query_caps) { + av_log(avctx, AV_LOG_ERROR, + "VAAPI driver doesn't support HDR to SDR\n"); + return AVERROR(EINVAL); + } } hdrtm_param.type = VAProcFilterHighDynamicRangeToneMapping; @@ -241,6 +337,8 @@ static int tonemap_vaapi_filter_frame(AVFilterLink *inlink, AVFrame *input_frame VAProcPipelineParameterBuffer params; int err; + VAHdrMetaData out_hdr_metadata; + av_log(avctx, AV_LOG_DEBUG, "Filter input: %s, %ux%u (%"PRId64").\n", av_get_pix_fmt_name(input_frame->format), input_frame->width, input_frame->height, input_frame->pts); @@ -278,10 +376,15 @@ static int tonemap_vaapi_filter_frame(AVFilterLink *inlink, AVFrame *input_frame if (err < 0) goto fail; - /* Use BT709 by default for HDR to SDR output frame */ - output_frame->color_primaries = AVCOL_PRI_BT709; - output_frame->color_trc = AVCOL_TRC_BT709; - output_frame->colorspace = AVCOL_SPC_BT709; + av_frame_remove_side_data(output_frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL); + av_frame_remove_side_data(output_frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA); + + if (!ctx->mastering_display) { + /* Use BT709 by default for HDR to SDR output frame */ + output_frame->color_primaries = AVCOL_PRI_BT709; + output_frame->color_trc = AVCOL_TRC_BT709; + output_frame->colorspace = AVCOL_SPC_BT709; + } if (ctx->color_primaries != AVCOL_PRI_UNSPECIFIED) output_frame->color_primaries = ctx->color_primaries; @@ -292,11 +395,24 @@ static int tonemap_vaapi_filter_frame(AVFilterLink *inlink, AVFrame *input_frame if (ctx->color_matrix != AVCOL_SPC_UNSPECIFIED) output_frame->colorspace = ctx->color_matrix; + if (ctx->mastering_display) { + err = tonemap_vaapi_update_sidedata(avctx, output_frame); + if (err < 0) + goto fail; + } + err = ff_vaapi_vpp_init_params(avctx, ¶ms, input_frame, output_frame); if (err < 0) goto fail; + if (ctx->mastering_display) { + out_hdr_metadata.metadata_type = VAProcHighDynamicRangeMetadataHDR10; + out_hdr_metadata.metadata = &ctx->out_metadata; + out_hdr_metadata.metadata_size = sizeof(VAHdrMetaDataHDR10); + params.output_hdr_metadata = &out_hdr_metadata; + } + if (vpp_ctx->nb_filter_buffers) { params.filters = &vpp_ctx->filter_buffers[0]; params.num_filters = vpp_ctx->nb_filter_buffers; @@ -312,9 +428,6 @@ static int tonemap_vaapi_filter_frame(AVFilterLink *inlink, AVFrame *input_frame av_get_pix_fmt_name(output_frame->format), output_frame->width, output_frame->height, output_frame->pts); - av_frame_remove_side_data(output_frame, AV_FRAME_DATA_CONTENT_LIGHT_LEVEL); - av_frame_remove_side_data(output_frame, AV_FRAME_DATA_MASTERING_DISPLAY_METADATA); - return ff_filter_frame(outlink, output_frame); fail: @@ -335,8 +448,13 @@ static av_cold int tonemap_vaapi_init(AVFilterContext *avctx) if (ctx->output_format_string) { vpp_ctx->output_format = av_get_pix_fmt(ctx->output_format_string); } else { - vpp_ctx->output_format = AV_PIX_FMT_NV12; - av_log(avctx, AV_LOG_VERBOSE, "Output format not set, use default format NV12 for HDR to SDR tone mapping.\n"); + if (ctx->mastering_display) { + vpp_ctx->output_format = AV_PIX_FMT_P010; + av_log(avctx, AV_LOG_VERBOSE, "Output format not set, use default format P010 for HDR to HDR tone mapping.\n"); + } else { + vpp_ctx->output_format = AV_PIX_FMT_NV12; + av_log(avctx, AV_LOG_VERBOSE, "Output format not set, use default format NV12 for HDR to SDR tone mapping.\n"); + } } #define STRING_OPTION(var_name, func_name, default_value) do { \ @@ -356,6 +474,37 @@ static av_cold int tonemap_vaapi_init(AVFilterContext *avctx) STRING_OPTION(color_transfer, color_transfer, AVCOL_TRC_UNSPECIFIED); STRING_OPTION(color_matrix, color_space, AVCOL_SPC_UNSPECIFIED); + if (ctx->mastering_display) { + if (10 != sscanf(ctx->mastering_display, + "%hu %hu|%hu %hu|%hu %hu|%hu %hu|%u %u", + &ctx->out_metadata.display_primaries_x[0], + &ctx->out_metadata.display_primaries_y[0], + &ctx->out_metadata.display_primaries_x[1], + &ctx->out_metadata.display_primaries_y[1], + &ctx->out_metadata.display_primaries_x[2], + &ctx->out_metadata.display_primaries_y[2], + &ctx->out_metadata.white_point_x, + &ctx->out_metadata.white_point_y, + &ctx->out_metadata.min_display_mastering_luminance, + &ctx->out_metadata.max_display_mastering_luminance)) { + av_log(avctx, AV_LOG_ERROR, + "Option mastering-display input invalid\n"); + return AVERROR(EINVAL); + } + + if (!ctx->content_light) { + ctx->out_metadata.max_content_light_level = 0; + ctx->out_metadata.max_pic_average_light_level = 0; + } else if (2 != sscanf(ctx->content_light, + "%hu %hu", + &ctx->out_metadata.max_content_light_level, + &ctx->out_metadata.max_pic_average_light_level)) { + av_log(avctx, AV_LOG_ERROR, + "Option content-light input invalid\n"); + return AVERROR(EINVAL); + } + } + return 0; } @@ -381,6 +530,12 @@ static const AVOption tonemap_vaapi_options[] = { { "t", "Output color transfer characteristics set", OFFSET(color_transfer_string), AV_OPT_TYPE_STRING, { .str = NULL }, .flags = FLAGS, .unit = "transfer" }, + { "display", "set mastering display colour volume", + OFFSET(mastering_display), AV_OPT_TYPE_STRING, + { .str = NULL }, .flags = FLAGS }, + { "light", "set content light level information", + OFFSET(content_light), AV_OPT_TYPE_STRING, + { .str = NULL }, .flags = FLAGS }, { NULL } };