From 27038693bb4a0ade33be70f06970631a159578d4 Mon Sep 17 00:00:00 2001 From: Philip Langdale Date: Fri, 25 Nov 2016 11:11:45 -0800 Subject: [PATCH] avcodec/nvenc: Delay identification of underlying format of cuda frames When input surfaces are cuda frames, we will not know what the actual underlying format (nv12, p010, etc) is at surface allocation time. On the other hand, we will know when the input frames are actually registered and associated with a surface. So, let's delay format discovery until registration time, which is actually how we handle other frame properties, such as dimensions. By itself, this change doesn't allow for transcoding of 10bit content from cuvid, but it reduces the problem to the hardcoding of the sw format in ffmpeg_cuvid.c Signed-off-by: Philip Langdale Signed-off-by: Timo Rothenpieler --- libavcodec/nvenc.c | 73 ++++++++++++++++++++++++---------------------- 1 file changed, 38 insertions(+), 35 deletions(-) diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c index d24d2780e5..2353161e06 100644 --- a/libavcodec/nvenc.c +++ b/libavcodec/nvenc.c @@ -28,6 +28,7 @@ #include "libavutil/imgutils.h" #include "libavutil/avassert.h" #include "libavutil/mem.h" +#include "libavutil/pixdesc.h" #include "internal.h" #define NVENC_CAP 0x30 @@ -1009,6 +1010,28 @@ static av_cold int nvenc_setup_encoder(AVCodecContext *avctx) return 0; } +static NV_ENC_BUFFER_FORMAT nvenc_map_buffer_format(enum AVPixelFormat pix_fmt) +{ + switch (pix_fmt) { + case AV_PIX_FMT_YUV420P: + return NV_ENC_BUFFER_FORMAT_YV12_PL; + case AV_PIX_FMT_NV12: + return NV_ENC_BUFFER_FORMAT_NV12_PL; + case AV_PIX_FMT_P010: + return NV_ENC_BUFFER_FORMAT_YUV420_10BIT; + case AV_PIX_FMT_YUV444P: + return NV_ENC_BUFFER_FORMAT_YUV444_PL; + case AV_PIX_FMT_YUV444P16: + return NV_ENC_BUFFER_FORMAT_YUV444_10BIT; + case AV_PIX_FMT_0RGB32: + return NV_ENC_BUFFER_FORMAT_ARGB; + case AV_PIX_FMT_0BGR32: + return NV_ENC_BUFFER_FORMAT_ABGR; + default: + return NV_ENC_BUFFER_FORMAT_UNDEFINED; + } +} + static av_cold int nvenc_alloc_surface(AVCodecContext *avctx, int idx) { NvencContext *ctx = avctx->priv_data; @@ -1019,46 +1042,20 @@ static av_cold int nvenc_alloc_surface(AVCodecContext *avctx, int idx) NV_ENC_CREATE_BITSTREAM_BUFFER allocOut = { 0 }; allocOut.version = NV_ENC_CREATE_BITSTREAM_BUFFER_VER; - switch (ctx->data_pix_fmt) { - case AV_PIX_FMT_YUV420P: - ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_YV12_PL; - break; - - case AV_PIX_FMT_NV12: - ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_NV12_PL; - break; - - case AV_PIX_FMT_P010: - ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_YUV420_10BIT; - break; - - case AV_PIX_FMT_YUV444P: - ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_YUV444_PL; - break; - - case AV_PIX_FMT_YUV444P16: - ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_YUV444_10BIT; - break; - - case AV_PIX_FMT_0RGB32: - ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_ARGB; - break; - - case AV_PIX_FMT_0BGR32: - ctx->surfaces[idx].format = NV_ENC_BUFFER_FORMAT_ABGR; - break; - - default: - av_log(avctx, AV_LOG_FATAL, "Invalid input pixel format\n"); - return AVERROR(EINVAL); - } - if (avctx->pix_fmt == AV_PIX_FMT_CUDA) { ctx->surfaces[idx].in_ref = av_frame_alloc(); if (!ctx->surfaces[idx].in_ref) return AVERROR(ENOMEM); } else { NV_ENC_CREATE_INPUT_BUFFER allocSurf = { 0 }; + + ctx->surfaces[idx].format = nvenc_map_buffer_format(ctx->data_pix_fmt); + if (ctx->surfaces[idx].format == NV_ENC_BUFFER_FORMAT_UNDEFINED) { + av_log(avctx, AV_LOG_FATAL, "Invalid input pixel format: %s\n", + av_get_pix_fmt_name(ctx->data_pix_fmt)); + return AVERROR(EINVAL); + } + allocSurf.version = NV_ENC_CREATE_INPUT_BUFFER_VER; allocSurf.width = (avctx->width + 31) & ~31; allocSurf.height = (avctx->height + 31) & ~31; @@ -1351,10 +1348,16 @@ static int nvenc_register_frame(AVCodecContext *avctx, const AVFrame *frame) reg.resourceType = NV_ENC_INPUT_RESOURCE_TYPE_CUDADEVICEPTR; reg.width = frames_ctx->width; reg.height = frames_ctx->height; - reg.bufferFormat = ctx->surfaces[0].format; reg.pitch = frame->linesize[0]; reg.resourceToRegister = frame->data[0]; + reg.bufferFormat = nvenc_map_buffer_format(frames_ctx->sw_format); + if (reg.bufferFormat == NV_ENC_BUFFER_FORMAT_UNDEFINED) { + av_log(avctx, AV_LOG_FATAL, "Invalid input pixel format: %s\n", + av_get_pix_fmt_name(frames_ctx->sw_format)); + return AVERROR(EINVAL); + } + ret = p_nvenc->nvEncRegisterResource(ctx->nvencoder, ®); if (ret != NV_ENC_SUCCESS) { nvenc_print_error(avctx, ret, "Error registering an input resource");