From 504c60660d3194758823ddd45ceddb86e35d806f Mon Sep 17 00:00:00 2001 From: Aman Karmani Date: Tue, 29 Jun 2021 14:43:16 -0700 Subject: [PATCH] avutil/hwcontext_videotoolbox: implement hwupload to convert AVFrame to CVPixelBuffer Teach AV_HWDEVICE_TYPE_VIDEOTOOLBOX to be able to create AVFrames of type AV_PIX_FMT_VIDEOTOOLBOX. This can be used to hwupload a regular AVFrame into its CVPixelBuffer equivalent. ffmpeg -init_hw_device videotoolbox -f lavfi -i color=black:640x480 -vf hwupload -c:v h264_videotoolbox -f null -y /dev/null Signed-off-by: Aman Karmani --- libavutil/hwcontext_videotoolbox.c | 418 ++++++++++++++++++++++++++++- libavutil/hwcontext_videotoolbox.h | 7 +- libavutil/version.h | 2 +- 3 files changed, 421 insertions(+), 6 deletions(-) diff --git a/libavutil/hwcontext_videotoolbox.c b/libavutil/hwcontext_videotoolbox.c index bded9873fe..58095a1fc9 100644 --- a/libavutil/hwcontext_videotoolbox.c +++ b/libavutil/hwcontext_videotoolbox.c @@ -24,6 +24,7 @@ #include #include "buffer.h" +#include "buffer_internal.h" #include "common.h" #include "hwcontext.h" #include "hwcontext_internal.h" @@ -32,6 +33,10 @@ #include "pixfmt.h" #include "pixdesc.h" +typedef struct VTFramesContext { + CVPixelBufferPoolRef pool; +} VTFramesContext; + static const struct { uint32_t cv_fmt; bool full_range; @@ -50,6 +55,39 @@ static const struct { #endif }; +static const enum AVPixelFormat supported_formats[] = { + AV_PIX_FMT_NV12, + AV_PIX_FMT_YUV420P, + AV_PIX_FMT_UYVY422, + AV_PIX_FMT_P010, + AV_PIX_FMT_BGRA, +}; + +static int vt_frames_get_constraints(AVHWDeviceContext *ctx, + const void *hwconfig, + AVHWFramesConstraints *constraints) +{ + int i; + + constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1, + sizeof(*constraints->valid_sw_formats)); + if (!constraints->valid_sw_formats) + return AVERROR(ENOMEM); + + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) + constraints->valid_sw_formats[i] = supported_formats[i]; + constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = AV_PIX_FMT_NONE; + + constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats)); + if (!constraints->valid_hw_formats) + return AVERROR(ENOMEM); + + constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX; + constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE; + + return 0; +} + enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt) { int i; @@ -75,11 +113,134 @@ uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, boo return 0; } +static int vt_pool_alloc(AVHWFramesContext *ctx) +{ + VTFramesContext *fctx = ctx->internal->priv; + CVReturn err; + CFNumberRef w, h, pixfmt; + uint32_t cv_pixfmt; + CFMutableDictionaryRef attributes, iosurface_properties; + + attributes = CFDictionaryCreateMutable( + NULL, + 2, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + + cv_pixfmt = av_map_videotoolbox_format_from_pixfmt(ctx->sw_format); + pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt); + CFDictionarySetValue( + attributes, + kCVPixelBufferPixelFormatTypeKey, + pixfmt); + CFRelease(pixfmt); + + iosurface_properties = CFDictionaryCreateMutable( + NULL, + 0, + &kCFTypeDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + CFDictionarySetValue(attributes, kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties); + CFRelease(iosurface_properties); + + w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width); + h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height); + CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w); + CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h); + CFRelease(w); + CFRelease(h); + + err = CVPixelBufferPoolCreate( + NULL, + NULL, + attributes, + &fctx->pool); + CFRelease(attributes); + + if (err == kCVReturnSuccess) + return 0; + + av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n", err); + return AVERROR_EXTERNAL; +} + +static AVBufferRef *vt_dummy_pool_alloc(void *opaque, size_t size) +{ + return NULL; +} + +static void vt_frames_uninit(AVHWFramesContext *ctx) +{ + VTFramesContext *fctx = ctx->internal->priv; + if (fctx->pool) { + CVPixelBufferPoolRelease(fctx->pool); + fctx->pool = NULL; + } +} + +static int vt_frames_init(AVHWFramesContext *ctx) +{ + int i, ret; + + for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) { + if (ctx->sw_format == supported_formats[i]) + break; + } + if (i == FF_ARRAY_ELEMS(supported_formats)) { + av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n", + av_get_pix_fmt_name(ctx->sw_format)); + return AVERROR(ENOSYS); + } + + // create a dummy pool so av_hwframe_get_buffer doesn't EINVAL + if (!ctx->pool) { + ctx->internal->pool_internal = av_buffer_pool_init2(0, ctx, vt_dummy_pool_alloc, NULL); + if (!ctx->internal->pool_internal) + return AVERROR(ENOMEM); + } + + ret = vt_pool_alloc(ctx); + if (ret < 0) + return ret; + + return 0; +} + +static void videotoolbox_buffer_release(void *opaque, uint8_t *data) +{ + CVPixelBufferRelease((CVPixelBufferRef)data); +} + static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame) { - frame->buf[0] = av_buffer_pool_get(ctx->pool); - if (!frame->buf[0]) - return AVERROR(ENOMEM); + VTFramesContext *fctx = ctx->internal->priv; + + if (ctx->pool && ctx->pool->size != 0) { + frame->buf[0] = av_buffer_pool_get(ctx->pool); + if (!frame->buf[0]) + return AVERROR(ENOMEM); + } else { + CVPixelBufferRef pixbuf; + AVBufferRef *buf = NULL; + CVReturn err; + + err = CVPixelBufferPoolCreatePixelBuffer( + NULL, + fctx->pool, + &pixbuf + ); + if (err != kCVReturnSuccess) { + av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from pool: %d\n", err); + return AVERROR_EXTERNAL; + } + + buf = av_buffer_create((uint8_t *)pixbuf, 1, videotoolbox_buffer_release, NULL, 0); + if (!buf) { + CVPixelBufferRelease(pixbuf); + return AVERROR(ENOMEM); + } + frame->buf[0] = buf; + } frame->data[3] = frame->buf[0]->data; frame->format = AV_PIX_FMT_VIDEOTOOLBOX; @@ -111,6 +272,248 @@ static void vt_unmap(AVHWFramesContext *ctx, HWMapDescriptor *hwmap) CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv); } +static int vt_pixbuf_set_par(AVHWFramesContext *hwfc, + CVPixelBufferRef pixbuf, const AVFrame *src) +{ + CFMutableDictionaryRef par = NULL; + CFNumberRef num = NULL, den = NULL; + AVRational avpar = src->sample_aspect_ratio; + + if (avpar.num == 0) + return 0; + + av_reduce(&avpar.num, &avpar.den, + avpar.num, avpar.den, + 0xFFFFFFFF); + + num = CFNumberCreate(kCFAllocatorDefault, + kCFNumberIntType, + &avpar.num); + + den = CFNumberCreate(kCFAllocatorDefault, + kCFNumberIntType, + &avpar.den); + + par = CFDictionaryCreateMutable(kCFAllocatorDefault, + 2, + &kCFCopyStringDictionaryKeyCallBacks, + &kCFTypeDictionaryValueCallBacks); + + if (!par || !num || !den) { + if (par) CFRelease(par); + if (num) CFRelease(num); + if (den) CFRelease(den); + return AVERROR(ENOMEM); + } + + CFDictionarySetValue( + par, + kCVImageBufferPixelAspectRatioHorizontalSpacingKey, + num); + CFDictionarySetValue( + par, + kCVImageBufferPixelAspectRatioVerticalSpacingKey, + den); + + CVBufferSetAttachment( + pixbuf, + kCVImageBufferPixelAspectRatioKey, + par, + kCVAttachmentMode_ShouldPropagate + ); + + CFRelease(par); + CFRelease(num); + CFRelease(den); + + return 0; +} + +static int vt_pixbuf_set_chromaloc(AVHWFramesContext *hwfc, + CVPixelBufferRef pixbuf, const AVFrame *src) +{ + CFStringRef loc = NULL; + + switch (src->chroma_location) { + case AVCHROMA_LOC_LEFT: + loc = kCVImageBufferChromaLocation_Left; + break; + case AVCHROMA_LOC_CENTER: + loc = kCVImageBufferChromaLocation_Center; + break; + case AVCHROMA_LOC_TOP: + loc = kCVImageBufferChromaLocation_Top; + break; + case AVCHROMA_LOC_BOTTOM: + loc = kCVImageBufferChromaLocation_Bottom; + break; + case AVCHROMA_LOC_TOPLEFT: + loc = kCVImageBufferChromaLocation_TopLeft; + break; + case AVCHROMA_LOC_BOTTOMLEFT: + loc = kCVImageBufferChromaLocation_BottomLeft; + break; + } + + if (loc) { + CVBufferSetAttachment( + pixbuf, + kCVImageBufferChromaLocationTopFieldKey, + loc, + kCVAttachmentMode_ShouldPropagate); + } + + return 0; +} + +static int vt_pixbuf_set_colorspace(AVHWFramesContext *hwfc, + CVPixelBufferRef pixbuf, const AVFrame *src) +{ + CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL; + Float32 gamma = 0; + + switch (src->colorspace) { + case AVCOL_SPC_BT2020_CL: + case AVCOL_SPC_BT2020_NCL: + if (__builtin_available(macOS 10.11, *)) + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_2020; + else + colormatrix = CFSTR("ITU_R_2020"); + break; + case AVCOL_SPC_BT470BG: + case AVCOL_SPC_SMPTE170M: + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4; + break; + case AVCOL_SPC_BT709: + colormatrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2; + break; + case AVCOL_SPC_SMPTE240M: + colormatrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995; + break; + case AVCOL_SPC_UNSPECIFIED: + break; + default: + av_log(hwfc, AV_LOG_WARNING, "Color space %s is not supported.\n", av_color_space_name(src->colorspace)); + } + + switch (src->color_primaries) { + case AVCOL_PRI_BT2020: + if (__builtin_available(macOS 10.11, *)) + colorpri = kCVImageBufferColorPrimaries_ITU_R_2020; + else + colorpri = CFSTR("ITU_R_2020"); + break; + case AVCOL_PRI_BT709: + colorpri = kCVImageBufferColorPrimaries_ITU_R_709_2; + break; + case AVCOL_PRI_SMPTE170M: + colorpri = kCVImageBufferColorPrimaries_SMPTE_C; + break; + case AVCOL_PRI_BT470BG: + colorpri = kCVImageBufferColorPrimaries_EBU_3213; + break; + case AVCOL_PRI_UNSPECIFIED: + break; + default: + av_log(hwfc, AV_LOG_WARNING, "Color primaries %s is not supported.\n", av_color_primaries_name(src->color_primaries)); + } + + switch (src->color_trc) { + case AVCOL_TRC_SMPTE2084: + if (__builtin_available(macOS 10.13, *)) + colortrc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ; + else + colortrc = CFSTR("SMPTE_ST_2084_PQ"); + break; + case AVCOL_TRC_BT2020_10: + case AVCOL_TRC_BT2020_12: + if (__builtin_available(macOS 10.11, *)) + colortrc = kCVImageBufferTransferFunction_ITU_R_2020; + else + colortrc = CFSTR("ITU_R_2020"); + break; + case AVCOL_TRC_BT709: + colortrc = kCVImageBufferTransferFunction_ITU_R_709_2; + break; + case AVCOL_TRC_SMPTE240M: + colortrc = kCVImageBufferTransferFunction_SMPTE_240M_1995; + break; + case AVCOL_TRC_SMPTE428: + if (__builtin_available(macOS 10.12, *)) + colortrc = kCVImageBufferTransferFunction_SMPTE_ST_428_1; + else + colortrc = CFSTR("SMPTE_ST_428_1"); + break; + case AVCOL_TRC_ARIB_STD_B67: + if (__builtin_available(macOS 10.13, *)) + colortrc = kCVImageBufferTransferFunction_ITU_R_2100_HLG; + else + colortrc = CFSTR("ITU_R_2100_HLG"); + break; + case AVCOL_TRC_GAMMA22: + gamma = 2.2; + colortrc = kCVImageBufferTransferFunction_UseGamma; + break; + case AVCOL_TRC_GAMMA28: + gamma = 2.8; + colortrc = kCVImageBufferTransferFunction_UseGamma; + break; + case AVCOL_TRC_UNSPECIFIED: + break; + default: + av_log(hwfc, AV_LOG_WARNING, "Color transfer function %s is not supported.\n", av_color_transfer_name(src->color_trc)); + } + + if (colormatrix) { + CVBufferSetAttachment( + pixbuf, + kCVImageBufferYCbCrMatrixKey, + colormatrix, + kCVAttachmentMode_ShouldPropagate); + } + if (colorpri) { + CVBufferSetAttachment( + pixbuf, + kCVImageBufferColorPrimariesKey, + colorpri, + kCVAttachmentMode_ShouldPropagate); + } + if (colortrc) { + CVBufferSetAttachment( + pixbuf, + kCVImageBufferTransferFunctionKey, + colortrc, + kCVAttachmentMode_ShouldPropagate); + } + if (gamma != 0) { + CFNumberRef gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma); + CVBufferSetAttachment( + pixbuf, + kCVImageBufferGammaLevelKey, + gamma_level, + kCVAttachmentMode_ShouldPropagate); + CFRelease(gamma_level); + } + + return 0; +} + +static int vt_pixbuf_set_attachments(AVHWFramesContext *hwfc, + CVPixelBufferRef pixbuf, const AVFrame *src) +{ + int ret; + ret = vt_pixbuf_set_par(hwfc, pixbuf, src); + if (ret < 0) + return ret; + ret = vt_pixbuf_set_colorspace(hwfc, pixbuf, src); + if (ret < 0) + return ret; + ret = vt_pixbuf_set_chromaloc(hwfc, pixbuf, src); + if (ret < 0) + return ret; + return 0; +} + static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src, int flags) { @@ -223,6 +626,10 @@ static int vt_transfer_data_to(AVHWFramesContext *hwfc, if (err) goto fail; + err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3], src); + if (err) + goto fail; + err = 0; fail: av_frame_free(&map); @@ -244,8 +651,13 @@ const HWContextType ff_hwcontext_type_videotoolbox = { .type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX, .name = "videotoolbox", + .frames_priv_size = sizeof(VTFramesContext), + .device_create = vt_device_create, + .frames_init = vt_frames_init, .frames_get_buffer = vt_get_buffer, + .frames_get_constraints = vt_frames_get_constraints, + .frames_uninit = vt_frames_uninit, .transfer_get_formats = vt_transfer_get_formats, .transfer_data_to = vt_transfer_data_to, .transfer_data_from = vt_transfer_data_from, diff --git a/libavutil/hwcontext_videotoolbox.h b/libavutil/hwcontext_videotoolbox.h index 5074d79e68..62cde07c51 100644 --- a/libavutil/hwcontext_videotoolbox.h +++ b/libavutil/hwcontext_videotoolbox.h @@ -29,11 +29,14 @@ * @file * An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX. * - * This API currently does not support frame allocation, as the raw VideoToolbox - * API does allocation, and FFmpeg itself never has the need to allocate frames. + * This API supports frame allocation using a native CVPixelBufferPool + * instead of an AVBufferPool. * * If the API user sets a custom pool, AVHWFramesContext.pool must return * AVBufferRefs whose data pointer is a CVImageBufferRef or CVPixelBufferRef. + * Note that the underlying CVPixelBuffer could be retained by OS frameworks + * depending on application usage, so it is preferable to let CoreVideo manage + * the pool using the default implementation. * * Currently AVHWDeviceContext.hwctx and AVHWFramesContext.hwctx are always * NULL. diff --git a/libavutil/version.h b/libavutil/version.h index e11eaa20d0..8312727aca 100644 --- a/libavutil/version.h +++ b/libavutil/version.h @@ -80,7 +80,7 @@ #define LIBAVUTIL_VERSION_MAJOR 57 #define LIBAVUTIL_VERSION_MINOR 0 -#define LIBAVUTIL_VERSION_MICRO 100 +#define LIBAVUTIL_VERSION_MICRO 101 #define LIBAVUTIL_VERSION_INT AV_VERSION_INT(LIBAVUTIL_VERSION_MAJOR, \ LIBAVUTIL_VERSION_MINOR, \