/* * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "config.h" #include #include #include #include "buffer.h" #include "buffer_internal.h" #include "common.h" #include "hwcontext.h" #include "hwcontext_internal.h" #include "hwcontext_videotoolbox.h" #include "mem.h" #include "pixfmt.h" #include "pixdesc.h" typedef struct VTFramesContext { /** * The public AVVTFramesContext. See hwcontext_videotoolbox.h for it. */ AVVTFramesContext p; CVPixelBufferPoolRef pool; } VTFramesContext; static const struct { uint32_t cv_fmt; bool full_range; enum AVPixelFormat pix_fmt; } cv_pix_fmts[] = { { kCVPixelFormatType_420YpCbCr8Planar, false, AV_PIX_FMT_YUV420P }, { kCVPixelFormatType_420YpCbCr8PlanarFullRange, true, AV_PIX_FMT_YUV420P }, { kCVPixelFormatType_422YpCbCr8, false, AV_PIX_FMT_UYVY422 }, { kCVPixelFormatType_32BGRA, true, AV_PIX_FMT_BGRA }, #ifdef kCFCoreFoundationVersionNumber10_7 { kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV12 }, { kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV12 }, { kCVPixelFormatType_4444AYpCbCr16, false, AV_PIX_FMT_AYUV64 }, #endif #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P010 }, { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P010 }, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE { kCVPixelFormatType_422YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV16 }, { kCVPixelFormatType_422YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV16 }, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE { kCVPixelFormatType_422YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P210 }, { kCVPixelFormatType_422YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P210 }, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR16BIPLANARVIDEORANGE { kCVPixelFormatType_422YpCbCr16BiPlanarVideoRange, false, AV_PIX_FMT_P216 }, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE { kCVPixelFormatType_444YpCbCr8BiPlanarVideoRange, false, AV_PIX_FMT_NV24 }, { kCVPixelFormatType_444YpCbCr8BiPlanarFullRange, true, AV_PIX_FMT_NV24 }, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE { kCVPixelFormatType_444YpCbCr10BiPlanarVideoRange, false, AV_PIX_FMT_P410 }, { kCVPixelFormatType_444YpCbCr10BiPlanarFullRange, true, AV_PIX_FMT_P410 }, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE { kCVPixelFormatType_444YpCbCr16BiPlanarVideoRange, false, AV_PIX_FMT_P416 }, #endif }; static const enum AVPixelFormat supported_formats[] = { #ifdef kCFCoreFoundationVersionNumber10_7 AV_PIX_FMT_NV12, AV_PIX_FMT_AYUV64, #endif AV_PIX_FMT_YUV420P, AV_PIX_FMT_UYVY422, #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE AV_PIX_FMT_P010, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE AV_PIX_FMT_NV16, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE AV_PIX_FMT_P210, #endif #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR16BIPLANARVIDEORANGE AV_PIX_FMT_P216, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE AV_PIX_FMT_NV24, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE AV_PIX_FMT_P410, #endif #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE AV_PIX_FMT_P416, #endif AV_PIX_FMT_BGRA, }; static int vt_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints) { int i; constraints->valid_sw_formats = av_malloc_array(FF_ARRAY_ELEMS(supported_formats) + 1, sizeof(*constraints->valid_sw_formats)); if (!constraints->valid_sw_formats) return AVERROR(ENOMEM); for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) constraints->valid_sw_formats[i] = supported_formats[i]; constraints->valid_sw_formats[FF_ARRAY_ELEMS(supported_formats)] = AV_PIX_FMT_NONE; constraints->valid_hw_formats = av_malloc_array(2, sizeof(*constraints->valid_hw_formats)); if (!constraints->valid_hw_formats) return AVERROR(ENOMEM); constraints->valid_hw_formats[0] = AV_PIX_FMT_VIDEOTOOLBOX; constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE; return 0; } enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt) { int i; for (i = 0; i < FF_ARRAY_ELEMS(cv_pix_fmts); i++) { if (cv_pix_fmts[i].cv_fmt == cv_fmt) return cv_pix_fmts[i].pix_fmt; } return AV_PIX_FMT_NONE; } static uint32_t vt_format_from_pixfmt(enum AVPixelFormat pix_fmt, enum AVColorRange range) { for (int i = 0; i < FF_ARRAY_ELEMS(cv_pix_fmts); i++) { if (cv_pix_fmts[i].pix_fmt == pix_fmt) { int full_range = (range == AVCOL_RANGE_JPEG); // Don't care if unspecified if (range == AVCOL_RANGE_UNSPECIFIED) return cv_pix_fmts[i].cv_fmt; if (cv_pix_fmts[i].full_range == full_range) return cv_pix_fmts[i].cv_fmt; } } return 0; } uint32_t av_map_videotoolbox_format_from_pixfmt(enum AVPixelFormat pix_fmt) { return av_map_videotoolbox_format_from_pixfmt2(pix_fmt, false); } uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, bool full_range) { return vt_format_from_pixfmt(pix_fmt, full_range ? AVCOL_RANGE_JPEG : AVCOL_RANGE_MPEG); } static int vt_pool_alloc(AVHWFramesContext *ctx) { VTFramesContext *fctx = ctx->hwctx; AVVTFramesContext *hw_ctx = &fctx->p; CVReturn err; CFNumberRef w, h, pixfmt; uint32_t cv_pixfmt; CFMutableDictionaryRef attributes, iosurface_properties; attributes = CFDictionaryCreateMutable( NULL, 2, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); cv_pixfmt = vt_format_from_pixfmt(ctx->sw_format, hw_ctx->color_range); pixfmt = CFNumberCreate(NULL, kCFNumberSInt32Type, &cv_pixfmt); CFDictionarySetValue( attributes, kCVPixelBufferPixelFormatTypeKey, pixfmt); CFRelease(pixfmt); iosurface_properties = CFDictionaryCreateMutable( NULL, 0, &kCFTypeDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); CFDictionarySetValue(attributes, kCVPixelBufferIOSurfacePropertiesKey, iosurface_properties); CFRelease(iosurface_properties); w = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->width); h = CFNumberCreate(NULL, kCFNumberSInt32Type, &ctx->height); CFDictionarySetValue(attributes, kCVPixelBufferWidthKey, w); CFDictionarySetValue(attributes, kCVPixelBufferHeightKey, h); CFRelease(w); CFRelease(h); err = CVPixelBufferPoolCreate( NULL, NULL, attributes, &fctx->pool); CFRelease(attributes); if (err == kCVReturnSuccess) return 0; av_log(ctx, AV_LOG_ERROR, "Error creating CVPixelBufferPool: %d\n", err); return AVERROR_EXTERNAL; } static void videotoolbox_buffer_release(void *opaque, uint8_t *data) { CVPixelBufferRelease((CVPixelBufferRef)data); } static AVBufferRef *vt_pool_alloc_buffer(void *opaque, size_t size) { CVPixelBufferRef pixbuf; AVBufferRef *buf; CVReturn err; AVHWFramesContext *ctx = opaque; VTFramesContext *fctx = ctx->hwctx; err = CVPixelBufferPoolCreatePixelBuffer( NULL, fctx->pool, &pixbuf ); if (err != kCVReturnSuccess) { av_log(ctx, AV_LOG_ERROR, "Failed to create pixel buffer from pool: %d\n", err); return NULL; } buf = av_buffer_create((uint8_t *)pixbuf, size, videotoolbox_buffer_release, NULL, 0); if (!buf) { CVPixelBufferRelease(pixbuf); return NULL; } return buf; } static void vt_frames_uninit(AVHWFramesContext *ctx) { VTFramesContext *fctx = ctx->hwctx; if (fctx->pool) { CVPixelBufferPoolRelease(fctx->pool); fctx->pool = NULL; } } static int vt_frames_init(AVHWFramesContext *ctx) { int i, ret; for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++) { if (ctx->sw_format == supported_formats[i]) break; } if (i == FF_ARRAY_ELEMS(supported_formats)) { av_log(ctx, AV_LOG_ERROR, "Pixel format '%s' is not supported\n", av_get_pix_fmt_name(ctx->sw_format)); return AVERROR(ENOSYS); } if (!ctx->pool) { ffhwframesctx(ctx)->pool_internal = av_buffer_pool_init2( sizeof(CVPixelBufferRef), ctx, vt_pool_alloc_buffer, NULL); if (!ffhwframesctx(ctx)->pool_internal) return AVERROR(ENOMEM); } ret = vt_pool_alloc(ctx); if (ret < 0) return ret; return 0; } static int vt_get_buffer(AVHWFramesContext *ctx, AVFrame *frame) { frame->buf[0] = av_buffer_pool_get(ctx->pool); if (!frame->buf[0]) return AVERROR(ENOMEM); frame->data[3] = frame->buf[0]->data; frame->format = AV_PIX_FMT_VIDEOTOOLBOX; frame->width = ctx->width; frame->height = ctx->height; return 0; } static int vt_transfer_get_formats(AVHWFramesContext *ctx, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats) { enum AVPixelFormat *fmts = av_malloc_array(2, sizeof(*fmts)); if (!fmts) return AVERROR(ENOMEM); fmts[0] = ctx->sw_format; fmts[1] = AV_PIX_FMT_NONE; *formats = fmts; return 0; } static void vt_unmap(AVHWFramesContext *ctx, HWMapDescriptor *hwmap) { CVPixelBufferRef pixbuf = (CVPixelBufferRef)hwmap->source->data[3]; CVPixelBufferUnlockBaseAddress(pixbuf, (uintptr_t)hwmap->priv); } static int vt_pixbuf_set_par(void *log_ctx, CVPixelBufferRef pixbuf, const AVFrame *src) { CFMutableDictionaryRef par = NULL; CFNumberRef num = NULL, den = NULL; AVRational avpar = src->sample_aspect_ratio; if (avpar.num == 0) { CVBufferRemoveAttachment(pixbuf, kCVImageBufferPixelAspectRatioKey); return 0; } av_reduce(&avpar.num, &avpar.den, avpar.num, avpar.den, 0xFFFFFFFF); num = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &avpar.num); den = CFNumberCreate(kCFAllocatorDefault, kCFNumberIntType, &avpar.den); par = CFDictionaryCreateMutable(kCFAllocatorDefault, 2, &kCFCopyStringDictionaryKeyCallBacks, &kCFTypeDictionaryValueCallBacks); if (!par || !num || !den) { if (par) CFRelease(par); if (num) CFRelease(num); if (den) CFRelease(den); return AVERROR(ENOMEM); } CFDictionarySetValue( par, kCVImageBufferPixelAspectRatioHorizontalSpacingKey, num); CFDictionarySetValue( par, kCVImageBufferPixelAspectRatioVerticalSpacingKey, den); CVBufferSetAttachment( pixbuf, kCVImageBufferPixelAspectRatioKey, par, kCVAttachmentMode_ShouldPropagate ); CFRelease(par); CFRelease(num); CFRelease(den); return 0; } CFStringRef av_map_videotoolbox_chroma_loc_from_av(enum AVChromaLocation loc) { switch (loc) { case AVCHROMA_LOC_LEFT: return kCVImageBufferChromaLocation_Left; case AVCHROMA_LOC_CENTER: return kCVImageBufferChromaLocation_Center; case AVCHROMA_LOC_TOP: return kCVImageBufferChromaLocation_Top; case AVCHROMA_LOC_BOTTOM: return kCVImageBufferChromaLocation_Bottom; case AVCHROMA_LOC_TOPLEFT: return kCVImageBufferChromaLocation_TopLeft; case AVCHROMA_LOC_BOTTOMLEFT: return kCVImageBufferChromaLocation_BottomLeft; default: return NULL; } } static int vt_pixbuf_set_chromaloc(void *log_ctx, CVPixelBufferRef pixbuf, const AVFrame *src) { CFStringRef loc = av_map_videotoolbox_chroma_loc_from_av(src->chroma_location); if (loc) { CVBufferSetAttachment( pixbuf, kCVImageBufferChromaLocationTopFieldKey, loc, kCVAttachmentMode_ShouldPropagate); } else CVBufferRemoveAttachment( pixbuf, kCVImageBufferChromaLocationTopFieldKey); return 0; } CFStringRef av_map_videotoolbox_color_matrix_from_av(enum AVColorSpace space) { switch (space) { case AVCOL_SPC_BT2020_CL: case AVCOL_SPC_BT2020_NCL: #if HAVE_KCVIMAGEBUFFERYCBCRMATRIX_ITU_R_2020 if (__builtin_available(macOS 10.11, iOS 9, *)) return kCVImageBufferYCbCrMatrix_ITU_R_2020; #endif return CFSTR("ITU_R_2020"); case AVCOL_SPC_BT470BG: case AVCOL_SPC_SMPTE170M: return kCVImageBufferYCbCrMatrix_ITU_R_601_4; case AVCOL_SPC_BT709: return kCVImageBufferYCbCrMatrix_ITU_R_709_2; case AVCOL_SPC_SMPTE240M: return kCVImageBufferYCbCrMatrix_SMPTE_240M_1995; default: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG if (__builtin_available(macOS 10.13, iOS 11, tvOS 11, watchOS 4, *)) return CVYCbCrMatrixGetStringForIntegerCodePoint(space); #endif case AVCOL_SPC_UNSPECIFIED: return NULL; } } CFStringRef av_map_videotoolbox_color_primaries_from_av(enum AVColorPrimaries pri) { switch (pri) { case AVCOL_PRI_BT2020: #if HAVE_KCVIMAGEBUFFERCOLORPRIMARIES_ITU_R_2020 if (__builtin_available(macOS 10.11, iOS 9, *)) return kCVImageBufferColorPrimaries_ITU_R_2020; #endif return CFSTR("ITU_R_2020"); case AVCOL_PRI_BT709: return kCVImageBufferColorPrimaries_ITU_R_709_2; case AVCOL_PRI_SMPTE170M: return kCVImageBufferColorPrimaries_SMPTE_C; case AVCOL_PRI_BT470BG: return kCVImageBufferColorPrimaries_EBU_3213; default: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG if (__builtin_available(macOS 10.13, iOS 11, tvOS 11, watchOS 4, *)) return CVColorPrimariesGetStringForIntegerCodePoint(pri); #endif case AVCOL_PRI_UNSPECIFIED: return NULL; } } CFStringRef av_map_videotoolbox_color_trc_from_av(enum AVColorTransferCharacteristic trc) { switch (trc) { case AVCOL_TRC_SMPTE2084: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ if (__builtin_available(macOS 10.13, iOS 11, *)) return kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ; #endif return CFSTR("SMPTE_ST_2084_PQ"); case AVCOL_TRC_BT2020_10: case AVCOL_TRC_BT2020_12: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2020 if (__builtin_available(macOS 10.11, iOS 9, *)) return kCVImageBufferTransferFunction_ITU_R_2020; #endif return CFSTR("ITU_R_2020"); case AVCOL_TRC_BT709: return kCVImageBufferTransferFunction_ITU_R_709_2; case AVCOL_TRC_SMPTE240M: return kCVImageBufferTransferFunction_SMPTE_240M_1995; case AVCOL_TRC_SMPTE428: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_428_1 if (__builtin_available(macOS 10.12, iOS 10, *)) return kCVImageBufferTransferFunction_SMPTE_ST_428_1; #endif return CFSTR("SMPTE_ST_428_1"); case AVCOL_TRC_ARIB_STD_B67: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG if (__builtin_available(macOS 10.13, iOS 11, *)) return kCVImageBufferTransferFunction_ITU_R_2100_HLG; #endif return CFSTR("ITU_R_2100_HLG"); case AVCOL_TRC_GAMMA22: return kCVImageBufferTransferFunction_UseGamma; case AVCOL_TRC_GAMMA28: return kCVImageBufferTransferFunction_UseGamma; default: #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG if (__builtin_available(macOS 10.13, iOS 11, tvOS 11, watchOS 4, *)) return CVTransferFunctionGetStringForIntegerCodePoint(trc); #endif case AVCOL_TRC_UNSPECIFIED: return NULL; } } static int vt_pixbuf_set_colorspace(void *log_ctx, CVPixelBufferRef pixbuf, const AVFrame *src) { CGColorSpaceRef colorspace = NULL; CFStringRef colormatrix = NULL, colorpri = NULL, colortrc = NULL; Float32 gamma = 0; colormatrix = av_map_videotoolbox_color_matrix_from_av(src->colorspace); if (colormatrix) CVBufferSetAttachment(pixbuf, kCVImageBufferYCbCrMatrixKey, colormatrix, kCVAttachmentMode_ShouldPropagate); else { CVBufferRemoveAttachment(pixbuf, kCVImageBufferYCbCrMatrixKey); if (src->colorspace != AVCOL_SPC_UNSPECIFIED) av_log(log_ctx, AV_LOG_WARNING, "Color space %s is not supported.\n", av_color_space_name(src->colorspace)); } colorpri = av_map_videotoolbox_color_primaries_from_av(src->color_primaries); if (colorpri) CVBufferSetAttachment(pixbuf, kCVImageBufferColorPrimariesKey, colorpri, kCVAttachmentMode_ShouldPropagate); else { CVBufferRemoveAttachment(pixbuf, kCVImageBufferColorPrimariesKey); if (src->color_primaries != AVCOL_SPC_UNSPECIFIED) av_log(log_ctx, AV_LOG_WARNING, "Color primaries %s is not supported.\n", av_color_primaries_name(src->color_primaries)); } colortrc = av_map_videotoolbox_color_trc_from_av(src->color_trc); if (colortrc) CVBufferSetAttachment(pixbuf, kCVImageBufferTransferFunctionKey, colorpri, kCVAttachmentMode_ShouldPropagate); else { CVBufferRemoveAttachment(pixbuf, kCVImageBufferTransferFunctionKey); if (src->color_trc != AVCOL_TRC_UNSPECIFIED) av_log(log_ctx, AV_LOG_WARNING, "Color transfer function %s is not supported.\n", av_color_transfer_name(src->color_trc)); } if (src->color_trc == AVCOL_TRC_GAMMA22) gamma = 2.2; else if (src->color_trc == AVCOL_TRC_GAMMA28) gamma = 2.8; if (gamma != 0) { CFNumberRef gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma); CVBufferSetAttachment(pixbuf, kCVImageBufferGammaLevelKey, gamma_level, kCVAttachmentMode_ShouldPropagate); CFRelease(gamma_level); } else CVBufferRemoveAttachment(pixbuf, kCVImageBufferGammaLevelKey); #if (TARGET_OS_OSX && __MAC_OS_X_VERSION_MAX_ALLOWED >= 100800) || \ (TARGET_OS_IOS && __IPHONE_OS_VERSION_MAX_ALLOWED >= 100000) if (__builtin_available(macOS 10.8, iOS 10, *)) { CFDictionaryRef attachments = NULL; if (__builtin_available(macOS 12.0, iOS 15.0, *)) attachments = CVBufferCopyAttachments(pixbuf, kCVAttachmentMode_ShouldPropagate); #if (TARGET_OS_OSX && __MAC_OS_X_VERSION_MIN_REQUIRED <= 120000) || \ (TARGET_OS_IOS && __IPHONE_OS_VERSION_MIN_REQUIRED <= 150000) else { CFDictionaryRef tmp = CVBufferGetAttachments(pixbuf, kCVAttachmentMode_ShouldPropagate); if (tmp) attachments = CFDictionaryCreateCopy(NULL, tmp); } #endif if (attachments) { colorspace = CVImageBufferCreateColorSpaceFromAttachments(attachments); CFRelease(attachments); } } #endif if (colorspace) { CVBufferSetAttachment(pixbuf, kCVImageBufferCGColorSpaceKey, colorspace, kCVAttachmentMode_ShouldPropagate); CFRelease(colorspace); } else CVBufferRemoveAttachment(pixbuf, kCVImageBufferCGColorSpaceKey); return 0; } static int vt_pixbuf_set_attachments(void *log_ctx, CVPixelBufferRef pixbuf, const AVFrame *src) { int ret; ret = vt_pixbuf_set_par(log_ctx, pixbuf, src); if (ret < 0) return ret; ret = vt_pixbuf_set_colorspace(log_ctx, pixbuf, src); if (ret < 0) return ret; ret = vt_pixbuf_set_chromaloc(log_ctx, pixbuf, src); if (ret < 0) return ret; return 0; } int av_vt_pixbuf_set_attachments(void *log_ctx, CVPixelBufferRef pixbuf, const AVFrame *src) { return vt_pixbuf_set_attachments(log_ctx, pixbuf, src); } static int vt_map_frame(AVHWFramesContext *ctx, AVFrame *dst, const AVFrame *src, int flags) { CVPixelBufferRef pixbuf = (CVPixelBufferRef)src->data[3]; OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf); CVReturn err; uint32_t map_flags = 0; int ret; int i; enum AVPixelFormat format; format = av_map_videotoolbox_format_to_pixfmt(pixel_format); if (dst->format != format) { av_log(ctx, AV_LOG_ERROR, "Unsupported or mismatching pixel format: %s\n", av_fourcc2str(pixel_format)); return AVERROR_UNKNOWN; } if (CVPixelBufferGetWidth(pixbuf) != ctx->width || CVPixelBufferGetHeight(pixbuf) != ctx->height) { av_log(ctx, AV_LOG_ERROR, "Inconsistent frame dimensions.\n"); return AVERROR_UNKNOWN; } if (flags == AV_HWFRAME_MAP_READ) map_flags = kCVPixelBufferLock_ReadOnly; err = CVPixelBufferLockBaseAddress(pixbuf, map_flags); if (err != kCVReturnSuccess) { av_log(ctx, AV_LOG_ERROR, "Error locking the pixel buffer.\n"); return AVERROR_UNKNOWN; } if (CVPixelBufferIsPlanar(pixbuf)) { int planes = CVPixelBufferGetPlaneCount(pixbuf); for (i = 0; i < planes; i++) { dst->data[i] = CVPixelBufferGetBaseAddressOfPlane(pixbuf, i); dst->linesize[i] = CVPixelBufferGetBytesPerRowOfPlane(pixbuf, i); } } else { dst->data[0] = CVPixelBufferGetBaseAddress(pixbuf); dst->linesize[0] = CVPixelBufferGetBytesPerRow(pixbuf); } ret = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, vt_unmap, (void *)(uintptr_t)map_flags); if (ret < 0) goto unlock; return 0; unlock: CVPixelBufferUnlockBaseAddress(pixbuf, map_flags); return ret; } static int vt_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src) { AVFrame *map; int err; if (dst->width > hwfc->width || dst->height > hwfc->height) return AVERROR(EINVAL); map = av_frame_alloc(); if (!map) return AVERROR(ENOMEM); map->format = dst->format; err = vt_map_frame(hwfc, map, src, AV_HWFRAME_MAP_READ); if (err) goto fail; map->width = dst->width; map->height = dst->height; err = av_frame_copy(dst, map); if (err) goto fail; err = 0; fail: av_frame_free(&map); return err; } static int vt_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src) { AVFrame *map; int err; if (src->width > hwfc->width || src->height > hwfc->height) return AVERROR(EINVAL); map = av_frame_alloc(); if (!map) return AVERROR(ENOMEM); map->format = src->format; err = vt_map_frame(hwfc, map, dst, AV_HWFRAME_MAP_WRITE | AV_HWFRAME_MAP_OVERWRITE); if (err) goto fail; map->width = src->width; map->height = src->height; err = av_frame_copy(map, src); if (err) goto fail; err = vt_pixbuf_set_attachments(hwfc, (CVPixelBufferRef)dst->data[3], src); if (err) goto fail; err = 0; fail: av_frame_free(&map); return err; } static int vt_map_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags) { int err; if (dst->format == AV_PIX_FMT_NONE) dst->format = hwfc->sw_format; else if (dst->format != hwfc->sw_format) return AVERROR(ENOSYS); err = vt_map_frame(hwfc, dst, src, flags); if (err) return err; dst->width = src->width; dst->height = src->height; err = av_frame_copy_props(dst, src); if (err) return err; return 0; } static int vt_device_create(AVHWDeviceContext *ctx, const char *device, AVDictionary *opts, int flags) { if (device && device[0]) { av_log(ctx, AV_LOG_ERROR, "Device selection unsupported.\n"); return AVERROR_UNKNOWN; } return 0; } const HWContextType ff_hwcontext_type_videotoolbox = { .type = AV_HWDEVICE_TYPE_VIDEOTOOLBOX, .name = "videotoolbox", .frames_hwctx_size = sizeof(VTFramesContext), .device_create = vt_device_create, .frames_init = vt_frames_init, .frames_get_buffer = vt_get_buffer, .frames_get_constraints = vt_frames_get_constraints, .frames_uninit = vt_frames_uninit, .transfer_get_formats = vt_transfer_get_formats, .transfer_data_to = vt_transfer_data_to, .transfer_data_from = vt_transfer_data_from, .map_from = vt_map_from, .pix_fmts = (const enum AVPixelFormat[]){ AV_PIX_FMT_VIDEOTOOLBOX, AV_PIX_FMT_NONE }, };