videotoolbox: change how videotoolbox format is managed

The underlying intention of this code is to make changing
--videotoolbox-format at runtime work. For this reason, the format can't
just be statically setup, but must be read from the option at runtime.

This means the format is not fixed anymore, and we have to make sure the
renderer is property reinitialized if the format changes. There is
currently no way to trigger reinit on this level, which is why the
mp_image_params.hw_subfmt field was introduced.

One sketchy thing remains: normally, the renderer is supposed to be
involved with VO format negotiation, which would ensure that the VO
can take the format at all. Since the hw_subfmt is not part of this
format negotiation, it's implied the get_vt_fmt() callback only
returns formats supported by the renderer. This is not necessarily
clear because vo_opengl checks this with converted_imgfmt separately.
None of this matters in practice though, because we know all formats
are always supported.

(This still requires somehow triggering decoder reinit to make the
change effective.)

video/decode/videotoolbox.c

@@ -31,7 +31,7 @@ static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *codec)
 {
     hwdec_request_api(info, "videotoolbox");
-    if (!info || !info->hwctx || info->hwctx->type != HWDEC_VIDEOTOOLBOX)
+    if (!info || !info->hwctx || !info->hwctx->get_vt_fmt)
         return HWDEC_ERR_NO_CTX;
     switch (mp_codec_to_av_codec_id(codec)) {
     case AV_CODEC_ID_H264:
@@ -88,9 +88,11 @@ static int init_decoder(struct lavc_ctx *ctx, int w, int h)
     av_videotoolbox_default_free(ctx->avctx);
 
     AVVideotoolboxContext *vtctx = av_videotoolbox_alloc_context();
-    vtctx->cv_pix_fmt_type = (uintptr_t)ctx->hwdec_info->hwctx->priv;
-    int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
 
+    struct mp_hwdec_ctx *hwctx = ctx->hwdec_info->hwctx;
+    vtctx->cv_pix_fmt_type = hwctx->get_vt_fmt(hwctx);
+
+    int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
     if (err < 0) {
         print_videotoolbox_error(ctx->log, MSGL_ERR, "failed to init videotoolbox decoder", err);
         return -1;
@@ -105,6 +107,15 @@ static void uninit(struct lavc_ctx *ctx)
         av_videotoolbox_default_free(ctx->avctx);
 }
 
+static struct mp_image *process_image(struct lavc_ctx *ctx, struct mp_image *img)
+{
+    if (img->imgfmt == IMGFMT_VIDEOTOOLBOX) {
+        CVPixelBufferRef pbuf = (CVPixelBufferRef)img->planes[3];
+        img->params.hw_subfmt = CVPixelBufferGetPixelFormatType(pbuf);
+    }
+    return img;
+}
+
 const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox = {
     .type = HWDEC_VIDEOTOOLBOX,
     .image_format = IMGFMT_VIDEOTOOLBOX,
@@ -112,4 +123,5 @@ const struct vd_lavc_hwdec mp_vd_lavc_videotoolbox = {
     .init = init,
     .uninit = uninit,
     .init_decoder = init_decoder,
+    .process_image = process_image,
 };

video/hwdec.h

@@ -32,6 +32,7 @@ struct mp_hwdec_ctx {
     struct mp_vdpau_ctx *vdpau_ctx;
     struct mp_vaapi_ctx *vaapi_ctx;
     struct mp_d3d_ctx *d3d_ctx;
+    uint32_t (*get_vt_fmt)(struct mp_hwdec_ctx *ctx);
 
     // Optional.
     // Allocates a software image from the pool, downloads the hw image from

video/out/opengl/hwdec.h

@@ -39,6 +39,7 @@ struct gl_hwdec_driver {
     // Prepare for rendering video. (E.g. create textures.)
     // Called on initialization, and every time the video size changes.
     // *params must be set to the format the hw textures return.
+    // This also can update hw->converted_imgfmt.
     int (*reinit)(struct gl_hwdec *hw, struct mp_image_params *params);
     // Return textures that contain a copy or reference of the given hw_image.
     int (*map_image)(struct gl_hwdec *hw, struct mp_image *hw_image,

video/out/opengl/hwdec_osx.c

@@ -146,14 +146,21 @@ static bool check_hwdec(struct gl_hwdec *hw)
     return true;
 }
 
+static uint32_t get_vt_fmt(struct mp_hwdec_ctx *ctx)
+{
+    struct gl_hwdec *hw = ctx->priv;
+    struct vt_format *f =
+        vt_get_gl_format_from_imgfmt(hw->global->opts->videotoolbox_format);
+    return f ? f->cvpixfmt : (uint32_t)-1;
+}
+
 static int create(struct gl_hwdec *hw)
 {
     if (!check_hwdec(hw))
         return -1;
 
     struct priv *p = talloc_zero(hw, struct priv);
-    struct vt_format *f =
-        vt_get_gl_format_from_imgfmt(hw->global->opts->videotoolbox_format);
+    struct vt_format *f = vt_get_gl_format_from_imgfmt(IMGFMT_NV12);
     if (!f)
         return -1;
 
@@ -162,17 +169,26 @@ static int create(struct gl_hwdec *hw)
     hw->hwctx = &p->hwctx;
     hw->hwctx->download_image = download_image;
     hw->hwctx->type = HWDEC_VIDEOTOOLBOX;
-    hw->hwctx->priv = (void *)(uintptr_t)f->cvpixfmt;
+    hw->hwctx->get_vt_fmt = get_vt_fmt;
 
     hw->gl_texture_target = GL_TEXTURE_RECTANGLE;
     hw->gl->GenTextures(MP_MAX_PLANES, p->gl_planes);
 
+    hw->hwctx->priv = hw;
     return 0;
 }
 
 static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 {
     assert(params->imgfmt == hw->driver->imgfmt);
+
+    struct vt_format *f = vt_get_gl_format(params->hw_subfmt);
+    if (!f) {
+        MP_ERR(hw, "Unsupported CVPixelBuffer format.\n");
+        return -1;
+    }
+
+    hw->converted_imgfmt = f->imgfmt;
     return 0;
 }