video: fix VideoToolbox/VDA autodetection

This affects vo_opengl_cb in particular: it'll most likely auto-load
VDA, and then the VideoToolbox decoder won't work. And everything fails.

This is mainly caused by FFmpeg using separate pixfmts for the _same_
thing (CVPixelBuffers), simply because libavcodec's architecture demands
that hwaccel backends are selected by pixfmts. (Which makes no sense,
but now we have the mess.)

So instead of duplicating FFmpeg's misdesign, just change the format to
our own canonical one on the image output by the decoder. Now the GL
interop code is exactly the same for VDA and VT, and we use the VT name
only.

video/decode/vda.c

@@ -26,11 +26,10 @@
 #include "video/decode/lavc.h"
 #include "config.h"
 
-
 static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *decoder)
 {
-    hwdec_request_api(info, "vda");
+    hwdec_request_api(info, "videotoolbox");
     if (!info || !info->hwctx)
         return HWDEC_ERR_NO_CTX;
     if (mp_codec_to_av_codec_id(decoder) != AV_CODEC_ID_H264)
@@ -98,6 +97,16 @@ static void uninit(struct lavc_ctx *ctx)
         av_vda_default_free(ctx->avctx);
 }
 
+static struct mp_image *process_image(struct lavc_ctx *ctx, struct mp_image *img)
+{
+    // Same representation. IMGFMT_VDA is only needed to select the libavcodec
+    // hwaccel driver.
+    if (img->imgfmt == IMGFMT_VDA)
+        mp_image_setfmt(img, IMGFMT_VIDEOTOOLBOX);
+
+    return img;
+}
+
 const struct vd_lavc_hwdec mp_vd_lavc_vda = {
     .type = HWDEC_VDA,
     .image_format = IMGFMT_VDA,
@@ -105,4 +114,5 @@ const struct vd_lavc_hwdec mp_vd_lavc_vda = {
     .init = init,
     .uninit = uninit,
     .init_decoder = init_decoder,
+    .process_image = process_image,
 };

video/img_format.h

@@ -202,11 +202,14 @@ enum mp_imgfmt {
     // structures, instead of pixel data.
     IMGFMT_VDPAU,           // VdpVideoSurface
     IMGFMT_VDPAU_OUTPUT,    // VdpOutputSurface
-    IMGFMT_VDA,
-    IMGFMT_VIDEOTOOLBOX,
     IMGFMT_VAAPI,
     IMGFMT_DXVA2,           // IDirect3DSurface9 (NV12)
     IMGFMT_MMAL,            // MMAL_BUFFER_HEADER_T
+    // These use the same underlying format, but FFmpeg requires us to keep
+    // them separate. The VDA decoder will change the format to
+    // IMGFMT_VIDEOTOOLBOX, though.
+    IMGFMT_VIDEOTOOLBOX,
+    IMGFMT_VDA,
 
     // Generic pass-through of AV_PIX_FMT_*. Used for formats which don't have
     // a corresponding IMGFMT_ value.

video/out/gl_hwdec.c

@@ -42,14 +42,11 @@ static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = {
 #if HAVE_VDPAU_GL_X11
     &gl_hwdec_vdpau,
 #endif
-#if HAVE_VDA_GL
-    &gl_hwdec_vda,
+#if HAVE_VIDEOTOOLBOX_VDA_GL
+    &gl_hwdec_videotoolbox,
 #endif
 #if HAVE_DXVA2_HWACCEL
     &gl_hwdec_dxva2,
-#endif
-#if HAVE_VIDEOTOOLBOX_GL
-    &gl_hwdec_videotoolbox,
 #endif
     NULL
 };

video/out/gl_hwdec_vda.c

@@ -17,6 +17,8 @@
  * with mpv.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+// Note: handles both VDA and VideoToolbox
+
 #include <IOSurface/IOSurface.h>
 #include <CoreVideo/CoreVideo.h>
 #include <OpenGL/OpenGL.h>
@@ -116,11 +118,6 @@ static struct mp_image *download_image(struct mp_hwdec_ctx *ctx,
 
 static bool check_hwdec(struct gl_hwdec *hw)
 {
-    if (hw->gl_texture_target != GL_TEXTURE_RECTANGLE) {
-        MP_ERR(hw, "must use rectangle video textures with VDA\n");
-        return false;
-    }
-
     if (!CGLGetCurrentContext()) {
         MP_ERR(hw, "need cocoa opengl backend to be active");
         return false;
@@ -150,9 +147,9 @@ static int create_common(struct gl_hwdec *hw, struct vda_format *format)
     return 0;
 }
 
-#if HAVE_VDA_GL
-static int create_vda(struct gl_hwdec *hw)
+static int create(struct gl_hwdec *hw)
 {
+    // For videotoolbox, we always request NV12.
 #if HAVE_VDA_DEFAULT_INIT2
     struct vda_format *f = vda_get_gl_format_from_imgfmt(IMGFMT_NV12);
 #else
@@ -161,24 +158,10 @@ static int create_vda(struct gl_hwdec *hw)
     if (create_common(hw, f))
         return -1;
 
-    hw->hwctx->type = HWDEC_VDA;
-
-    return 0;
-}
-#endif
-
-#if HAVE_VIDEOTOOLBOX_GL
-static int create_videotoolbox(struct gl_hwdec *hw)
-{
-    struct vda_format *f = vda_get_gl_format_from_imgfmt(IMGFMT_NV12);
-    if (create_common(hw, f))
-        return -1;
-
     hw->hwctx->type = HWDEC_VIDEOTOOLBOX;
 
     return 0;
 }
-#endif
 
 static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 {
@@ -242,24 +225,11 @@ static void destroy(struct gl_hwdec *hw)
     gl->DeleteTextures(MP_MAX_PLANES, p->gl_planes);
 }
 
-#if HAVE_VDA_GL
-const struct gl_hwdec_driver gl_hwdec_vda = {
-    .api_name = "vda",
-    .imgfmt = IMGFMT_VDA,
-    .create = create_vda,
-    .reinit = reinit,
-    .map_image = map_image,
-    .destroy = destroy,
-};
-#endif
-
-#if HAVE_VIDEOTOOLBOX_GL
 const struct gl_hwdec_driver gl_hwdec_videotoolbox = {
     .api_name = "videotoolbox",
     .imgfmt = IMGFMT_VIDEOTOOLBOX,
-    .create = create_videotoolbox,
+    .create = create,
     .reinit = reinit,
     .map_image = map_image,
     .destroy = destroy,
 };
-#endif