vo: hwdec: do hwdec interop lookup by image format

It turns out that it's generally more useful to look up hwdecs by image
format, rather than device type. In the situations where we need to
find one, we generally know the image format we're dealing with. Doing
this avoids us having to create mappings from image format to device
type.

The most significant part of this change is filling in the image format
for the various hw interops. There is a hw_imgfmt field today today, but
only a couple of the interops fill it in, and that seems to be because
we've never actually used this piece of metadata before. Well, now we
have a good use for it.

filters/f_hwtransfer.c

@@ -198,10 +198,10 @@ static const struct mp_filter_info hwupload_filter = {
 // So filter out all not explicitly supported formats.
 static bool vo_supports(struct mp_hwdec_ctx *ctx, int hw_fmt, int sw_fmt)
 {
-    if (!ctx->hw_imgfmt)
-        return true; // if unset, all formats are allowed
     if (ctx->hw_imgfmt != hw_fmt)
         return false;
+    if (!ctx->supported_formats)
+        return true; // if unset, all formats are allowed
 
     for (int i = 0; ctx->supported_formats &&  ctx->supported_formats[i]; i++) {
         if (ctx->supported_formats[i] == sw_fmt)

filters/filter.c

@@ -685,32 +685,19 @@ struct mp_stream_info *mp_filter_find_stream_info(struct mp_filter *f)
     return NULL;
 }
 
-struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int avtype)
+struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt)
 {
     struct mp_stream_info *info = mp_filter_find_stream_info(f);
     if (!info || !info->hwdec_devs)
         return NULL;
 
-    int imgfmt = IMGFMT_NONE;
-    switch (avtype) {
-    case AV_HWDEVICE_TYPE_VAAPI:
-        imgfmt = IMGFMT_VAAPI;
-        break;
-    case AV_HWDEVICE_TYPE_VDPAU:
-        imgfmt = IMGFMT_VDPAU;
-        break;
-    default:
-        MP_WARN(f,
-               "Unrecognised HW Device type requested. Loading all devices\n");
-    }
-
     struct hwdec_imgfmt_request params = {
         .imgfmt = imgfmt,
         .probing = false,
     };
     hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
 
-    return hwdec_devices_get_lavc(info->hwdec_devs, avtype);
+    return hwdec_devices_get_imgfmt(info->hwdec_devs, imgfmt);
 }
 
 static void filter_wakeup(struct mp_filter *f, bool mark_only)

filters/filter.h

@@ -409,7 +409,7 @@ struct mp_stream_info {
 struct mp_stream_info *mp_filter_find_stream_info(struct mp_filter *f);
 
 struct AVBufferRef;
-struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int avtype);
+struct AVBufferRef *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt);
 
 // Perform filtering. This runs until the filter graph is blocked (due to
 // missing external input or unread output). It returns whether any outside

video/decode/vd_lavc.c

@@ -435,12 +435,16 @@ static AVBufferRef *hwdec_create_dev(struct mp_filter *vd,
             return ref;
         }
     } else if (ctx->hwdec_devs) {
+        int imgfmt = pixfmt2imgfmt(hwdec->pix_fmt);
         struct hwdec_imgfmt_request params = {
-            .imgfmt = pixfmt2imgfmt(hwdec->pix_fmt),
+            .imgfmt = imgfmt,
             .probing = autoprobe,
         };
         hwdec_devices_request_for_img_fmt(ctx->hwdec_devs, &params);
-        return hwdec_devices_get_lavc(ctx->hwdec_devs, hwdec->lavc_device);
+
+        const struct mp_hwdec_ctx *hw_ctx =
+            hwdec_devices_get_by_imgfmt(ctx->hwdec_devs, imgfmt);
+        return hw_ctx ? av_buffer_ref(hw_ctx->av_device_ref) : NULL;
     }
 
     return NULL;

video/filter/vf_d3d11vpp.c

@@ -435,7 +435,7 @@ static struct mp_filter *vf_d3d11vpp_create(struct mp_filter *parent,
     hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
 
     struct mp_hwdec_ctx *hwctx =
-        hwdec_devices_get_by_lavc(info->hwdec_devs, AV_HWDEVICE_TYPE_D3D11VA);
+        hwdec_devices_get_by_imgfmt(info->hwdec_devs, IMGFMT_D3D11);
     if (!hwctx || !hwctx->av_device_ref)
         goto fail;
     AVHWDeviceContext *avhwctx = (void *)hwctx->av_device_ref->data;

video/filter/vf_vavpp.c

@@ -448,7 +448,7 @@ static struct mp_filter *vf_vavpp_create(struct mp_filter *parent, void *options
 
     p->queue = mp_refqueue_alloc(f);
 
-    p->av_device_ref = mp_filter_load_hwdec_device(f, AV_HWDEVICE_TYPE_VAAPI);
+    p->av_device_ref = mp_filter_load_hwdec_device(f, IMGFMT_VAAPI);
     if (!p->av_device_ref)
         goto error;
 

video/filter/vf_vdpaupp.c

@@ -136,7 +136,7 @@ static struct mp_filter *vf_vdpaupp_create(struct mp_filter *parent, void *optio
 
     p->queue = mp_refqueue_alloc(f);
 
-    AVBufferRef *ref = mp_filter_load_hwdec_device(f, AV_HWDEVICE_TYPE_VDPAU);
+    AVBufferRef *ref = mp_filter_load_hwdec_device(f, IMGFMT_VDPAU);
     if (!ref)
         goto error;
     p->ctx = mp_vdpau_get_ctx_from_av(ref);

video/hwdec.c

@@ -35,34 +35,22 @@ void hwdec_devices_destroy(struct mp_hwdec_devices *devs)
     talloc_free(devs);
 }
 
-struct mp_hwdec_ctx *hwdec_devices_get_by_lavc(struct mp_hwdec_devices *devs,
+struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt(struct mp_hwdec_devices *devs,
-                                               int av_hwdevice_type)
+                                                 int hw_imgfmt)
 {
     struct mp_hwdec_ctx *res = NULL;
     pthread_mutex_lock(&devs->lock);
     for (int n = 0; n < devs->num_hwctxs; n++) {
         struct mp_hwdec_ctx *dev = devs->hwctxs[n];
-        if (dev->av_device_ref) {
+        if (dev->hw_imgfmt == hw_imgfmt) {
-            AVHWDeviceContext *hwctx = (void *)dev->av_device_ref->data;
+            res = dev;
-            if (hwctx->type == av_hwdevice_type) {
+            break;
-                res = dev;
-                break;
-            }
         }
     }
     pthread_mutex_unlock(&devs->lock);
     return res;
 }
 
-struct AVBufferRef *hwdec_devices_get_lavc(struct mp_hwdec_devices *devs,
-                                           int av_hwdevice_type)
-{
-    struct mp_hwdec_ctx *ctx = hwdec_devices_get_by_lavc(devs, av_hwdevice_type);
-    if (!ctx)
-        return NULL;
-    return av_buffer_ref(ctx->av_device_ref);
-}
-
 struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs)
 {
     return hwdec_devices_get_n(devs, 0);

video/hwdec.h

@@ -13,9 +13,10 @@ struct mp_hwdec_ctx {
     // libavutil-wrapped context, if available.
     struct AVBufferRef *av_device_ref; // AVHWDeviceContext*
 
-    // List of IMGFMT_s, terminated with 0. NULL if N/A.
+    // List of allowed IMGFMT_s, terminated with 0.
+    // If NULL, all software formats are considered to be supported.
     const int *supported_formats;
-    // HW format for which above hw_subfmts are valid.
+    // HW format used by the hwdec
     int hw_imgfmt;
 };
 
@@ -25,17 +26,8 @@ struct mp_hwdec_devices;
 struct mp_hwdec_devices *hwdec_devices_create(void);
 void hwdec_devices_destroy(struct mp_hwdec_devices *devs);
 
-// Return the device context for the given API type. Returns NULL if none
+struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt(struct mp_hwdec_devices *devs,
-// available. Logically, the returned pointer remains valid until VO
+                                                 int hw_imgfmt);
-// uninitialization is started (all users of it must be uninitialized before).
-// hwdec_devices_request() may be used before this to lazily load devices.
-// Contains a wrapped AVHWDeviceContext.
-// Beware that this creates a _new_ reference.
-struct AVBufferRef *hwdec_devices_get_lavc(struct mp_hwdec_devices *devs,
-                                           int av_hwdevice_type);
-
-struct mp_hwdec_ctx *hwdec_devices_get_by_lavc(struct mp_hwdec_devices *devs,
-                                               int av_hwdevice_type);
 
 // For code which still strictly assumes there is 1 (or none) device.
 struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs);

video/out/d3d11/hwdec_dxva2dxgi.c

@@ -134,6 +134,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->dev9),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
 

video/out/hwdec/hwdec_cuda.c

@@ -122,6 +122,7 @@ static int cuda_init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx) {
         .driver_name = hw->driver->name,
         .av_device_ref = hw_device_ctx,
+        .hw_imgfmt = IMGFMT_CUDA,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
     return 0;

video/out/opengl/hwdec_drmprime_drm.c

@@ -298,6 +298,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx) {
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_DRMPRIME,
     };
     if (!av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_DRM,
                                 drmGetDeviceNameFromFd2(p->ctx->fd), NULL, 0)) {

video/out/opengl/hwdec_dxva2egl.c

@@ -181,6 +181,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->device9ex),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
 

video/out/opengl/hwdec_dxva2gldx.c

@@ -80,6 +80,7 @@ static int init(struct ra_hwdec *hw)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
         .av_device_ref = d3d9_wrap_device_ref((IDirect3DDevice9 *)p->device),
+        .hw_imgfmt = IMGFMT_DXVA2,
     };
     hwdec_devices_add(hw->devs, &p->hwctx);
     return 0;

video/out/opengl/hwdec_ios.m

@@ -70,6 +70,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_VIDEOTOOLBOX,
     };
 
     av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX,

video/out/opengl/hwdec_osx.c

@@ -71,6 +71,7 @@ static int init(struct ra_hwdec *hw)
 
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = hw->driver->name,
+        .hw_imgfmt = IMGFMT_VIDEOTOOLBOX,
     };
 
     av_hwdevice_ctx_create(&p->hwctx.av_device_ref, AV_HWDEVICE_TYPE_VIDEOTOOLBOX,

video/out/opengl/hwdec_vdpau.c

@@ -61,6 +61,7 @@ static int init(struct ra_hwdec *hw)
     if (hw->probing && mp_vdpau_guess_if_emulated(p->ctx))
         return -1;
     p->ctx->hwctx.driver_name = hw->driver->name;
+    p->ctx->hwctx.hw_imgfmt = IMGFMT_VDPAU;
     hwdec_devices_add(hw->devs, &p->ctx->hwctx);
     return 0;
 }

video/out/vo_mediacodec_embed.c

@@ -52,6 +52,7 @@ static int preinit(struct vo *vo)
     p->hwctx = (struct mp_hwdec_ctx){
         .driver_name = "mediacodec_embed",
         .av_device_ref = create_mediacodec_device_ref(vo),
+        .hw_imgfmt = IMGFMT_MEDIACODEC,
     };
     hwdec_devices_add(vo->hwdec_devs, &p->hwctx);
     return 0;