video: hwdec: extend look up of hwdecs to consider device type as well

Now that we have identified the device types associated with each hwdec, we
can take advantage of this to ensure that get the right hwdec is chosen when
decoding frames and there are multiple choices available with different device
types.

This is also plumbed into the filter initialisation logic, although in practice
the decision on the hwdec has already been made at this point - still it's
easier than retaining the ambiguous look up logic and worrying that it might
lead to an inconsistent result.

filters/f_lavfi.c

@@ -556,7 +556,9 @@ static void init_graph(struct lavfi *c)
             if (c->hwdec_interop) {
                 int imgfmt =
                     ra_hwdec_driver_get_imgfmt_for_name(c->hwdec_interop);
-                hwdec_ctx = mp_filter_load_hwdec_device(c->f, imgfmt);
+                enum AVHWDeviceType device_type =
+                    ra_hwdec_driver_get_device_type_for_name(c->hwdec_interop);
+                hwdec_ctx = mp_filter_load_hwdec_device(c->f, imgfmt, device_type);
             } else {
                 hwdec_ctx = hwdec_devices_get_first(info->hwdec_devs);
             }

filters/filter.c

@@ -688,7 +688,8 @@ struct mp_stream_info *mp_filter_find_stream_info(struct mp_filter *f)
     return NULL;
 }
 
-struct mp_hwdec_ctx *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt)
+struct mp_hwdec_ctx *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt,
+                                                 enum AVHWDeviceType device_type)
 {
     struct mp_stream_info *info = mp_filter_find_stream_info(f);
     if (!info || !info->hwdec_devs)
@@ -700,7 +701,7 @@ struct mp_hwdec_ctx *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt
     };
     hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
 
-    return hwdec_devices_get_by_imgfmt(info->hwdec_devs, imgfmt);
+    return hwdec_devices_get_by_imgfmt_and_type(info->hwdec_devs, imgfmt, device_type);
 }
 
 static void filter_wakeup(struct mp_filter *f, bool mark_only)

filters/filter.h

@@ -1,6 +1,7 @@
 #pragma once
 
 #include <stdbool.h>
+#include <libavutil/hwcontext.h>
 
 #include "frame.h"
 
@@ -410,7 +411,8 @@ struct mp_stream_info {
 // Search for a parent filter (including f) that has this set, and return it.
 struct mp_stream_info *mp_filter_find_stream_info(struct mp_filter *f);
 
-struct mp_hwdec_ctx *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt);
+struct mp_hwdec_ctx *mp_filter_load_hwdec_device(struct mp_filter *f, int imgfmt,
+                                                 enum AVHWDeviceType device_type);
 
 // Perform filtering. This runs until the filter graph is blocked (due to
 // missing external input or unread output). It returns whether any outside

video/decode/vd_lavc.c

@@ -456,7 +456,8 @@ static AVBufferRef *hwdec_create_dev(struct mp_filter *vd,
         hwdec_devices_request_for_img_fmt(ctx->hwdec_devs, &params);
 
         const struct mp_hwdec_ctx *hw_ctx =
-            hwdec_devices_get_by_imgfmt(ctx->hwdec_devs, imgfmt);
+            hwdec_devices_get_by_imgfmt_and_type(ctx->hwdec_devs, imgfmt,
+                                                 hwdec->lavc_device);
 
         if (hw_ctx && hw_ctx->av_device_ref)
             return av_buffer_ref(hw_ctx->av_device_ref);

video/filter/vf_d3d11vpp.c

@@ -435,7 +435,8 @@ static struct mp_filter *vf_d3d11vpp_create(struct mp_filter *parent,
     hwdec_devices_request_for_img_fmt(info->hwdec_devs, &params);
 
     struct mp_hwdec_ctx *hwctx =
-        hwdec_devices_get_by_imgfmt(info->hwdec_devs, IMGFMT_D3D11);
+        hwdec_devices_get_by_imgfmt_and_type(info->hwdec_devs, IMGFMT_D3D11,
+                                             AV_HWDEVICE_TYPE_D3D11VA);
     if (!hwctx || !hwctx->av_device_ref)
         goto fail;
     AVHWDeviceContext *avhwctx = (void *)hwctx->av_device_ref->data;

video/filter/vf_vavpp.c

@@ -451,7 +451,7 @@ static struct mp_filter *vf_vavpp_create(struct mp_filter *parent, void *options
     p->queue = mp_refqueue_alloc(f);
 
     struct mp_hwdec_ctx *hwdec_ctx =
-        mp_filter_load_hwdec_device(f, IMGFMT_VAAPI);
+        mp_filter_load_hwdec_device(f, IMGFMT_VAAPI, AV_HWDEVICE_TYPE_VAAPI);
     if (!hwdec_ctx || !hwdec_ctx->av_device_ref)
         goto error;
     p->av_device_ref = av_buffer_ref(hwdec_ctx->av_device_ref);

video/filter/vf_vdpaupp.c

@@ -137,7 +137,7 @@ static struct mp_filter *vf_vdpaupp_create(struct mp_filter *parent, void *optio
     p->queue = mp_refqueue_alloc(f);
 
     struct mp_hwdec_ctx *hwdec_ctx =
-        mp_filter_load_hwdec_device(f, IMGFMT_VDPAU);
+        mp_filter_load_hwdec_device(f, IMGFMT_VDPAU, AV_HWDEVICE_TYPE_VDPAU);
     if (!hwdec_ctx || !hwdec_ctx->av_device_ref)
         goto error;
     p->ctx = mp_vdpau_get_ctx_from_av(hwdec_ctx->av_device_ref);

video/hwdec.c

@@ -34,14 +34,18 @@ void hwdec_devices_destroy(struct mp_hwdec_devices *devs)
     talloc_free(devs);
 }
 
-struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt(struct mp_hwdec_devices *devs,
+struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt_and_type(struct mp_hwdec_devices *devs,
-                                                 int hw_imgfmt)
+                                                          int hw_imgfmt,
+                                                          enum AVHWDeviceType device_type)
 {
     struct mp_hwdec_ctx *res = NULL;
     mp_mutex_lock(&devs->lock);
     for (int n = 0; n < devs->num_hwctxs; n++) {
         struct mp_hwdec_ctx *dev = devs->hwctxs[n];
-        if (dev->hw_imgfmt == hw_imgfmt) {
+        AVHWDeviceContext *hw_device_ctx =
+            dev->av_device_ref ? (AVHWDeviceContext *)dev->av_device_ref->data : NULL;
+        if (dev->hw_imgfmt == hw_imgfmt &&
+            (!hw_device_ctx || hw_device_ctx->type == device_type)) {
             res = dev;
             break;
         }

video/hwdec.h

@@ -2,6 +2,7 @@
 #define MP_HWDEC_H_
 
 #include <libavutil/buffer.h>
+#include <libavutil/hwcontext.h>
 
 #include "options/m_option.h"
 
@@ -35,8 +36,9 @@ struct mp_hwdec_devices;
 struct mp_hwdec_devices *hwdec_devices_create(void);
 void hwdec_devices_destroy(struct mp_hwdec_devices *devs);
 
-struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt(struct mp_hwdec_devices *devs,
+struct mp_hwdec_ctx *hwdec_devices_get_by_imgfmt_and_type(struct mp_hwdec_devices *devs,
-                                                 int hw_imgfmt);
+                                                          int hw_imgfmt,
+                                                          enum AVHWDeviceType device_type);
 
 // For code which still strictly assumes there is 1 (or none) device.
 struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs);