diff --git a/player/command.c b/player/command.c
index 8ff2914c66..c8394c403c 100644
--- a/player/command.c
+++ b/player/command.c
@@ -2198,14 +2198,14 @@ static int mp_property_hwdec_interop(void *ctx, struct m_property *prop,
                                      int action, void *arg)
 {
     MPContext *mpctx = ctx;
-    if (!mpctx->video_out)
+    if (!mpctx->video_out || !mpctx->video_out->hwdec_devs)
         return M_PROPERTY_UNAVAILABLE;
 
-    struct mp_hwdec_info *hwdec_info = NULL;
-    vo_control(mpctx->video_out, VOCTRL_GET_HWDEC_INFO, &hwdec_info);
-    struct mp_hwdec_ctx *hwctx = hwdec_info ? hwdec_info->hwctx : NULL;
+    struct mp_hwdec_ctx *hwctx =
+        hwdec_devices_get_first(mpctx->video_out->hwdec_devs);
+
     const char *name = hwctx ? hwctx->driver_name : NULL;
-    if (!name && hwctx && hwctx->type != HWDEC_NONE && hwctx->type != HWDEC_AUTO)
+    if (!name && hwctx)
         name = m_opt_choice_str(mp_hwdec_names, hwctx->type);
 
     return m_property_strdup_ro(action, arg, name);
@@ -2244,8 +2244,11 @@ static int mp_property_detected_hwdec(void *ctx, struct m_property *prop,
         if (vd)
             video_vd_control(vd, VDCTRL_GET_HWDEC, &current);
 
-        if (current <= 0 && vd && vd->hwdec_info && vd->hwdec_info->hwctx)
-            current = vd->hwdec_info->hwctx->type;
+        if (current <= 0 && vd && vd->hwdec_devs) {
+            struct mp_hwdec_ctx *hwctx = hwdec_devices_get_first(vd->hwdec_devs);
+            if (hwctx)
+                current = hwctx->type;
+        }
 
         // In case of the "-copy" ones, which are "detected" every time the
         // decoder is opened, return "no" if no decoding is active.
diff --git a/player/core.h b/player/core.h
index 489d1f8d3f..3a5689b9f3 100644
--- a/player/core.h
+++ b/player/core.h
@@ -153,7 +153,7 @@ struct track {
 struct vo_chain {
     struct mp_log *log;
 
-    struct mp_hwdec_info *hwdec_info;
+    struct mp_hwdec_devices *hwdec_devs;
     double container_fps;
 
     struct vf_chain *vf;
diff --git a/player/screenshot.c b/player/screenshot.c
index 33b972bb25..13532ec1a3 100644
--- a/player/screenshot.c
+++ b/player/screenshot.c
@@ -346,8 +346,9 @@ static struct mp_image *screenshot_get(struct MPContext *mpctx, int mode)
         }
     }
 
-    if (image && mpctx->vo_chain && mpctx->vo_chain->hwdec_info) {
-        struct mp_hwdec_ctx *ctx = mpctx->vo_chain->hwdec_info->hwctx;
+    if (image && mpctx->vo_chain && mpctx->vo_chain->hwdec_devs) {
+        struct mp_hwdec_ctx *ctx =
+            hwdec_devices_get_first(mpctx->vo_chain->hwdec_devs);
         struct mp_image *nimage = NULL;
         if (ctx && ctx->download_image && (image->fmt.flags & MP_IMGFLAG_HWACCEL))
             nimage = ctx->download_image(ctx, image, NULL);
diff --git a/player/video.c b/player/video.c
index de3eb963fa..1d2dc29fc6 100644
--- a/player/video.c
+++ b/player/video.c
@@ -204,7 +204,7 @@ static void recreate_video_filters(struct MPContext *mpctx)
 
     vf_destroy(vo_c->vf);
     vo_c->vf = vf_new(mpctx->global);
-    vo_c->vf->hwdec = vo_c->hwdec_info;
+    vo_c->vf->hwdec_devs = vo_c->hwdec_devs;
     vo_c->vf->wakeup_callback = wakeup_playloop;
     vo_c->vf->wakeup_callback_ctx = mpctx;
     vo_c->vf->container_fps = vo_c->container_fps;
@@ -334,7 +334,7 @@ int init_video_decoder(struct MPContext *mpctx, struct track *track)
     d_video->codec = track->stream->codec;
     d_video->fps = d_video->header->codec->fps;
     if (mpctx->vo_chain)
-        d_video->hwdec_info = mpctx->vo_chain->hwdec_info;
+        d_video->hwdec_devs = mpctx->vo_chain->hwdec_devs;
 
     MP_VERBOSE(d_video, "Container reported FPS: %f\n", d_video->fps);
 
@@ -404,7 +404,7 @@ int reinit_video_chain_src(struct MPContext *mpctx, struct lavfi_pad *src)
     vo_c->vo = mpctx->video_out;
     vo_c->vf = vf_new(mpctx->global);
 
-    vo_control(vo_c->vo, VOCTRL_GET_HWDEC_INFO, &vo_c->hwdec_info);
+    vo_c->hwdec_devs = vo_c->vo->hwdec_devs;
 
     vo_c->filter_src = src;
     if (!vo_c->filter_src) {
diff --git a/video/d3d.h b/video/d3d.h
deleted file mode 100644
index b5cf365f7f..0000000000
--- a/video/d3d.h
+++ /dev/null
@@ -1,15 +0,0 @@
-#ifndef MP_D3D_H_
-#define MP_D3D_H_
-
-#include <d3d9.h>
-#include <d3d11.h>
-
-#include "hwdec.h"
-
-struct mp_d3d_ctx {
-    struct mp_hwdec_ctx hwctx;
-    IDirect3DDevice9 *d3d9_device;
-    ID3D11Device *d3d11_device;
-};
-
-#endif
diff --git a/video/decode/d3d11va.c b/video/decode/d3d11va.c
index 246a35ec9a..1626626767 100644
--- a/video/decode/d3d11va.c
+++ b/video/decode/d3d11va.c
@@ -26,7 +26,6 @@
 #include "video/mp_image_pool.h"
 #include "video/hwdec.h"
 
-#include "video/d3d.h"
 #include "d3d.h"
 
 #define ADDITIONAL_SURFACES (4 + HWDEC_DELAY_QUEUE_COUNT)
@@ -492,9 +491,7 @@ static int d3d11va_init(struct lavc_ctx *s)
         p->sw_pool = talloc_steal(p, mp_image_pool_new(17));
     }
 
-    if (s->hwdec_info && s->hwdec_info->hwctx && s->hwdec_info->hwctx->d3d_ctx)
-        p->device = s->hwdec_info->hwctx->d3d_ctx->d3d11_device;
-
+    p->device = hwdec_devices_load(s->hwdec_devs, s->hwdec->type);
     if (p->device) {
         ID3D11Device_AddRef(p->device);
         ID3D11Device_GetImmediateContext(p->device, &p->device_ctx);
@@ -539,15 +536,12 @@ fail:
     return -1;
 }
 
-static int d3d11va_probe(struct vd_lavc_hwdec *hwdec,
-                         struct mp_hwdec_info *info,
+static int d3d11va_probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                          const char *codec)
 {
-    hwdec_request_api(info, "d3d11va");
     // d3d11va-copy can do without external context; dxva2 requires it.
     if (hwdec->type != HWDEC_D3D11VA_COPY) {
-        if (!info || !info->hwctx || !info->hwctx->d3d_ctx ||
-            !info->hwctx->d3d_ctx->d3d11_device)
+        if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_D3D11VA))
             return HWDEC_ERR_NO_CTX;
     }
     return d3d_probe_codec(codec);
diff --git a/video/decode/dec_video.h b/video/decode/dec_video.h
index f4646a97d0..1030973e1c 100644
--- a/video/decode/dec_video.h
+++ b/video/decode/dec_video.h
@@ -32,7 +32,7 @@ struct dec_video {
     struct mpv_global *global;
     struct MPOpts *opts;
     const struct vd_functions *vd_driver;
-    struct mp_hwdec_info *hwdec_info; // video output hwdec handles
+    struct mp_hwdec_devices *hwdec_devs; // video output hwdec handles
     struct sh_stream *header;
     struct mp_codec_params *codec;
 
diff --git a/video/decode/dxva2.c b/video/decode/dxva2.c
index 70623c8fb4..5d3afda86c 100644
--- a/video/decode/dxva2.c
+++ b/video/decode/dxva2.c
@@ -32,7 +32,6 @@
 #include "video/mp_image_pool.h"
 #include "video/hwdec.h"
 
-#include "video/d3d.h"
 #include "video/dxva2.h"
 #include "d3d.h"
 
@@ -406,9 +405,7 @@ static int dxva2_init(struct lavc_ctx *s)
         p->sw_pool = talloc_steal(p, mp_image_pool_new(17));
     }
 
-    if (s->hwdec_info && s->hwdec_info->hwctx && s->hwdec_info->hwctx->d3d_ctx)
-        p->device = s->hwdec_info->hwctx->d3d_ctx->d3d9_device;
-
+    p->device = hwdec_devices_load(s->hwdec_devs, s->hwdec->type);
     if (p->device) {
         IDirect3D9_AddRef(p->device);
         MP_VERBOSE(p, "Using VO-supplied device %p.\n", p->device);
@@ -477,16 +474,15 @@ fail:
     return -1;
 }
 
-static int dxva2_probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int dxva2_probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                        const char *codec)
 {
-    hwdec_request_api(info, "dxva2");
     // dxva2-copy can do without external context; dxva2 requires it.
-    if (hwdec->type != HWDEC_DXVA2_COPY) {
-        if (!info || !info->hwctx || !info->hwctx->d3d_ctx ||
-            info->hwctx->type == HWDEC_DXVA2_COPY ||
-            !info->hwctx->d3d_ctx->d3d9_device)
+    if (hwdec->type == HWDEC_DXVA2) {
+        if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_DXVA2))
             return HWDEC_ERR_NO_CTX;
+    } else {
+        hwdec_devices_load(ctx->hwdec_devs, HWDEC_DXVA2_COPY);
     }
     return d3d_probe_codec(codec);
 }
diff --git a/video/decode/lavc.h b/video/decode/lavc.h
index 73243e16c4..dbefe79ad9 100644
--- a/video/decode/lavc.h
+++ b/video/decode/lavc.h
@@ -30,7 +30,7 @@ typedef struct lavc_ctx {
     int max_delay_queue;
 
     // From VO
-    struct mp_hwdec_info *hwdec_info;
+    struct mp_hwdec_devices *hwdec_devs;
 
     // For free use by hwdec implementation
     void *hwdec_priv;
@@ -54,7 +54,7 @@ struct vd_lavc_hwdec {
     // efficiency by not blocking on the hardware pipeline by reading back
     // immediately after decoding.
     int delay_queue;
-    int (*probe)(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+    int (*probe)(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                  const char *codec);
     int (*init)(struct lavc_ctx *ctx);
     int (*init_decoder)(struct lavc_ctx *ctx, int w, int h);
diff --git a/video/decode/vaapi.c b/video/decode/vaapi.c
index 169564d6c4..4b098a8804 100644
--- a/video/decode/vaapi.c
+++ b/video/decode/vaapi.c
@@ -357,7 +357,7 @@ static void destroy_va_dummy_ctx(struct priv *p)
 
 // Creates a "private" VADisplay, disconnected from the VO. We just create a
 // new X connection, because that's simpler. (We could also pass the X
-// connection along with struct mp_hwdec_info, if we wanted.)
+// connection along with struct mp_hwdec_devices, if we wanted.)
 static bool create_va_dummy_ctx(struct priv *p)
 {
     for (int n = 0; native_displays[n]; n++) {
@@ -399,21 +399,23 @@ static void uninit(struct lavc_ctx *ctx)
     ctx->hwdec_priv = NULL;
 }
 
-static int init_with_vactx(struct lavc_ctx *ctx, struct mp_vaapi_ctx *vactx)
+static int init(struct lavc_ctx *ctx, bool direct)
 {
     struct priv *p = talloc_ptrtype(NULL, p);
     *p = (struct priv) {
         .log = mp_log_new(p, ctx->log, "vaapi"),
-        .ctx = vactx,
         .va_context = &p->va_context_storage,
         .rt_format = VA_RT_FORMAT_YUV420
     };
 
-    if (!p->ctx)
+    if (direct) {
+        p->ctx = hwdec_devices_get(ctx->hwdec_devs, HWDEC_VAAPI)->ctx;
+    } else {
         create_va_dummy_ctx(p);
-    if (!p->ctx) {
-        talloc_free(p);
-        return -1;
+        if (!p->ctx) {
+            talloc_free(p);
+            return -1;
+        }
     }
 
     p->display = p->ctx->display;
@@ -431,25 +433,22 @@ static int init_with_vactx(struct lavc_ctx *ctx, struct mp_vaapi_ctx *vactx)
     return 0;
 }
 
-static int init(struct lavc_ctx *ctx)
+static int init_direct(struct lavc_ctx *ctx)
 {
-    return init_with_vactx(ctx, ctx->hwdec_info->hwctx->vaapi_ctx);
+    return init(ctx, true);
 }
 
-static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                  const char *codec)
 {
-    hwdec_request_api(info, "vaapi");
-    if (!info || !info->hwctx || !info->hwctx->vaapi_ctx)
+    if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_VAAPI))
         return HWDEC_ERR_NO_CTX;
     if (!hwdec_check_codec_support(codec, profiles))
         return HWDEC_ERR_NO_CODEC;
-    if (va_guess_if_emulated(info->hwctx->vaapi_ctx))
-        return HWDEC_ERR_EMULATED;
     return 0;
 }
 
-static int probe_copy(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int probe_copy(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                       const char *codec)
 {
     struct priv dummy = {mp_null_log};
@@ -466,7 +465,7 @@ static int probe_copy(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
 
 static int init_copy(struct lavc_ctx *ctx)
 {
-    return init_with_vactx(ctx, NULL);
+    return init(ctx, false);
 }
 
 static struct mp_image *copy_image(struct lavc_ctx *ctx, struct mp_image *img)
@@ -497,7 +496,7 @@ const struct vd_lavc_hwdec mp_vd_lavc_vaapi = {
     .type = HWDEC_VAAPI,
     .image_format = IMGFMT_VAAPI,
     .probe = probe,
-    .init = init,
+    .init = init_direct,
     .uninit = uninit,
     .init_decoder = init_decoder,
     .allocate_image = allocate_image,
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index ff97e3446c..bc70b7d348 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -280,18 +280,13 @@ static bool hwdec_is_wrapper(struct vd_lavc_hwdec *hwdec, const char *decoder)
     return bstr_endswith0(bstr0(decoder), hwdec->lavc_suffix);
 }
 
-void hwdec_request_api(struct mp_hwdec_info *info, const char *api_name)
-{
-    if (info && info->load_api)
-        info->load_api(info, api_name);
-}
-
-static int hwdec_probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int hwdec_probe(struct dec_video *vd, struct vd_lavc_hwdec *hwdec,
                        const char *codec)
 {
+    vd_ffmpeg_ctx *ctx = vd->priv;
     int r = 0;
     if (hwdec->probe)
-        r = hwdec->probe(hwdec, info, codec);
+        r = hwdec->probe(ctx, hwdec, codec);
     if (r >= 0) {
         if (hwdec->lavc_suffix && !hwdec_find_decoder(codec, hwdec->lavc_suffix))
             return HWDEC_ERR_NO_CODEC;
@@ -309,7 +304,7 @@ static struct vd_lavc_hwdec *probe_hwdec(struct dec_video *vd, bool autoprobe,
         MP_VERBOSE(vd, "Requested hardware decoder not compiled.\n");
         return NULL;
     }
-    int r = hwdec_probe(hwdec, vd->hwdec_info, codec);
+    int r = hwdec_probe(vd, hwdec, codec);
     if (r == HWDEC_ERR_EMULATED) {
         if (autoprobe)
             return NULL;
@@ -412,6 +407,7 @@ static int init(struct dec_video *vd, const char *decoder)
     ctx->log = vd->log;
     ctx->opts = vd->opts;
     ctx->decoder = talloc_strdup(ctx, decoder);
+    ctx->hwdec_devs = vd->hwdec_devs;
 
     reinit(vd);
 
@@ -441,8 +437,6 @@ static void init_avctx(struct dec_video *vd, const char *decoder,
     if (!lavc_codec)
         return;
 
-    ctx->hwdec_info = vd->hwdec_info;
-
     ctx->codec_timebase = (AVRational){0};
     if (strstr(decoder, "_mmal") || strstr(decoder, "_mediacodec"))
         ctx->codec_timebase = (AVRational){1, 1000000};
diff --git a/video/decode/vdpau.c b/video/decode/vdpau.c
index 313fabff76..2aba10c13b 100644
--- a/video/decode/vdpau.c
+++ b/video/decode/vdpau.c
@@ -75,7 +75,7 @@ static int init(struct lavc_ctx *ctx)
     struct priv *p = talloc_ptrtype(NULL, p);
     *p = (struct priv) {
         .log = mp_log_new(p, ctx->log, "vdpau"),
-        .mpvdp = ctx->hwdec_info->hwctx->vdpau_ctx,
+        .mpvdp = hwdec_devices_get(ctx->hwdec_devs, HWDEC_VDPAU)->ctx,
     };
     ctx->hwdec_priv = p;
 
@@ -83,14 +83,11 @@ static int init(struct lavc_ctx *ctx)
     return 0;
 }
 
-static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                  const char *codec)
 {
-    hwdec_request_api(info, "vdpau");
-    if (!info || !info->hwctx || !info->hwctx->vdpau_ctx)
+    if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_VDPAU))
         return HWDEC_ERR_NO_CTX;
-    if (mp_vdpau_guess_if_emulated(info->hwctx->vdpau_ctx))
-        return HWDEC_ERR_EMULATED;
     return 0;
 }
 
diff --git a/video/decode/videotoolbox.c b/video/decode/videotoolbox.c
index 2d2f5f735c..c69d5e89e6 100644
--- a/video/decode/videotoolbox.c
+++ b/video/decode/videotoolbox.c
@@ -27,11 +27,10 @@
 #include "config.h"
 
 
-static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+static int probe(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
                  const char *codec)
 {
-    hwdec_request_api(info, "videotoolbox");
-    if (!info || !info->hwctx || !info->hwctx->get_vt_fmt)
+    if (!hwdec_devices_load(ctx->hwdec_devs, HWDEC_VIDEOTOOLBOX))
         return HWDEC_ERR_NO_CTX;
     switch (mp_codec_to_av_codec_id(codec)) {
     case AV_CODEC_ID_H264:
@@ -89,8 +88,8 @@ static int init_decoder(struct lavc_ctx *ctx, int w, int h)
 
     AVVideotoolboxContext *vtctx = av_videotoolbox_alloc_context();
 
-    struct mp_hwdec_ctx *hwctx = ctx->hwdec_info->hwctx;
-    vtctx->cv_pix_fmt_type = hwctx->get_vt_fmt(hwctx);
+    struct mp_vt_ctx *vt = hwdec_devices_load(ctx->hwdec_devs, HWDEC_VIDEOTOOLBOX);
+    vtctx->cv_pix_fmt_type = vt->get_vt_fmt(vt);
 
     int err = av_videotoolbox_default_init2(ctx->avctx, vtctx);
     if (err < 0) {
diff --git a/video/filter/vf.c b/video/filter/vf.c
index d8e7f6b4c8..7ca1b08f34 100644
--- a/video/filter/vf.c
+++ b/video/filter/vf.c
@@ -244,7 +244,7 @@ static struct vf_instance *vf_open(struct vf_chain *c, const char *name,
     *vf = (vf_instance_t) {
         .info = desc.p,
         .log = mp_log_new(vf, c->log, name),
-        .hwdec = c->hwdec,
+        .hwdec_devs = c->hwdec_devs,
         .query_format = vf_default_query_format,
         .out_pool = talloc_steal(vf, mp_image_pool_new(16)),
         .chain = c,
diff --git a/video/filter/vf.h b/video/filter/vf.h
index c982b612e1..02f6f2c8fd 100644
--- a/video/filter/vf.h
+++ b/video/filter/vf.h
@@ -92,7 +92,7 @@ typedef struct vf_instance {
     struct mp_image_pool *out_pool;
     struct vf_priv_s *priv;
     struct mp_log *log;
-    struct mp_hwdec_info *hwdec;
+    struct mp_hwdec_devices *hwdec_devs;
 
     struct mp_image **out_queued;
     int num_out_queued;
@@ -120,7 +120,7 @@ struct vf_chain {
     struct mp_log *log;
     struct MPOpts *opts;
     struct mpv_global *global;
-    struct mp_hwdec_info *hwdec;
+    struct mp_hwdec_devices *hwdec_devs;
 
     // Call when the filter chain wants new processing (for filters with
     // asynchronous behavior) - must be immutable once filters are created,
diff --git a/video/filter/vf_vavpp.c b/video/filter/vf_vavpp.c
index 9dab15e2b9..554ddc1a05 100644
--- a/video/filter/vf_vavpp.c
+++ b/video/filter/vf_vavpp.c
@@ -481,19 +481,17 @@ static bool initialize(struct vf_instance *vf)
 
 static int vf_open(vf_instance_t *vf)
 {
+    struct vf_priv_s *p = vf->priv;
+
     vf->reconfig = reconfig;
     vf->filter_ext = filter_ext;
     vf->query_format = query_format;
     vf->uninit = uninit;
     vf->control = control;
 
-    struct vf_priv_s *p = vf->priv;
-    if (!vf->hwdec)
-        return false;
-    hwdec_request_api(vf->hwdec, "vaapi");
-    p->va = vf->hwdec->hwctx ? vf->hwdec->hwctx->vaapi_ctx : NULL;
-    if (!p->va || !p->va->display)
-        return false;
+    p->va = hwdec_devices_load(vf->hwdec_devs, HWDEC_VAAPI);
+    if (!p->va)
+        return 0;
     p->display = p->va->display;
     if (initialize(vf))
         return true;
diff --git a/video/filter/vf_vdpaupp.c b/video/filter/vf_vdpaupp.c
index 882b80d9e2..23afeafe6a 100644
--- a/video/filter/vf_vdpaupp.c
+++ b/video/filter/vf_vdpaupp.c
@@ -223,10 +223,7 @@ static int vf_open(vf_instance_t *vf)
     vf->control = control;
     vf->uninit = uninit;
 
-    if (!vf->hwdec)
-        return 0;
-    hwdec_request_api(vf->hwdec, "vdpau");
-    p->ctx = vf->hwdec->hwctx ? vf->hwdec->hwctx->vdpau_ctx : NULL;
+    p->ctx = hwdec_devices_load(vf->hwdec_devs, HWDEC_VDPAU);
     if (!p->ctx)
         return 0;
 
diff --git a/video/filter/vf_vdpaurb.c b/video/filter/vf_vdpaurb.c
index 62f7f34af1..92dfa52486 100644
--- a/video/filter/vf_vdpaurb.c
+++ b/video/filter/vf_vdpaurb.c
@@ -101,14 +101,9 @@ static int vf_open(vf_instance_t *vf)
     vf->reconfig = reconfig;
     vf->query_format = query_format;
 
-    if (!vf->hwdec) {
+    p->ctx = hwdec_devices_load(vf->hwdec_devs, HWDEC_VDPAU);
+    if (!p->ctx)
         return 0;
-    }
-    hwdec_request_api(vf->hwdec, "vdpau");
-    p->ctx = vf->hwdec->hwctx ? vf->hwdec->hwctx->vdpau_ctx : NULL;
-    if (!p->ctx) {
-        return 0;
-    }
 
     return 1;
 }
diff --git a/video/hwdec.c b/video/hwdec.c
new file mode 100644
index 0000000000..6db8d57869
--- /dev/null
+++ b/video/hwdec.c
@@ -0,0 +1,90 @@
+#include <pthread.h>
+#include <assert.h>
+
+#include "hwdec.h"
+
+struct mp_hwdec_devices {
+    pthread_mutex_t lock;
+
+    struct mp_hwdec_ctx *hwctx;
+
+    void (*load_api)(void *ctx, enum hwdec_type type);
+    void *load_api_ctx;
+};
+
+struct mp_hwdec_devices *hwdec_devices_create(void)
+{
+    struct mp_hwdec_devices *devs = talloc_zero(NULL, struct mp_hwdec_devices);
+    pthread_mutex_init(&devs->lock, NULL);
+    return devs;
+}
+
+void hwdec_devices_destroy(struct mp_hwdec_devices *devs)
+{
+    if (!devs)
+        return;
+    assert(!devs->hwctx); // must have been hwdec_devices_remove()ed
+    assert(!devs->load_api); // must have been unset
+    pthread_mutex_destroy(&devs->lock);
+    talloc_free(devs);
+}
+
+struct mp_hwdec_ctx *hwdec_devices_get(struct mp_hwdec_devices *devs,
+                                       enum hwdec_type type)
+{
+    struct mp_hwdec_ctx *res = NULL;
+    pthread_mutex_lock(&devs->lock);
+    if (devs->hwctx && devs->hwctx->type == type)
+        res = devs->hwctx;
+    pthread_mutex_unlock(&devs->lock);
+    return res;
+}
+
+struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs)
+{
+    pthread_mutex_lock(&devs->lock);
+    struct mp_hwdec_ctx *res = devs->hwctx;
+    pthread_mutex_unlock(&devs->lock);
+    return res;
+}
+
+void hwdec_devices_add(struct mp_hwdec_devices *devs, struct mp_hwdec_ctx *ctx)
+{
+    pthread_mutex_lock(&devs->lock);
+    // We support only 1 device; ignore the rest.
+    if (!devs->hwctx)
+        devs->hwctx = ctx;
+    pthread_mutex_unlock(&devs->lock);
+}
+
+void hwdec_devices_remove(struct mp_hwdec_devices *devs, struct mp_hwdec_ctx *ctx)
+{
+    pthread_mutex_lock(&devs->lock);
+    if (devs->hwctx == ctx)
+        devs->hwctx = NULL;
+    pthread_mutex_unlock(&devs->lock);
+}
+
+void hwdec_devices_set_loader(struct mp_hwdec_devices *devs,
+    void (*load_api)(void *ctx, enum hwdec_type type), void *load_api_ctx)
+{
+    devs->load_api = load_api;
+    devs->load_api_ctx = load_api_ctx;
+}
+
+// Cause VO to lazily load the requested device, and will block until this is
+// done (even if not available).
+void hwdec_devices_request(struct mp_hwdec_devices *devs, enum hwdec_type type)
+{
+    if (devs->load_api && !hwdec_devices_get_first(devs))
+        devs->load_api(devs->load_api_ctx, type);
+}
+
+void *hwdec_devices_load(struct mp_hwdec_devices *devs, enum hwdec_type type)
+{
+    if (!devs)
+        return NULL;
+    hwdec_devices_request(devs, type);
+    struct mp_hwdec_ctx *hwctx = hwdec_devices_get(devs, type);
+    return hwctx ? hwctx->ctx : NULL;
+}
diff --git a/video/hwdec.h b/video/hwdec.h
index 94667774e7..48ec6a2a21 100644
--- a/video/hwdec.h
+++ b/video/hwdec.h
@@ -25,16 +25,17 @@ enum hwdec_type {
 extern const struct m_opt_choice_alternatives mp_hwdec_names[];
 
 struct mp_hwdec_ctx {
-    enum hwdec_type type;
+    enum hwdec_type type; // (never HWDEC_NONE or HWDEC_AUTO)
     const char *driver_name; // NULL if unknown/not loaded
 
-    void *priv; // for free use by hwdec implementation
-
-    // API-specific, not needed by all backends.
-    struct mp_vdpau_ctx *vdpau_ctx;
-    struct mp_vaapi_ctx *vaapi_ctx;
-    struct mp_d3d_ctx *d3d_ctx;
-    uint32_t (*get_vt_fmt)(struct mp_hwdec_ctx *ctx);
+    // This is never NULL. Its meaning depends on the .type field:
+    //  HWDEC_VDPAU:            struct mp_vaapi_ctx*
+    //  HWDEC_VIDEOTOOLBOX:     struct mp_vt_ctx*
+    //  HWDEC_VAAPI:            struct mp_vaapi_ctx*
+    //  HWDEC_D3D11VA:          ID3D11Device*
+    //  HWDEC_DXVA2:            IDirect3DDevice9*
+    //  HWDEC_DXVA2_COPY:       IDirect3DDevice9*
+    void *ctx;
 
     // Optional.
     // Allocates a software image from the pool, downloads the hw image from
@@ -46,24 +47,50 @@ struct mp_hwdec_ctx {
                                        struct mp_image_pool *swpool);
 };
 
-// Used to communicate hardware decoder API handles from VO to video decoder.
-// The VO can set the context pointer for supported APIs.
-struct mp_hwdec_info {
-    // (Since currently only 1 hwdec API is loaded at a time, this pointer
-    // simply maps to the loaded one.)
-    struct mp_hwdec_ctx *hwctx;
-
-    // Can be used to lazily load a requested API.
-    // api_name is e.g. "vdpau" (like the fields above, without "_ctx")
-    // Can be NULL, is idempotent, caller checks hwctx fields for success/access.
-    // Due to threading, the callback is the only code that is allowed to
-    // change fields in this struct after initialization.
-    void (*load_api)(struct mp_hwdec_info *info, const char *api_name);
-    void *load_api_ctx;
+struct mp_vt_ctx {
+    void *priv;
+    uint32_t (*get_vt_fmt)(struct mp_vt_ctx *ctx);
 };
 
-// Trivial helper to call info->load_api().
-// Implemented in vd_lavc.c.
-void hwdec_request_api(struct mp_hwdec_info *info, const char *api_name);
+// Used to communicate hardware decoder device handles from VO to video decoder.
+struct mp_hwdec_devices;
+
+struct mp_hwdec_devices *hwdec_devices_create(void);
+void hwdec_devices_destroy(struct mp_hwdec_devices *devs);
+
+// Return the device context for the given API type. Returns NULL if none
+// available. Logically, the returned pointer remains valid until VO
+// uninitialization is started (all users of it must be uninitialized before).
+// hwdec_devices_request() may be used before this to lazily load devices.
+struct mp_hwdec_ctx *hwdec_devices_get(struct mp_hwdec_devices *devs,
+                                       enum hwdec_type type);
+
+// For code which still strictly assumes there is 1 (or none) device.
+struct mp_hwdec_ctx *hwdec_devices_get_first(struct mp_hwdec_devices *devs);
+
+// Add this to the list of internal devices. Adding the same pointer twice must
+// be avoided.
+void hwdec_devices_add(struct mp_hwdec_devices *devs, struct mp_hwdec_ctx *ctx);
+
+// Remove this from the list of internal devices. Idempotent/ignores entries
+// not added yet.
+void hwdec_devices_remove(struct mp_hwdec_devices *devs, struct mp_hwdec_ctx *ctx);
+
+// Can be used to enable lazy loading of an API with hwdec_devices_request().
+// If used at all, this must be set/unset during initialization/uninitialization,
+// as concurrent use with hwdec_devices_request() is a race condition.
+void hwdec_devices_set_loader(struct mp_hwdec_devices *devs,
+    void (*load_api)(void *ctx, enum hwdec_type type), void *load_api_ctx);
+
+// Cause VO to lazily load the requested device, and will block until this is
+// done (even if not available).
+void hwdec_devices_request(struct mp_hwdec_devices *devs, enum hwdec_type type);
+
+// Convenience function:
+// - return NULL if devs==NULL
+// - call hwdec_devices_request(devs, type)
+// - call hwdec_devices_get(devs, type)
+// - then return the mp_hwdec_ctx.ctx field
+void *hwdec_devices_load(struct mp_hwdec_devices *devs, enum hwdec_type type);
 
 #endif
diff --git a/video/out/opengl/hwdec.c b/video/out/opengl/hwdec.c
index 9c3bec1a0f..b4b5c23580 100644
--- a/video/out/opengl/hwdec.c
+++ b/video/out/opengl/hwdec.c
@@ -61,6 +61,7 @@ static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = {
 
 static struct gl_hwdec *load_hwdec_driver(struct mp_log *log, GL *gl,
                                           struct mpv_global *global,
+                                          struct mp_hwdec_devices *devs,
                                           const struct gl_hwdec_driver *drv,
                                           bool is_auto)
 {
@@ -70,6 +71,7 @@ static struct gl_hwdec *load_hwdec_driver(struct mp_log *log, GL *gl,
         .log = mp_log_new(hwdec, log, drv->name),
         .global = global,
         .gl = gl,
+        .devs = devs,
         .gl_texture_target = GL_TEXTURE_2D,
         .probing = is_auto,
     };
@@ -79,19 +81,19 @@ static struct gl_hwdec *load_hwdec_driver(struct mp_log *log, GL *gl,
         mp_verbose(log, "Loading failed.\n");
         return NULL;
     }
-    if (hwdec->hwctx && !hwdec->hwctx->driver_name)
-        hwdec->hwctx->driver_name = hwdec->driver->name;
     return hwdec;
 }
 
-struct gl_hwdec *gl_hwdec_load_api_id(struct mp_log *log, GL *gl,
-                                      struct mpv_global *g, int id)
+struct gl_hwdec *gl_hwdec_load_api(struct mp_log *log, GL *gl,
+                                   struct mpv_global *g,
+                                   struct mp_hwdec_devices *devs,
+                                   enum hwdec_type api)
 {
-    bool is_auto = id == HWDEC_AUTO;
+    bool is_auto = api == HWDEC_AUTO;
     for (int n = 0; mpgl_hwdec_drivers[n]; n++) {
         const struct gl_hwdec_driver *drv = mpgl_hwdec_drivers[n];
-        if (is_auto || id == drv->api) {
-            struct gl_hwdec *r = load_hwdec_driver(log, gl, g, drv, is_auto);
+        if (is_auto || api == drv->api) {
+            struct gl_hwdec *r = load_hwdec_driver(log, gl, g, devs, drv, is_auto);
             if (r)
                 return r;
         }
@@ -99,19 +101,6 @@ struct gl_hwdec *gl_hwdec_load_api_id(struct mp_log *log, GL *gl,
     return NULL;
 }
 
-// Like gl_hwdec_load_api_id(), but use option names.
-struct gl_hwdec *gl_hwdec_load_api(struct mp_log *log, GL *gl,
-                                   struct mpv_global *g, const char *api_name)
-{
-    int id = HWDEC_NONE;
-    for (const struct m_opt_choice_alternatives *c = mp_hwdec_names; c->name; c++)
-    {
-        if (strcmp(c->name, api_name) == 0)
-            id = c->value;
-    }
-    return gl_hwdec_load_api_id(log, gl, g, id);
-}
-
 void gl_hwdec_uninit(struct gl_hwdec *hwdec)
 {
     if (hwdec)
diff --git a/video/out/opengl/hwdec.h b/video/out/opengl/hwdec.h
index a9d524a1d7..fcc6d3c11e 100644
--- a/video/out/opengl/hwdec.h
+++ b/video/out/opengl/hwdec.h
@@ -4,14 +4,12 @@
 #include "common.h"
 #include "video/hwdec.h"
 
-struct mp_hwdec_info;
-
 struct gl_hwdec {
     const struct gl_hwdec_driver *driver;
     struct mp_log *log;
     struct mpv_global *global;
     GL *gl;
-    struct mp_hwdec_ctx *hwctx;
+    struct mp_hwdec_devices *devs;
     // For free use by hwdec driver
     void *priv;
     // For working around the vdpau vs. vaapi mess.
@@ -33,7 +31,7 @@ struct gl_hwdec_driver {
     enum hwdec_type api;
     // The hardware surface IMGFMT_ that must be passed to map_image later.
     int imgfmt;
-    // Create the hwdec device. It must fill in hw->info, if applicable.
+    // Create the hwdec device. It must add it to hw->devs, if applicable.
     // This also must set hw->converted_imgfmt.
     int (*create)(struct gl_hwdec *hw);
     // Prepare for rendering video. (E.g. create textures.)
@@ -49,9 +47,9 @@ struct gl_hwdec_driver {
 };
 
 struct gl_hwdec *gl_hwdec_load_api(struct mp_log *log, GL *gl,
-                                   struct mpv_global *g, const char *api_name);
-struct gl_hwdec *gl_hwdec_load_api_id(struct mp_log *log, GL *gl,
-                                      struct mpv_global *g, int id);
+                                   struct mpv_global *g,
+                                   struct mp_hwdec_devices *devs,
+                                   enum hwdec_type api);
 
 void gl_hwdec_uninit(struct gl_hwdec *hwdec);
 
diff --git a/video/out/opengl/hwdec_d3d11egl.c b/video/out/opengl/hwdec_d3d11egl.c
index de872d1396..caf27e1dc4 100644
--- a/video/out/opengl/hwdec_d3d11egl.c
+++ b/video/out/opengl/hwdec_d3d11egl.c
@@ -27,11 +27,10 @@
 #include "osdep/timer.h"
 #include "osdep/windows_utils.h"
 #include "hwdec.h"
-#include "video/d3d.h"
 #include "video/hwdec.h"
 
 struct priv {
-    struct mp_d3d_ctx ctx;
+    struct mp_hwdec_ctx hwctx;
 
     ID3D11Device *d3d11_device;
     ID3D11VideoDevice *video_dev;
@@ -94,6 +93,8 @@ static void destroy(struct gl_hwdec *hw)
 
     destroy_objects(hw);
 
+    hwdec_devices_remove(hw->devs, &p->hwctx);
+
     if (p->video_ctx)
         ID3D11VideoContext_Release(p->video_ctx);
     p->video_ctx = NULL;
@@ -109,9 +110,6 @@ static void destroy(struct gl_hwdec *hw)
 
 static int create(struct gl_hwdec *hw)
 {
-    if (hw->hwctx)
-        return -1;
-
     EGLDisplay egl_display = eglGetCurrentDisplay();
     if (!egl_display)
         return -1;
@@ -199,11 +197,13 @@ static int create(struct gl_hwdec *hw)
 
     hw->converted_imgfmt = IMGFMT_RGB0;
 
-    p->ctx.d3d11_device = p->d3d11_device;
-    p->ctx.hwctx.type = HWDEC_D3D11VA;
-    p->ctx.hwctx.d3d_ctx = &p->ctx;
+    p->hwctx = (struct mp_hwdec_ctx){
+        .type = HWDEC_D3D11VA,
+        .driver_name = hw->driver->name,
+        .ctx = p->d3d11_device,
+    };
+    hwdec_devices_add(hw->devs, &p->hwctx);
 
-    hw->hwctx = &p->ctx.hwctx;
     return 0;
 fail:
     destroy(hw);
diff --git a/video/out/opengl/hwdec_dxva2.c b/video/out/opengl/hwdec_dxva2.c
index f72c817a20..35d091f14f 100644
--- a/video/out/opengl/hwdec_dxva2.c
+++ b/video/out/opengl/hwdec_dxva2.c
@@ -1,8 +1,9 @@
+#include <d3d9.h>
+
 #include "common/common.h"
 
 #include "hwdec.h"
 #include "utils.h"
-#include "video/d3d.h"
 #include "video/hwdec.h"
 
 // This does not provide real (zero-copy) interop - it merely exists for
@@ -10,35 +11,39 @@
 // may help with OpenGL fullscreen mode.
 
 struct priv {
-    struct mp_d3d_ctx ctx;
+    struct mp_hwdec_ctx hwctx;
 };
 
 static void destroy(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
-    if (p->ctx.d3d9_device)
-        IDirect3DDevice9_Release(p->ctx.d3d9_device);
+    hwdec_devices_remove(hw->devs, &p->hwctx);
+    if (p->hwctx.ctx)
+        IDirect3DDevice9_Release((IDirect3DDevice9 *)p->hwctx.ctx);
 }
 
 static int create(struct gl_hwdec *hw)
 {
     GL *gl = hw->gl;
-    if (hw->hwctx || !gl->MPGetNativeDisplay)
+    if (!gl->MPGetNativeDisplay)
         return -1;
 
     struct priv *p = talloc_zero(hw, struct priv);
     hw->priv = p;
 
-    p->ctx.d3d9_device = gl->MPGetNativeDisplay("IDirect3DDevice9");
-    if (!p->ctx.d3d9_device)
+    IDirect3DDevice9 *d3d = gl->MPGetNativeDisplay("IDirect3DDevice9");
+    if (!d3d)
         return -1;
 
-    p->ctx.hwctx.type = HWDEC_DXVA2_COPY;
-    p->ctx.hwctx.d3d_ctx = &p->ctx;
+    MP_VERBOSE(hw, "Using libmpv supplied device %p.\n", d3d);
 
-    MP_VERBOSE(hw, "Using libmpv supplied device %p.\n", p->ctx.d3d9_device);
+    p->hwctx = (struct mp_hwdec_ctx){
+        .type = HWDEC_DXVA2_COPY,
+        .driver_name = hw->driver->name,
+        .ctx = d3d,
+    };
+    hwdec_devices_add(hw->devs, &p->hwctx);
 
-    hw->hwctx = &p->ctx.hwctx;
     hw->converted_imgfmt = 0;
     return 0;
 }
diff --git a/video/out/opengl/hwdec_dxva2egl.c b/video/out/opengl/hwdec_dxva2egl.c
index ed1a6e66b7..1384e2bb47 100644
--- a/video/out/opengl/hwdec_dxva2egl.c
+++ b/video/out/opengl/hwdec_dxva2egl.c
@@ -17,6 +17,8 @@
 
 #include <assert.h>
 #include <windows.h>
+#include <d3d9.h>
+
 #include <EGL/egl.h>
 #include <EGL/eglext.h>
 
@@ -25,11 +27,10 @@
 #include "osdep/windows_utils.h"
 #include "hwdec.h"
 #include "video/dxva2.h"
-#include "video/d3d.h"
 #include "video/hwdec.h"
 
 struct priv {
-    struct mp_d3d_ctx ctx;
+    struct mp_hwdec_ctx hwctx;
 
     HMODULE             d3d9_dll;
     IDirect3D9Ex       *d3d9ex;
@@ -77,6 +78,8 @@ static void destroy(struct gl_hwdec *hw)
 
     destroy_textures(hw);
 
+    hwdec_devices_remove(hw->devs, &p->hwctx);
+
     if (p->query9)
         IDirect3DQuery9_Release(p->query9);
 
@@ -92,9 +95,6 @@ static void destroy(struct gl_hwdec *hw)
 
 static int create(struct gl_hwdec *hw)
 {
-    if (hw->hwctx)
-        return -1;
-
     EGLDisplay egl_display = eglGetCurrentDisplay();
     if (!egl_display)
         return -1;
@@ -207,11 +207,13 @@ static int create(struct gl_hwdec *hw)
 
     hw->converted_imgfmt = IMGFMT_RGB0;
 
-    p->ctx.d3d9_device = (IDirect3DDevice9 *)p->device9ex;
-    p->ctx.hwctx.type = HWDEC_DXVA2;
-    p->ctx.hwctx.d3d_ctx = &p->ctx;
+    p->hwctx = (struct mp_hwdec_ctx){
+        .type = HWDEC_DXVA2,
+        .driver_name = hw->driver->name,
+        .ctx = (IDirect3DDevice9 *)p->device9ex,
+    };
+    hwdec_devices_add(hw->devs, &p->hwctx);
 
-    hw->hwctx = &p->ctx.hwctx;
     return 0;
 fail:
     destroy(hw);
diff --git a/video/out/opengl/hwdec_dxva2gldx.c b/video/out/opengl/hwdec_dxva2gldx.c
index 69be0ccd18..97f1918a2c 100644
--- a/video/out/opengl/hwdec_dxva2gldx.c
+++ b/video/out/opengl/hwdec_dxva2gldx.c
@@ -15,13 +15,13 @@
  * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include <d3d9.h>
 #include <assert.h>
 
 #include "common/common.h"
 #include "osdep/windows_utils.h"
 #include "hwdec.h"
 #include "video/hwdec.h"
-#include "video/d3d.h"
 #include "video/dxva2.h"
 
 // for  WGL_ACCESS_READ_ONLY_NV
@@ -30,7 +30,7 @@
 #define SHARED_SURFACE_D3DFMT D3DFMT_X8R8G8B8
 #define SHARED_SURFACE_MPFMT  IMGFMT_RGB0
 struct priv {
-    struct mp_d3d_ctx ctx;
+    struct mp_hwdec_ctx hwctx;
     IDirect3DDevice9Ex *device;
     HANDLE device_h;
 
@@ -74,6 +74,8 @@ static void destroy(struct gl_hwdec *hw)
     struct priv *p = hw->priv;
     destroy_objects(hw);
 
+    hwdec_devices_remove(hw->devs, &p->hwctx);
+
     if (p->device)
         IDirect3DDevice9Ex_Release(p->device);
 }
@@ -81,10 +83,8 @@ static void destroy(struct gl_hwdec *hw)
 static int create(struct gl_hwdec *hw)
 {
     GL *gl = hw->gl;
-    if (hw->hwctx || !gl->MPGetNativeDisplay ||
-        !(gl->mpgl_caps & MPGL_CAP_DXINTEROP)) {
+    if (!gl->MPGetNativeDisplay || !(gl->mpgl_caps & MPGL_CAP_DXINTEROP))
         return -1;
-    }
 
     struct priv *p = talloc_zero(hw, struct priv);
     hw->priv = p;
@@ -100,12 +100,14 @@ static int create(struct gl_hwdec *hw)
     if (!p->device)
         return -1;
     IDirect3DDevice9Ex_AddRef(p->device);
-    p->ctx.d3d9_device = (IDirect3DDevice9 *)p->device;
 
-    p->ctx.hwctx.type = HWDEC_DXVA2;
-    p->ctx.hwctx.d3d_ctx = &p->ctx;
+    p->hwctx = (struct mp_hwdec_ctx){
+        .type = HWDEC_DXVA2,
+        .driver_name = hw->driver->name,
+        .ctx = (IDirect3DDevice9 *)p->device,
+    };
+    hwdec_devices_add(hw->devs, &p->hwctx);
 
-    hw->hwctx = &p->ctx.hwctx;
     hw->converted_imgfmt = SHARED_SURFACE_MPFMT;
     return 0;
 }
diff --git a/video/out/opengl/hwdec_osx.c b/video/out/opengl/hwdec_osx.c
index addc16f404..5aa4d3dcd4 100644
--- a/video/out/opengl/hwdec_osx.c
+++ b/video/out/opengl/hwdec_osx.c
@@ -43,9 +43,11 @@ struct vt_format {
 };
 
 struct priv {
+    struct mp_hwdec_ctx hwctx;
+    struct mp_vt_ctx vtctx;
+
     CVPixelBufferRef pbuf;
     GLuint gl_planes[MP_MAX_PLANES];
-    struct mp_hwdec_ctx hwctx;
 };
 
 static struct vt_format vt_formats[] = {
@@ -147,9 +149,9 @@ static bool check_hwdec(struct gl_hwdec *hw)
     return true;
 }
 
-static uint32_t get_vt_fmt(struct mp_hwdec_ctx *ctx)
+static uint32_t get_vt_fmt(struct mp_vt_ctx *vtctx)
 {
-    struct gl_hwdec *hw = ctx->priv;
+    struct gl_hwdec *hw = vtctx->priv;
     struct vt_format *f =
         vt_get_gl_format_from_imgfmt(hw->global->opts->videotoolbox_format);
     return f ? f->cvpixfmt : (uint32_t)-1;
@@ -167,15 +169,21 @@ static int create(struct gl_hwdec *hw)
 
     hw->priv = p;
     hw->converted_imgfmt = f->imgfmt;
-    hw->hwctx = &p->hwctx;
-    hw->hwctx->download_image = download_image;
-    hw->hwctx->type = HWDEC_VIDEOTOOLBOX;
-    hw->hwctx->get_vt_fmt = get_vt_fmt;
 
     hw->gl_texture_target = GL_TEXTURE_RECTANGLE;
     hw->gl->GenTextures(MP_MAX_PLANES, p->gl_planes);
 
-    hw->hwctx->priv = hw;
+    p->vtctx = (struct mp_vt_ctx){
+        .priv = hw,
+        .get_vt_fmt = get_vt_fmt,
+    };
+    p->hwctx = (struct mp_hwdec_ctx){
+        .type = HWDEC_VIDEOTOOLBOX,
+        .download_image = download_image,
+        .ctx = &p->vtctx,
+    };
+    hwdec_devices_add(hw->devs, &p->hwctx);
+
     return 0;
 }
 
@@ -251,6 +259,8 @@ static void destroy(struct gl_hwdec *hw)
 
     CVPixelBufferRelease(p->pbuf);
     gl->DeleteTextures(MP_MAX_PLANES, p->gl_planes);
+
+    hwdec_devices_remove(hw->devs, &p->hwctx);
 }
 
 const struct gl_hwdec_driver gl_hwdec_videotoolbox = {
diff --git a/video/out/opengl/hwdec_vaegl.c b/video/out/opengl/hwdec_vaegl.c
index 6356ec4e8c..84e6e83a73 100644
--- a/video/out/opengl/hwdec_vaegl.c
+++ b/video/out/opengl/hwdec_vaegl.c
@@ -169,6 +169,8 @@ static void destroy(struct gl_hwdec *hw)
     struct priv *p = hw->priv;
     unref_image(hw);
     destroy_textures(hw);
+    if (p->ctx)
+        hwdec_devices_remove(hw->devs, &p->ctx->hwctx);
     va_destroy(p->ctx);
 }
 
@@ -181,8 +183,6 @@ static int create(struct gl_hwdec *hw)
     p->current_image.buf = p->current_image.image_id = VA_INVALID_ID;
     p->log = hw->log;
 
-    if (hw->hwctx)
-        return -1;
     if (!eglGetCurrentContext())
         return -1;
 
@@ -229,7 +229,8 @@ static int create(struct gl_hwdec *hw)
         return -1;
     }
 
-    hw->hwctx = &p->ctx->hwctx;
+    p->ctx->hwctx.driver_name = hw->driver->name;
+    hwdec_devices_add(hw->devs, &p->ctx->hwctx);
     return 0;
 }
 
diff --git a/video/out/opengl/hwdec_vaglx.c b/video/out/opengl/hwdec_vaglx.c
index 77b1f27c51..a9c5d03af4 100644
--- a/video/out/opengl/hwdec_vaglx.c
+++ b/video/out/opengl/hwdec_vaglx.c
@@ -64,13 +64,13 @@ static void destroy(struct gl_hwdec *hw)
 {
     struct priv *p = hw->priv;
     destroy_texture(hw);
+    if (p->ctx)
+        hwdec_devices_remove(hw->devs, &p->ctx->hwctx);
     va_destroy(p->ctx);
 }
 
 static int create(struct gl_hwdec *hw)
 {
-    if (hw->hwctx)
-        return -1;
     Display *x11disp = glXGetCurrentDisplay();
     if (!x11disp)
         return -1;
@@ -126,7 +126,8 @@ static int create(struct gl_hwdec *hw)
         return -1;
     }
 
-    hw->hwctx = &p->ctx->hwctx;
+    p->ctx->hwctx.driver_name = hw->driver->name;
+    hwdec_devices_add(hw->devs, &p->ctx->hwctx);
     hw->converted_imgfmt = IMGFMT_RGB0;
     return 0;
 }
diff --git a/video/out/opengl/hwdec_vdpau.c b/video/out/opengl/hwdec_vdpau.c
index 99e5a1414a..e3b69941bf 100644
--- a/video/out/opengl/hwdec_vdpau.c
+++ b/video/out/opengl/hwdec_vdpau.c
@@ -92,14 +92,14 @@ static void destroy(struct gl_hwdec *hw)
 
     destroy_objects(hw);
     mp_vdpau_mixer_destroy(p->mixer);
+    if (p->ctx)
+        hwdec_devices_remove(hw->devs, &p->ctx->hwctx);
     mp_vdpau_destroy(p->ctx);
 }
 
 static int create(struct gl_hwdec *hw)
 {
     GL *gl = hw->gl;
-    if (hw->hwctx)
-        return -1;
     Display *x11disp = glXGetCurrentDisplay();
     if (!x11disp)
         return -1;
@@ -119,7 +119,8 @@ static int create(struct gl_hwdec *hw)
         destroy(hw);
         return -1;
     }
-    hw->hwctx = &p->ctx->hwctx;
+    p->ctx->hwctx.driver_name = hw->driver->name;
+    hwdec_devices_add(hw->devs, &p->ctx->hwctx);
     hw->converted_imgfmt = IMGFMT_RGB0;
     return 0;
 }
diff --git a/video/out/vo.c b/video/out/vo.c
index 3e7999a698..3390f364f7 100644
--- a/video/out/vo.c
+++ b/video/out/vo.c
@@ -43,6 +43,7 @@
 #include "options/m_config.h"
 #include "common/msg.h"
 #include "common/global.h"
+#include "video/hwdec.h"
 #include "video/mp_image.h"
 #include "sub/osd.h"
 #include "osdep/io.h"
diff --git a/video/out/vo.h b/video/out/vo.h
index f6bc270afd..f417d5b522 100644
--- a/video/out/vo.h
+++ b/video/out/vo.h
@@ -61,9 +61,8 @@ enum mp_voctrl {
     VOCTRL_SET_EQUALIZER,               // struct voctrl_set_equalizer_args*
     VOCTRL_GET_EQUALIZER,               // struct voctrl_get_equalizer_args*
 
-    /* for hardware decoding */
-    VOCTRL_GET_HWDEC_INFO,              // struct mp_hwdec_info**
-    VOCTRL_LOAD_HWDEC_API,              // private to vo_opengl
+    /* private to vo_opengl */
+    VOCTRL_LOAD_HWDEC_API,
 
     // Redraw the image previously passed to draw_image() (basically, repeat
     // the previous draw_image call). If this is handled, the OSD should also
@@ -297,6 +296,7 @@ struct vo {
     struct vo_w32_state *w32;
     struct vo_cocoa_state *cocoa;
     struct vo_wayland_state *wayland;
+    struct mp_hwdec_devices *hwdec_devs;
     struct input_ctx *input_ctx;
     struct osd_state *osd;
     struct encode_lavc_context *encode_lavc_ctx;
diff --git a/video/out/vo_opengl.c b/video/out/vo_opengl.c
index dfef6ec500..306e893f3b 100644
--- a/video/out/vo_opengl.c
+++ b/video/out/vo_opengl.c
@@ -59,7 +59,6 @@ struct gl_priv {
     struct gl_lcms *cms;
 
     struct gl_hwdec *hwdec;
-    struct mp_hwdec_info hwdec_info;
 
     // Options
     struct gl_video_opts *renderer_opts;
@@ -196,25 +195,23 @@ static int reconfig(struct vo *vo, struct mp_image_params *params)
     return 0;
 }
 
-static void request_hwdec_api(struct gl_priv *p, const char *api_name)
+static void request_hwdec_api(struct vo *vo, void *api)
 {
+    struct gl_priv *p = vo->priv;
+
     if (p->hwdec)
         return;
 
-    p->hwdec = gl_hwdec_load_api(p->vo->log, p->gl, p->vo->global, api_name);
+    p->hwdec = gl_hwdec_load_api(p->vo->log, p->gl, p->vo->global,
+                                 vo->hwdec_devs, (intptr_t)api);
     gl_video_set_hwdec(p->renderer, p->hwdec);
-    if (p->hwdec)
-        p->hwdec_info.hwctx = p->hwdec->hwctx;
 }
 
-static void call_request_hwdec_api(struct mp_hwdec_info *info,
-                                   const char *api_name)
+static void call_request_hwdec_api(void *ctx, enum hwdec_type type)
 {
-    struct vo *vo = info->load_api_ctx;
-    assert(&((struct gl_priv *)vo->priv)->hwdec_info == info);
     // Roundabout way to run hwdec loading on the VO thread.
     // Redirects to request_hwdec_api().
-    vo_control(vo, VOCTRL_LOAD_HWDEC_API, (void *)api_name);
+    vo_control(ctx, VOCTRL_LOAD_HWDEC_API, (void *)(intptr_t)type);
 }
 
 static void get_and_update_icc_profile(struct gl_priv *p, int *events)
@@ -325,13 +322,8 @@ static int control(struct vo *vo, uint32_t request, void *data)
         *(struct mp_image **)data = screen;
         return true;
     }
-    case VOCTRL_GET_HWDEC_INFO: {
-        struct mp_hwdec_info **arg = data;
-        *arg = &p->hwdec_info;
-        return true;
-    }
     case VOCTRL_LOAD_HWDEC_API:
-        request_hwdec_api(p, data);
+        request_hwdec_api(vo, data);
         return true;
     case VOCTRL_SET_COMMAND_LINE: {
         char *arg = data;
@@ -373,6 +365,8 @@ static void uninit(struct vo *vo)
 
     gl_video_uninit(p->renderer);
     gl_hwdec_uninit(p->hwdec);
+    hwdec_devices_set_loader(vo->hwdec_devs, NULL, NULL);
+    hwdec_devices_destroy(vo->hwdec_devs);
     mpgl_uninit(p->glctx);
 }
 
@@ -424,17 +418,17 @@ static int preinit(struct vo *vo)
     gl_lcms_set_options(p->cms, p->icc_opts);
     get_and_update_icc_profile(p, &(int){0});
 
-    p->hwdec_info.load_api = call_request_hwdec_api;
-    p->hwdec_info.load_api_ctx = vo;
+    vo->hwdec_devs = hwdec_devices_create();
+
+    hwdec_devices_set_loader(vo->hwdec_devs, call_request_hwdec_api, vo);
 
     int hwdec = vo->opts->hwdec_preload_api;
     if (hwdec == HWDEC_NONE)
         hwdec = vo->global->opts->hwdec_api;
     if (hwdec != HWDEC_NONE) {
-        p->hwdec = gl_hwdec_load_api_id(p->vo->log, p->gl, vo->global, hwdec);
+        p->hwdec = gl_hwdec_load_api(p->vo->log, p->gl, vo->global,
+                                     vo->hwdec_devs, hwdec);
         gl_video_set_hwdec(p->renderer, p->hwdec);
-        if (p->hwdec)
-            p->hwdec_info.hwctx = p->hwdec->hwctx;
     }
 
     return 0;
diff --git a/video/out/vo_opengl_cb.c b/video/out/vo_opengl_cb.c
index 40930fbcae..06e0013a96 100644
--- a/video/out/vo_opengl_cb.c
+++ b/video/out/vo_opengl_cb.c
@@ -89,13 +89,16 @@ struct mpv_opengl_cb_context {
     struct vo *active;
     int hwdec_api;
 
+    // --- This is only mutable while initialized=false, during which nothing
+    //     except the OpenGL context manager is allowed to access it.
+    struct mp_hwdec_devices *hwdec_devs;
+
     // --- All of these can only be accessed from the thread where the host
     //     application's OpenGL context is current - i.e. only while the
     //     host application is calling certain mpv_opengl_cb_* APIs.
     GL *gl;
     struct gl_video *renderer;
     struct gl_hwdec *hwdec;
-    struct mp_hwdec_info hwdec_info; // it's also semi-immutable after init
 };
 
 static void update(struct vo_priv *p);
@@ -180,11 +183,10 @@ int mpv_opengl_cb_init_gl(struct mpv_opengl_cb_context *ctx, const char *exts,
     if (!ctx->renderer)
         return MPV_ERROR_UNSUPPORTED;
 
-    ctx->hwdec = gl_hwdec_load_api_id(ctx->log, ctx->gl, ctx->global,
-                                      ctx->hwdec_api);
+    ctx->hwdec_devs = hwdec_devices_create();
+    ctx->hwdec = gl_hwdec_load_api(ctx->log, ctx->gl, ctx->global,
+                                   ctx->hwdec_devs, ctx->hwdec_api);
     gl_video_set_hwdec(ctx->renderer, ctx->hwdec);
-    if (ctx->hwdec)
-        ctx->hwdec_info.hwctx = ctx->hwdec->hwctx;
 
     pthread_mutex_lock(&ctx->lock);
     // We don't know the exact caps yet - use a known superset
@@ -222,6 +224,8 @@ int mpv_opengl_cb_uninit_gl(struct mpv_opengl_cb_context *ctx)
     ctx->renderer = NULL;
     gl_hwdec_uninit(ctx->hwdec);
     ctx->hwdec = NULL;
+    hwdec_devices_destroy(ctx->hwdec_devs);
+    ctx->hwdec_devs = NULL;
     talloc_free(ctx->gl);
     ctx->gl = NULL;
     talloc_free(ctx->new_opts_cfg);
@@ -514,11 +518,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
         char *arg = data;
         return reparse_cmdline(p, arg);
     }
-    case VOCTRL_GET_HWDEC_INFO: {
-        struct mp_hwdec_info **arg = data;
-        *arg = p->ctx ? &p->ctx->hwdec_info : NULL;
-        return true;
-    }
     }
 
     return VO_NOTIMPL;
@@ -561,6 +560,8 @@ static int preinit(struct vo *vo)
     p->ctx->eq_changed = true;
     pthread_mutex_unlock(&p->ctx->lock);
 
+    vo->hwdec_devs = p->ctx->hwdec_devs;
+
     return 0;
 }
 
diff --git a/video/out/vo_vaapi.c b/video/out/vo_vaapi.c
index 5275d4d28d..dc8aaacf9e 100644
--- a/video/out/vo_vaapi.c
+++ b/video/out/vo_vaapi.c
@@ -68,7 +68,6 @@ struct priv {
     struct vo               *vo;
     VADisplay                display;
     struct mp_vaapi_ctx     *mpvaapi;
-    struct mp_hwdec_info     hwdec_info;
 
     struct mp_image_params   image_params;
     struct mp_rect           src_rect;
@@ -515,11 +514,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
     struct priv *p = vo->priv;
 
     switch (request) {
-    case VOCTRL_GET_HWDEC_INFO: {
-        struct mp_hwdec_info **arg = data;
-        *arg = &p->hwdec_info;
-        return true;
-    }
     case VOCTRL_SET_EQUALIZER: {
         struct voctrl_set_equalizer_args *eq = data;
         return set_equalizer(p, eq->name, eq->value);
@@ -561,6 +555,11 @@ static void uninit(struct vo *vo)
         free_subpicture(p, &part->image);
     }
 
+    if (vo->hwdec_devs) {
+        hwdec_devices_remove(vo->hwdec_devs, &p->mpvaapi->hwctx);
+        hwdec_devices_destroy(vo->hwdec_devs);
+    }
+
     va_destroy(p->mpvaapi);
 
     vo_x11_uninit(vo);
@@ -591,8 +590,6 @@ static int preinit(struct vo *vo)
         goto fail;
     }
 
-    p->hwdec_info.hwctx = &p->mpvaapi->hwctx;
-
     if (va_guess_if_emulated(p->mpvaapi)) {
         MP_WARN(vo, "VA-API is most likely emulated via VDPAU.\n"
                     "It's better to use VDPAU directly with: --vo=vdpau\n");
@@ -645,6 +642,10 @@ static int preinit(struct vo *vo)
             p->va_num_display_attrs = 0;
         p->mp_display_attr = talloc_zero_array(vo, int, p->va_num_display_attrs);
     }
+
+    vo->hwdec_devs = hwdec_devices_create();
+    hwdec_devices_add(vo->hwdec_devs, &p->mpvaapi->hwctx);
+
     return 0;
 
 fail:
diff --git a/video/out/vo_vdpau.c b/video/out/vo_vdpau.c
index 1dc5dc3b36..15472b2189 100644
--- a/video/out/vo_vdpau.c
+++ b/video/out/vo_vdpau.c
@@ -71,7 +71,6 @@ struct vdpctx {
     struct vdp_functions              *vdp;
     VdpDevice                          vdp_device;
     uint64_t                           preemption_counter;
-    struct mp_hwdec_info               hwdec_info;
 
     struct m_color                     colorkey;
 
@@ -1028,6 +1027,9 @@ static void uninit(struct vo *vo)
 {
     struct vdpctx *vc = vo->priv;
 
+    hwdec_devices_remove(vo->hwdec_devs, &vc->mpvdp->hwctx);
+    hwdec_devices_destroy(vo->hwdec_devs);
+
     /* Destroy all vdpau objects */
     mp_vdpau_mixer_destroy(vc->video_mixer);
     destroy_vdpau_objects(vo);
@@ -1053,7 +1055,8 @@ static int preinit(struct vo *vo)
         return -1;
     }
 
-    vc->hwdec_info.hwctx = &vc->mpvdp->hwctx;
+    vo->hwdec_devs = hwdec_devices_create();
+    hwdec_devices_add(vo->hwdec_devs, &vc->mpvdp->hwctx);
 
     vc->video_mixer = mp_vdpau_mixer_create(vc->mpvdp, vo->log);
 
@@ -1117,11 +1120,6 @@ static int control(struct vo *vo, uint32_t request, void *data)
     check_preemption(vo);
 
     switch (request) {
-    case VOCTRL_GET_HWDEC_INFO: {
-        struct mp_hwdec_info **arg = data;
-        *arg = &vc->hwdec_info;
-        return true;
-    }
     case VOCTRL_GET_PANSCAN:
         return VO_TRUE;
     case VOCTRL_SET_PANSCAN:
diff --git a/video/vaapi.c b/video/vaapi.c
index 9a5820a98c..f8d0faba34 100644
--- a/video/vaapi.c
+++ b/video/vaapi.c
@@ -128,8 +128,7 @@ struct mp_vaapi_ctx *va_initialize(VADisplay *display, struct mp_log *plog,
         .display = display,
         .hwctx = {
             .type = HWDEC_VAAPI,
-            .priv = res,
-            .vaapi_ctx = res,
+            .ctx = res,
             .download_image = ctx_download_image,
         },
     };
diff --git a/video/vdpau.c b/video/vdpau.c
index 6398fa6e58..73aa844abd 100644
--- a/video/vdpau.c
+++ b/video/vdpau.c
@@ -32,7 +32,10 @@ static struct mp_image *download_image(struct mp_hwdec_ctx *hwctx,
                                        struct mp_image *mpi,
                                        struct mp_image_pool *swpool)
 {
-    struct mp_vdpau_ctx *ctx = hwctx->vdpau_ctx;
+    if (mpi->imgfmt != IMGFMT_VDPAU && mpi->imgfmt != IMGFMT_VDPAU_OUTPUT)
+        return NULL;
+
+    struct mp_vdpau_ctx *ctx = hwctx->ctx;
     struct vdp_functions *vdp = &ctx->vdp;
     VdpStatus vdp_st;
 
@@ -395,8 +398,7 @@ struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log, Display *x11
         .preemption_counter = 1,
         .hwctx = {
             .type = HWDEC_VDPAU,
-            .priv = ctx,
-            .vdpau_ctx = ctx,
+            .ctx = ctx,
             .download_image = download_image,
         },
         .getimg_surface = VDP_INVALID_HANDLE,
diff --git a/wscript_build.py b/wscript_build.py
index fda99c16cf..15637826ea 100644
--- a/wscript_build.py
+++ b/wscript_build.py
@@ -282,6 +282,7 @@ def build(ctx):
         ( "video/gpu_memcpy.c",                  "sse4-intrinsics" ),
         ( "video/image_writer.c" ),
         ( "video/img_format.c" ),
+        ( "video/hwdec.c" ),
         ( "video/mp_image.c" ),
         ( "video/mp_image_pool.c" ),
         ( "video/sws_utils.c" ),