diff --git a/video/decode/lavc.h b/video/decode/lavc.h
index 3c38e6878d..fe7f9937f4 100644
--- a/video/decode/lavc.h
+++ b/video/decode/lavc.h
@@ -7,6 +7,7 @@
 
 #include "demux/stheader.h"
 #include "video/mp_image.h"
+#include "video/mp_image_pool.h"
 #include "video/hwdec.h"
 
 #define HWDEC_DELAY_QUEUE_COUNT 2
@@ -61,6 +62,8 @@ typedef struct lavc_ctx {
 
     bool hwdec_request_reinit;
     int hwdec_fail_count;
+
+    struct mp_image_pool *hwdec_swpool;
 } vd_ffmpeg_ctx;
 
 struct vd_lavc_hwdec {
diff --git a/video/decode/vaapi.c b/video/decode/vaapi.c
index bd84fcc253..aeadeb488e 100644
--- a/video/decode/vaapi.c
+++ b/video/decode/vaapi.c
@@ -43,8 +43,6 @@ struct priv {
 
     AVBufferRef *device_ref;
     AVBufferRef *frames_ref;
-
-    struct mp_image_pool *sw_pool;
 };
 
 
@@ -123,7 +121,6 @@ static int init(struct lavc_ctx *ctx, bool direct)
     struct priv *p = talloc_ptrtype(NULL, p);
     *p = (struct priv) {
         .log = mp_log_new(p, ctx->log, "vaapi"),
-        .sw_pool = talloc_steal(p, mp_image_pool_new(17)),
     };
 
     if (direct) {
@@ -185,15 +182,6 @@ static int init_copy(struct lavc_ctx *ctx)
     return init(ctx, false);
 }
 
-static struct mp_image *copy_image(struct lavc_ctx *ctx, struct mp_image *img)
-{
-    struct priv *p = ctx->hwdec_priv;
-
-    struct mp_image *simg = va_surface_download(img, p->sw_pool);
-    talloc_free(img);
-    return simg;
-}
-
 static void intel_shit_lock(struct lavc_ctx *ctx)
 {
     struct priv *p = ctx->hwdec_priv;
@@ -227,6 +215,5 @@ const struct vd_lavc_hwdec mp_vd_lavc_vaapi_copy = {
     .init = init_copy,
     .uninit = uninit,
     .init_decoder = init_decoder,
-    .process_image = copy_image,
     .delay_queue = HWDEC_DELAY_QUEUE_COUNT,
 };
diff --git a/video/decode/vd_lavc.c b/video/decode/vd_lavc.c
index 8a2070df79..d53513ad55 100644
--- a/video/decode/vd_lavc.c
+++ b/video/decode/vd_lavc.c
@@ -452,6 +452,7 @@ static int init(struct dec_video *vd, const char *decoder)
     ctx->opts = vd->opts;
     ctx->decoder = talloc_strdup(ctx, decoder);
     ctx->hwdec_devs = vd->hwdec_devs;
+    ctx->hwdec_swpool = talloc_steal(ctx, mp_image_pool_new(17));
 
     reinit(vd);
 
@@ -920,6 +921,18 @@ static bool receive_frame(struct dec_video *vd, struct mp_image **out_image)
     if (!res)
         return progress;
 
+    if (ctx->hwdec && ctx->hwdec->copying && (res->fmt.flags & MP_IMGFLAG_HWACCEL))
+    {
+        struct mp_image *sw = mp_image_hw_download(res, ctx->hwdec_swpool);
+        mp_image_unrefp(&res);
+        res = sw;
+        if (!res) {
+            MP_ERR(vd, "Could not copy back hardware decoded frame.\n");
+            handle_err(vd);
+            return NULL;
+        }
+    }
+
     if (!ctx->hwdec_notified && vd->opts->hwdec_api != HWDEC_NONE) {
         if (ctx->hwdec) {
             MP_INFO(vd, "Using hardware decoding (%s).\n",