vaapi: use libavutil functions for copying hw surfaces to memory

Makes va_surface_download() call mp_image_hw_download() for
libavutil-allocated surfaces, which in turn calls
av_hwframe_transfer_data().

mp_image_hw_download() is actually not specific to vaapi, and can be
used for any hw surface allocated by libavutil.

video/mp_image_pool.c

@@ -23,12 +23,14 @@
 #include <assert.h>
 
 #include <libavutil/buffer.h>
+#include <libavutil/hwcontext.h>
 
 #include "mpv_talloc.h"
 
 #include "common/common.h"
-#include "video/mp_image.h"
 
+#include "fmt-conversion.h"
+#include "mp_image.h"
 #include "mp_image_pool.h"
 
 static pthread_mutex_t pool_mutex = PTHREAD_MUTEX_INITIALIZER;
@@ -247,3 +249,62 @@ void mp_image_pool_set_lru(struct mp_image_pool *pool)
 {
     pool->use_lru = true;
 }
+
+
+// Copies the contents of the HW surface img to system memory and retuns it.
+// If swpool is not NULL, it's used to allocate the target image.
+// img must be a hw surface with a AVHWFramesContext attached. If not, you
+// must use the legacy mp_hwdec_ctx.download_image.
+// The returned image is cropped as needed.
+// Returns NULL on failure.
+struct mp_image *mp_image_hw_download(struct mp_image *src,
+                                      struct mp_image_pool *swpool)
+{
+    if (!src->hwctx)
+        return NULL;
+    AVHWFramesContext *fctx = (void *)src->hwctx->data;
+
+    // Try to find the first format which we can apparently use.
+    int imgfmt = 0;
+    enum AVPixelFormat *fmts;
+    if (av_hwframe_transfer_get_formats(src->hwctx,
+            AV_HWFRAME_TRANSFER_DIRECTION_FROM, &fmts, 0) < 0)
+        return NULL;
+    for (int n = 0; fmts[n] != AV_PIX_FMT_NONE; n++) {
+        imgfmt = pixfmt2imgfmt(fmts[n]);
+        if (imgfmt)
+            break;
+    }
+    av_free(fmts);
+
+    if (!imgfmt)
+        return NULL;
+
+    struct mp_image *dst =
+        mp_image_pool_get(swpool, imgfmt, fctx->width, fctx->height);
+    if (!dst)
+        return NULL;
+
+    // Target image must be writable, so unref it.
+    AVFrame *dstav = mp_image_to_av_frame_and_unref(dst);
+    if (!dstav)
+        return NULL;
+
+    AVFrame *srcav = mp_image_to_av_frame(src);
+    if (!srcav) {
+        av_frame_unref(dstav);
+        return NULL;
+    }
+
+    int res = av_hwframe_transfer_data(dstav, srcav, 0);
+    av_frame_unref(srcav);
+    dst = mp_image_from_av_frame(dstav);
+    av_frame_unref(dstav);
+    if (res >= 0) {
+        mp_image_set_size(dst, src->w, src->h);
+        mp_image_copy_attributes(dst, src);
+    } else {
+        mp_image_unrefp(&dst);
+    }
+    return dst;
+}

video/mp_image_pool.h

@@ -26,4 +26,7 @@ struct mp_image *mp_image_pool_new_copy(struct mp_image_pool *pool,
 bool mp_image_pool_make_writeable(struct mp_image_pool *pool,
                                   struct mp_image *img);
 
+struct mp_image *mp_image_hw_download(struct mp_image *img,
+                                      struct mp_image_pool *swpool);
+
 #endif

video/vaapi.c

@@ -514,25 +514,10 @@ struct mp_image *va_surface_download(struct mp_image *src,
     if (!src || src->imgfmt != IMGFMT_VAAPI)
         return NULL;
     struct va_surface *p = va_surface_in_mp_image(src);
-    struct va_surface tmp_p;
     if (!p) {
         // We might still be able to get to the cheese if this is a surface
         // produced by libavutil's vaapi glue code.
-        if (!src->hwctx)
-            return NULL;
-        AVHWFramesContext *fctx = (void *)src->hwctx->data;
-        // as set by video/decode/vaapi.c
-        struct mp_vaapi_ctx *ctx = fctx->user_opaque;
-        tmp_p = (struct va_surface){
-            .ctx = ctx,
-            .id = va_surface_id(src),
-            .rt_format = VA_RT_FORMAT_YUV420,
-            .w = fctx->width,
-            .h = fctx->height,
-            .display = ctx->display,
-            .image = { .image_id = VA_INVALID_ID, .buf = VA_INVALID_ID },
-        };
-        p = &tmp_p;
+        return mp_image_hw_download(src, pool);
     }
     struct mp_image *mpi = NULL;
     struct mp_vaapi_ctx *ctx = p->ctx;
@@ -563,11 +548,6 @@ struct mp_image *va_surface_download(struct mp_image *src,
 
 done:
 
-    if (p == &tmp_p) {
-        if (p->image.image_id != VA_INVALID_ID)
-            vaDestroyImage(p->display, p->image.image_id);
-    }
-
     if (!mpi)
         MP_ERR(ctx, "failed to get surface data.\n");
     return mpi;