vaapi: allow GPU read-back with --hwdec=vaapi-copy

This code is actually quite inefficient: it reuses the (slow, simple)
screenshot code. It uses an inefficient method to read the image
(vaGetImage() instead of vaDeriveImage()), allocates new memory for
each frame that is read, and it tries all image formats again each
time.

Also, in my tests it always picked NV12 as image format, which is not
ideal if you actually want to filter the video, and vo_xv can't handle
this format without conversion either.

However, a user confirmed that it worked for him, so everything is fine.

DOCS/man/en/options.rst

@@ -1083,6 +1083,7 @@
     :auto:      see below
     :vdpau:     requires ``--vo=vdpau`` (Linux only)
     :vaapi:     requires ``--vo=vaapi`` (Linux with Intel GPUs only)
+    :vaapi-copy: copies video back into system RAM (Linux with Intel GPUs only)
     :vda:       requires ``--vo=corevideo`` (OSX only)
     :crystalhd: Broadcom Crystal HD
 
@@ -1092,6 +1093,9 @@
     Also note that if the first found method doesn't actually work, it will
     always fall back to software decoding, instead of trying the next method.
 
+    The ``vaapi-copy`` function allows you to use vaapi with any VO. Because
+    this copies the decoded video back to system RAM, it's quite inefficient.
+
 ``--hwdec-codecs=<codec1,codec2,...|all>``
     Allow hardware decoding for a given list of codecs only. The default is the
     special value ``all``, which always allows all codecs.

mpvcore/options.c

@@ -479,7 +479,8 @@ const m_option_t mp_opts[] = {
                 {"vdpau", 1},
                 {"vda", 2},
                 {"crystalhd", 3},
-                {"vaapi", 4})),
+                {"vaapi", 4},
+                {"vaapi-copy", 5})),
     OPT_STRING("hwdec-codecs", hwdec_codecs, 0),
 
     // postprocessing:

video/decode/lavc.h

@@ -18,6 +18,7 @@ enum hwdec_type {
     HWDEC_VDA = 2,
     HWDEC_CRYSTALHD = 3,
     HWDEC_VAAPI = 4,
+    HWDEC_VAAPI_COPY = 5,
 };
 
 typedef struct lavc_ctx {

video/decode/vaapi.c

@@ -25,6 +25,8 @@
 #include <libavcodec/vaapi.h>
 #include <libavutil/common.h>
 
+#include <X11/Xlib.h>
+
 #include "lavc.h"
 #include "mpvcore/mp_common.h"
 #include "mpvcore/av_common.h"
@@ -54,6 +56,7 @@
 struct priv {
     struct mp_vaapi_ctx *ctx;
     VADisplay display;
+    Display *x11_display;
 
     // libavcodec shared struct
     struct vaapi_context *va_context;
@@ -64,6 +67,8 @@ struct priv {
 
     struct va_surface_pool *pool;
     int rt_format;
+
+    bool printed_readback_warning;
 };
 
 struct profile_entry {
@@ -342,6 +347,38 @@ static struct mp_image *allocate_image(struct lavc_ctx *ctx, int format,
     return NULL;
 }
 
+
+static void destroy_va_dummy_ctx(struct priv *p)
+{
+    if (p->x11_display)
+        XCloseDisplay(p->x11_display);
+    p->x11_display = NULL;
+    va_destroy(p->ctx);
+    p->ctx = NULL;
+}
+
+// Creates a "private" VADisplay, disconnected from the VO. We just create a
+// new X connection, because that's simpler. (We could also pass the X
+// connection along with struct mp_hwdec_info, if we wanted.)
+static bool create_va_dummy_ctx(struct priv *p)
+{
+    p->x11_display = XOpenDisplay(NULL);
+    if (!p->x11_display)
+        goto destroy_ctx;
+    VADisplay *display = vaGetDisplay(p->x11_display);
+    if (!display)
+        goto destroy_ctx;
+    p->ctx = va_initialize(display);
+    if (!p->ctx) {
+        vaTerminate(display);
+        goto destroy_ctx;
+    }
+    return true;
+destroy_ctx:
+    destroy_va_dummy_ctx(p);
+    return false;
+}
+
 static void uninit(struct lavc_ctx *ctx)
 {
     struct priv *p = ctx->hwdec_priv;
@@ -350,21 +387,31 @@ static void uninit(struct lavc_ctx *ctx)
         return;
 
     destroy_decoder(ctx);
-
     va_surface_pool_release(p->pool);
+
+    if (p->x11_display)
+        destroy_va_dummy_ctx(p);
+
     talloc_free(p);
     ctx->hwdec_priv = NULL;
 }
 
-static int init(struct lavc_ctx *ctx)
+static int init_with_vactx(struct lavc_ctx *ctx, struct mp_vaapi_ctx *vactx)
 {
     struct priv *p = talloc_ptrtype(NULL, p);
     *p = (struct priv) {
-        .ctx = ctx->hwdec_info->vaapi_ctx,
+        .ctx = vactx,
         .va_context = &p->va_context_storage,
         .rt_format = VA_RT_FORMAT_YUV420
     };
 
+    if (!p->ctx)
+        create_va_dummy_ctx(p);
+    if (!p->ctx) {
+        talloc_free(p);
+        return -1;
+    }
+
     p->display = p->ctx->display;
     p->pool = va_surface_pool_alloc(p->display, p->rt_format);
 
@@ -378,6 +425,12 @@ static int init(struct lavc_ctx *ctx)
     return 0;
 }
 
+static int init(struct lavc_ctx *ctx)
+{
+    if (!ctx->hwdec_info->vaapi_ctx)
+        return -1;
+    return init_with_vactx(ctx, ctx->hwdec_info->vaapi_ctx);
+}
 
 static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *decoder)
@@ -389,6 +442,44 @@ static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
     return 0;
 }
 
+static int probe_copy(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
+                      const char *decoder)
+{
+    struct priv dummy = {0};
+    if (!create_va_dummy_ctx(&dummy))
+        return HWDEC_ERR_NO_CTX;
+    destroy_va_dummy_ctx(&dummy);
+    if (!find_codec(mp_codec_to_av_codec_id(decoder)))
+        return HWDEC_ERR_NO_CODEC;
+    return 0;
+}
+
+static int init_copy(struct lavc_ctx *ctx)
+{
+    return init_with_vactx(ctx, NULL);
+}
+
+static struct mp_image *copy_image(struct lavc_ctx *ctx, struct mp_image *img)
+{
+    struct priv *p = ctx->hwdec_priv;
+
+    struct va_surface *surface = va_surface_in_mp_image(img);
+    if (surface) {
+        struct mp_image *simg =
+            va_surface_download(surface, p->ctx->image_formats);
+        if (simg) {
+            if (!p->printed_readback_warning) {
+                mp_msg(MSGT_VO, MSGL_WARN, "[vaapi] Using GPU readback. This "
+                       "is usually inefficient.\n");
+                p->printed_readback_warning = true;
+            }
+            talloc_free(img);
+            return simg;
+        }
+    }
+    return img;
+}
+
 const struct vd_lavc_hwdec mp_vd_lavc_vaapi = {
     .type = HWDEC_VAAPI,
     .image_formats = (const int[]) {IMGFMT_VAAPI, IMGFMT_VAAPI_MPEG2_IDCT,
@@ -398,3 +489,14 @@ const struct vd_lavc_hwdec mp_vd_lavc_vaapi = {
     .uninit = uninit,
     .allocate_image = allocate_image,
 };
+
+const struct vd_lavc_hwdec mp_vd_lavc_vaapi_copy = {
+    .type = HWDEC_VAAPI_COPY,
+    .image_formats = (const int[]) {IMGFMT_VAAPI, IMGFMT_VAAPI_MPEG2_IDCT,
+                                    IMGFMT_VAAPI_MPEG2_MOCO, 0},
+    .probe = probe_copy,
+    .init = init_copy,
+    .uninit = uninit,
+    .allocate_image = allocate_image,
+    .process_image = copy_image,
+};

video/decode/vd_lavc.c

@@ -86,6 +86,7 @@ const struct vd_lavc_hwdec mp_vd_lavc_vdpau;
 const struct vd_lavc_hwdec mp_vd_lavc_vdpau_old;
 const struct vd_lavc_hwdec mp_vd_lavc_vda;
 const struct vd_lavc_hwdec mp_vd_lavc_vaapi;
+const struct vd_lavc_hwdec mp_vd_lavc_vaapi_copy;
 
 static const struct vd_lavc_hwdec mp_vd_lavc_crystalhd = {
     .type = HWDEC_CRYSTALHD,
@@ -114,6 +115,7 @@ static const struct vd_lavc_hwdec *hwdec_list[] = {
     &mp_vd_lavc_crystalhd,
 #if CONFIG_VAAPI
     &mp_vd_lavc_vaapi,
+    &mp_vd_lavc_vaapi_copy,
 #endif
     NULL
 };