vo_opengl: add support for VA-API OpenGL interop

VA-API's OpenGL/GLX interop is pretty bad and perhaps slow (renders a
X11 pixmap into a FBO, and has to go over X11, probably involves one or
more copies), and this code serves more as an example, rather than for
serious use. On the other hand, this might be work much better than
vo_vaapi, even if slightly slower.

DOCS/man/en/options.rst

@@ -1118,7 +1118,7 @@ OPTIONS
     :no:        always use software decoding (default)
     :auto:      see below
     :vdpau:     requires ``--vo=vdpau`` (Linux only)
-    :vaapi:     requires ``--vo=vaapi`` (Linux with Intel GPUs only)
+    :vaapi:     requires ``--vo=opengl`` or ``--vo=vaapi`` (Linux with Intel GPUs only)
     :vaapi-copy: copies video back into system RAM (Linux with Intel GPUs only)
     :vda:       requires ``--vo=corevideo`` (OSX only)
     :crystalhd: Broadcom Crystal HD

Makefile

@@ -121,6 +121,7 @@ SOURCES-$(VAAPI)                += video/out/vo_vaapi.c \
                                    video/decode/vaapi.c \
                                    video/vaapi.c
 SOURCES-$(VAAPI_VPP)            += video/filter/vf_vavpp.c
+SOURCES-$(VAAPI_GLX)            += video/out/gl_hwdec_vaglx.c
 
 SOURCES-$(X11)                  += video/out/vo_x11.c video/out/x11_common.c
 SOURCES-$(XV)                   += video/out/vo_xv.c

configure

@@ -1860,6 +1860,8 @@ echores "$_vdpau"
 echocheck "VAAPI"
 _vaapi_vpp=no
 def_vaapi_vpp='#define CONFIG_VAAPI_VPP 0'
+_vaapi_glx=no
+def_vaapi_glx='#define CONFIG_VAAPI_GLX 0'
 if test "$_vaapi" = auto && test "$_x11" = yes ; then
   _vaapi=no
   if test "$_dl" = yes ; then
@@ -1881,7 +1883,13 @@ if test "$_vaapi" = yes ; then
     _vaapi_vpp=yes
     def_vaapi_vpp='#define CONFIG_VAAPI_VPP 1'
   fi
-  echores "$_vaapi_vpp"
+  echores "$_vaapi_glx"
+  echocheck "VAAPI GLX"
+  if pkg_config_add 'libva-glx >= 0.32.0' ; then
+    _vaapi_glx=yes
+    def_vaapi_glx='#define CONFIG_VAAPI_GLX 1'
+  fi
+  echores "$_vaapi_glx"
 fi
 
 
@@ -3297,6 +3305,7 @@ VDA = $_vda
 VDA_REFCOUNTING = $_vda_refcounting
 VAAPI = $_vaapi
 VAAPI_VPP = $_vaapi_vpp
+VAAPI_GLX = $_vaapi_glx
 WIN32 = $_win32
 X11 = $_x11
 WAYLAND = $_wayland
@@ -3475,6 +3484,7 @@ $def_vda
 $def_vda_refcounting
 $def_vaapi
 $def_vaapi_vpp
+$def_vaapi_glx
 $def_vm
 $def_x11
 $def_wayland

video/decode/vaapi.c

@@ -421,6 +421,7 @@ static int init(struct lavc_ctx *ctx)
 static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *decoder)
 {
+    hwdec_request_api(info, "vaapi");
     if (!info || !info->vaapi_ctx)
         return HWDEC_ERR_NO_CTX;
     if (!hwdec_check_codec_support(decoder, profiles))

video/out/gl_common.c

@@ -1009,6 +1009,11 @@ void mp_log_source(struct mp_log *log, int lev, const char *src)
     }
 }
 
+extern const struct gl_hwdec_driver gl_hwdec_vaglx;
+
 const struct gl_hwdec_driver *mpgl_hwdec_drivers[] = {
+#if CONFIG_VAAPI_GLX
+    &gl_hwdec_vaglx,
+#endif
     NULL
 };

video/out/gl_common.h

@@ -185,6 +185,7 @@ struct gl_hwdec_driver {
     // Test whether the given IMGFMT_ is supported.
     bool (*query_format)(int imgfmt);
     // Create the hwdec device. It must fill in hw->info, if applicable.
+    // This also must set hw->converted_imgfmt.
     int (*create)(struct gl_hwdec *hw);
     // Prepare for rendering video. (E.g. create textures.)
     // Called on initialization, and every time the video size changes.

video/out/gl_hwdec_vaglx.c

@@ -0,0 +1,140 @@
+/*
+ * This file is part of mpv.
+ *
+ * Parts based on the MPlayer VA-API patch (see vo_vaapi.c).
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stddef.h>
+#include <assert.h>
+
+#include <GL/glx.h>
+#include <va/va_glx.h>
+
+#include "x11_common.h"
+#include "gl_common.h"
+#include "video/vaapi.h"
+#include "video/decode/dec_video.h"
+
+struct priv {
+    struct mp_vaapi_ctx *ctx;
+    VADisplay *display;
+    GLuint gl_texture;
+    void *vaglx_surface;
+};
+
+static bool query_format(int imgfmt)
+{
+    return imgfmt == IMGFMT_VAAPI;
+}
+
+static void destroy_texture(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    VAStatus status;
+
+    if (p->vaglx_surface) {
+        status = vaDestroySurfaceGLX(p->display, p->vaglx_surface);
+        check_va_status(status, "vaDestroySurfaceGLX()");
+        p->vaglx_surface = NULL;
+    }
+
+    glDeleteTextures(1, &p->gl_texture);
+    p->gl_texture = 0;
+}
+
+static void destroy(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    destroy_texture(hw);
+    va_destroy(p->ctx);
+}
+
+static int create(struct gl_hwdec *hw)
+{
+    if (hw->info->vaapi_ctx)
+        return -1;
+    if (!hw->mpgl->vo->x11 || !glXGetCurrentContext())
+        return -1;
+    struct priv *p = talloc_zero(hw, struct priv);
+    hw->priv = p;
+    p->display = vaGetDisplayGLX(hw->mpgl->vo->x11->display);
+    if (!p->display)
+        return -1;
+    p->ctx = va_initialize(p->display);
+    if (!p->ctx) {
+        vaTerminate(p->display);
+        return -1;
+    }
+    hw->info->vaapi_ctx = p->ctx;
+    hw->converted_imgfmt = IMGFMT_RGBA;
+    return 0;
+}
+
+static int reinit(struct gl_hwdec *hw, int w, int h)
+{
+    struct priv *p = hw->priv;
+    GL *gl = hw->mpgl->gl;
+    VAStatus status;
+
+    destroy_texture(hw);
+
+    gl->GenTextures(1, &p->gl_texture);
+    gl->BindTexture(GL_TEXTURE_2D, p->gl_texture);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    gl->TexImage2D(GL_TEXTURE_2D, 0, GL_RGBA, w, h, 0,
+                   GL_RGBA, GL_UNSIGNED_BYTE, NULL);
+    gl->BindTexture(GL_TEXTURE_2D, 0);
+
+    status = vaCreateSurfaceGLX(p->display, GL_TEXTURE_2D,
+                                p->gl_texture, &p->vaglx_surface);
+    return check_va_status(status, "vaCreateSurfaceGLX()") ? 0 : -1;
+}
+
+static int load_image(struct gl_hwdec *hw, struct mp_image *hw_image,
+                      GLuint *out_textures)
+{
+    struct priv *p = hw->priv;
+    VAStatus status;
+
+    if (!p->vaglx_surface)
+        return -1;
+
+    status = vaCopySurfaceGLX(p->display, p->vaglx_surface,
+                              va_surface_id_in_mp_image(hw_image),
+                              va_get_colorspace_flag(hw_image->colorspace));
+    if (!check_va_status(status, "vaCopySurfaceGLX()"))
+        return -1;
+
+    out_textures[0] = p->gl_texture;
+    return 0;
+}
+
+static void unload_image(struct gl_hwdec *hw)
+{
+}
+
+const struct gl_hwdec_driver gl_hwdec_vaglx = {
+    .api_name = "vaapi",
+    .query_format = query_format,
+    .create = create,
+    .reinit = reinit,
+    .load_image = load_image,
+    .unload_image = unload_image,
+    .destroy = destroy,
+};