vaapi: remove dependency on X11

There are at least 2 ways of using VAAPI without X11 (Wayland, DRM).
Remove the X11 requirement from the decoder part and the EGL interop.
This will be used by a following commit, which adds Wayland support.

The worst about this is the decoder part, which includes a bad hack for
using the decoder without any VO interop (also known as "vaapi-copy"
mode). Separate the X11 parts so that they're self-contained. For the
EGL interop code we do something similar (it's kept slightly simpler,
because it essentially only has to translate between our silly
MPGetNativeDisplay abstraction and the vaGetDisplay...() call).

TOOLS/old-configure

@@ -610,6 +610,7 @@ define_yes_no $_vdpau HAVE_VDPAU_HWACCEL
 check_pkg_config "VAAPI" $_vaapi VAAPI 'libva >= 0.32.0 libva-x11 >= 0.32.0'
 _vaapi=$(defretval)
 define_yes_no $_vaapi HAVE_VAAPI_HWACCEL
+define_yes_no $_vaapi HAVE_VAAPI_X11
 
 if test "$_vaapi" = yes ; then
     check_pkg_config "VAAPI VPP" auto VAAPI_VPP 'libva >= 0.34.0'
@@ -706,6 +707,7 @@ check_pkg_config "VAAPI with OpenGL/X11" $_vaapi_glx VAAPI_GLX 'libva-glx >= 0.3
 _vaapi_x_egl=no
 (test "$_gl_x11_egl" = yes && test "$_vaapi" = yes) && _vaapi_x_egl=yes
 check_yes_no $_vaapi_x_egl VAAPI_X_EGL
+check_yes_no $_vaapi_x_egl VAAPI_EGL
 
 check_pkg_config "SDL 2.0" $_sdl2 SDL2 'sdl2'
 

video/decode/vaapi.c

@@ -25,7 +25,7 @@
 #include <libavcodec/vaapi.h>
 #include <libavutil/common.h>
 
-#include <X11/Xlib.h>
+#include "config.h"
 
 #include "lavc.h"
 #include "common/common.h"
@@ -53,7 +53,9 @@ struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
     VADisplay display;
-    Display *x11_display;
+
+    const struct va_native_display *native_display_fns;
+    void *native_display;
 
     // libavcodec shared struct
     struct vaapi_context *va_context;
@@ -65,6 +67,47 @@ struct priv {
     struct mp_image_pool *sw_pool;
 };
 
+struct va_native_display {
+    void (*create)(struct priv *p);
+    void (*destroy)(struct priv *p);
+};
+
+static const struct va_native_display disp_x11;
+
+static const struct va_native_display *const native_displays[] = {
+#if HAVE_VAAPI_X11
+    &disp_x11,
+#endif
+    NULL
+};
+
+#if HAVE_VAAPI_X11
+#include <X11/Xlib.h>
+#include <va/va_x11.h>
+
+static void x11_destroy(struct priv *p)
+{
+    if (p->native_display)
+        XCloseDisplay(p->native_display);
+    p->native_display = NULL;
+}
+
+static void x11_create(struct priv *p)
+{
+    p->native_display = XOpenDisplay(NULL);
+    if (!p->native_display)
+        return;
+    p->display = vaGetDisplay(p->native_display);
+    if (!p->display)
+        x11_destroy(p);
+}
+
+static const struct va_native_display disp_x11 = {
+    .create = x11_create,
+    .destroy = x11_destroy,
+};
+#endif
+
 #define HAS_HEVC VA_CHECK_VERSION(0, 38, 0)
 
 #define PE(av_codec_id, ff_profile, vdp_profile)                \
@@ -295,9 +338,8 @@ static void destroy_va_dummy_ctx(struct priv *p)
     va_destroy(p->ctx);
     p->ctx = NULL;
     p->display = NULL;
-    if (p->x11_display)
+    if (p->native_display_fns)
-        XCloseDisplay(p->x11_display);
+        p->native_display_fns->destroy(p);
-    p->x11_display = NULL;
 }
 
 // Creates a "private" VADisplay, disconnected from the VO. We just create a
@@ -305,15 +347,18 @@ static void destroy_va_dummy_ctx(struct priv *p)
 // connection along with struct mp_hwdec_info, if we wanted.)
 static bool create_va_dummy_ctx(struct priv *p)
 {
-    p->x11_display = XOpenDisplay(NULL);
+    for (int n = 0; native_displays[n]; n++) {
-    if (!p->x11_display)
+        native_displays[n]->create(p);
+        if (p->display) {
+            p->native_display_fns = native_displays[n];
+            break;
+        }
+    }
+    if (!p->display)
         goto destroy_ctx;
-    VADisplay *display = vaGetDisplay(p->x11_display);
+    p->ctx = va_initialize(p->display, p->log, true);
-    if (!display)
-        goto destroy_ctx;
-    p->ctx = va_initialize(display, p->log, true);
     if (!p->ctx) {
-        vaTerminate(display);
+        vaTerminate(p->display);
         goto destroy_ctx;
     }
     return true;
@@ -334,7 +379,7 @@ static void uninit(struct lavc_ctx *ctx)
     talloc_free(p->pool);
     p->pool = NULL;
 
-    if (p->x11_display)
+    if (p->native_display_fns)
         destroy_va_dummy_ctx(p);
 
     talloc_free(p);

video/out/opengl/hwdec.c

@@ -37,7 +37,7 @@ extern const struct gl_hwdec_driver gl_hwdec_vdpau;
 extern const struct gl_hwdec_driver gl_hwdec_dxva2;
 
 static const struct gl_hwdec_driver *const mpgl_hwdec_drivers[] = {
-#if HAVE_VAAPI_X_EGL
+#if HAVE_VAAPI_EGL
     &gl_hwdec_vaegl,
 #endif
 #if HAVE_VAAPI_GLX

video/out/opengl/hwdec_vaegl.c

@@ -26,7 +26,6 @@
 
 #include <va/va_drmcommon.h>
 
-#include "video/out/x11_common.h"
 #include "hwdec.h"
 #include "video/vaapi.h"
 #include "video/img_fourcc.h"
@@ -47,6 +46,29 @@ typedef void *EGLImageKHR;
 #define EGL_DMA_BUF_PLANE0_PITCH_EXT      0x3274
 #endif
 
+#if HAVE_VAAPI_X11
+#include <va/va_x11.h>
+
+static VADisplay *create_x11_va_display(GL *gl)
+{
+    Display *x11 = gl->MPGetNativeDisplay("x11");
+    return x11 ? vaGetDisplay(x11) : NULL;
+}
+#endif
+
+static VADisplay *create_native_va_display(GL *gl)
+{
+    if (!gl->MPGetNativeDisplay)
+        return NULL;
+    VADisplay *display = NULL;
+#if HAVE_VAAPI_X11
+    display = create_x11_va_display(gl);
+    if (display)
+        return display;
+#endif
+    return display;
+}
+
 struct priv {
     struct mp_log *log;
     struct mp_vaapi_ctx *ctx;
@@ -152,10 +174,6 @@ static int create(struct gl_hwdec *hw)
     if (!eglGetCurrentDisplay())
         return -1;
 
-    p->xdisplay =
-        hw->gl->MPGetNativeDisplay ? hw->gl->MPGetNativeDisplay("x11") : NULL;
-    if (!p->xdisplay)
-        return -1;
     if (!strstr(gl->extensions, "EXT_image_dma_buf_import") ||
         !strstr(gl->extensions, "EGL_KHR_image_base") ||
         !strstr(gl->extensions, "GL_OES_EGL_image") ||
@@ -173,7 +191,7 @@ static int create(struct gl_hwdec *hw)
         !p->EGLImageTargetTexture2DOES)
         return -1;
 
-    p->display = vaGetDisplay(p->xdisplay);
+    p->display = create_native_va_display(gl);
     if (!p->display)
         return -1;
 

video/out/opengl/hwdec_vaglx.c

@@ -22,6 +22,7 @@
 #include <assert.h>
 
 #include <GL/glx.h>
+#include <va/va_x11.h>
 
 #include "video/out/x11_common.h"
 #include "hwdec.h"

video/vaapi.h

@@ -22,7 +22,6 @@
 #include <inttypes.h>
 #include <pthread.h>
 #include <va/va.h>
-#include <va/va_x11.h>
 
 #ifndef VA_FOURCC_I420
 #define VA_FOURCC_I420 VA_FOURCC('I', '4', '2', '0')

wscript

@@ -639,9 +639,14 @@ video_output_features = [
     }, {
         'name': '--vaapi',
         'desc': 'VAAPI acceleration',
-        'deps': [ 'x11', 'libdl' ],
+        'deps': [ 'libdl' ],
-        'func': check_pkg_config(
+        'deps_any': [ 'x11' ],
-            'libva', '>= 0.34.0', 'libva-x11', '>= 0.34.0'),
+        'func': check_pkg_config('libva', '>= 0.34.0'),
+    }, {
+        'name': '--vaapi-x11',
+        'desc': 'VAAPI (X11 support)',
+        'deps': [ 'vaapi', 'x11' ],
+        'func': check_pkg_config('libva-x11', '>= 0.34.0'),
     }, {
         'name': '--vaapi-vpp',
         'desc': 'VAAPI VPP',
@@ -650,12 +655,17 @@ video_output_features = [
     }, {
         'name': '--vaapi-glx',
         'desc': 'VAAPI GLX',
-        'deps': [ 'vaapi', 'gl-x11' ],
+        'deps': [ 'vaapi-x11', 'gl-x11' ],
         'func': check_true,
     }, {
         'name': '--vaapi-x-egl',
         'desc': 'VAAPI EGL on X11',
-        'deps': [ 'vaapi', 'egl-x11' ],
+        'deps': [ 'vaapi-x11', 'egl-x11' ],
+        'func': check_true,
+    }, {
+        'name': 'vaapi-egl',
+        'desc': 'VAAPI EGL',
+        'deps_any': [ 'vaapi-x-egl' ],
         'func': check_true,
     }, {
         'name': '--caca',

wscript_build.py

@@ -320,7 +320,7 @@ def build(ctx):
         ( "video/out/opengl/rpi.c",              "rpi" ),
         ( "video/out/opengl/hwdec.c",            "gl" ),
         ( "video/out/opengl/hwdec_dxva2.c",      "gl-win32" ),
-        ( "video/out/opengl/hwdec_vaegl.c",      "vaapi-x-egl" ),
+        ( "video/out/opengl/hwdec_vaegl.c",      "vaapi-egl" ),
         ( "video/out/opengl/hwdec_vaglx.c",      "vaapi-glx" ),
         ( "video/out/opengl/hwdec_vda.c",        "videotoolbox-vda-gl" ),
         ( "video/out/opengl/hwdec_vdpau.c",      "vdpau-gl-x11" ),
@@ -344,7 +344,7 @@ def build(ctx):
         ( "video/out/vo_opengl.c",               "gl" ),
         ( "video/out/vo_opengl_cb.c",            "gl" ),
         ( "video/out/vo_sdl.c",                  "sdl2" ),
-        ( "video/out/vo_vaapi.c",                "vaapi" ),
+        ( "video/out/vo_vaapi.c",                "vaapi-x11" ),
         ( "video/out/vo_vdpau.c",                "vdpau" ),
         ( "video/out/vo_wayland.c",              "wayland" ),
         ( "video/out/vo_xv.c",                   "xv" ),