vo_gpu: x11: remove special vdpau probing, use EGL by default

Originally, vo_gpu/vo_opengl considered the case of Nvidia proprietary
drivers, which required vdpau/GLX, and Intel open source drivers, which
require vaapi/EGL. Since window creation and GPU context creation are
inseparable in mpv's internal API, it had to pick the correct API very
early, or hardware decoding wouldn't work. "x11probe" was introduced for
this reason. It created a GLX context (without showing the window yet),
and checked whether vdpau was available. If yes, it used GLX, if not, it
continued probing x11/EGL. (Obviously it couldn't always fail on GLX
without vdpau, which is why it was a separate "probe" backend.)

Years passed, and now the situation is different. Vdpau is dead. Nvidia
drivers and libavcodec now provide CUDA interop, which requires EGL, and
fixes some of the vdpau problems. AMD drivers now provide vaapi, which
generally works better than vdpau. Intel didn't change.

In particular, vaapi provides working HEVC Main10 support. In theory, it
should work on vdpau too, with quality reduction (no 10 bit surfaces),
but I couldn't get it to work.

So always prefer EGL. And suddenly hardware decoding works. This is
actually rather important, because HEVC is unfortunately on the rise,
despite shitty encoders and unoptimized decoders. The latter may mean
that hardware decoding works better than libavcodec.

This should have been done a long, long time ago.

DOCS/interface-changes.rst

@@ -85,6 +85,9 @@ Interface changes
     - Remove all "tv-..." options and properties, along with the classic Linux
       analog TV support.
     - remove "program" property (no replacement)
+    - always prefer EGL over GLX, which helps with AMD/vaapi, but will break
+      vdpau with --vo=gpu - use --gpu-context=x11 to be able to use vdpau. This
+      does not affect --vo=vdpau or --hwdec=vdpau-copy.
  --- mpv 0.29.0 ---
     - drop --opensles-sample-rate, as --audio-samplerate should be used if desired
     - drop deprecated --videotoolbox-format, --ff-aid, --ff-vid, --ff-sid,

DOCS/man/options.rst

@@ -4814,9 +4814,6 @@ The following video options are currently all specific to ``--vo=gpu`` and
         X11/GLX
     x11vk
         VK_KHR_xlib_surface
-    x11probe
-        For internal autoprobing, equivalent to ``x11`` otherwise. Don't use
-        directly, it could be removed without warning as autoprobing is changed.
     wayland
         Wayland/EGL
     waylandvk

video/out/gpu/context.c

@@ -35,7 +35,6 @@
 
 /* OpenGL */
 extern const struct ra_ctx_fns ra_ctx_glx;
-extern const struct ra_ctx_fns ra_ctx_glx_probe;
 extern const struct ra_ctx_fns ra_ctx_x11_egl;
 extern const struct ra_ctx_fns ra_ctx_drm_egl;
 extern const struct ra_ctx_fns ra_ctx_cocoa;
@@ -83,9 +82,6 @@ static const struct ra_ctx_fns *contexts[] = {
 #if HAVE_GL_WAYLAND
     &ra_ctx_wayland_egl,
 #endif
-#if HAVE_GL_X11
-    &ra_ctx_glx_probe,
-#endif
 #if HAVE_EGL_X11
     &ra_ctx_x11_egl,
 #endif

video/out/opengl/context_glx.c

@@ -346,20 +346,6 @@ uninit:
     return false;
 }
 
-static bool glx_init_probe(struct ra_ctx *ctx)
-{
-    if (!glx_init(ctx))
-        return false;
-
-    struct priv *p = ctx->priv;
-    if (!(p->gl.mpgl_caps & MPGL_CAP_VDPAU)) {
-        MP_VERBOSE(ctx, "No vdpau support found - probing more things.\n");
-        glx_uninit(ctx);
-        return false;
-    }
-
-    return true;
-}
 
 static void resize(struct ra_ctx *ctx)
 {
@@ -401,14 +387,3 @@ const struct ra_ctx_fns ra_ctx_glx = {
     .init           = glx_init,
     .uninit         = glx_uninit,
 };
-
-const struct ra_ctx_fns ra_ctx_glx_probe = {
-    .type           = "opengl",
-    .name           = "x11probe",
-    .reconfig       = glx_reconfig,
-    .control        = glx_control,
-    .wakeup         = glx_wakeup,
-    .wait_events    = glx_wait_events,
-    .init           = glx_init_probe,
-    .uninit         = glx_uninit,
-};