diff --git a/configure b/configure
index 1bf8674403..b702d22960 100755
--- a/configure
+++ b/configure
@@ -2078,7 +2078,6 @@ HAVE_LIST="
     vaapi_x11
     vdpau_x11
     winrt
-    xlib
 "
 
 # options emitted with CONFIG_ prefix but not available on the command line
@@ -6076,18 +6075,16 @@ enabled vaapi &&
     check_lib "va/va.h va/va_drm.h" vaGetDisplayDRM -lva -lva-drm &&
     enable vaapi_drm
 
+enabled vaapi &&
+    check_lib "va/va.h va/va_x11.h" vaGetDisplay -lva -lva-x11 -lX11 &&
+    enable vaapi_x11
+
 enabled vdpau &&
     check_cpp_condition vdpau/vdpau.h "defined VDP_DECODER_PROFILE_MPEG4_PART2_ASP" ||
     disable vdpau
 
-enabled_any vaapi vdpau && check_lib X11/Xlib.h XOpenDisplay -lX11 && enable xlib
-
-enabled vaapi && enabled xlib &&
-    check_lib "va/va.h va/va_x11.h" vaGetDisplay -lva -lva-x11 &&
-    enable vaapi_x11
-
-enabled vdpau && enabled xlib &&
-    check_lib "vdpau/vdpau.h vdpau/vdpau_x11.h" vdp_device_create_x11 -lvdpau &&
+enabled vdpau &&
+    check_lib "vdpau/vdpau.h vdpau/vdpau_x11.h" vdp_device_create_x11 -lvdpau -lX11 &&
     enable vdpau_x11
 
 if enabled x86; then