vo_opengl: support for vdpau hardware decoding

This uses vdpau OpenGL interop to convert a vdpau surface to a texture.

Note that this is a bit weak and primitive. Deinterlacing (or any other
form of vdpau postprocessing) is not supported. vo_opengl chroma scaling
and chroma sample position are not supported. Internally, the vdpau
video surfaces are converted to a RGBA surface first, because using the
video surfaces directly is too complicated. (These surfaces are always
split into separate fields, and the vo_opengl core expects progressive
frames or frames with weaved fields.)

DOCS/man/en/options.rst

@@ -1117,7 +1117,7 @@ OPTIONS
 
     :no:        always use software decoding (default)
     :auto:      see below
-    :vdpau:     requires ``--vo=vdpau`` (Linux only)
+    :vdpau:     requires ``--vo=vdpau`` or ``--vo=opengl`` (Linux only)
     :vaapi:     requires ``--vo=opengl`` or ``--vo=vaapi`` (Linux with Intel GPUs only)
     :vaapi-copy: copies video back into system RAM (Linux with Intel GPUs only)
     :vda:       requires ``--vo=corevideo`` (OSX only)

Makefile

@@ -114,6 +114,7 @@ SOURCES-$(PORTAUDIO)            += audio/out/ao_portaudio.c
 SOURCES-$(RSOUND)               += audio/out/ao_rsound.c
 SOURCES-$(SNDIO)                += audio/out/ao_sndio.c
 SOURCES-$(VDPAU)                += video/vdpau.c video/out/vo_vdpau.c
+SOURCES-$(VDPAU_GL_X11)         += video/out/gl_hwdec_vdpau.c
 SOURCES-$(VDA)                  += video/decode/vda.c
 SOURCES-$(VDPAU_DEC)            += video/decode/vdpau.c
 SOURCES-$(VDPAU_DEC_OLD)        += video/decode/vdpau_old.c

configure

@@ -2142,6 +2142,16 @@ fi
 echores "$_gl"
 
 
+echocheck "VDPAU with OpenGL/X11"
+if test "$_gl_x11" = yes && test "$_vdpau" = yes ; then
+  def_vdpau_gl_x11='#define HAVE_VDPAU_GL_X11 1'
+  _vdpau_gl_x11=yes
+else
+  def_vdpau_gl_x11='#define HAVE_VDPAU_GL_X11 0'
+  _vdpau_gl_x11=no
+fi
+echores "$_vdpau_gl_x11"
+
 
 if win32; then
 
@@ -3328,6 +3338,7 @@ VCD = $_vcd
 VDPAU = $_vdpau
 VDPAU_DEC = $_vdpau_dec
 VDPAU_DEC_OLD = $_vdpau_dec_old
+VDPAU_GL_X11 = $_vdpau_gl_x11
 VDA = $_vda
 VDA_REFCOUNTING = $_vda_refcounting
 VAAPI = $_vaapi
@@ -3509,6 +3520,7 @@ $def_v4l2
 $def_vdpau
 $def_vdpau_dec
 $def_vdpau_dec_old
+$def_vdpau_gl_x11
 $def_vda
 $def_vda_refcounting
 $def_vaapi

video/decode/vdpau.c

@@ -221,6 +221,7 @@ static int init(struct lavc_ctx *ctx)
 static int probe(struct vd_lavc_hwdec *hwdec, struct mp_hwdec_info *info,
                  const char *decoder)
 {
+    hwdec_request_api(info, "vdpau");
     if (!info || !info->vdpau_ctx)
         return HWDEC_ERR_NO_CTX;
     if (!hwdec_check_codec_support(decoder, profiles))

video/out/gl_common.c

@@ -445,6 +445,23 @@ struct gl_functions gl_functions[] = {
             {0}
         },
     },
+    // For gl_hwdec_vdpau.c
+    // http://www.opengl.org/registry/specs/NV/vdpau_interop.txt
+    {
+        .extension = "GL_NV_vdpau_interop",
+        .provides = MPGL_CAP_VDPAU,
+        .functions = (struct gl_function[]) {
+            // (only functions needed by us)
+            DEF_FN(VDPAUInitNV),
+            DEF_FN(VDPAUFiniNV),
+            DEF_FN(VDPAURegisterOutputSurfaceNV),
+            DEF_FN(VDPAUUnregisterSurfaceNV),
+            DEF_FN(VDPAUSurfaceAccessNV),
+            DEF_FN(VDPAUMapSurfacesNV),
+            DEF_FN(VDPAUUnmapSurfacesNV),
+            {0}
+        },
+    },
 };
 
 #undef FN_OFFS
@@ -1010,10 +1027,14 @@ void mp_log_source(struct mp_log *log, int lev, const char *src)
 }
 
 extern const struct gl_hwdec_driver gl_hwdec_vaglx;
+extern const struct gl_hwdec_driver gl_hwdec_vdpau;
 
 const struct gl_hwdec_driver *mpgl_hwdec_drivers[] = {
 #if HAVE_VAAPI_GLX
     &gl_hwdec_vaglx,
+#endif
+#if HAVE_VDPAU_GL_X11
+    &gl_hwdec_vdpau,
 #endif
     NULL
 };

video/out/gl_common.h

@@ -87,6 +87,7 @@ enum {
     MPGL_CAP_SRGB_FB            = (1 << 8),
     MPGL_CAP_FLOAT_TEX          = (1 << 9),
     MPGL_CAP_TEX_RG             = (1 << 10),    // GL_ARB_texture_rg / GL 3.x
+    MPGL_CAP_VDPAU              = (1 << 11),    // GL_NV_vdpau_interop
     MPGL_CAP_NO_SW              = (1 << 30),    // used to block sw. renderers
 };
 
@@ -355,6 +356,15 @@ struct GL {
                                         const GLfloat *);
     void (GLAPIENTRY *UniformMatrix4x3fv)(GLint, GLsizei, GLboolean,
                                           const GLfloat *);
+
+    void (GLAPIENTRY *VDPAUInitNV)(const GLvoid *, const GLvoid *);
+    void (GLAPIENTRY *VDPAUFiniNV)(void);
+    GLvdpauSurfaceNV (GLAPIENTRY *VDPAURegisterOutputSurfaceNV)
+        (GLvoid *, GLenum, GLsizei, const GLuint *);
+    void (GLAPIENTRY *VDPAUUnregisterSurfaceNV)(GLvdpauSurfaceNV);
+    void (GLAPIENTRY *VDPAUSurfaceAccessNV)(GLvdpauSurfaceNV, GLenum);
+    void (GLAPIENTRY *VDPAUMapSurfacesNV)(GLsizei, const GLvdpauSurfaceNV *);
+    void (GLAPIENTRY *VDPAUUnmapSurfacesNV)(GLsizei, const GLvdpauSurfaceNV *);
 };
 
 #endif /* MPLAYER_GL_COMMON_H */

video/out/gl_header_fixes.h

@@ -249,6 +249,10 @@
 #endif
 #endif
 
+#ifdef GL_NV_vdpau_interop
+#define GLvdpauSurfaceNV GLintptr
+#endif
+
 #undef MP_GET_GL_WORKAROUNDS
 
 #endif

video/out/gl_hwdec_vdpau.c

@@ -0,0 +1,258 @@
+/*
+ * This file is part of mpv.
+ *
+ * Parts based on the MPlayer VA-API patch (see vo_vaapi.c).
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stddef.h>
+#include <assert.h>
+
+#include "gl_common.h"
+#include "video/vdpau.h"
+#include "video/decode/dec_video.h"
+
+static int reinit(struct gl_hwdec *hw, const struct mp_image_params *params);
+
+struct priv {
+    struct mp_vdpau_ctx *ctx;
+    uint64_t preemption_counter;
+    struct mp_image_params image_params;
+    GLuint gl_texture;
+    GLvdpauSurfaceNV vdpgl_surface;
+    VdpOutputSurface vdp_surface;
+    VdpVideoMixer video_mixer;
+};
+
+static bool query_format(int imgfmt)
+{
+    return IMGFMT_IS_VDPAU(imgfmt);
+}
+
+static void mark_vdpau_objects_uninitialized(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+
+    p->vdp_surface = VDP_INVALID_HANDLE;
+    p->video_mixer = VDP_INVALID_HANDLE;
+}
+
+static int handle_preemption(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+
+    if (!mp_vdpau_status_ok(p->ctx)) {
+        mark_vdpau_objects_uninitialized(hw);
+        return -1;
+    }
+
+    if (p->preemption_counter == p->ctx->preemption_counter)
+        return 0;
+
+    mark_vdpau_objects_uninitialized(hw);
+
+    p->preemption_counter = p->ctx->preemption_counter;
+
+    if (reinit(hw, &p->image_params) < 0)
+        return -1;
+
+    return 1;
+}
+
+static void destroy_objects(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    GL *gl = hw->mpgl->gl;
+    struct vdp_functions *vdp = p->ctx->vdp;
+    VdpStatus vdp_st;
+
+    if (p->vdpgl_surface)
+        gl->VDPAUUnregisterSurfaceNV(p->vdpgl_surface);
+    p->vdpgl_surface = 0;
+
+    glDeleteTextures(1, &p->gl_texture);
+    p->gl_texture = 0;
+
+    if (p->vdp_surface != VDP_INVALID_HANDLE) {
+        vdp_st = vdp->output_surface_destroy(p->vdp_surface);
+        CHECK_ST_WARNING("Error when calling vdp_output_surface_destroy");
+    }
+
+    if (p->video_mixer != VDP_INVALID_HANDLE) {
+        vdp_st = vdp->video_mixer_destroy(p->video_mixer);
+        CHECK_ST_WARNING("Error when calling vdp_video_mixer_destroy");
+    }
+
+    glCheckError(gl, hw->log, "Before uninitializing OpenGL interop");
+
+    gl->VDPAUFiniNV();
+
+    // If the GL/vdpau state is not initialized, above calls raises an error.
+    while (1) {
+        if (gl->GetError() == GL_NO_ERROR)
+            break;
+    }
+
+    mark_vdpau_objects_uninitialized(hw);
+}
+
+static void destroy(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+
+    destroy_objects(hw);
+    mp_vdpau_destroy(p->ctx);
+}
+
+static int create(struct gl_hwdec *hw)
+{
+    GL *gl = hw->mpgl->gl;
+    if (hw->info->vdpau_ctx)
+        return -1;
+    if (!hw->mpgl->vo->x11)
+        return -1;
+    if (!(gl->mpgl_caps & MPGL_CAP_VDPAU))
+        return -1;
+    struct priv *p = talloc_zero(hw, struct priv);
+    hw->priv = p;
+    p->ctx = mp_vdpau_create_device_x11(hw->log, hw->mpgl->vo->x11);
+    if (!p->ctx)
+        return -1;
+    p->preemption_counter = p->ctx->preemption_counter;
+    mark_vdpau_objects_uninitialized(hw);
+    hw->info->vdpau_ctx = p->ctx;
+    hw->converted_imgfmt = IMGFMT_RGB0;
+    return 0;
+}
+
+static int reinit(struct gl_hwdec *hw, const struct mp_image_params *params)
+{
+    struct priv *p = hw->priv;
+    GL *gl = hw->mpgl->gl;
+    struct vdp_functions *vdp = p->ctx->vdp;
+    VdpStatus vdp_st;
+
+    destroy_objects(hw);
+
+    p->image_params = *params;
+
+    if (!mp_vdpau_status_ok(p->ctx))
+        return -1;
+
+    gl->VDPAUInitNV((void *)p->ctx->vdp_device, p->ctx->get_proc_address);
+
+#define VDP_NUM_MIXER_PARAMETER 3
+    static const VdpVideoMixerParameter parameters[VDP_NUM_MIXER_PARAMETER] = {
+        VDP_VIDEO_MIXER_PARAMETER_VIDEO_SURFACE_WIDTH,
+        VDP_VIDEO_MIXER_PARAMETER_VIDEO_SURFACE_HEIGHT,
+        VDP_VIDEO_MIXER_PARAMETER_CHROMA_TYPE,
+    };
+
+    const void *const parameter_values[VDP_NUM_MIXER_PARAMETER] = {
+        &(uint32_t){params->w},
+        &(uint32_t){params->h},
+        &(VdpChromaType){VDP_CHROMA_TYPE_420},
+    };
+    vdp_st = vdp->video_mixer_create(p->ctx->vdp_device, 0, NULL,
+                                     VDP_NUM_MIXER_PARAMETER,
+                                     parameters, parameter_values,
+                                     &p->video_mixer);
+    CHECK_ST_ERROR("Error when calling vdp_video_mixer_create");
+
+    struct mp_csp_params cparams = MP_CSP_PARAMS_DEFAULTS;
+    cparams.colorspace.levels_in = params->colorlevels;
+    cparams.colorspace.format = params->colorspace;
+    // VdpCSCMatrix happens to be compatible with mpv's CSC matrix type
+    // both are float[3][4]
+    VdpCSCMatrix matrix;
+    mp_get_yuv2rgb_coeffs(&cparams, matrix);
+    VdpVideoMixerAttribute csc_attr = VDP_VIDEO_MIXER_ATTRIBUTE_CSC_MATRIX;
+    vdp_st = vdp->video_mixer_set_attribute_values(p->video_mixer, 1, &csc_attr,
+                                                   &(const void *){matrix});
+    CHECK_ST_WARNING("Error when setting vdpau colorspace conversion matrix");
+
+    vdp_st = vdp->output_surface_create(p->ctx->vdp_device,
+                                        VDP_RGBA_FORMAT_B8G8R8A8,
+                                        params->w, params->h, &p->vdp_surface);
+    CHECK_ST_ERROR("Error when calling vdp_output_surface_create");
+
+    gl->GenTextures(1, &p->gl_texture);
+    gl->BindTexture(GL_TEXTURE_2D, p->gl_texture);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE);
+    gl->TexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE);
+    gl->BindTexture(GL_TEXTURE_2D, 0);
+
+    p->vdpgl_surface = gl->VDPAURegisterOutputSurfaceNV((void *)p->vdp_surface,
+                                                        GL_TEXTURE_2D,
+                                                        1, &p->gl_texture);
+    if (!p->vdpgl_surface)
+        return -1;
+
+    gl->VDPAUSurfaceAccessNV(p->vdpgl_surface, GL_READ_ONLY);
+
+    glCheckError(gl, hw->log, "After initializing vdpau OpenGL interop");
+
+    return 0;
+}
+
+static int map_image(struct gl_hwdec *hw, struct mp_image *hw_image,
+                     GLuint *out_textures)
+{
+    struct priv *p = hw->priv;
+    GL *gl = hw->mpgl->gl;
+    struct vdp_functions *vdp = p->ctx->vdp;
+    VdpStatus vdp_st;
+
+    assert(hw_image && IMGFMT_IS_VDPAU(hw_image->imgfmt));
+    VdpVideoSurface video_surface = (intptr_t)hw_image->planes[3];
+
+    if (handle_preemption(hw) < 0)
+        return -1;
+
+    if (!p->vdpgl_surface)
+        return -1;
+
+    VdpRect *video_rect = NULL;
+    vdp_st = vdp->video_mixer_render(p->video_mixer, VDP_INVALID_HANDLE,
+                                     0, VDP_VIDEO_MIXER_PICTURE_STRUCTURE_FRAME,
+                                     0, NULL, video_surface, 0, NULL,
+                                     video_rect, p->vdp_surface,
+                                     NULL, NULL, 0, NULL);
+    CHECK_ST_ERROR("Error when calling vdp_video_mixer_render");
+
+    gl->VDPAUMapSurfacesNV(1, &p->vdpgl_surface);
+    out_textures[0] = p->gl_texture;
+    return 0;
+}
+
+static void unmap_image(struct gl_hwdec *hw)
+{
+    struct priv *p = hw->priv;
+    GL *gl = hw->mpgl->gl;
+
+    gl->VDPAUUnmapSurfacesNV(1, &p->vdpgl_surface);
+}
+
+const struct gl_hwdec_driver gl_hwdec_vdpau = {
+    .api_name = "vdpau",
+    .query_format = query_format,
+    .create = create,
+    .reinit = reinit,
+    .map_image = map_image,
+    .unmap_image = unmap_image,
+    .destroy = destroy,
+};