vdpau: use libavutil for surface allocation during decoding

Use the libavutil vdpau frame allocation code instead of our own "old"
code. This also uses its code for copying a video surface to normal
memory (used by vdpau-copy).

Since vdpau doesn't really have an internal pixel format, 4:2:0 can be
accessed as both nv12 and yuv420p - and libavutil prefers to report
yuv420p. The OpenGL interop has to be adjusted accordingly.

Preemption is a potential problem, but it doesn't break it more than it
already is.

This requires a bug fix to FFmpeg's vdpau code, or vdpau-copy (as well
as taking screenshots) will fail. Libav has fixed this bug ages ago.

video/decode/hw_vdpau.c

@@ -18,6 +18,7 @@
 #include <libavcodec/avcodec.h>
 #include <libavcodec/vdpau.h>
 #include <libavutil/common.h>
+#include <libavutil/hwcontext.h>
 
 #include "lavc.h"
 #include "common/common.h"
@@ -31,12 +32,15 @@ struct priv {
     uint64_t                    preemption_counter;
     // vdpau-copy
     Display                    *display;
-    struct mp_image_pool       *sw_pool;
 };
 
 static int init_decoder(struct lavc_ctx *ctx, int w, int h)
 {
     struct priv *p = ctx->hwdec_priv;
+    int sw_format = ctx->avctx->sw_pix_fmt;
+
+    if (hwdec_setup_hw_frames_ctx(ctx, p->mpvdp->av_device_ref, sw_format, 0) < 0)
+        return -1;
 
     // During preemption, pretend everything is ok.
     if (mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter) < 0)
@@ -48,34 +52,6 @@ static int init_decoder(struct lavc_ctx *ctx, int w, int h)
                                  AV_HWACCEL_FLAG_ALLOW_HIGH_DEPTH);
 }
 
-static struct mp_image *allocate_image(struct lavc_ctx *ctx, int w, int h)
-{
-    struct priv *p = ctx->hwdec_priv;
-
-    // In case of preemption, reinit the decoder. Setting hwdec_request_reinit
-    // will cause init_decoder() to be called again.
-    if (mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter) == 0)
-        ctx->hwdec_request_reinit = true;
-
-    VdpChromaType chroma = 0;
-    uint32_t s_w = w, s_h = h;
-    if (av_vdpau_get_surface_parameters(ctx->avctx, &chroma, &s_w, &s_h) < 0)
-        return NULL;
-
-    return mp_vdpau_get_video_surface(p->mpvdp, chroma, s_w, s_h);
-}
-
-static struct mp_image *update_format(struct lavc_ctx *ctx, struct mp_image *img)
-{
-    VdpChromaType chroma = 0;
-    uint32_t s_w, s_h;
-    if (av_vdpau_get_surface_parameters(ctx->avctx, &chroma, &s_w, &s_h) >= 0) {
-        if (chroma == VDP_CHROMA_TYPE_420)
-            img->params.hw_subfmt = IMGFMT_NV12;
-    }
-    return img;
-}
-
 static void uninit(struct lavc_ctx *ctx)
 {
     struct priv *p = ctx->hwdec_priv;
@@ -128,8 +104,6 @@ static int init_copy(struct lavc_ctx *ctx)
     if (!p->mpvdp)
         goto error;
 
-    p->sw_pool = talloc_steal(p, mp_image_pool_new(17));
-
     ctx->hwdec_priv = p;
 
     mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter);
@@ -156,15 +130,6 @@ static int probe_copy(struct lavc_ctx *ctx, struct vd_lavc_hwdec *hwdec,
     return r;
 }
 
-static struct mp_image *copy_image(struct lavc_ctx *ctx, struct mp_image *img)
-{
-    struct priv *p = ctx->hwdec_priv;
-    struct mp_hwdec_ctx *hwctx = &p->mpvdp->hwctx;
-    struct mp_image *out = hwctx->download_image(hwctx, img, p->sw_pool);
-    talloc_free(img);
-    return out;
-}
-
 const struct vd_lavc_hwdec mp_vd_lavc_vdpau = {
     .type = HWDEC_VDPAU,
     .image_format = IMGFMT_VDPAU,
@@ -172,8 +137,7 @@ const struct vd_lavc_hwdec mp_vd_lavc_vdpau = {
     .init = init,
     .uninit = uninit,
     .init_decoder = init_decoder,
-    .allocate_image = allocate_image,
+    .volatile_context = true,
-    .process_image = update_format,
 };
 
 const struct vd_lavc_hwdec mp_vd_lavc_vdpau_copy = {
@@ -184,6 +148,5 @@ const struct vd_lavc_hwdec mp_vd_lavc_vdpau_copy = {
     .init = init_copy,
     .uninit = uninit,
     .init_decoder = init_decoder,
-    .allocate_image = allocate_image,
+    .volatile_context = true,
-    .process_image = copy_image,
 };

video/out/opengl/hwdec_vdpau.c

@@ -158,7 +158,8 @@ static int reinit(struct gl_hwdec *hw, struct mp_image_params *params)
 
     p->vdpgl_initialized = true;
 
-    p->direct_mode = params->hw_subfmt == IMGFMT_NV12;
+    p->direct_mode = params->hw_subfmt == IMGFMT_NV12 ||
+                     params->hw_subfmt == IMGFMT_420P;
 
     gl->GenTextures(4, p->gl_textures);
     for (int n = 0; n < 4; n++) {

video/vdpau.c

@@ -17,6 +17,9 @@
 
 #include <assert.h>
 
+#include <libavutil/hwcontext.h>
+#include <libavutil/hwcontext_vdpau.h>
+
 #include "vdpau.h"
 
 #include "osdep/threads.h"
@@ -32,47 +35,10 @@ static struct mp_image *download_image_yuv(struct mp_hwdec_ctx *hwctx,
                                            struct mp_image *mpi,
                                            struct mp_image_pool *swpool)
 {
-    struct mp_vdpau_ctx *ctx = hwctx->ctx;
-    struct vdp_functions *vdp = &ctx->vdp;
-    VdpStatus vdp_st;
-
     if (mpi->imgfmt != IMGFMT_VDPAU || mp_vdpau_mixed_frame_get(mpi))
         return NULL;
 
-    VdpVideoSurface surface = (uintptr_t)mpi->planes[3];
+    return mp_image_hw_download(mpi, swpool);
-
-    VdpChromaType s_chroma_type;
-    uint32_t s_w, s_h;
-    vdp_st = vdp->video_surface_get_parameters(surface, &s_chroma_type, &s_w, &s_h);
-    CHECK_VDP_ERROR_NORETURN(ctx,
-                    "Error when calling vdp_video_surface_get_parameters");
-    if (vdp_st != VDP_STATUS_OK)
-        return NULL;
-
-    // Don't bother supporting other types for now.
-    if (s_chroma_type != VDP_CHROMA_TYPE_420)
-        return NULL;
-
-    // The allocation needs to be uncropped, because get_bits writes to it.
-    struct mp_image *out = mp_image_pool_get(swpool, IMGFMT_NV12, s_w, s_h);
-    if (!out)
-        return NULL;
-
-    mp_image_set_size(out, mpi->w, mpi->h);
-    mp_image_copy_attributes(out, mpi);
-
-    vdp_st = vdp->video_surface_get_bits_y_cb_cr(surface,
-                                                 VDP_YCBCR_FORMAT_NV12,
-                                                 (void * const *)out->planes,
-                                                 out->stride);
-    CHECK_VDP_ERROR_NORETURN(ctx,
-                "Error when calling vdp_output_surface_get_bits_y_cb_cr");
-    if (vdp_st != VDP_STATUS_OK) {
-        talloc_free(out);
-        return NULL;
-    }
-
-    return out;
 }
 
 static struct mp_image *download_image(struct mp_hwdec_ctx *hwctx,
@@ -439,6 +405,26 @@ struct mp_image *mp_vdpau_get_video_surface(struct mp_vdpau_ctx *ctx,
     return mp_vdpau_get_surface(ctx, chroma, 0, false, w, h);
 }
 
+static bool open_lavu_vdpau_device(struct mp_vdpau_ctx *ctx)
+{
+    ctx->av_device_ref = av_hwdevice_ctx_alloc(AV_HWDEVICE_TYPE_VDPAU);
+    if (!ctx->av_device_ref)
+        return false;
+
+    AVHWDeviceContext *hwctx = (void *)ctx->av_device_ref->data;
+    AVVDPAUDeviceContext *vdctx = hwctx->hwctx;
+
+    vdctx->device = ctx->vdp_device;
+    vdctx->get_proc_address = ctx->get_proc_address;
+
+    if (av_hwdevice_ctx_init(ctx->av_device_ref) < 0)
+        av_buffer_unref(&ctx->av_device_ref);
+
+    ctx->hwctx.av_device_ref = ctx->av_device_ref;
+
+    return !!ctx->av_device_ref;
+}
+
 struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log, Display *x11,
                                                 bool probing)
 {
@@ -463,6 +449,10 @@ struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log, Display *x11
         mp_vdpau_destroy(ctx);
         return NULL;
     }
+    if (!open_lavu_vdpau_device(ctx)) {
+        mp_vdpau_destroy(ctx);
+        return NULL;
+    }
     return ctx;
 }
 

video/vdpau.h

@@ -48,6 +48,7 @@ struct mp_vdpau_ctx {
     Display *x11;
 
     struct mp_hwdec_ctx hwctx;
+    struct AVBufferRef *av_device_ref;
 
     // These are mostly immutable, except on preemption. We don't really care
     // to synchronize the preemption case fully correctly, because it's an