vdpau: handle display preemption during decoding

This was broken for some time, and it didn't recover correctly.

Redo decoder display preemption. Instead of trying to reinitialize the
hw decoder, simply fallback to software decoding. I consider display
preemption a bug in the vdpau API, so being able to _somehow_ recover
playback is good enough.

The approach taking here will probably also make it easier to handle
multithreading.

video/decode/vdpau.c

@@ -33,7 +33,6 @@ struct priv {
     struct mp_log              *log;
     struct mp_vdpau_ctx        *mpvdp;
     struct vdp_functions       *vdp;
-    VdpDevice                   vdp_device;
     uint64_t                    preemption_counter;
 
     AVVDPAUContext              context;
@@ -61,41 +60,15 @@ static const struct hwdec_profile_entry profiles[] = {
     {0}
 };
 
-static void mark_uninitialized(struct lavc_ctx *ctx)
-{
-    struct priv *p = ctx->hwdec_priv;
-
-    p->vdp_device = VDP_INVALID_HANDLE;
-    p->context.decoder = VDP_INVALID_HANDLE;
-}
-
-static int handle_preemption(struct lavc_ctx *ctx)
-{
-    struct priv *p = ctx->hwdec_priv;
-
-    if (!mp_vdpau_status_ok(p->mpvdp))
-        return -1;
-
-    // Mark objects as destroyed if preemption+reinit occured
-    if (p->preemption_counter < p->mpvdp->preemption_counter) {
-        p->preemption_counter = p->mpvdp->preemption_counter;
-        mark_uninitialized(ctx);
-    }
-
-    p->vdp_device = p->mpvdp->vdp_device;
-    p->vdp = &p->mpvdp->vdp;
-
-    return 0;
-}
-
 static int init_decoder(struct lavc_ctx *ctx, int fmt, int w, int h)
 {
     struct priv *p = ctx->hwdec_priv;
     struct vdp_functions *vdp = &p->mpvdp->vdp;
+    VdpDevice vdp_device = p->mpvdp->vdp_device;
     VdpStatus vdp_st;
 
-    if (handle_preemption(ctx) < 0)
-        return -1;
+    if (mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter) < 1)
+        goto fail;
 
     if (p->context.decoder != VDP_INVALID_HANDLE)
         vdp->decoder_destroy(p->context.decoder);
@@ -108,7 +81,7 @@ static int init_decoder(struct lavc_ctx *ctx, int fmt, int w, int h)
 
     VdpBool supported;
     uint32_t maxl, maxm, maxw, maxh;
-    vdp_st = vdp->decoder_query_capabilities(p->vdp_device, pe->hw_profile,
+    vdp_st = vdp->decoder_query_capabilities(vdp_device, pe->hw_profile,
                                              &supported, &maxl, &maxm,
                                              &maxw, &maxh);
     CHECK_VDP_WARNING(p, "Querying VDPAU decoder capabilities");
@@ -124,8 +97,8 @@ static int init_decoder(struct lavc_ctx *ctx, int fmt, int w, int h)
 
     int maxrefs = hwdec_get_max_refs(ctx);
 
-    vdp_st = vdp->decoder_create(p->vdp_device, pe->hw_profile,
-                                 w, h, maxrefs, &p->context.decoder);
+    vdp_st = vdp->decoder_create(vdp_device, pe->hw_profile, w, h, maxrefs,
+                                 &p->context.decoder);
     CHECK_VDP_WARNING(p, "Failed creating VDPAU decoder");
     if (vdp_st != VDP_STATUS_OK)
         goto fail;
@@ -141,7 +114,11 @@ static struct mp_image *allocate_image(struct lavc_ctx *ctx, int fmt,
 {
     struct priv *p = ctx->hwdec_priv;
 
-    handle_preemption(ctx);
+    // Trigger software fallback, but only _after_ recovery. This way,
+    // vo_reconfig will not fail (which it will during preemption on systems
+    // with nvidia binary drivers).
+    if (mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter) == 0)
+        return NULL;
 
     VdpChromaType chroma;
     mp_vdpau_get_format(IMGFMT_VDPAU, &chroma, NULL);
@@ -175,11 +152,9 @@ static int init(struct lavc_ctx *ctx)
 
     p->vdp = &p->mpvdp->vdp;
     p->context.render = p->vdp->decoder_render;
+    p->context.decoder = VDP_INVALID_HANDLE;
 
-    p->preemption_counter = p->mpvdp->preemption_counter;
-    mark_uninitialized(ctx);
-
-    if (handle_preemption(ctx) < 0)
+    if (mp_vdpau_handle_preemption(p->mpvdp, &p->preemption_counter) < 1)
         return -1;
 
     ctx->avctx->hwaccel_context = &p->context;

video/vdpau.c

@@ -127,6 +127,30 @@ bool mp_vdpau_status_ok(struct mp_vdpau_ctx *ctx)
     return handle_preemption(ctx) >= 0;
 }
 
+// Check whether vdpau display preemption happened. The caller provides a
+// preemption counter, which contains the logical timestamp of the last
+// preemption handled by the caller. The counter can be 0 for init.
+// Return values:
+//  -1: the display is currently preempted, and vdpau can't be used
+//   0: a preemption event happened, and the caller must recover
+//      (*counter is updated, and a second call will report status ok)
+//   1: everything is fine, no preemption happened
+int mp_vdpau_handle_preemption(struct mp_vdpau_ctx *ctx, uint64_t *counter)
+{
+    // First time init
+    if (!*counter)
+        *counter = ctx->preemption_counter;
+
+    if (handle_preemption(ctx) < 0)
+        return -1;
+
+    if (*counter < ctx->preemption_counter) {
+        *counter = ctx->preemption_counter;
+        return 0; // signal recovery after preemption
+    }
+    return 1;
+}
+
 static void release_decoder_surface(void *ptr)
 {
     bool *in_use_ptr = ptr;
@@ -205,6 +229,7 @@ struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log,
     *ctx = (struct mp_vdpau_ctx) {
         .log = log,
         .x11 = x11,
+        .preemption_counter = 1,
     };
 
     mark_vdpau_objects_uninitialized(ctx);

video/vdpau.h

@@ -62,6 +62,7 @@ struct mp_vdpau_ctx *mp_vdpau_create_device_x11(struct mp_log *log,
 void mp_vdpau_destroy(struct mp_vdpau_ctx *ctx);
 
 bool mp_vdpau_status_ok(struct mp_vdpau_ctx *ctx);
+int mp_vdpau_handle_preemption(struct mp_vdpau_ctx *ctx, uint64_t *counter);
 
 struct mp_image *mp_vdpau_get_video_surface(struct mp_vdpau_ctx *ctx,
                                             VdpChromaType chroma, int w, int h);