vf_vdpaurb: Add a new filter for reading back vdpau decoded frames

Normally, vdpau decoded frames are passed directly to a suitable
vo (vo_vdpau or vo_opengl) without ever touching system memory. This
is efficient for output purposes, but prevents any of the regular
filters from being used with such frames.

This new filter implements a read-back step to pull the frames back
into system memory where they can be acted on by other filters.
Eventually the frames will be sent to the vo as if they were normal
software-decoded frames.

Note that a vdpau compatible vo must still be used to ensure that
the decoder is properly initialised.

Signed-off-by: wm4 <wm4@nowhere>

DOCS/man/vf.rst

@@ -878,6 +878,13 @@ Available filters are:
         1-9
             Apply high quality VDPAU scaling (needs capable hardware).
 
+``vdpaurb``
+    VDPAU video read back. Works with ``--vo=vdpau`` and ``--vo=opengl`` only.
+    This filter will read back frames decoded by VDPAU so that other filters,
+    which are not normally compatible with VDPAU, can be used like normal.
+    This filter must be specified before ``vdpaupp`` in the filter chain if
+    ``vdpaupp`` is used.
+
 ``buffer=<num>``
     Buffer ``<num>`` frames in the filter chain. This filter is probably pretty
     useless, except for debugging. (Note that this won't help smoothing out

TOOLS/old-makefile

@@ -80,7 +80,8 @@ SOURCES-$(RSOUND)               += audio/out/ao_rsound.c
 SOURCES-$(SNDIO)                += audio/out/ao_sndio.c
 SOURCES-$(VDPAU)                += video/vdpau.c video/vdpau_mixer.c \
                                    video/out/vo_vdpau.c video/decode/vdpau.c \
-                                   video/filter/vf_vdpaupp.c
+                                   video/filter/vf_vdpaupp.c \
+                                   video/filter/vf_vdpaurb.c
 SOURCES-$(VDPAU_GL_X11)         += video/out/gl_hwdec_vdpau.c
 SOURCES-$(VAAPI)                += video/out/vo_vaapi.c \
                                    video/decode/vaapi.c \

video/filter/vf.c

@@ -63,6 +63,7 @@ extern const vf_info_t vf_info_vaapi;
 extern const vf_info_t vf_info_vapoursynth;
 extern const vf_info_t vf_info_vapoursynth_lazy;
 extern const vf_info_t vf_info_vdpaupp;
+extern const vf_info_t vf_info_vdpaurb;
 extern const vf_info_t vf_info_buffer;
 
 // list of available filters:
@@ -106,6 +107,7 @@ static const vf_info_t *const filter_list[] = {
 #endif
 #if HAVE_VDPAU
     &vf_info_vdpaupp,
+    &vf_info_vdpaurb,
 #endif
     NULL
 };

video/filter/vf_vdpaurb.c

@@ -0,0 +1,112 @@
+/*
+ * This file is part of mpv.
+ *
+ * mpv is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * mpv is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License along
+ * with mpv.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <assert.h>
+
+#include "video/vdpau.h"
+#include "video/vdpau_mixer.h"
+#include "vf.h"
+
+// This filter will read back decoded frames that have been decoded by vdpau
+// so they can be post-processed by regular filters. As vdpau is still doing
+// the decoding, a vdpau compatible vo must always be used.
+//
+// NB: This filter assumes the video surface will have a 420 chroma type and
+// can always be read back in NV12 format. This is a safe assumption at the
+// time of writing, but may not always remain true.
+
+struct vf_priv_s {
+    struct mp_vdpau_ctx *ctx;
+};
+
+static int filter_ext(struct vf_instance *vf, struct mp_image *mpi)
+{
+    VdpStatus vdp_st;
+    struct vf_priv_s *p = vf->priv;
+    struct mp_vdpau_ctx *ctx = p->ctx;
+    struct vdp_functions *vdp = &ctx->vdp;
+
+    if (mp_vdpau_mixed_frame_get(mpi)) {
+        MP_ERR(vf, "Can't apply vdpaurb filter after vdpaupp filter.\n");
+        mp_image_unrefp(&mpi);
+        return -1;
+    }
+
+    struct mp_image *out = vf_alloc_out_image(vf);
+    if (!out) {
+        mp_image_unrefp(&mpi);
+        return -1;
+    }
+    mp_image_copy_attributes(out, mpi);
+
+    VdpVideoSurface surface = (uintptr_t)mpi->planes[3];
+    assert(surface > 0);
+
+    vdp_st = vdp->video_surface_get_bits_y_cb_cr(surface,
+                                                 VDP_YCBCR_FORMAT_NV12,
+                                                 (void * const *)out->planes,
+                                                 out->stride);
+    CHECK_VDP_WARNING(vf, "Error when calling vdp_output_surface_get_bits_y_cb_cr");
+
+    vf_add_output_frame(vf, out);
+    mp_image_unrefp(&mpi);
+    return 0;
+}
+
+static int reconfig(struct vf_instance *vf, struct mp_image_params *in,
+                    struct mp_image_params *out)
+{
+    *out = *in;
+    out->imgfmt = IMGFMT_NV12;
+    return 0;
+}
+
+static int query_format(struct vf_instance *vf, unsigned int fmt)
+{
+    if (fmt == IMGFMT_VDPAU) {
+        return vf_next_query_format(vf, IMGFMT_NV12);
+    }
+    return 0;
+}
+
+static int vf_open(vf_instance_t *vf)
+{
+    struct vf_priv_s *p = vf->priv;
+
+    vf->filter_ext = filter_ext;
+    vf->filter = NULL;
+    vf->reconfig = reconfig;
+    vf->query_format = query_format;
+
+    if (!vf->hwdec) {
+        return 0;
+    }
+    hwdec_request_api(vf->hwdec, "vdpau");
+    p->ctx = vf->hwdec->hwctx ? vf->hwdec->hwctx->vdpau_ctx : NULL;
+    if (!p->ctx) {
+        return 0;
+    }
+
+    return 1;
+}
+
+const vf_info_t vf_info_vdpaurb = {
+    .description = "vdpau readback",
+    .name = "vdpaurb",
+    .open = vf_open,
+    .priv_size = sizeof(struct vf_priv_s),
+};

wscript_build.py

@@ -321,6 +321,7 @@ def build(ctx):
         ( "video/filter/vf_vapoursynth.c",       "vapoursynth-core" ),
         ( "video/filter/vf_vavpp.c",             "vaapi-vpp"),
         ( "video/filter/vf_vdpaupp.c",           "vdpau" ),
+        ( "video/filter/vf_vdpaurb.c",           "vdpau" ),
         ( "video/filter/vf_yadif.c",             "libavfilter"),
         ( "video/out/aspect.c" ),
         ( "video/out/bitmap_packer.c" ),