From 872b068cb75d6d3b173b3d60bf0b1cd375a6d20d Mon Sep 17 00:00:00 2001
From: Philip Langdale <philipl@overt.org>
Date: Mon, 27 Feb 2023 21:11:39 -0800
Subject: [PATCH] f_hwtransfer: disable vulkan multiplane images when uploading
 from cuda

Although we can support vulkan multiplane images, cuda lacks any such
support, and so cannot natively import such images for interop. It's
possible that we can do separate exports for each plane in the image
and have it work, but for now, we can selectively disable multiplane
when we know that we'll be consuming cuda frames.

As a reminder, even though cuda is the frame source, interop is one way
so the vulkan images have to be imported to cuda before we copy the
frame contents over.

This logic here is slightly more complex than I'd like but you can't
just set the flag blindly, as it will cause hwframes ctx creation to
fail if the format is packed or if it's planar rgb. Oh well.
---
 filters/f_hwtransfer.c  |  5 +++--
 video/filter/vf_vavpp.c |  3 ++-
 video/mp_image_pool.c   | 21 ++++++++++++++++++++-
 video/mp_image_pool.h   |  3 ++-
 4 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/filters/f_hwtransfer.c b/filters/f_hwtransfer.c
index 9488c5536c..b281a15838 100644
--- a/filters/f_hwtransfer.c
+++ b/filters/f_hwtransfer.c
@@ -198,7 +198,8 @@ static void process(struct mp_filter *f)
     }
 
     if (!mp_update_av_hw_frames_pool(&p->hw_pool, p->av_device_ctx, p->hw_imgfmt,
-                                     p->last_hw_output_fmt, src->w, src->h))
+                                     p->last_hw_output_fmt, src->w, src->h,
+                                     src->imgfmt == IMGFMT_CUDA))
     {
         MP_ERR(f, "failed to create frame pool\n");
         goto error;
@@ -353,7 +354,7 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
         // Creates an AVHWFramesContexts with the given parameters.
         AVBufferRef *frames = NULL;
         if (!mp_update_av_hw_frames_pool(&frames, ctx->av_device_ref,
-                                         hw_imgfmt, imgfmt, 128, 128))
+                                         hw_imgfmt, imgfmt, 128, 128, false))
         {
             MP_WARN(u->f, "failed to allocate pool\n");
             continue;
diff --git a/video/filter/vf_vavpp.c b/video/filter/vf_vavpp.c
index 5b30c46163..52be148111 100644
--- a/video/filter/vf_vavpp.c
+++ b/video/filter/vf_vavpp.c
@@ -164,7 +164,8 @@ static struct mp_image *alloc_out(struct mp_filter *vf)
     int src_h = hw_frames->height;
 
     if (!mp_update_av_hw_frames_pool(&p->hw_pool, p->av_device_ref,
-                                     IMGFMT_VAAPI, IMGFMT_NV12, src_w, src_h))
+                                     IMGFMT_VAAPI, IMGFMT_NV12, src_w, src_h,
+                                     false))
     {
         MP_ERR(vf, "Failed to create hw pool.\n");
         return NULL;
diff --git a/video/mp_image_pool.c b/video/mp_image_pool.c
index 2f0259146a..8f982b414c 100644
--- a/video/mp_image_pool.c
+++ b/video/mp_image_pool.c
@@ -15,6 +15,8 @@
  * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "config.h"
+
 #include <stddef.h>
 #include <stdbool.h>
 #include <pthread.h>
@@ -22,7 +24,11 @@
 
 #include <libavutil/buffer.h>
 #include <libavutil/hwcontext.h>
+#if HAVE_VULKAN_INTEROP
+#include <libavutil/hwcontext_vulkan.h>
+#endif
 #include <libavutil/mem.h>
+#include <libavutil/pixdesc.h>
 
 #include "mpv_talloc.h"
 
@@ -354,7 +360,8 @@ done:
 
 bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
                                  struct AVBufferRef *hw_device_ctx,
-                                 int imgfmt, int sw_imgfmt, int w, int h)
+                                 int imgfmt, int sw_imgfmt, int w, int h,
+                                 bool disable_multiplane)
 {
     enum AVPixelFormat format = imgfmt2pixfmt(imgfmt);
     enum AVPixelFormat sw_format = imgfmt2pixfmt(sw_imgfmt);
@@ -385,6 +392,18 @@ bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
         hw_frames->sw_format = sw_format;
         hw_frames->width = w;
         hw_frames->height = h;
+
+#if HAVE_VULKAN_INTEROP
+        if (format == AV_PIX_FMT_VULKAN && disable_multiplane) {
+            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sw_format);
+            if ((desc->flags & AV_PIX_FMT_FLAG_PLANAR) &&
+                !(desc->flags & AV_PIX_FMT_FLAG_RGB)) {
+                AVVulkanFramesContext *vk_frames = hw_frames->hwctx;
+                vk_frames->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE;
+            }
+        }
+#endif
+
         if (av_hwframe_ctx_init(*hw_frames_ctx) < 0) {
             av_buffer_unref(hw_frames_ctx);
             return false;
diff --git a/video/mp_image_pool.h b/video/mp_image_pool.h
index ab9065e6e9..8cb2a5f995 100644
--- a/video/mp_image_pool.h
+++ b/video/mp_image_pool.h
@@ -36,7 +36,8 @@ bool mp_image_hw_upload(struct mp_image *hw_img, struct mp_image *src);
 struct AVBufferRef;
 bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
                                  struct AVBufferRef *hw_device_ctx,
-                                 int imgfmt, int sw_imgfmt, int w, int h);
+                                 int imgfmt, int sw_imgfmt, int w, int h,
+                                 bool disable_multiplane);
 
 struct mp_image *mp_av_pool_image_hw_upload(struct AVBufferRef *hw_frames_ctx,
                                             struct mp_image *src);