f_hwtransfer: disable vulkan multiplane images when uploading from cuda

Although we can support vulkan multiplane images, cuda lacks any such
support, and so cannot natively import such images for interop. It's
possible that we can do separate exports for each plane in the image
and have it work, but for now, we can selectively disable multiplane
when we know that we'll be consuming cuda frames.

As a reminder, even though cuda is the frame source, interop is one way
so the vulkan images have to be imported to cuda before we copy the
frame contents over.

This logic here is slightly more complex than I'd like but you can't
just set the flag blindly, as it will cause hwframes ctx creation to
fail if the format is packed or if it's planar rgb. Oh well.

filters/f_hwtransfer.c

@@ -198,7 +198,8 @@ static void process(struct mp_filter *f)
     }
 
     if (!mp_update_av_hw_frames_pool(&p->hw_pool, p->av_device_ctx, p->hw_imgfmt,
-                                     p->last_hw_output_fmt, src->w, src->h))
+                                     p->last_hw_output_fmt, src->w, src->h,
+                                     src->imgfmt == IMGFMT_CUDA))
     {
         MP_ERR(f, "failed to create frame pool\n");
         goto error;
@@ -353,7 +354,7 @@ static bool probe_formats(struct mp_hwupload *u, int hw_imgfmt)
         // Creates an AVHWFramesContexts with the given parameters.
         AVBufferRef *frames = NULL;
         if (!mp_update_av_hw_frames_pool(&frames, ctx->av_device_ref,
-                                         hw_imgfmt, imgfmt, 128, 128))
+                                         hw_imgfmt, imgfmt, 128, 128, false))
         {
             MP_WARN(u->f, "failed to allocate pool\n");
             continue;

video/filter/vf_vavpp.c

@@ -164,7 +164,8 @@ static struct mp_image *alloc_out(struct mp_filter *vf)
     int src_h = hw_frames->height;
 
     if (!mp_update_av_hw_frames_pool(&p->hw_pool, p->av_device_ref,
-                                     IMGFMT_VAAPI, IMGFMT_NV12, src_w, src_h))
+                                     IMGFMT_VAAPI, IMGFMT_NV12, src_w, src_h,
+                                     false))
     {
         MP_ERR(vf, "Failed to create hw pool.\n");
         return NULL;

video/mp_image_pool.c

@@ -15,6 +15,8 @@
  * License along with mpv.  If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "config.h"
+
 #include <stddef.h>
 #include <stdbool.h>
 #include <pthread.h>
@@ -22,7 +24,11 @@
 
 #include <libavutil/buffer.h>
 #include <libavutil/hwcontext.h>
+#if HAVE_VULKAN_INTEROP
+#include <libavutil/hwcontext_vulkan.h>
+#endif
 #include <libavutil/mem.h>
+#include <libavutil/pixdesc.h>
 
 #include "mpv_talloc.h"
 
@@ -354,7 +360,8 @@ done:
 
 bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
                                  struct AVBufferRef *hw_device_ctx,
-                                 int imgfmt, int sw_imgfmt, int w, int h)
+                                 int imgfmt, int sw_imgfmt, int w, int h,
+                                 bool disable_multiplane)
 {
     enum AVPixelFormat format = imgfmt2pixfmt(imgfmt);
     enum AVPixelFormat sw_format = imgfmt2pixfmt(sw_imgfmt);
@@ -385,6 +392,18 @@ bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
         hw_frames->sw_format = sw_format;
         hw_frames->width = w;
         hw_frames->height = h;
+
+#if HAVE_VULKAN_INTEROP
+        if (format == AV_PIX_FMT_VULKAN && disable_multiplane) {
+            const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(sw_format);
+            if ((desc->flags & AV_PIX_FMT_FLAG_PLANAR) &&
+                !(desc->flags & AV_PIX_FMT_FLAG_RGB)) {
+                AVVulkanFramesContext *vk_frames = hw_frames->hwctx;
+                vk_frames->flags = AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE;
+            }
+        }
+#endif
+
         if (av_hwframe_ctx_init(*hw_frames_ctx) < 0) {
             av_buffer_unref(hw_frames_ctx);
             return false;

video/mp_image_pool.h

@@ -36,7 +36,8 @@ bool mp_image_hw_upload(struct mp_image *hw_img, struct mp_image *src);
 struct AVBufferRef;
 bool mp_update_av_hw_frames_pool(struct AVBufferRef **hw_frames_ctx,
                                  struct AVBufferRef *hw_device_ctx,
-                                 int imgfmt, int sw_imgfmt, int w, int h);
+                                 int imgfmt, int sw_imgfmt, int w, int h,
+                                 bool disable_multiplane);
 
 struct mp_image *mp_av_pool_image_hw_upload(struct AVBufferRef *hw_frames_ctx,
                                             struct mp_image *src);