From f66ac83c22d8c088832ce8df922b08a0caa11833 Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Thu, 23 Apr 2020 18:08:14 +0100
Subject: [PATCH] overlay_vulkan: add support for overlaying images with an
 alpha channel

---
 libavfilter/vf_overlay_vulkan.c | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/libavfilter/vf_overlay_vulkan.c b/libavfilter/vf_overlay_vulkan.c
index 7cedcc6e88..83cfae40e2 100644
--- a/libavfilter/vf_overlay_vulkan.c
+++ b/libavfilter/vf_overlay_vulkan.c
@@ -59,11 +59,27 @@ static const char overlay_noalpha[] = {
     C(0, }                                                                     )
 };
 
+static const char overlay_alpha[] = {
+    C(0, void overlay_alpha_opaque(int i, ivec2 pos)                           )
+    C(0, {                                                                     )
+    C(1,     vec4 res = texture(main_img[i], pos);                             )
+    C(1,     if ((o_offset[i].x <= pos.x) && (o_offset[i].y <= pos.y) &&
+                 (pos.x < (o_offset[i].x + o_size[i].x)) &&
+                 (pos.y < (o_offset[i].y + o_size[i].y))) {                    )
+    C(2,         vec4 ovr = texture(overlay_img[i], pos - o_offset[i]);        )
+    C(2,         res = ovr * ovr.a + res * (1.0f - ovr.a);                     )
+    C(2,         res.a = 1.0f;                                                 )
+    C(2,         imageStore(output_img[i], pos, res);                          )
+    C(1,     }                                                                 )
+    C(1,     imageStore(output_img[i], pos, res);                              )
+    C(0, }                                                                     )
+};
+
 static av_cold int init_filter(AVFilterContext *ctx)
 {
     int err;
     OverlayVulkanContext *s = ctx->priv;
-    VkSampler *sampler = ff_vk_init_sampler(ctx, 1, VK_FILTER_LINEAR);
+    VkSampler *sampler = ff_vk_init_sampler(ctx, 1, VK_FILTER_NEAREST);
     if (!sampler)
         return AVERROR_EXTERNAL;
 
@@ -73,6 +89,7 @@ static av_cold int init_filter(AVFilterContext *ctx)
 
     { /* Create the shader */
         const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
+        const int ialpha = av_pix_fmt_desc_get(s->vkctx.input_format)->flags & AV_PIX_FMT_FLAG_ALPHA;
 
         VulkanDescriptorSetBinding desc_i[3] = {
             {
@@ -126,12 +143,16 @@ static av_cold int init_filter(AVFilterContext *ctx)
         RET(ff_vk_add_descriptor_set(ctx, s->pl, shd, &desc_b, 1, 0)); /* set 1 */
 
         GLSLD(   overlay_noalpha                                              );
+        GLSLD(   overlay_alpha                                                );
         GLSLC(0, void main()                                                  );
         GLSLC(0, {                                                            );
         GLSLC(1,     ivec2 pos = ivec2(gl_GlobalInvocationID.xy);             );
         GLSLF(1,     int planes = %i;                                  ,planes);
         GLSLC(1,     for (int i = 0; i < planes; i++) {                       );
-        GLSLC(2,         overlay_noalpha(i, pos);                             );
+        if (ialpha)
+            GLSLC(2,         overlay_alpha_opaque(i, pos);                    );
+        else
+            GLSLC(2,         overlay_noalpha(i, pos);                         );
         GLSLC(1,     }                                                        );
         GLSLC(0, }                                                            );