From 2942ca802ae401a1c3b6358dffaa5bfcc7cff5f0 Mon Sep 17 00:00:00 2001
From: Lynne <dev@lynne.ee>
Date: Sun, 22 Sep 2024 09:15:25 +0200
Subject: [PATCH] hwcontext_vulkan: forward debug_mode to check_extensions()
 for devices

This allows disabling of certain extensions when debug mode is turned on.
---
 libavutil/hwcontext_vulkan.c | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/libavutil/hwcontext_vulkan.c b/libavutil/hwcontext_vulkan.c
index 6604dffd30..5e56a215e8 100644
--- a/libavutil/hwcontext_vulkan.c
+++ b/libavutil/hwcontext_vulkan.c
@@ -798,11 +798,11 @@ end:
 }
 
 /* Creates a VkInstance */
-static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts)
+static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts,
+                           enum FFVulkanDebugMode *debug_mode)
 {
     int err = 0;
     VkResult ret;
-    enum FFVulkanDebugMode debug_mode;
     VulkanDevicePriv *p = ctx->hwctx;
     AVVulkanDeviceContext *hwctx = &p->p;
     FFVulkanFunctions *vk = &p->vkctx.vkfn;
@@ -839,20 +839,20 @@ static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts)
     }
 
     err = check_layers(ctx, opts, &inst_props.ppEnabledLayerNames,
-                       &inst_props.enabledLayerCount, &debug_mode);
+                       &inst_props.enabledLayerCount, debug_mode);
     if (err)
         goto fail;
 
     /* Check for present/missing extensions */
     err = check_extensions(ctx, 0, opts, &inst_props.ppEnabledExtensionNames,
-                           &inst_props.enabledExtensionCount, debug_mode);
+                           &inst_props.enabledExtensionCount, *debug_mode);
     hwctx->enabled_inst_extensions = inst_props.ppEnabledExtensionNames;
     hwctx->nb_enabled_inst_extensions = inst_props.enabledExtensionCount;
     if (err < 0)
         goto fail;
 
     /* Enable debug features if needed */
-    if (debug_mode == FF_VULKAN_DEBUG_VALIDATE) {
+    if (*debug_mode == FF_VULKAN_DEBUG_VALIDATE) {
         static const VkValidationFeatureEnableEXT feat_list_validate[] = {
             VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
             VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
@@ -861,7 +861,7 @@ static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts)
         validation_features.pEnabledValidationFeatures = feat_list_validate;
         validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_validate);
         inst_props.pNext = &validation_features;
-    } else if (debug_mode == FF_VULKAN_DEBUG_PRINTF) {
+    } else if (*debug_mode == FF_VULKAN_DEBUG_PRINTF) {
         static const VkValidationFeatureEnableEXT feat_list_debug[] = {
             VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
             VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
@@ -870,7 +870,7 @@ static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts)
         validation_features.pEnabledValidationFeatures = feat_list_debug;
         validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_debug);
         inst_props.pNext = &validation_features;
-    } else if (debug_mode == FF_VULKAN_DEBUG_PRACTICES) {
+    } else if (*debug_mode == FF_VULKAN_DEBUG_PRACTICES) {
         static const VkValidationFeatureEnableEXT feat_list_practices[] = {
             VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
             VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT,
@@ -908,9 +908,9 @@ static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts)
     }
 
     /* Setup debugging callback if needed */
-    if ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
-        (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
-        (debug_mode == FF_VULKAN_DEBUG_PRACTICES)) {
+    if ((*debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
+        (*debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
+        (*debug_mode == FF_VULKAN_DEBUG_PRACTICES)) {
         VkDebugUtilsMessengerCreateInfoEXT dbg = {
             .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
             .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT |
@@ -1403,6 +1403,7 @@ static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
     VulkanDevicePriv *p = ctx->hwctx;
     AVVulkanDeviceContext *hwctx = &p->p;
     FFVulkanFunctions *vk = &p->vkctx.vkfn;
+    enum FFVulkanDebugMode debug_mode = FF_VULKAN_DEBUG_NONE;
 
     /*
      * VkPhysicalDeviceVulkan12Features has a timelineSemaphore field, but
@@ -1460,7 +1461,7 @@ static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
     ctx->free = vulkan_device_free;
 
     /* Create an instance if not given one */
-    if ((err = create_instance(ctx, opts)))
+    if ((err = create_instance(ctx, opts, &debug_mode)))
         goto end;
 
     /* Find a device (if not given one) */
@@ -1529,7 +1530,7 @@ static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
 
     /* Find and enable extensions */
     if ((err = check_extensions(ctx, 1, opts, &dev_info.ppEnabledExtensionNames,
-                                &dev_info.enabledExtensionCount, 0))) {
+                                &dev_info.enabledExtensionCount, debug_mode))) {
         for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
             av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
         av_free((void *)dev_info.pQueueCreateInfos);