fix free of descriptor sets in cvk_kernel_argument_values

We used to free descriptor sets when cvk_kernel_argument_values is
destroyed. But as cvk_kernel has a ref on it, it is only freed when
clReleaseKernel is called.

To be able to free them earlier, keep a refcount between
retain_resources and release_resources. When the counter reach zero,
free the descriptor sets, and set m_is_enqueued to false to make sure
we reallocate the descriptor set if we need to use them once again.

As descriptor sets can be allocated after being freed, reset
them to VK_NULL_HANDLE.

src/kernel.hpp

@@ -168,23 +168,17 @@ struct cvk_kernel_argument_values {
           m_args(m_entry_point->args()), m_pod_arg(nullptr),
           m_kernel_resources(m_entry_point->num_resource_slots()),
           m_local_args_size(m_entry_point->args().size(), 0),
-          m_args_set(m_args.size(), false), m_descriptor_sets{VK_NULL_HANDLE} {}
+          m_args_set(m_args.size(), false), m_descriptor_sets{VK_NULL_HANDLE},
+          m_descriptor_sets_refcount(0) {}
 
     cvk_kernel_argument_values(const cvk_kernel_argument_values& other)
         : m_entry_point(other.m_entry_point), m_is_enqueued(false),
           m_args(m_entry_point->args()), m_pod_arg(nullptr),
           m_kernel_resources(other.m_kernel_resources),
           m_local_args_size(other.m_local_args_size),
           m_specialization_constants(other.m_specialization_constants),
-          m_args_set(other.m_args_set), m_descriptor_sets{VK_NULL_HANDLE} {}
-
-    ~cvk_kernel_argument_values() {
-        for (auto ds : m_descriptor_sets) {
-            if (ds != VK_NULL_HANDLE) {
-                m_entry_point->free_descriptor_set(ds);
-            }
-        }
-    }
+          m_args_set(other.m_args_set), m_descriptor_sets{VK_NULL_HANDLE},
+          m_descriptor_sets_refcount(0) {}
 
     static std::shared_ptr<cvk_kernel_argument_values>
     create(cvk_entry_point* entry_point) {
@@ -342,6 +336,8 @@ struct cvk_kernel_argument_values {
             if (resource)
                 resource->retain();
         }
+        std::lock_guard<std::mutex> lock(m_lock);
+        m_descriptor_sets_refcount++;
     }
 
     // Release all resources owned resources.
@@ -350,6 +346,16 @@ struct cvk_kernel_argument_values {
             if (resource)
                 resource->release();
         }
+        std::lock_guard<std::mutex> lock(m_lock);
+        if (--m_descriptor_sets_refcount == 0) {
+            m_is_enqueued = false;
+            for (auto ds : m_descriptor_sets) {
+                if (ds != VK_NULL_HANDLE) {
+                    m_entry_point->free_descriptor_set(ds);
+                    ds = VK_NULL_HANDLE;
+                }
+            }
+        }
     }
 
     const std::vector<cvk_mem*> memory_objects() const {
@@ -397,4 +403,5 @@ struct cvk_kernel_argument_values {
     std::unique_ptr<cvk_buffer> m_pod_buffer;
     std::array<VkDescriptorSet, spir_binary::MAX_DESCRIPTOR_SETS>
         m_descriptor_sets;
+    uint32_t m_descriptor_sets_refcount;
 };