From c69704203431978a525fcfc9172cb5dd7dbe9f72 Mon Sep 17 00:00:00 2001 From: Hugues Evrard Date: Fri, 23 Nov 2018 14:54:25 +0000 Subject: [PATCH] Prepare command buffers only once for several renderings and exports (#129) * Cache export image command buffers * Build graphics pipeline once, render several times * Add log marker for perf measurement --- .../src/android/src/main/cpp/main.cc | 2 +- vulkan-worker/src/common/vulkan_worker.cc | 268 ++++++++++-------- vulkan-worker/src/common/vulkan_worker.h | 9 +- vulkan-worker/src/linux/main.cc | 2 +- 4 files changed, 162 insertions(+), 119 deletions(-) diff --git a/vulkan-worker/src/android/src/main/cpp/main.cc b/vulkan-worker/src/android/src/main/cpp/main.cc index c004475a9..b1b139f0e 100644 --- a/vulkan-worker/src/android/src/main/cpp/main.cc +++ b/vulkan-worker/src/android/src/main/cpp/main.cc @@ -50,7 +50,7 @@ void ProcessAppCmd (struct android_app *app, int32_t cmd) { assert(app_data->vertex_file != nullptr); assert(app_data->fragment_file != nullptr); assert(app_data->uniform_file != nullptr); - app_data->vulkan_worker->Render(app_data->vertex_file, app_data->fragment_file, app_data->uniform_file, FLAGS_skip_render); + app_data->vulkan_worker->RunTest(app_data->vertex_file, app_data->fragment_file, app_data->uniform_file, FLAGS_skip_render); ANativeActivity_finish(app->activity); } break; diff --git a/vulkan-worker/src/common/vulkan_worker.cc b/vulkan-worker/src/common/vulkan_worker.cc index c8bd36f45..196db8506 100644 --- a/vulkan-worker/src/common/vulkan_worker.cc +++ b/vulkan-worker/src/common/vulkan_worker.cc @@ -90,9 +90,11 @@ VulkanWorker::VulkanWorker(PlatformData *platform_data) { BindDepthImageMemory(); CreateDepthImageView(); PrepareVertexBufferObject(); + PrepareExport(); } VulkanWorker::~VulkanWorker() { + CleanExport(); CleanVertexBufferObject(); DestroyDepthResources(); DestroySwapchainImageViews(); @@ -1123,7 +1125,7 @@ void VulkanWorker::PresentToDisplay() { VKCHECK(vkQueuePresentKHR(queue_, &present_info)); } -void VulkanWorker::UpdateImageLayout(VkImage image, VkImageLayout old_image_layout, VkImageLayout new_image_layout, VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dest_stage_mask) { +void VulkanWorker::UpdateImageLayout(VkCommandBuffer command_buffer, VkImage image, VkImageLayout old_image_layout, VkImageLayout new_image_layout, VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dest_stage_mask) { VkImageMemoryBarrier image_memory_barrier = {}; // generic @@ -1187,103 +1189,102 @@ void VulkanWorker::UpdateImageLayout(VkImage image, VkImageLayout old_image_layo break; } - VKLOG(vkCmdPipelineBarrier(command_buffer_, src_stage_mask, dest_stage_mask, 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier)); + VKLOG(vkCmdPipelineBarrier(command_buffer, src_stage_mask, dest_stage_mask, 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier)); } void VulkanWorker::PrepareExport() { - VkImageCreateInfo export_image_create_info = {}; - export_image_create_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; - export_image_create_info.pNext = nullptr; - export_image_create_info.flags = 0; - export_image_create_info.imageType = VK_IMAGE_TYPE_2D; - export_image_create_info.format = format_; - export_image_create_info.extent.width = width_; - export_image_create_info.extent.height = height_; - export_image_create_info.extent.depth = 1; - export_image_create_info.mipLevels = 1; - export_image_create_info.arrayLayers = 1; - export_image_create_info.samples = VK_SAMPLE_COUNT_1_BIT; - export_image_create_info.tiling = VK_IMAGE_TILING_LINEAR; - export_image_create_info.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT; - export_image_create_info.queueFamilyIndexCount = 0; - export_image_create_info.pQueueFamilyIndices = nullptr; - export_image_create_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; - export_image_create_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; - VKCHECK(vkCreateImage(device_, &export_image_create_info, nullptr, &export_image_)); - - VKLOG(vkGetImageMemoryRequirements(device_, export_image_, &export_image_memory_requirements_)); - - VkMemoryAllocateInfo export_image_memory_allocate_info = {}; - export_image_memory_allocate_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; - export_image_memory_allocate_info.pNext = nullptr; - export_image_memory_allocate_info.allocationSize = export_image_memory_requirements_.size; - export_image_memory_allocate_info.memoryTypeIndex = GetMemoryTypeIndex(export_image_memory_requirements_.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); - VKCHECK(vkAllocateMemory(device_, &export_image_memory_allocate_info, nullptr, &(export_image_memory_))); - - VKCHECK(vkBindImageMemory(device_, export_image_, export_image_memory_, 0)); - - VKCHECK(vkResetCommandBuffer(command_buffer_, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT)); - - VkCommandBufferBeginInfo export_command_buffer_begin_info = {}; - export_command_buffer_begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; - export_command_buffer_begin_info.pNext = nullptr; - export_command_buffer_begin_info.flags = 0; - export_command_buffer_begin_info.pInheritanceInfo = nullptr; - VKCHECK(vkBeginCommandBuffer(command_buffer_, &export_command_buffer_begin_info)); - - UpdateImageLayout(export_image_, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - UpdateImageLayout(images_[swapchain_image_index_], VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); - - VkImageCopy export_image_copy = {}; - export_image_copy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - export_image_copy.srcSubresource.mipLevel = 0; - export_image_copy.srcSubresource.baseArrayLayer = 0; - export_image_copy.srcSubresource.layerCount = 1; - export_image_copy.srcOffset.x = 0; - export_image_copy.srcOffset.y = 0; - export_image_copy.srcOffset.z = 0; - export_image_copy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; - export_image_copy.dstSubresource.mipLevel = 0; - export_image_copy.dstSubresource.baseArrayLayer = 0; - export_image_copy.dstSubresource.layerCount = 1; - export_image_copy.dstOffset.x = 0; - export_image_copy.dstOffset.y = 0; - export_image_copy.dstOffset.z = 0; - export_image_copy.extent.width = width_; - export_image_copy.extent.height = height_; - export_image_copy.extent.depth = 1; - VKLOG(vkCmdCopyImage(command_buffer_, images_[swapchain_image_index_], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, export_image_, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &export_image_copy)); - - UpdateImageLayout(export_image_, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT); - VKCHECK(vkEndCommandBuffer(command_buffer_)); - - VKCHECK(vkResetFences(device_, 1, &fence_)); - - const VkCommandBuffer command_buffers[1] = {command_buffer_}; - VkSubmitInfo submit_info[1] = {}; - submit_info[0].sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - submit_info[0].pNext = nullptr; - submit_info[0].waitSemaphoreCount = 0; - submit_info[0].pWaitSemaphores = nullptr; - submit_info[0].pWaitDstStageMask = nullptr; - submit_info[0].commandBufferCount = 1; - submit_info[0].pCommandBuffers = command_buffers; - submit_info[0].signalSemaphoreCount = 0; - submit_info[0].pSignalSemaphores = nullptr; + { + // Prepare export image + VkImageCreateInfo export_image_create_info = {}; + export_image_create_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO; + export_image_create_info.pNext = nullptr; + export_image_create_info.flags = 0; + export_image_create_info.imageType = VK_IMAGE_TYPE_2D; + export_image_create_info.format = format_; + export_image_create_info.extent.width = width_; + export_image_create_info.extent.height = height_; + export_image_create_info.extent.depth = 1; + export_image_create_info.mipLevels = 1; + export_image_create_info.arrayLayers = 1; + export_image_create_info.samples = VK_SAMPLE_COUNT_1_BIT; + export_image_create_info.tiling = VK_IMAGE_TILING_LINEAR; + export_image_create_info.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT; + export_image_create_info.queueFamilyIndexCount = 0; + export_image_create_info.pQueueFamilyIndices = nullptr; + export_image_create_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + export_image_create_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + VKCHECK(vkCreateImage(device_, &export_image_create_info, nullptr, &export_image_)); + + VKLOG(vkGetImageMemoryRequirements(device_, export_image_, &export_image_memory_requirements_)); + + VkMemoryAllocateInfo export_image_memory_allocate_info = {}; + export_image_memory_allocate_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + export_image_memory_allocate_info.pNext = nullptr; + export_image_memory_allocate_info.allocationSize = export_image_memory_requirements_.size; + export_image_memory_allocate_info.memoryTypeIndex = GetMemoryTypeIndex(export_image_memory_requirements_.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + VKCHECK(vkAllocateMemory(device_, &export_image_memory_allocate_info, nullptr, &(export_image_memory_))); + + VKCHECK(vkBindImageMemory(device_, export_image_, export_image_memory_, 0)); + } - VKCHECK(vkQueueSubmit(queue_, 1, submit_info, fence_)); + { + // Prepare export command buffers, one per swapchain image + + uint32_t num_swapchain_images = images_.size(); + export_command_buffers_.resize(num_swapchain_images); + + VkCommandBufferAllocateInfo export_command_buffer_allocate_info = {}; + export_command_buffer_allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + export_command_buffer_allocate_info.pNext = nullptr; + export_command_buffer_allocate_info.commandPool = command_pool_; + export_command_buffer_allocate_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + export_command_buffer_allocate_info.commandBufferCount = images_.size(); + VKCHECK(vkAllocateCommandBuffers(device_, &export_command_buffer_allocate_info, export_command_buffers_.data())); + + for (uint32_t i = 0; i < num_swapchain_images; i++) { + VkCommandBuffer export_command_buffer = export_command_buffers_[i]; + + VkCommandBufferBeginInfo export_command_buffer_begin_info = {}; + export_command_buffer_begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + export_command_buffer_begin_info.pNext = nullptr; + export_command_buffer_begin_info.flags = 0; + export_command_buffer_begin_info.pInheritanceInfo = nullptr; + VKCHECK(vkBeginCommandBuffer(export_command_buffer, &export_command_buffer_begin_info)); + + UpdateImageLayout(export_command_buffer, export_image_, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + UpdateImageLayout(export_command_buffer, images_[i], VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT); + + VkImageCopy export_image_copy = {}; + export_image_copy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + export_image_copy.srcSubresource.mipLevel = 0; + export_image_copy.srcSubresource.baseArrayLayer = 0; + export_image_copy.srcSubresource.layerCount = 1; + export_image_copy.srcOffset.x = 0; + export_image_copy.srcOffset.y = 0; + export_image_copy.srcOffset.z = 0; + export_image_copy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + export_image_copy.dstSubresource.mipLevel = 0; + export_image_copy.dstSubresource.baseArrayLayer = 0; + export_image_copy.dstSubresource.layerCount = 1; + export_image_copy.dstOffset.x = 0; + export_image_copy.dstOffset.y = 0; + export_image_copy.dstOffset.z = 0; + export_image_copy.extent.width = width_; + export_image_copy.extent.height = height_; + export_image_copy.extent.depth = 1; + VKLOG(vkCmdCopyImage(export_command_buffer, images_[i], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, export_image_, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &export_image_copy)); + + UpdateImageLayout(export_command_buffer, export_image_, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT); + + VKCHECK(vkEndCommandBuffer(export_command_buffer)); + } + } - VkResult result = VK_TIMEOUT; - do { - // Do not use VKCHECK as VK_TIMEOUT is a valid result - result = vkWaitForFences(device_, 1, &fence_, VK_TRUE, fence_timeout_nanoseconds_); - log("vkWaitForFences(): %s", getVkResultString(result)); - } while (result == VK_TIMEOUT); - assert(result == VK_SUCCESS); } void VulkanWorker::CleanExport() { + VKLOG(vkFreeCommandBuffers(device_, command_pool_, export_command_buffers_.size(), export_command_buffers_.data())); VKLOG(vkFreeMemory(device_, export_image_memory_, nullptr)); VKLOG(vkDestroyImage(device_, export_image_, nullptr)); } @@ -1452,6 +1453,32 @@ void VulkanWorker::LoadUniforms(const char *uniforms_string) { } void VulkanWorker::ExportPNG(const char *png_filename) { + log("EXPORTPNG START"); + + VKCHECK(vkResetFences(device_, 1, &fence_)); + + const VkCommandBuffer command_buffers[1] = {export_command_buffers_[swapchain_image_index_]}; + VkSubmitInfo submit_info[1] = {}; + submit_info[0].sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submit_info[0].pNext = nullptr; + submit_info[0].waitSemaphoreCount = 0; + submit_info[0].pWaitSemaphores = nullptr; + submit_info[0].pWaitDstStageMask = nullptr; + submit_info[0].commandBufferCount = 1; + submit_info[0].pCommandBuffers = command_buffers; + submit_info[0].signalSemaphoreCount = 0; + submit_info[0].pSignalSemaphores = nullptr; + + VKCHECK(vkQueueSubmit(queue_, 1, submit_info, fence_)); + + VkResult result = VK_TIMEOUT; + do { + // Do not use VKCHECK as VK_TIMEOUT is a valid result + result = vkWaitForFences(device_, 1, &fence_, VK_TRUE, fence_timeout_nanoseconds_); + log("vkWaitForFences(): %s", getVkResultString(result)); + } while (result == VK_TIMEOUT); + assert(result == VK_SUCCESS); + // Get export image binary blob in whatever format the device exposes unsigned char *source_image_blob = (unsigned char *)malloc(export_image_memory_requirements_.size); assert(source_image_blob != nullptr); @@ -1519,10 +1546,12 @@ void VulkanWorker::ExportPNG(const char *png_filename) { free(source_image_blob); free(rgba_blob); free(png); + + log("EXPORTPNG END"); } -void VulkanWorker::DoRender(std::vector &vertex_spv, std::vector &fragment_spv, const char *uniforms_string, const char *png_filename, bool skip_render) { - log("RENDER START"); +void VulkanWorker::PrepareTest(std::vector &vertex_spv, std::vector &fragment_spv, const char *uniforms_string) { + log("PREPARETEST START"); vertex_shader_spv_ = vertex_spv; fragment_shader_spv_ = fragment_spv; @@ -1544,10 +1573,31 @@ void VulkanWorker::DoRender(std::vector &vertex_spv, std::vector 0) { + FreeDescriptorSet(); + DestroyDescriptorPool(); + DestroyDescriptorSetLayout(); + } + + DestroyPipelineLayout(); + DestroyUniformResources(); +} + +void VulkanWorker::DrawTest(const char *png_filename, bool skip_render) { + log("DRAWTEST START"); + if (skip_render) { log("SKIP_RENDER"); } else { - log("DRAW"); CreateSemaphore(); AcquireNextImage(); PrepareCommandBuffer(); @@ -1555,35 +1605,21 @@ void VulkanWorker::DoRender(std::vector &vertex_spv, std::vector 0) { - FreeDescriptorSet(); - DestroyDescriptorPool(); - DestroyDescriptorSetLayout(); - } - - DestroyPipelineLayout(); - DestroyUniformResources(); - - log("RENDER END"); + log("DRAWTEST END"); } -void VulkanWorker::Render(FILE *vertex_file, FILE *fragment_file, FILE *uniforms_file, bool skip_render) { +void VulkanWorker::RunTest(FILE *vertex_file, FILE *fragment_file, FILE *uniforms_file, bool skip_render) { // Sanity before - DoRender(sanity_vertex_shader_spv_, sanity_fragment_shader_spv_, sanity_unifoms_string, FLAGS_sanity_before.c_str(), false); + PrepareTest(sanity_vertex_shader_spv_, sanity_fragment_shader_spv_, sanity_unifoms_string); + DrawTest(FLAGS_sanity_before.c_str(), false); + CleanTest(); // Test workload std::vector vertex_spv; @@ -1592,16 +1628,20 @@ void VulkanWorker::Render(FILE *vertex_file, FILE *fragment_file, FILE *uniforms LoadSpirvFromFile(fragment_file, fragment_spv); char *uniforms_string = GetFileContent(uniforms_file); + PrepareTest(vertex_spv, fragment_spv, uniforms_string); + for (int i = 0; i < FLAGS_num_render; i++) { std::string png_filename = FLAGS_png_template + "_" + std::to_string(i) + ".png"; - DoRender(vertex_spv, fragment_spv, uniforms_string, png_filename.c_str(), skip_render); + DrawTest(png_filename.c_str(), skip_render); } + CleanTest(); free(uniforms_string); // Sanity after - DoRender(sanity_vertex_shader_spv_, sanity_fragment_shader_spv_, sanity_unifoms_string, FLAGS_sanity_after.c_str(), false); - + PrepareTest(sanity_vertex_shader_spv_, sanity_fragment_shader_spv_, sanity_unifoms_string); + DrawTest(FLAGS_sanity_after.c_str(), false); + CleanTest(); } // DumpWorkerInfo() is static to be callable without creating a full-blown worker. diff --git a/vulkan-worker/src/common/vulkan_worker.h b/vulkan-worker/src/common/vulkan_worker.h index d7e255c04..3a1e1b51b 100644 --- a/vulkan-worker/src/common/vulkan_worker.h +++ b/vulkan-worker/src/common/vulkan_worker.h @@ -68,6 +68,7 @@ class VulkanWorker { VkDevice device_; VkCommandPool command_pool_; VkCommandBuffer command_buffer_; + std::vector export_command_buffers_; VkSurfaceKHR surface_; VkFormat format_; VkSwapchainKHR swapchain_; @@ -159,8 +160,10 @@ class VulkanWorker { void PrepareExport(); void CleanExport(); void ExportPNG(const char *png_filename); - void UpdateImageLayout(VkImage image, VkImageLayout old_image_layout, VkImageLayout new_image_layout, VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dest_stage_mask); - void DoRender(std::vector &vertex_spv, std::vector &fragment_spv, const char *uniforms_string, const char *png_filename, bool skip_render); + void UpdateImageLayout(VkCommandBuffer command_buffer, VkImage image, VkImageLayout old_image_layout, VkImageLayout new_image_layout, VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dest_stage_mask); + void PrepareTest(std::vector &vertex_spv, std::vector &fragment_spv, const char *uniforms_string); + void CleanTest(); + void DrawTest(const char *png_filename, bool skip_render); uint32_t GetMemoryTypeIndex(uint32_t memory_requirements_type_bits, VkMemoryPropertyFlags required_properties); char *GetFileContent(FILE *file); @@ -170,7 +173,7 @@ class VulkanWorker { public: VulkanWorker(PlatformData *platform_data); ~VulkanWorker(); - void Render(FILE *vertex_file, FILE *fragment_file, FILE *uniforms_file, bool skip_render); + void RunTest(FILE *vertex_file, FILE *fragment_file, FILE *uniforms_file, bool skip_render); static void DumpWorkerInfo(const char *worker_info_filename); }; diff --git a/vulkan-worker/src/linux/main.cc b/vulkan-worker/src/linux/main.cc index 2bebc4bce..a7a4adc2e 100644 --- a/vulkan-worker/src/linux/main.cc +++ b/vulkan-worker/src/linux/main.cc @@ -65,7 +65,7 @@ int main(int argc, char **argv) { platform_data.window = glfwCreateWindow(WIDTH, HEIGHT, "VulkanWorker", nullptr, nullptr); VulkanWorker* vulkan_worker = new VulkanWorker(&platform_data); - vulkan_worker->Render(vertex_file, fragment_file, uniform_file, FLAGS_skip_render); + vulkan_worker->RunTest(vertex_file, fragment_file, uniform_file, FLAGS_skip_render); delete vulkan_worker; fclose(vertex_file);