Skip to content

Commit

Permalink
Prepare command buffers only once for several renderings and exports (#…
Browse files Browse the repository at this point in the history 
…129)

* Cache export image command buffers

* Build graphics pipeline once, render several times

* Add log marker for perf measurement
  • Loading branch information
@hevrard
hevrard authored Nov 23, 2018
1 parent 4eb8193 commit c697042
Show file tree
Hide file tree
Showing 4 changed files with 162 additions and 119 deletions.
2 changes: 1 addition & 1 deletion vulkan-worker/src/android/src/main/cpp/main.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ void ProcessAppCmd (struct android_app *app, int32_t cmd) {
assert(app_data->vertex_file != nullptr);
assert(app_data->fragment_file != nullptr);
assert(app_data->uniform_file != nullptr);
app_data->vulkan_worker->Render(app_data->vertex_file, app_data->fragment_file, app_data->uniform_file, FLAGS_skip_render);
app_data->vulkan_worker->RunTest(app_data->vertex_file, app_data->fragment_file, app_data->uniform_file, FLAGS_skip_render);
ANativeActivity_finish(app->activity);
}
break;
Expand Down
268 changes: 154 additions & 114 deletions vulkan-worker/src/common/vulkan_worker.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,11 @@ VulkanWorker::VulkanWorker(PlatformData *platform_data) {
BindDepthImageMemory();
CreateDepthImageView();
PrepareVertexBufferObject();
PrepareExport();
}

VulkanWorker::~VulkanWorker() {
CleanExport();
CleanVertexBufferObject();
DestroyDepthResources();
DestroySwapchainImageViews();
Expand Down Expand Up @@ -1123,7 +1125,7 @@ void VulkanWorker::PresentToDisplay() {
VKCHECK(vkQueuePresentKHR(queue_, &present_info));
}

void VulkanWorker::UpdateImageLayout(VkImage image, VkImageLayout old_image_layout, VkImageLayout new_image_layout, VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dest_stage_mask) {
void VulkanWorker::UpdateImageLayout(VkCommandBuffer command_buffer, VkImage image, VkImageLayout old_image_layout, VkImageLayout new_image_layout, VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dest_stage_mask) {

VkImageMemoryBarrier image_memory_barrier = {};
// generic
Expand Down Expand Up @@ -1187,103 +1189,102 @@ void VulkanWorker::UpdateImageLayout(VkImage image, VkImageLayout old_image_layo
break;
}

VKLOG(vkCmdPipelineBarrier(command_buffer_, src_stage_mask, dest_stage_mask, 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier));
VKLOG(vkCmdPipelineBarrier(command_buffer, src_stage_mask, dest_stage_mask, 0, 0, nullptr, 0, nullptr, 1, &image_memory_barrier));
}

void VulkanWorker::PrepareExport() {
VkImageCreateInfo export_image_create_info = {};
export_image_create_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
export_image_create_info.pNext = nullptr;
export_image_create_info.flags = 0;
export_image_create_info.imageType = VK_IMAGE_TYPE_2D;
export_image_create_info.format = format_;
export_image_create_info.extent.width = width_;
export_image_create_info.extent.height = height_;
export_image_create_info.extent.depth = 1;
export_image_create_info.mipLevels = 1;
export_image_create_info.arrayLayers = 1;
export_image_create_info.samples = VK_SAMPLE_COUNT_1_BIT;
export_image_create_info.tiling = VK_IMAGE_TILING_LINEAR;
export_image_create_info.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT;
export_image_create_info.queueFamilyIndexCount = 0;
export_image_create_info.pQueueFamilyIndices = nullptr;
export_image_create_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
export_image_create_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
VKCHECK(vkCreateImage(device_, &export_image_create_info, nullptr, &export_image_));

VKLOG(vkGetImageMemoryRequirements(device_, export_image_, &export_image_memory_requirements_));

VkMemoryAllocateInfo export_image_memory_allocate_info = {};
export_image_memory_allocate_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
export_image_memory_allocate_info.pNext = nullptr;
export_image_memory_allocate_info.allocationSize = export_image_memory_requirements_.size;
export_image_memory_allocate_info.memoryTypeIndex = GetMemoryTypeIndex(export_image_memory_requirements_.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
VKCHECK(vkAllocateMemory(device_, &export_image_memory_allocate_info, nullptr, &(export_image_memory_)));

VKCHECK(vkBindImageMemory(device_, export_image_, export_image_memory_, 0));

VKCHECK(vkResetCommandBuffer(command_buffer_, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT));

VkCommandBufferBeginInfo export_command_buffer_begin_info = {};
export_command_buffer_begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
export_command_buffer_begin_info.pNext = nullptr;
export_command_buffer_begin_info.flags = 0;
export_command_buffer_begin_info.pInheritanceInfo = nullptr;
VKCHECK(vkBeginCommandBuffer(command_buffer_, &export_command_buffer_begin_info));

UpdateImageLayout(export_image_, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
UpdateImageLayout(images_[swapchain_image_index_], VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);

VkImageCopy export_image_copy = {};
export_image_copy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
export_image_copy.srcSubresource.mipLevel = 0;
export_image_copy.srcSubresource.baseArrayLayer = 0;
export_image_copy.srcSubresource.layerCount = 1;
export_image_copy.srcOffset.x = 0;
export_image_copy.srcOffset.y = 0;
export_image_copy.srcOffset.z = 0;
export_image_copy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
export_image_copy.dstSubresource.mipLevel = 0;
export_image_copy.dstSubresource.baseArrayLayer = 0;
export_image_copy.dstSubresource.layerCount = 1;
export_image_copy.dstOffset.x = 0;
export_image_copy.dstOffset.y = 0;
export_image_copy.dstOffset.z = 0;
export_image_copy.extent.width = width_;
export_image_copy.extent.height = height_;
export_image_copy.extent.depth = 1;
VKLOG(vkCmdCopyImage(command_buffer_, images_[swapchain_image_index_], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, export_image_, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &export_image_copy));

UpdateImageLayout(export_image_, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT);

VKCHECK(vkEndCommandBuffer(command_buffer_));

VKCHECK(vkResetFences(device_, 1, &fence_));

const VkCommandBuffer command_buffers[1] = {command_buffer_};
VkSubmitInfo submit_info[1] = {};
submit_info[0].sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit_info[0].pNext = nullptr;
submit_info[0].waitSemaphoreCount = 0;
submit_info[0].pWaitSemaphores = nullptr;
submit_info[0].pWaitDstStageMask = nullptr;
submit_info[0].commandBufferCount = 1;
submit_info[0].pCommandBuffers = command_buffers;
submit_info[0].signalSemaphoreCount = 0;
submit_info[0].pSignalSemaphores = nullptr;
{
// Prepare export image
VkImageCreateInfo export_image_create_info = {};
export_image_create_info.sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO;
export_image_create_info.pNext = nullptr;
export_image_create_info.flags = 0;
export_image_create_info.imageType = VK_IMAGE_TYPE_2D;
export_image_create_info.format = format_;
export_image_create_info.extent.width = width_;
export_image_create_info.extent.height = height_;
export_image_create_info.extent.depth = 1;
export_image_create_info.mipLevels = 1;
export_image_create_info.arrayLayers = 1;
export_image_create_info.samples = VK_SAMPLE_COUNT_1_BIT;
export_image_create_info.tiling = VK_IMAGE_TILING_LINEAR;
export_image_create_info.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT;
export_image_create_info.queueFamilyIndexCount = 0;
export_image_create_info.pQueueFamilyIndices = nullptr;
export_image_create_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE;
export_image_create_info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED;
VKCHECK(vkCreateImage(device_, &export_image_create_info, nullptr, &export_image_));

VKLOG(vkGetImageMemoryRequirements(device_, export_image_, &export_image_memory_requirements_));

VkMemoryAllocateInfo export_image_memory_allocate_info = {};
export_image_memory_allocate_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
export_image_memory_allocate_info.pNext = nullptr;
export_image_memory_allocate_info.allocationSize = export_image_memory_requirements_.size;
export_image_memory_allocate_info.memoryTypeIndex = GetMemoryTypeIndex(export_image_memory_requirements_.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
VKCHECK(vkAllocateMemory(device_, &export_image_memory_allocate_info, nullptr, &(export_image_memory_)));

VKCHECK(vkBindImageMemory(device_, export_image_, export_image_memory_, 0));
}

VKCHECK(vkQueueSubmit(queue_, 1, submit_info, fence_));
{
// Prepare export command buffers, one per swapchain image

uint32_t num_swapchain_images = images_.size();
export_command_buffers_.resize(num_swapchain_images);

VkCommandBufferAllocateInfo export_command_buffer_allocate_info = {};
export_command_buffer_allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
export_command_buffer_allocate_info.pNext = nullptr;
export_command_buffer_allocate_info.commandPool = command_pool_;
export_command_buffer_allocate_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
export_command_buffer_allocate_info.commandBufferCount = images_.size();
VKCHECK(vkAllocateCommandBuffers(device_, &export_command_buffer_allocate_info, export_command_buffers_.data()));

for (uint32_t i = 0; i < num_swapchain_images; i++) {
VkCommandBuffer export_command_buffer = export_command_buffers_[i];

VkCommandBufferBeginInfo export_command_buffer_begin_info = {};
export_command_buffer_begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
export_command_buffer_begin_info.pNext = nullptr;
export_command_buffer_begin_info.flags = 0;
export_command_buffer_begin_info.pInheritanceInfo = nullptr;
VKCHECK(vkBeginCommandBuffer(export_command_buffer, &export_command_buffer_begin_info));

UpdateImageLayout(export_command_buffer, export_image_, VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
UpdateImageLayout(export_command_buffer, images_[i], VK_IMAGE_LAYOUT_PRESENT_SRC_KHR, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);

VkImageCopy export_image_copy = {};
export_image_copy.srcSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
export_image_copy.srcSubresource.mipLevel = 0;
export_image_copy.srcSubresource.baseArrayLayer = 0;
export_image_copy.srcSubresource.layerCount = 1;
export_image_copy.srcOffset.x = 0;
export_image_copy.srcOffset.y = 0;
export_image_copy.srcOffset.z = 0;
export_image_copy.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT;
export_image_copy.dstSubresource.mipLevel = 0;
export_image_copy.dstSubresource.baseArrayLayer = 0;
export_image_copy.dstSubresource.layerCount = 1;
export_image_copy.dstOffset.x = 0;
export_image_copy.dstOffset.y = 0;
export_image_copy.dstOffset.z = 0;
export_image_copy.extent.width = width_;
export_image_copy.extent.height = height_;
export_image_copy.extent.depth = 1;
VKLOG(vkCmdCopyImage(export_command_buffer, images_[i], VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, export_image_, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 1, &export_image_copy));

UpdateImageLayout(export_command_buffer, export_image_, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_GENERAL, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT);

VKCHECK(vkEndCommandBuffer(export_command_buffer));
}
}

VkResult result = VK_TIMEOUT;
do {
// Do not use VKCHECK as VK_TIMEOUT is a valid result
result = vkWaitForFences(device_, 1, &fence_, VK_TRUE, fence_timeout_nanoseconds_);
log("vkWaitForFences(): %s", getVkResultString(result));
} while (result == VK_TIMEOUT);
assert(result == VK_SUCCESS);
}

void VulkanWorker::CleanExport() {
VKLOG(vkFreeCommandBuffers(device_, command_pool_, export_command_buffers_.size(), export_command_buffers_.data()));
VKLOG(vkFreeMemory(device_, export_image_memory_, nullptr));
VKLOG(vkDestroyImage(device_, export_image_, nullptr));
}
Expand Down Expand Up @@ -1452,6 +1453,32 @@ void VulkanWorker::LoadUniforms(const char *uniforms_string) {
}

void VulkanWorker::ExportPNG(const char *png_filename) {
log("EXPORTPNG START");

VKCHECK(vkResetFences(device_, 1, &fence_));

const VkCommandBuffer command_buffers[1] = {export_command_buffers_[swapchain_image_index_]};
VkSubmitInfo submit_info[1] = {};
submit_info[0].sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
submit_info[0].pNext = nullptr;
submit_info[0].waitSemaphoreCount = 0;
submit_info[0].pWaitSemaphores = nullptr;
submit_info[0].pWaitDstStageMask = nullptr;
submit_info[0].commandBufferCount = 1;
submit_info[0].pCommandBuffers = command_buffers;
submit_info[0].signalSemaphoreCount = 0;
submit_info[0].pSignalSemaphores = nullptr;

VKCHECK(vkQueueSubmit(queue_, 1, submit_info, fence_));

VkResult result = VK_TIMEOUT;
do {
// Do not use VKCHECK as VK_TIMEOUT is a valid result
result = vkWaitForFences(device_, 1, &fence_, VK_TRUE, fence_timeout_nanoseconds_);
log("vkWaitForFences(): %s", getVkResultString(result));
} while (result == VK_TIMEOUT);
assert(result == VK_SUCCESS);

// Get export image binary blob in whatever format the device exposes
unsigned char *source_image_blob = (unsigned char *)malloc(export_image_memory_requirements_.size);
assert(source_image_blob != nullptr);
Expand Down Expand Up @@ -1519,10 +1546,12 @@ void VulkanWorker::ExportPNG(const char *png_filename) {
free(source_image_blob);
free(rgba_blob);
free(png);

log("EXPORTPNG END");
}

void VulkanWorker::DoRender(std::vector<uint32_t> &vertex_spv, std::vector<uint32_t> &fragment_spv, const char *uniforms_string, const char *png_filename, bool skip_render) {
log("RENDER START");
void VulkanWorker::PrepareTest(std::vector<uint32_t> &vertex_spv, std::vector<uint32_t> &fragment_spv, const char *uniforms_string) {
log("PREPARETEST START");

vertex_shader_spv_ = vertex_spv;
fragment_shader_spv_ = fragment_spv;
Expand All @@ -1544,46 +1573,53 @@ void VulkanWorker::DoRender(std::vector<uint32_t> &vertex_spv, std::vector<uint3
CreateFramebuffers();
CreateGraphicsPipeline();

log("PREPARETEST END");
}

void VulkanWorker::CleanTest() {
DestroyGraphicsPipeline();
DestroyFramebuffers();
DestroyShaderModules();
DestroyRenderPass();

if (uniform_entries_.size() > 0) {
FreeDescriptorSet();
DestroyDescriptorPool();
DestroyDescriptorSetLayout();
}

DestroyPipelineLayout();
DestroyUniformResources();
}

void VulkanWorker::DrawTest(const char *png_filename, bool skip_render) {
log("DRAWTEST START");

if (skip_render) {
log("SKIP_RENDER");
} else {
log("DRAW");
CreateSemaphore();
AcquireNextImage();
PrepareCommandBuffer();
CreateFence();

SubmitCommandBuffer();
PresentToDisplay();
PrepareExport();
ExportPNG(png_filename);
CleanExport();

DestroyFence();
DestroySemaphore();
}

DestroyGraphicsPipeline();
DestroyFramebuffers();
DestroyShaderModules();
DestroyRenderPass();

if (uniform_entries_.size() > 0) {
FreeDescriptorSet();
DestroyDescriptorPool();
DestroyDescriptorSetLayout();
}

DestroyPipelineLayout();
DestroyUniformResources();

log("RENDER END");
log("DRAWTEST END");
}

void VulkanWorker::Render(FILE *vertex_file, FILE *fragment_file, FILE *uniforms_file, bool skip_render) {
void VulkanWorker::RunTest(FILE *vertex_file, FILE *fragment_file, FILE *uniforms_file, bool skip_render) {

// Sanity before
DoRender(sanity_vertex_shader_spv_, sanity_fragment_shader_spv_, sanity_unifoms_string, FLAGS_sanity_before.c_str(), false);
PrepareTest(sanity_vertex_shader_spv_, sanity_fragment_shader_spv_, sanity_unifoms_string);
DrawTest(FLAGS_sanity_before.c_str(), false);
CleanTest();

// Test workload
std::vector<uint32_t> vertex_spv;
Expand All @@ -1592,16 +1628,20 @@ void VulkanWorker::Render(FILE *vertex_file, FILE *fragment_file, FILE *uniforms
LoadSpirvFromFile(fragment_file, fragment_spv);
char *uniforms_string = GetFileContent(uniforms_file);

PrepareTest(vertex_spv, fragment_spv, uniforms_string);

for (int i = 0; i < FLAGS_num_render; i++) {
std::string png_filename = FLAGS_png_template + "_" + std::to_string(i) + ".png";
DoRender(vertex_spv, fragment_spv, uniforms_string, png_filename.c_str(), skip_render);
DrawTest(png_filename.c_str(), skip_render);
}

CleanTest();
free(uniforms_string);

// Sanity after
DoRender(sanity_vertex_shader_spv_, sanity_fragment_shader_spv_, sanity_unifoms_string, FLAGS_sanity_after.c_str(), false);

PrepareTest(sanity_vertex_shader_spv_, sanity_fragment_shader_spv_, sanity_unifoms_string);
DrawTest(FLAGS_sanity_after.c_str(), false);
CleanTest();
}

// DumpWorkerInfo() is static to be callable without creating a full-blown worker.
Expand Down
9 changes: 6 additions & 3 deletions vulkan-worker/src/common/vulkan_worker.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ class VulkanWorker {
VkDevice device_;
VkCommandPool command_pool_;
VkCommandBuffer command_buffer_;
std::vector<VkCommandBuffer> export_command_buffers_;
VkSurfaceKHR surface_;
VkFormat format_;
VkSwapchainKHR swapchain_;
Expand Down Expand Up @@ -159,8 +160,10 @@ class VulkanWorker {
void PrepareExport();
void CleanExport();
void ExportPNG(const char *png_filename);
void UpdateImageLayout(VkImage image, VkImageLayout old_image_layout, VkImageLayout new_image_layout, VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dest_stage_mask);
void DoRender(std::vector<uint32_t> &vertex_spv, std::vector<uint32_t> &fragment_spv, const char *uniforms_string, const char *png_filename, bool skip_render);
void UpdateImageLayout(VkCommandBuffer command_buffer, VkImage image, VkImageLayout old_image_layout, VkImageLayout new_image_layout, VkPipelineStageFlags src_stage_mask, VkPipelineStageFlags dest_stage_mask);
void PrepareTest(std::vector<uint32_t> &vertex_spv, std::vector<uint32_t> &fragment_spv, const char *uniforms_string);
void CleanTest();
void DrawTest(const char *png_filename, bool skip_render);

uint32_t GetMemoryTypeIndex(uint32_t memory_requirements_type_bits, VkMemoryPropertyFlags required_properties);
char *GetFileContent(FILE *file);
Expand All @@ -170,7 +173,7 @@ class VulkanWorker {
public:
VulkanWorker(PlatformData *platform_data);
~VulkanWorker();
void Render(FILE *vertex_file, FILE *fragment_file, FILE *uniforms_file, bool skip_render);
void RunTest(FILE *vertex_file, FILE *fragment_file, FILE *uniforms_file, bool skip_render);
static void DumpWorkerInfo(const char *worker_info_filename);
};

Expand Down
Loading

0 comments on commit c697042

Please sign in to comment.