Shader lazy load

.gitignore

@@ -9,11 +9,19 @@ Engine/ThirdPartyProjects
 ARM64
 
 # tmp
-Projects/Shared
+Projects/Shared/*
 Projects/Test/*
 !Projects/Test/*/
 Projects/Test/Textures/*
 !Projects/Test/Textures/*/
 
+# Precompiled Shader
+!Projects/Shared/BuiltInShaders
+Projects/Shared/BuiltInShaders/*
+!Projects/Shared/BuiltInShaders/Direct3D11
+Projects/Shared/BuiltInShaders/Direct3D11/*
+!Projects/Shared/BuiltInShaders/Direct3D11/vs_imgui.bin
+!Projects/Shared/BuiltInShaders/Direct3D11/fs_imgui.bin
+
 # Commercial SKD
 Engine/Auto/commercial_sdk_locations.bat

Engine/BuiltInShaders/common/Envirnoment.sh

@@ -10,7 +10,7 @@
 
 #if defined(ATM)
 
-#include "../atm/atm_functions.sh"
+#include "atm_functions.sh"
 
 uniform vec4 u_LightDir;
 uniform vec4 u_HeightOffsetAndshadowLength;

Engine/BuiltInShaders/atm/cs_ComputeDirectIrradiance.sc → Engine/BuiltInShaders/shaders/cs_ComputeDirectIrradiance.sc

@@ -1,7 +1,7 @@
 #include "../common/bgfx_compute.sh"
 
 #define COMPUTE
-#include "atm_functions.sh"
+#include "../common/atm_functions.sh"
 
 IMAGE2D_WR(s_delta_irradiance, rgba32f, 0);
 IMAGE2D_WR(s_irradiance, rgba32f, 1);

Engine/BuiltInShaders/atm/cs_ComputeIndirectIrradiance.sc → Engine/BuiltInShaders/shaders/cs_ComputeIndirectIrradiance.sc

@@ -1,7 +1,7 @@
 #include "../common/bgfx_compute.sh"
 
 #define COMPUTE
-#include "atm_functions.sh"
+#include "../common/atm_functions.sh"
 
 uniform vec4 u_numScatteringOrders;
 

Engine/BuiltInShaders/atm/cs_ComputeMultipleScattering.sc → Engine/BuiltInShaders/shaders/cs_ComputeMultipleScattering.sc

@@ -1,7 +1,7 @@
 #include "../common/bgfx_compute.sh"
 
 #define COMPUTE
-#include "atm_functions.sh"
+#include "../common/atm_functions.sh"
 
 IMAGE3D_WR(s_delta_multiple_scattering, rgba32f, 0);
 IMAGE3D_RW(s_scattering, rgba32f, 1);

Engine/BuiltInShaders/atm/cs_ComputeScatteringDensity.sc → Engine/BuiltInShaders/shaders/cs_ComputeScatteringDensity.sc

@@ -1,7 +1,7 @@
 #include "../common/bgfx_compute.sh"
 
 #define COMPUTE
-#include "atm_functions.sh"
+#include "../common/atm_functions.sh"
 
 uniform vec4 u_numScatteringOrders;
 

Engine/BuiltInShaders/atm/cs_ComputeSingleScattering.sc → Engine/BuiltInShaders/shaders/cs_ComputeSingleScattering.sc

@@ -1,7 +1,7 @@
 #include "../common/bgfx_compute.sh"
 
 #define COMPUTE
-#include "atm_functions.sh"
+#include "../common/atm_functions.sh"
 
 IMAGE3D_WR(s_delta_rayleigh_scattering, rgba32f, 0);
 IMAGE3D_WR(s_delta_mie_scattering, rgba32f, 1);

Engine/BuiltInShaders/atm/cs_ComputeTransmittance.sc → Engine/BuiltInShaders/shaders/cs_ComputeTransmittance.sc

@@ -1,7 +1,7 @@
 #include "../common/bgfx_compute.sh"
 
 #define COMPUTE
-#include "atm_functions.sh"
+#include "../common/atm_functions.sh"
 
 IMAGE2D_WR(s_transmittance, rgba32f, 0);
 

Engine/BuiltInShaders/atm/fs_PrecomputedAtmosphericScattering_LUT.sc → Engine/BuiltInShaders/shaders/fs_PrecomputedAtmosphericScattering_LUT.sc

@@ -2,7 +2,7 @@ $input v_worldPos
 
 #include "../common/common.sh"
 #include "../common/Camera.sh"
-#include "atm_functions.sh"
+#include "../common/atm_functions.sh"
 
 uniform vec4 u_LightDir;
 uniform vec4 u_HeightOffset;

Engine/Source/Editor/EditorApp.cpp

@@ -25,13 +25,13 @@
 #include "Rendering/RenderContext.h"
 #include "Rendering/SkeletonRenderer.h"
 #include "Rendering/SkyboxRenderer.h"
+#include "Rendering/ShaderCollections.h"
 #include "Rendering/TerrainRenderer.h"
 #include "Rendering/WorldRenderer.h"
 #include "Rendering/ParticleRenderer.h"
 #include "Resources/FileWatcher.h"
 #include "Resources/ResourceBuilder.h"
 #include "Resources/ShaderBuilder.h"
-#include "Resources/ShaderLoader.h"
 #include "Scene/SceneDatabase.h"
 #include "UILayers/AssetBrowser.h"
 #include "UILayers/EntityList.h"
@@ -88,15 +88,19 @@ void EditorApp::Init(engine::EngineInitArgs initArgs)
 
 	// Init graphics backend
 	InitRenderContext(m_initArgs.backend, pSplashWindow->GetNativeHandle());
+
 	pSplashWindow->OnResize.Bind<engine::RenderContext, &engine::RenderContext::OnResize>(m_pRenderContext.get());
 	AddWindow(cd::MoveTemp(pSplashWindow));
 
 	InitEditorRenderers();
+	EditorRenderersWarmup();
 	InitEditorImGuiContext(m_initArgs.language);
 
 	InitECWorld();
 	m_pEditorImGuiContext->SetSceneWorld(m_pSceneWorld.get());
 
+	InitEngineRenderers();
+
 	// Add shader build tasks and create a thread to update tasks.
 	InitShaderPrograms();
 	m_pEditorImGuiContext->AddStaticLayer(std::make_unique<Splash>("Splash"));
@@ -108,6 +112,7 @@ void EditorApp::Init(engine::EngineInitArgs initArgs)
 	resourceThread.detach();
 
 	m_pFileWatcher = std::make_unique<FileWatcher>(CDENGINE_BUILTIN_SHADER_PATH);
+	m_pFileWatcher->EnableWatchSubTree();
 }
 
 void EditorApp::Shutdown()
@@ -245,7 +250,6 @@ void EditorApp::InitECWorld()
 	InitEditorCameraEntity();
 
 	InitSkyEntity();
-	InitShaderVariantCollectionEntity();
 
 #ifdef ENABLE_DDGI
 	m_pSceneWorld->InitDDGISDK();
@@ -324,19 +328,6 @@ void EditorApp::InitSkyEntity()
 	meshComponent.Submit();
 }
 
-void EditorApp::InitShaderVariantCollectionEntity()
-{
-	engine::World* pWorld = m_pSceneWorld->GetWorld();
-
-	engine::Entity shaderVariantCollectionEntity = pWorld->CreateEntity();
-	m_pSceneWorld->SetShaderVariantCollectionEntity(shaderVariantCollectionEntity);
-
-	auto& nameComponent = pWorld->CreateComponent<engine::NameComponent>(shaderVariantCollectionEntity);
-	nameComponent.SetName("ShaderVariantCollection");
-
-	auto& shaderVariantCollectionsComponent = pWorld->CreateComponent<engine::ShaderVariantCollectionsComponent>(shaderVariantCollectionEntity);
-}
-
 #ifdef ENABLE_DDGI
 void EditorApp::InitDDGIEntity()
 {
@@ -352,6 +343,29 @@ void EditorApp::InitDDGIEntity()
 }
 #endif
 
+void EditorApp::LazyCompileAndLoadShaders()
+{
+	bool doCompile = false;
+
+	for (const auto& info : m_pRenderContext->GetShaderCompileTasks())
+	{
+		ShaderBuilder::BuildShader(m_pRenderContext.get(), info);
+		doCompile = true;
+	}
+
+	if (doCompile)
+	{
+		ResourceBuilder::Get().Update(true);
+
+		for (auto& info : m_pRenderContext->GetShaderCompileTasks())
+		{
+			m_pRenderContext->UploadShaderProgram(info.m_programName, info.m_featuresCombine);
+		}
+
+		m_pRenderContext->ClearShaderCompileTasks();
+	}
+}
+
 void EditorApp::InitRenderContext(engine::GraphicsBackend backend, void* hwnd)
 {
 	CD_INFO("Init graphics backend : {}", nameof::nameof_enum(backend));
@@ -455,6 +469,22 @@ void EditorApp::InitEngineRenderers()
 	AddEngineRenderer(std::make_unique<engine::ImGuiRenderer>(m_pRenderContext->CreateView(), pSceneRenderTarget));
 }
 
+void EditorApp::EditorRenderersWarmup()
+{
+	for (std::unique_ptr<engine::Renderer>& pRenderer : m_pEditorRenderers)
+	{
+		pRenderer->Warmup();
+	}
+}
+
+void EditorApp::EngineRenderersWarmup()
+{
+	for (std::unique_ptr<engine::Renderer>& pRenderer : m_pEngineRenderers)
+	{
+		pRenderer->Warmup();
+	}
+}
+
 bool EditorApp::IsAtmosphericScatteringEnable() const
 {
 	engine::GraphicsBackend backend = engine::Path::GetGraphicsBackend();
@@ -466,18 +496,11 @@ bool EditorApp::IsAtmosphericScatteringEnable() const
 
 void EditorApp::InitShaderPrograms() const
 {
-	std::string nonUberBuildPath = CDENGINE_BUILTIN_SHADER_PATH;
-	ShaderBuilder::BuildNonUberShader(nonUberBuildPath + "shaders");
-	if (IsAtmosphericScatteringEnable())
-	{
-		std::string atmBuildPath = CDENGINE_BUILTIN_SHADER_PATH;
-		ShaderBuilder::BuildNonUberShader(atmBuildPath + "atm");
-	}
+	ShaderBuilder::PreBuildNonUberShaders(m_pRenderContext.get());
+	ShaderBuilder::PreBuildUberShaders(m_pRenderContext.get(), m_pSceneWorld->GetPBRMaterialType());
 
-	ShaderBuilder::BuildUberShader(m_pSceneWorld->GetPBRMaterialType());
-	ShaderBuilder::BuildUberShader(m_pSceneWorld->GetAnimationMaterialType());
 #ifdef ENABLE_DDGI
-	ShaderBuilder::BuildUberShader(m_pSceneWorld->GetDDGIMaterialType());
+	ShaderBuilder::PreBuildUberShaders(m_pSceneWorld->GetDDGIMaterialType());
 #endif
 }
 
@@ -510,11 +533,8 @@ bool EditorApp::Update(float deltaTime)
 	if (!m_bInitEditor && ResourceBuilder::Get().IsIdle())
 	{
 		m_bInitEditor = true;
-		engine::ShaderLoader::UploadUberShader(m_pSceneWorld->GetPBRMaterialType());
-		engine::ShaderLoader::UploadUberShader(m_pSceneWorld->GetAnimationMaterialType());
-#ifdef ENABLE_DDGI
-		engine::ShaderLoader::UploadUberShader(m_pSceneWorld->GetDDGIMaterialType());
-#endif
+
+		EngineRenderersWarmup();
 
 		// Phase 2 - Project Manager
 		//		* TODO : Show project selector
@@ -533,7 +553,6 @@ bool EditorApp::Update(float deltaTime)
 		GetMainWindow()->SetBordedLess(false);
 		GetMainWindow()->SetResizeable(true);
 
-		InitEngineRenderers();
 		m_pEditorImGuiContext->ClearUILayers();
 		InitEditorUILayers();
 
@@ -591,7 +610,7 @@ bool EditorApp::Update(float deltaTime)
 
 		for (std::unique_ptr<engine::Renderer>& pRenderer : m_pEngineRenderers)
 		{
-			if (pRenderer->IsEnable())
+			if (pRenderer->IsEnable() && pRenderer->CheckResources())
 			{
 				const float* pViewMatrix = pMainCameraComponent->GetViewMatrix().Begin();
 				const float* pProjectionMatrix = pMainCameraComponent->GetProjectionMatrix().Begin();
@@ -603,6 +622,8 @@ bool EditorApp::Update(float deltaTime)
 
 	m_pRenderContext->EndFrame();
 
+	LazyCompileAndLoadShaders();
+
 	engine::Input::Get().FlushInputs();
 
 	return !GetMainWindow()->ShouldClose();

Engine/Source/Editor/EditorApp.h

@@ -18,6 +18,7 @@ class Renderer;
 class AABBRenderer;
 class RenderTarget;
 class SceneWorld;
+class ShaderCollections;
 
 }
 
@@ -50,8 +51,13 @@ class EditorApp final : public engine::IApplication
 	void RemoveWindow(size_t index);
 
 	void InitRenderContext(engine::GraphicsBackend backend, void* hwnd = nullptr);
+
 	void InitEditorRenderers();
 	void InitEngineRenderers();
+
+	void EditorRenderersWarmup();
+	void EngineRenderersWarmup();
+
 	void InitShaderPrograms() const;
 	void AddEditorRenderer(std::unique_ptr<engine::Renderer> pRenderer);
 	void AddEngineRenderer(std::unique_ptr<engine::Renderer> pRenderer);
@@ -71,12 +77,13 @@ class EditorApp final : public engine::IApplication
 private:
 	void InitEditorCameraEntity();
 	void InitSkyEntity();
-	void InitShaderVariantCollectionEntity();
 
 #ifdef ENABLE_DDGI
 	void InitDDGIEntity();
 #endif
 
+	void LazyCompileAndLoadShaders();
+
 	bool m_bInitEditor = false;
 	engine::EngineInitArgs m_initArgs;
 
@@ -101,6 +108,7 @@ class EditorApp final : public engine::IApplication
 
 	// Rendering
 	std::unique_ptr<engine::RenderContext> m_pRenderContext;
+
 	std::vector<std::unique_ptr<engine::Renderer>> m_pEditorRenderers;
 	std::vector<std::unique_ptr<engine::Renderer>> m_pEngineRenderers;
 

Engine/Source/Editor/Resources/ResourceBuilder.cpp

@@ -150,7 +150,7 @@ bool ResourceBuilder::AddTask(Process process)
 	return true;
 }
 
-bool ResourceBuilder::AddShaderBuildTask(ShaderType shaderType, const char* pInputFilePath, const char* pOutputFilePath, const char* pShaderFeatures)
+bool ResourceBuilder::AddShaderBuildTask(engine::ShaderType shaderType, const char* pInputFilePath, const char* pOutputFilePath, const char* pShaderFeatures)
 {
 	if (s_SkipStatus & static_cast<uint8_t>(CheckFileStatus(pInputFilePath, pOutputFilePath)))
 	{
@@ -183,15 +183,15 @@ bool ResourceBuilder::AddShaderBuildTask(ShaderType shaderType, const char* pInp
 #endif
 
 	commandArguments.push_back("--type");
-	if (ShaderType::Compute == shaderType)
+	if (engine::ShaderType::Compute == shaderType)
 	{
 		commandArguments.push_back("c");
 	}
-	else if (ShaderType::Fragment == shaderType)
+	else if (engine::ShaderType::Fragment == shaderType)
 	{
 		commandArguments.push_back("f");
 	}
-	else if (ShaderType::Vertex == shaderType)
+	else if (engine::ShaderType::Vertex == shaderType)
 	{
 		commandArguments.push_back("v");
 	}