Add transparency support for LightmapGI

Co-authored-by: Guerro323 <kaltobattle@gmail.com>

doc/classes/ProjectSettings.xml

@@ -2596,6 +2596,9 @@
 		<member name="rendering/lightmapping/bake_performance/max_rays_per_probe_pass" type="int" setter="" getter="" default="64">
 			The maximum number of rays that can be thrown per pass when baking dynamic object lighting in [LightmapProbe]s with [LightmapGI]. Depending on the scene, adjusting this value may result in higher GPU utilization when baking lightmaps, leading to faster bake times.
 		</member>
+		<member name="rendering/lightmapping/bake_performance/max_transparency_rays" type="int" setter="" getter="" default="64">
+			The maximum number of retry rays that can be thrown per pass when hitting a transparent surface when baking lightmaps with [LightmapGI]. Depending on the scene, adjusting this value may result in higher GPU utilization when baking lightmaps, leading to faster bake times.
+		</member>
 		<member name="rendering/lightmapping/bake_performance/region_size" type="int" setter="" getter="" default="512">
 			The region size to use when baking lightmaps with [LightmapGI].
 		</member>

drivers/gles3/rasterizer_scene_gles3.cpp

@@ -977,7 +977,7 @@ void RasterizerSceneGLES3::_update_sky_radiance(RID p_env, const Projection &p_p
 		glBindFramebuffer(GL_FRAMEBUFFER, sky->radiance_framebuffer);
 
 		scene_state.reset_gl_state();
-		scene_state.set_gl_cull_mode(GLES3::SceneShaderData::CULL_DISABLED);
+		scene_state.set_gl_cull_mode(GLES3::ShaderData::CULL_DISABLED);
 		scene_state.enable_gl_blend(false);
 
 		for (int i = 0; i < 6; i++) {
@@ -1000,7 +1000,7 @@ void RasterizerSceneGLES3::_update_sky_radiance(RID p_env, const Projection &p_p
 	} else {
 		if (sky_mode == RS::SKY_MODE_INCREMENTAL && sky->processing_layer < max_processing_layer) {
 			scene_state.reset_gl_state();
-			scene_state.set_gl_cull_mode(GLES3::SceneShaderData::CULL_DISABLED);
+			scene_state.set_gl_cull_mode(GLES3::ShaderData::CULL_DISABLED);
 			scene_state.enable_gl_blend(false);
 
 			cubemap_filter->filter_radiance(sky->raw_radiance, sky->radiance, sky->radiance_framebuffer, sky->radiance_size, sky->mipmap_count, sky->processing_layer);
@@ -1439,6 +1439,10 @@ void RasterizerSceneGLES3::_fill_render_list(RenderListType p_render_list, const
 				if (surf->flags & GeometryInstanceSurface::FLAG_PASS_SHADOW) {
 					rl->add_element(surf);
 				}
+			} else if (p_pass_mode == PASS_MODE_MATERIAL) {
+				if (surf->flags & (GeometryInstanceSurface::FLAG_PASS_DEPTH | GeometryInstanceSurface::FLAG_PASS_OPAQUE | GeometryInstanceSurface::FLAG_PASS_ALPHA)) {
+					rl->add_element(surf);
+				}
 			} else {
 				if (surf->flags & (GeometryInstanceSurface::FLAG_PASS_DEPTH | GeometryInstanceSurface::FLAG_PASS_OPAQUE)) {
 					rl->add_element(surf);
@@ -2216,7 +2220,7 @@ void RasterizerSceneGLES3::_render_shadow_pass(RID p_light, RID p_shadow_atlas,
 	scene_state.enable_gl_depth_test(false);
 	scene_state.enable_gl_depth_draw(true);
 	glDisable(GL_CULL_FACE);
-	scene_state.cull_mode = GLES3::SceneShaderData::CULL_DISABLED;
+	scene_state.cull_mode = GLES3::ShaderData::CULL_DISABLED;
 	glBindFramebuffer(GL_FRAMEBUFFER, GLES3::TextureStorage::system_fbo);
 }
 
@@ -2593,7 +2597,7 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 		scene_state.enable_gl_depth_draw(false);
 		scene_state.enable_gl_depth_test(false);
 		scene_state.enable_gl_blend(false);
-		scene_state.set_gl_cull_mode(GLES3::SceneShaderData::CULL_BACK);
+		scene_state.set_gl_cull_mode(GLES3::ShaderData::CULL_BACK);
 
 		Ref<CameraFeed> feed = CameraServer::get_singleton()->get_feed_by_id(camera_feed_id);
 
@@ -2621,7 +2625,7 @@ void RasterizerSceneGLES3::render_scene(const Ref<RenderSceneBuffers> &p_render_
 
 		scene_state.enable_gl_depth_test(true);
 		scene_state.enable_gl_blend(false);
-		scene_state.set_gl_cull_mode(GLES3::SceneShaderData::CULL_BACK);
+		scene_state.set_gl_cull_mode(GLES3::ShaderData::CULL_BACK);
 
 		Transform3D transform = render_data.cam_transform;
 		Projection projection = render_data.cam_projection;
@@ -3105,19 +3109,19 @@ void RasterizerSceneGLES3::_render_list_template(RenderListParameters *p_params,
 			}
 
 			// Find cull variant.
-			GLES3::SceneShaderData::Cull cull_mode = shader->cull_mode;
+			GLES3::ShaderData::CullMode cull_mode = shader->cull_mode;
 
 			if (p_pass_mode == PASS_MODE_MATERIAL || (surf->flags & GeometryInstanceSurface::FLAG_USES_DOUBLE_SIDED_SHADOWS)) {
-				cull_mode = GLES3::SceneShaderData::CULL_DISABLED;
+				cull_mode = GLES3::ShaderData::CULL_DISABLED;
 			} else {
 				bool mirror = inst->mirror;
 				if (p_params->reverse_cull) {
 					mirror = !mirror;
 				}
-				if (cull_mode == GLES3::SceneShaderData::CULL_FRONT && mirror) {
-					cull_mode = GLES3::SceneShaderData::CULL_BACK;
-				} else if (cull_mode == GLES3::SceneShaderData::CULL_BACK && mirror) {
-					cull_mode = GLES3::SceneShaderData::CULL_FRONT;
+				if (cull_mode == GLES3::ShaderData::CULL_FRONT && mirror) {
+					cull_mode = GLES3::ShaderData::CULL_BACK;
+				} else if (cull_mode == GLES3::ShaderData::CULL_BACK && mirror) {
+					cull_mode = GLES3::ShaderData::CULL_FRONT;
 				}
 			}
 
@@ -3789,7 +3793,7 @@ void RasterizerSceneGLES3::_render_buffers_debug_draw(Ref<RenderSceneBuffersGLES
 			glActiveTexture(GL_TEXTURE0);
 			scene_state.enable_gl_depth_draw(true);
 			glDepthFunc(GL_ALWAYS);
-			scene_state.set_gl_cull_mode(GLES3::SceneShaderData::CULL_DISABLED);
+			scene_state.set_gl_cull_mode(GLES3::ShaderData::CULL_DISABLED);
 
 			// Loop through quadrants and copy shadows over.
 			for (int quadrant = 0; quadrant < 4; quadrant++) {

drivers/gles3/rasterizer_scene_gles3.h

@@ -461,7 +461,7 @@ class RasterizerSceneGLES3 : public RendererSceneRender {
 		bool used_depth_prepass = false;
 
 		GLES3::SceneShaderData::BlendMode current_blend_mode = GLES3::SceneShaderData::BLEND_MODE_MIX;
-		GLES3::SceneShaderData::Cull cull_mode = GLES3::SceneShaderData::CULL_BACK;
+		GLES3::ShaderData::CullMode cull_mode = GLES3::ShaderData::CULL_BACK;
 
 		bool current_blend_enabled = false;
 		bool current_depth_draw_enabled = false;
@@ -477,24 +477,24 @@ class RasterizerSceneGLES3 : public RendererSceneRender {
 
 			glCullFace(GL_BACK);
 			glEnable(GL_CULL_FACE);
-			cull_mode = GLES3::SceneShaderData::CULL_BACK;
+			cull_mode = GLES3::ShaderData::CULL_BACK;
 
 			glDepthMask(GL_FALSE);
 			current_depth_draw_enabled = false;
 			glDisable(GL_DEPTH_TEST);
 			current_depth_test_enabled = false;
 		}
 
-		void set_gl_cull_mode(GLES3::SceneShaderData::Cull p_mode) {
+		void set_gl_cull_mode(GLES3::ShaderData::CullMode p_mode) {
 			if (cull_mode != p_mode) {
-				if (p_mode == GLES3::SceneShaderData::CULL_DISABLED) {
+				if (p_mode == GLES3::ShaderData::CULL_DISABLED) {
 					glDisable(GL_CULL_FACE);
 				} else {
-					if (cull_mode == GLES3::SceneShaderData::CULL_DISABLED) {
+					if (cull_mode == GLES3::ShaderData::CULL_DISABLED) {
 						// Last time was disabled, so enable and set proper face.
 						glEnable(GL_CULL_FACE);
 					}
-					glCullFace(p_mode == GLES3::SceneShaderData::CULL_FRONT ? GL_FRONT : GL_BACK);
+					glCullFace(p_mode == GLES3::ShaderData::CULL_FRONT ? GL_FRONT : GL_BACK);
 				}
 				cull_mode = p_mode;
 			}

drivers/gles3/shaders/scene.glsl

@@ -1859,10 +1859,18 @@ void main() {
 #ifndef USE_SHADOW_TO_OPACITY
 
 #if defined(ALPHA_SCISSOR_USED)
+#ifdef RENDER_MATERIAL
+	if (alpha < alpha_scissor_threshold) {
+		alpha = 0.0;
+	} else {
+		alpha = 1.0;
+	}
+#else
 	if (alpha < alpha_scissor_threshold) {
 		discard;
 	}
 	alpha = 1.0;
+#endif // RENDER_MATERIAL
 #else
 #ifdef MODE_RENDER_DEPTH
 #ifdef USE_OPAQUE_PREPASS
@@ -2200,9 +2208,17 @@ void main() {
 	alpha = min(alpha, clamp(length(ambient_light), 0.0, 1.0));
 
 #if defined(ALPHA_SCISSOR_USED)
+#ifdef RENDER_MATERIAL
+	if (alpha < alpha_scissor_threshold) {
+		alpha = 0.0;
+	} else {
+		alpha = 1.0;
+	}
+#else
 	if (alpha < alpha_scissor_threshold) {
 		discard;
 	}
+#endif // RENDER_MATERIAL
 #endif // !ALPHA_SCISSOR_USED
 
 #endif // !MODE_RENDER_DEPTH

drivers/gles3/storage/material_storage.cpp

@@ -2421,6 +2421,15 @@ void MaterialStorage::material_set_shader(RID p_material, RID p_shader) {
 	_material_queue_update(material, true, true);
 }
 
+ShaderData *MaterialStorage::material_get_shader_data(RID p_material) {
+	const Material *material = MaterialStorage::get_singleton()->get_material(p_material);
+	if (material && material->shader && material->shader->data) {
+		return material->shader->data;
+	}
+
+	return nullptr;
+}
+
 void MaterialStorage::material_set_param(RID p_material, const StringName &p_param, const Variant &p_value) {
 	GLES3::Material *material = material_owner.get_or_null(p_material);
 	ERR_FAIL_NULL(material);
@@ -2635,6 +2644,10 @@ bool CanvasShaderData::casts_shadows() const {
 	return false;
 }
 
+ShaderData::CullMode CanvasShaderData::get_cull_mode() const {
+	return CULL_DISABLED;
+}
+
 RS::ShaderNativeSourceCode CanvasShaderData::get_native_source_code() const {
 	return MaterialStorage::get_singleton()->shaders.canvas_shader.version_get_native_source_code(version);
 }
@@ -2807,6 +2820,10 @@ bool SkyShaderData::casts_shadows() const {
 	return false;
 }
 
+ShaderData::CullMode SkyShaderData::get_cull_mode() const {
+	return CULL_DISABLED;
+}
+
 RS::ShaderNativeSourceCode SkyShaderData::get_native_source_code() const {
 	return MaterialStorage::get_singleton()->shaders.sky_shader.version_get_native_source_code(version);
 }
@@ -2990,7 +3007,7 @@ void SceneShaderData::set_code(const String &p_code) {
 	alpha_antialiasing_mode = AlphaAntiAliasing(alpha_antialiasing_modei);
 	depth_draw = DepthDraw(depth_drawi);
 	depth_test = DepthTest(depth_testi);
-	cull_mode = Cull(cull_modei);
+	cull_mode = CullMode(cull_modei);
 
 	vertex_input_mask = RS::ARRAY_FORMAT_VERTEX | RS::ARRAY_FORMAT_NORMAL; // We can always read vertices and normals.
 	vertex_input_mask |= uses_tangent << RS::ARRAY_TANGENT;
@@ -3080,6 +3097,10 @@ bool SceneShaderData::casts_shadows() const {
 	return !has_alpha || (uses_depth_prepass_alpha && !(depth_draw == DEPTH_DRAW_DISABLED || depth_test == DEPTH_TEST_DISABLED));
 }
 
+ShaderData::CullMode SceneShaderData::get_cull_mode() const {
+	return cull_mode;
+}
+
 RS::ShaderNativeSourceCode SceneShaderData::get_native_source_code() const {
 	return MaterialStorage::get_singleton()->shaders.scene_shader.version_get_native_source_code(version);
 }
@@ -3195,6 +3216,10 @@ bool ParticlesShaderData::casts_shadows() const {
 	return false;
 }
 
+ShaderData::CullMode ParticlesShaderData::get_cull_mode() const {
+	return CULL_DISABLED;
+}
+
 RS::ShaderNativeSourceCode ParticlesShaderData::get_native_source_code() const {
 	return MaterialStorage::get_singleton()->shaders.particles_process_shader.version_get_native_source_code(version);
 }

drivers/gles3/storage/material_storage.h

@@ -52,6 +52,12 @@ namespace GLES3 {
 /* Shader Structs */
 
 struct ShaderData {
+	enum CullMode {
+		CULL_DISABLED,
+		CULL_FRONT,
+		CULL_BACK
+	};
+
 	String path;
 	HashMap<StringName, ShaderLanguage::ShaderNode::Uniform> uniforms;
 	HashMap<StringName, HashMap<int, RID>> default_texture_params;
@@ -66,6 +72,7 @@ struct ShaderData {
 	virtual void set_code(const String &p_Code) = 0;
 	virtual bool is_animated() const = 0;
 	virtual bool casts_shadows() const = 0;
+	virtual CullMode get_cull_mode() const = 0;
 	virtual RS::ShaderNativeSourceCode get_native_source_code() const { return RS::ShaderNativeSourceCode(); }
 
 	virtual ~ShaderData() {}
@@ -175,6 +182,7 @@ struct CanvasShaderData : public ShaderData {
 	virtual void set_code(const String &p_Code);
 	virtual bool is_animated() const;
 	virtual bool casts_shadows() const;
+	virtual CullMode get_cull_mode() const;
 	virtual RS::ShaderNativeSourceCode get_native_source_code() const;
 
 	CanvasShaderData();
@@ -220,6 +228,7 @@ struct SkyShaderData : public ShaderData {
 	virtual void set_code(const String &p_Code);
 	virtual bool is_animated() const;
 	virtual bool casts_shadows() const;
+	virtual CullMode get_cull_mode() const;
 	virtual RS::ShaderNativeSourceCode get_native_source_code() const;
 	SkyShaderData();
 	virtual ~SkyShaderData();
@@ -263,12 +272,6 @@ struct SceneShaderData : public ShaderData {
 		DEPTH_TEST_ENABLED
 	};
 
-	enum Cull {
-		CULL_DISABLED,
-		CULL_FRONT,
-		CULL_BACK
-	};
-
 	enum AlphaAntiAliasing {
 		ALPHA_ANTIALIASING_OFF,
 		ALPHA_ANTIALIASING_ALPHA_TO_COVERAGE,
@@ -292,7 +295,7 @@ struct SceneShaderData : public ShaderData {
 	AlphaAntiAliasing alpha_antialiasing_mode;
 	DepthDraw depth_draw;
 	DepthTest depth_test;
-	Cull cull_mode;
+	CullMode cull_mode;
 
 	bool uses_point_size;
 	bool uses_alpha;
@@ -335,6 +338,7 @@ struct SceneShaderData : public ShaderData {
 	virtual void set_code(const String &p_Code);
 	virtual bool is_animated() const;
 	virtual bool casts_shadows() const;
+	virtual CullMode get_cull_mode() const;
 	virtual RS::ShaderNativeSourceCode get_native_source_code() const;
 
 	SceneShaderData();
@@ -387,6 +391,7 @@ struct ParticlesShaderData : public ShaderData {
 	virtual void set_code(const String &p_Code);
 	virtual bool is_animated() const;
 	virtual bool casts_shadows() const;
+	virtual CullMode get_cull_mode() const;
 	virtual RS::ShaderNativeSourceCode get_native_source_code() const;
 
 	ParticlesShaderData() {}
@@ -609,6 +614,7 @@ class MaterialStorage : public RendererMaterialStorage {
 	virtual void material_free(RID p_rid) override;
 
 	virtual void material_set_shader(RID p_material, RID p_shader) override;
+	ShaderData *material_get_shader_data(RID p_material);
 
 	virtual void material_set_param(RID p_material, const StringName &p_param, const Variant &p_value) override;
 	virtual Variant material_get_param(RID p_material, const StringName &p_param) const override;

modules/lightmapper_rd/lightmapper_rd.cpp

@@ -30,6 +30,7 @@
 
 #include "lightmapper_rd.h"
 
+#include "drivers/gles3/storage/material_storage.h"
 #include "lm_blendseams.glsl.gen.h"
 #include "lm_compute.glsl.gen.h"
 #include "lm_raster.glsl.gen.h"
@@ -470,7 +471,23 @@ void LightmapperRD::_create_acceleration_structures(RenderingDevice *rd, Size2i
 			t.max_bounds[0] = taabb.position.x + MAX(taabb.size.x, 0.0001);
 			t.max_bounds[1] = taabb.position.y + MAX(taabb.size.y, 0.0001);
 			t.max_bounds[2] = taabb.position.z + MAX(taabb.size.z, 0.0001);
-			t.pad0 = t.pad1 = 0; //make valgrind not complain
+
+			t.cull_mode = RendererRD::MaterialStorage::ShaderData::CULL_BACK;
+
+			RID material = mi.data.material[i];
+			if (material.is_valid()) {
+				// TODO: Switch based on rendering method.
+				if (OS::get_singleton()->get_current_rendering_method() == "gl_compatibility") {
+					GLES3::MaterialStorage *material_storage = GLES3::MaterialStorage::get_singleton();
+					GLES3::ShaderData *shader_data = material_storage->material_get_shader_data(material);
+					t.cull_mode = shader_data->get_cull_mode();
+				} else {
+					RendererRD::MaterialStorage *material_storage = RendererRD::MaterialStorage::get_singleton();
+					RendererRD::MaterialStorage::ShaderData *shader_data = material_storage->material_get_shader_data(mi.data.material[i]);
+					t.cull_mode = shader_data->get_cull_mode();
+				}
+			}
+			t.pad1 = 0; //make valgrind not complain
 			triangles.push_back(t);
 			slice_triangle_count.write[t.slice]++;
 		}
@@ -1260,6 +1277,8 @@ LightmapperRD::BakeError LightmapperRD::bake(BakeQuality p_quality, bool p_use_d
 	bake_parameters.exposure_normalization = p_exposure_normalization;
 	bake_parameters.bounces = p_bounces;
 	bake_parameters.bounce_indirect_energy = p_bounce_indirect_energy;
+	// Same number of rays for transparency regardless of quality (it's more of a retry rather than shooting new ones).
+	bake_parameters.transparency_rays = GLOBAL_GET("rendering/lightmapping/bake_performance/max_transparency_rays");
 
 	bake_parameters_buffer = rd->uniform_buffer_create(sizeof(BakeParameters));
 	rd->buffer_update(bake_parameters_buffer, 0, sizeof(BakeParameters), &bake_parameters);

modules/lightmapper_rd/lightmapper_rd.h

@@ -36,6 +36,8 @@
 #include "scene/resources/mesh.h"
 #include "servers/rendering/rendering_device.h"
 
+#include "servers/rendering/renderer_rd/storage_rd/material_storage.h"
+
 class RDShaderFile;
 class LightmapperRD : public Lightmapper {
 	GDCLASS(LightmapperRD, Lightmapper)
@@ -57,7 +59,8 @@ class LightmapperRD : public Lightmapper {
 		uint32_t bounces = 0;
 
 		float bounce_indirect_energy = 0.0f;
-		uint32_t pad[3] = {};
+		uint32_t transparency_rays = 0;
+		uint32_t pad[2] = {};
 	};
 
 	struct MeshInstance {
@@ -184,7 +187,7 @@ class LightmapperRD : public Lightmapper {
 		uint32_t indices[3] = {};
 		uint32_t slice = 0;
 		float min_bounds[3] = {};
-		float pad0 = 0.0;
+		uint32_t cull_mode = 0;
 		float max_bounds[3] = {};
 		float pad1 = 0.0;
 		bool operator<(const Triangle &p_triangle) const {