fix: skylighting alpha testing (#836)

* fix: skylighting alpha testing * style: 🎨 apply clang-format changes * chore: removed skylighting foliage hacks --------- Co-authored-by: doodlum <doodlum@users.noreply.github.com>
doodlum · Dec 9, 2024 · f950450 · f950450
1 parent 21f99f7
commit f950450
Show file tree

Hide file tree

Showing 4 changed files with 19 additions and 65 deletions.
diff --git a/package/Shaders/Utility.hlsl b/package/Shaders/Utility.hlsl
@@ -444,12 +444,6 @@ PS_OUTPUT main(PS_INPUT input)
 	}
 #	endif
 
-#	if defined(FOLIAGE)
-	float checkerboard = Random::InterleavedGradientNoise(input.PositionCS.xy, 0);
-	if (checkerboard > 0.75)
-		discard;
-#	endif
-
 	float2 baseTexCoord = 0;
 #	if !(defined(RENDER_DEPTH) && defined(RENDER_SHADOWMASK_ANY)) && SHADOWFILTER != 2
 #		if (defined(RENDER_DEPTH) || defined(RENDER_SHADOWMAP)) && defined(ALPHA_TEST) && !defined(RENDER_SHADOWMAP_PB)

diff --git a/src/Features/Skylighting.cpp b/src/Features/Skylighting.cpp
@@ -48,28 +48,6 @@ void Skylighting::DrawSettings()
 		ImGui::Text("Smaller angles creates more focused top-down shadow.");
 }
 
-ID3D11PixelShader* Skylighting::GetFoliagePS()
-{
-	if (!foliagePixelShader) {
-		logger::debug("Compiling Utility.hlsl");
-		foliagePixelShader = (ID3D11PixelShader*)Util::CompileShader(L"Data\\Shaders\\Utility.hlsl", { { "RENDER_DEPTH", "" }, { "FOLIAGE", "" } }, "ps_5_0");
-	}
-	return foliagePixelShader;
-}
-
-void Skylighting::SkylightingShaderHacks()
-{
-	if (inOcclusion) {
-		auto& context = State::GetSingleton()->context;
-
-		if (foliage) {
-			context->PSSetShader(GetFoliagePS(), NULL, NULL);
-		} else {
-			context->PSSetShader(nullptr, NULL, NULL);
-		}
-	}
-}
-
 void Skylighting::SetupResources()
 {
 	auto renderer = RE::BSGraphics::Renderer::GetSingleton();
@@ -156,10 +134,6 @@ void Skylighting::ClearShaderCache()
 		shader = nullptr;
 
 	CompileComputeShaders();
-	if (foliagePixelShader) {
-		foliagePixelShader->Release();
-		foliagePixelShader = nullptr;
-	}
 }
 
 void Skylighting::CompileComputeShaders()
@@ -259,7 +233,6 @@ void Skylighting::PostPostLoad()
 	stl::write_vfunc<0x2D, BSLightingShaderProperty_GetPrecipitationOcclusionMapRenderPassesImpl>(RE::VTABLE_BSLightingShaderProperty[0]);
 	stl::write_thunk_call<Main_Precipitation_RenderOcclusion>(REL::RelocationID(35560, 36559).address() + REL::Relocate(0x3A1, 0x3A1, 0x2FA));
 	stl::write_thunk_call<SetViewFrustum>(REL::RelocationID(25643, 26185).address() + REL::Relocate(0x5D9, 0x59D, 0x5DC));
-	stl::write_vfunc<0x6, BSUtilityShader_SetupGeometry>(RE::VTABLE_BSUtilityShader[0]);
 	MenuOpenCloseEventHandler::Register();
 }
 
@@ -368,13 +341,29 @@ RE::BSLightingShaderProperty::Data* Skylighting::BSLightingShaderProperty_GetPre
 			stl::enumeration<RE::BSUtilityShader::Flags> technique;
 			technique.set(RenderDepth);
 
-			auto pass = precipitationOcclusionMapRenderPassList->EmplacePass(
+			if (property->flags.any(kVertexColors)) {
+				technique.set(Vc);
+			}
+
+			const auto alphaProperty = static_cast<RE::NiAlphaProperty*>(geometry->GetGeometryRuntimeData().properties[0].get());
+			if (alphaProperty && alphaProperty->GetAlphaTesting()) {
+				technique.set(Texture);
+				technique.set(AlphaTest);
+			}
+
+			if (property->flags.any(kLODObjects, kHDLODObjects)) {
+				technique.set(LodObject);
+			}
+
+			if (property->flags.any(kTreeAnim)) {
+				technique.set(TreeAnim);
+			}
+
+			precipitationOcclusionMapRenderPassList->EmplacePass(
 				RE::BSUtilityShader::GetSingleton(),
 				property,
 				geometry,
 				technique.underlying() + static_cast<uint32_t>(ShaderTechnique::UtilityGeneralStart));
-			if (property->flags.any(kTreeAnim))
-				pass->accumulationHint = 11;
 		}
 	}
 	return precipitationOcclusionMapRenderPassList;
@@ -497,25 +486,13 @@ void Skylighting::Main_Precipitation_RenderOcclusion::thunk()
 					PrecipitationShaderDirection = originalParticleShaderDirection;
 
 					precipitation = precipitationCopy;
-
-					singleton->foliage = false;
 				}
 			}
 		}
 	}
 	State::GetSingleton()->EndPerfEvent();
 }
 
-void Skylighting::BSUtilityShader_SetupGeometry::thunk(RE::BSShader* This, RE::BSRenderPass* Pass, uint32_t RenderFlags)
-{
-	auto& feat = *GetSingleton();
-	if (feat.inOcclusion) {
-		feat.foliage = Pass->shaderProperty->flags.any(RE::BSShaderProperty::EShaderPropertyFlag::kTreeAnim);
-	}
-
-	func(This, Pass, RenderFlags);
-}
-
 void Skylighting::SetViewFrustum::thunk(RE::NiCamera* a_camera, RE::NiFrustum* a_frustum)
 {
 	if (GetSingleton()->inOcclusion) {

diff --git a/src/Features/Skylighting.h b/src/Features/Skylighting.h
@@ -34,9 +34,6 @@ struct Skylighting : Feature
 
 	virtual void PostPostLoad() override;
 
-	ID3D11PixelShader* GetFoliagePS();
-	void SkylightingShaderHacks();  // referenced in State.cpp
-
 	//////////////////////////////////////////////////////////////////////////////////
 
 	struct Settings
@@ -74,16 +71,13 @@ struct Skylighting : Feature
 
 	winrt::com_ptr<ID3D11ComputeShader> probeUpdateCompute = nullptr;
 
-	ID3D11PixelShader* foliagePixelShader = nullptr;
-
 	// misc parameters
 	bool doOcclusion = true;
 	uint probeArrayDims[3] = { 256, 256, 128 };
 	float occlusionDistance = 4096.f * 3.f;  // 3 cells
 
 	// cached variables
 	bool inOcclusion = false;
-	bool foliage = false;
 	REX::W32::XMFLOAT4X4 OcclusionTransform;
 	float4 OcclusionDir;
 	uint forceFrames = 255 * 4;
@@ -106,12 +100,6 @@ struct Skylighting : Feature
 		static inline REL::Relocation<decltype(thunk)> func;
 	};
 
-	struct BSUtilityShader_SetupGeometry
-	{
-		static void thunk(RE::BSShader* This, RE::BSRenderPass* Pass, uint32_t RenderFlags);
-		static inline REL::Relocation<decltype(thunk)> func;
-	};
-
 	struct SetViewFrustum
 	{
 		static void thunk(RE::NiCamera* a_camera, RE::NiFrustum* a_frustum);

diff --git a/src/State.cpp b/src/State.cpp
@@ -12,7 +12,6 @@
 #include "Util.h"
 
 #include "Deferred.h"
-#include "Features/Skylighting.h"
 #include "Features/TerrainBlending.h"
 #include "TruePBR.h"
 
@@ -29,10 +28,6 @@ void State::Draw()
 
 		TruePBR::GetSingleton()->SetShaderResouces();
 
-		auto skylighting = Skylighting::GetSingleton();
-		if (skylighting->loaded)
-			skylighting->SkylightingShaderHacks();
-
 		if (auto accumulator = RE::BSGraphics::BSShaderAccumulator::GetCurrentAccumulator()) {
 			// Set an unused bit to indicate if we are rendering an object in the main rendering pass
 			if (accumulator->GetRuntimeData().activeShadowSceneNode == RE::BSShaderManager::State::GetSingleton().shadowSceneNode[0]) {