Skip to content

Commit

Permalink
feat: recompile automatically for changed shaders
Browse files Browse the repository at this point in the history 
Adds feature where changes to the shader .hlsl files will automatically
recompile shaders.
Changes to RE::BSShader::Type files in Data\Shaders (e.g.,
Lighting.hlsl) will result in a type specific recompile.
All other modification to hlsl/hlsli files will recompile all files
since there isn't a way to detect what may be impacted.

closes doodlum#205
  • Loading branch information
@alandtse
alandtse committed Mar 11, 2024
1 parent ce7276b commit 90ec85a
Show file tree
Hide file tree
Showing 5 changed files with 157 additions and 15 deletions.
7 changes: 3 additions & 4 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ find_path(CLIB_UTIL_INCLUDE_DIRS "ClibUtil/utils.hpp")
find_package(pystring CONFIG REQUIRED)
find_package(cppwinrt CONFIG REQUIRED)
find_package(unordered_dense CONFIG REQUIRED)

find_package(efsw CONFIG REQUIRED)
target_include_directories(
${PROJECT_NAME}
PRIVATE
Expand All @@ -62,6 +62,7 @@ target_link_libraries(
Microsoft::DirectXTex
pystring::pystring
unordered_dense::unordered_dense
efsw::efsw
)

# https://gitlab.kitware.com/cmake/cmake/-/issues/24922#note_1371990
Expand All @@ -87,7 +88,6 @@ endif()
# #######################################################################################################################
# # Feature version detection
# #######################################################################################################################

file(GLOB_RECURSE FEATURE_CONFIG_FILES
LIST_DIRECTORIES false
CONFIGURE_DEPENDS
Expand All @@ -106,7 +106,7 @@ endforeach()

set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${FEATURE_CONFIG_FILES}")

string (REPLACE ";" ",\n" FEATURE_VERSIONS "${FEATURE_VERSIONS}")
string(REPLACE ";" ",\n" FEATURE_VERSIONS "${FEATURE_VERSIONS}")

configure_file(
${CMAKE_CURRENT_SOURCE_DIR}/cmake/FeatureVersions.h.in
Expand Down Expand Up @@ -189,7 +189,6 @@ endif()

# Create a AIO zip for easier testing
if(AIO_ZIP_TO_DIST)

if(NOT ZIP_TO_DIST)
add_custom_target(build-time-make-directory ALL
COMMAND ${CMAKE_COMMAND} -E remove_directory "${ZIP_DIR}" ${CMAKE_SOURCE_DIR}/dist
Expand Down
124 changes: 117 additions & 7 deletions src/ShaderCache.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@ namespace SIE
{
static void GetShaderDefines(RE::BSShader::Type, uint32_t, D3D_SHADER_MACRO*);
static std::string GetShaderString(ShaderClass, const RE::BSShader&, uint32_t, bool = false);
/**
@brief Get the BSShader::Type from the ShaderString
@param a_key The key generated from GetShaderString
@return A string with a valid BSShader::Type
*/
static std::string GetTypeFromShaderString(std::string);
constexpr const char* VertexShaderProfile = "vs_5_0";
constexpr const char* PixelShaderProfile = "ps_5_0";
constexpr const char* ComputeShaderProfile = "cs_5_0";
Expand Down Expand Up @@ -952,6 +958,15 @@ namespace SIE
return result;
}

std::string GetTypeFromShaderString(std::string a_key)
{
std::string type = "";
std::string::size_type pos = a_key.find(':');
if (pos != std::string::npos)
type = a_key.substr(0, pos);
return type;
}

static ID3DBlob* CompileShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, bool useDiskCache)
{
ID3DBlob* shaderBlob = nullptr;
Expand All @@ -971,7 +986,11 @@ namespace SIE

if (!shaderBlob && useDiskCache && std::filesystem::exists(diskPath)) {
shaderBlob = nullptr;
if (FAILED(D3DReadFileToBlob(diskPath.c_str(), &shaderBlob))) {
// check build time of cache
auto diskCacheTime = std::chrono::clock_cast<std::chrono::system_clock>(std::filesystem::last_write_time(diskPath));
if (cache.ShaderModifiedSince(shader.fxpFilename, diskCacheTime)) {
logger::debug("Diskcached shader {} older than {}", SIE::SShaderCache::GetShaderString(shaderClass, shader, descriptor, true), std::format("{:%Y%m%d%H%M}", diskCacheTime));
} else if (FAILED(D3DReadFileToBlob(diskPath.c_str(), &shaderBlob))) {
logger::error("Failed to load {} shader {}::{}", magic_enum::enum_name(shaderClass), magic_enum::enum_name(type), descriptor);

if (shaderBlob != nullptr) {
Expand Down Expand Up @@ -1010,10 +1029,19 @@ namespace SIE
defines[lastIndex] = { nullptr, nullptr }; // do final entry
GetShaderDefines(type, descriptor, &defines[lastIndex]);

logger::debug("Defines set for {}:{}:{:X} to {}", magic_enum::enum_name(type), magic_enum::enum_name(shaderClass), descriptor, MergeDefinesString(defines));
const std::wstring path = GetShaderPath(shader.fxpFilename);

// Set timestamp based on file timestamp
auto shaderSourceTime = std::chrono::clock_cast<std::chrono::system_clock>(std::filesystem::last_write_time(path));
cache.modifiedShaderMap.insert_or_assign(std::string(shader.fxpFilename), shaderSourceTime);

std::string strPath;
std::transform(path.begin(), path.end(), std::back_inserter(strPath), [](wchar_t c) {
return (char)c;
});
logger::debug("Compiling {} {}:{}:{:X} to {}", strPath, magic_enum::enum_name(type), magic_enum::enum_name(shaderClass), descriptor, MergeDefinesString(defines));

// compile shaders
const std::wstring path = GetShaderPath(shader.fxpFilename);
ID3DBlob* errorBlob = nullptr;
const uint32_t flags = D3DCOMPILE_OPTIMIZATION_LEVEL3;
const HRESULT compileResult = D3DCompileFromFile(path.c_str(), defines.data(), D3D_COMPILE_STANDARD_FILE_INCLUDE, "main",
Expand Down Expand Up @@ -1286,6 +1314,7 @@ namespace SIE
ShaderCache::~ShaderCache()
{
Clear();
fileWatcher->removeWatch(watchID);
}

void ShaderCache::Clear()
Expand All @@ -1308,23 +1337,48 @@ namespace SIE
shaderMap.clear();
}

void ShaderCache::Clear(RE::BSShader::Type a_type)
{
logger::debug("Clearing cache for {}", magic_enum::enum_name(a_type));
std::lock_guard lockGuardV(vertexShadersMutex);
{
for (auto& [id, shader] : vertexShaders[static_cast<size_t>(a_type)]) {
shader->shader->Release();
}
vertexShaders[static_cast<size_t>(a_type)].clear();
}
std::lock_guard lockGuardP(pixelShadersMutex);
{
for (auto& [id, shader] : pixelShaders[static_cast<size_t>(a_type)]) {
shader->shader->Release();
}
pixelShaders[static_cast<size_t>(a_type)].clear();
}
compilationSet.Clear();
}

bool ShaderCache::AddCompletedShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, ID3DBlob* a_blob)
{
auto key = SIE::SShaderCache::GetShaderString(shaderClass, shader, descriptor, true);
auto status = a_blob ? ShaderCompilationTask::Status::Completed : ShaderCompilationTask::Status::Failed;
std::unique_lock lock{ mapMutex };
logger::debug("Adding {} shader to map: {}", magic_enum ::enum_name(status), key);
shaderMap.insert_or_assign(key, std::pair(a_blob, status));
shaderMap.insert_or_assign(key, ShaderCacheResult{ a_blob, status, system_clock::now() });
return (bool)a_blob;
}

ID3DBlob* ShaderCache::GetCompletedShader(const std::string a_key)
{
std::string type = SIE::SShaderCache::GetTypeFromShaderString(a_key);
std::scoped_lock lock{ mapMutex };
if (!shaderMap.empty() && shaderMap.contains(a_key)) {
auto status = shaderMap.at(a_key).second;
if (ShaderModifiedSince(type, shaderMap.at(a_key).compileTime)) {
logger::debug("Shader {} compiled {} before changes at {}", a_key, std::format("{:%Y%m%d%H%M}", shaderMap.at(a_key).compileTime), std::format("{:%Y%m%d%H%M}", modifiedShaderMap.at(type)));
return nullptr;
}
auto status = shaderMap.at(a_key).status;
if (status != ShaderCompilationTask::Status::Pending)
return shaderMap.at(a_key).first;
return shaderMap.at(a_key).blob;
}
return nullptr;
}
Expand All @@ -1346,7 +1400,7 @@ namespace SIE
{
std::scoped_lock lock{ mapMutex };
if (!shaderMap.empty() && shaderMap.contains(a_key)) {
return shaderMap.at(a_key).second;
return shaderMap.at(a_key).status;
}
return ShaderCompilationTask::Status::Pending;
}
Expand Down Expand Up @@ -1452,6 +1506,29 @@ namespace SIE
compilationPool.push_task(&ShaderCache::ManageCompilationSet, this, ssource.get_token());
}

void ShaderCache::StartFileWatcher()
{
fileWatcher = new efsw::FileWatcher();
listener = new UpdateListener();
// Add a folder to watch, and get the efsw::WatchID
// Reporting the files and directories changes to the instance of the listener
watchID = fileWatcher->addWatch("Data\\Shaders", listener, true);
// Start watching asynchronously the directories
fileWatcher->watch();
std::string pathStr = "";
for (auto path : fileWatcher->directories()) {
pathStr += std::format("{}; ", path);
}
logger::debug("ShaderCache watching for changes in {}", pathStr);
}

bool ShaderCache::ShaderModifiedSince(std::string a_type, system_clock::time_point a_current)
{
return !a_type.empty() && magic_enum::enum_cast<RE::BSShader::Type>(a_type, magic_enum::case_insensitive).has_value() // type is valid
&& !modifiedShaderMap.empty() && modifiedShaderMap.contains(a_type) // map has Type
&& modifiedShaderMap.at(a_type) > a_current; //modification time is older than a_current
}

RE::BSGraphics::VertexShader* ShaderCache::MakeAndAddVertexShader(const RE::BSShader& shader,
uint32_t descriptor)
{
Expand Down Expand Up @@ -1726,4 +1803,37 @@ namespace SIE
GetHumanTime(totalMs),
GetHumanTime(GetEta() + totalMs));
}

void UpdateListener::handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string)
{
auto& cache = SIE::ShaderCache::Instance();
const std::filesystem::path filePath = std::filesystem::path(std::format("{}\\{}", dir, filename));
auto modifiedTime = std::chrono::clock_cast<std::chrono::system_clock>(std::filesystem::last_write_time(filePath));
std::string extension = filePath.extension().string();
std::string parentDir = filePath.parent_path().string();
std::string shaderTypeString = filePath.stem().string();
auto shaderType = magic_enum::enum_cast<RE::BSShader::Type>(shaderTypeString, magic_enum::case_insensitive);
switch (action) {
case efsw::Actions::Add:
break;
case efsw::Actions::Delete:
break;
case efsw::Actions::Modified:
logger::info("Detected changed file {}", filePath.string());
if (extension.starts_with(".hlsl") && parentDir.ends_with("Shaders") && shaderType.has_value()) { // TODO: Case insensitive checks
// Shader types, so only invalidate specific shader type (e.g,. Lighting)
cache.modifiedShaderMap.insert_or_assign(shaderTypeString, modifiedTime);
cache.Clear(shaderType.value());
} else if (extension.starts_with(".hlsl")) { // TODO: Case insensitive checks
// all other shaders, since we don't know what is using it, clear everything
cache.DeleteDiskCache();
cache.Clear();
}
break;
case efsw::Actions::Moved:
break;
default:
logger::error("Filewatcher received invalid action {}", magic_enum::enum_name(action));
}
}
}
36 changes: 34 additions & 2 deletions src/ShaderCache.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <RE/B/BSShader.h>

#include "BS_thread_pool.hpp"
#include "efsw/efsw.hpp"
#include <chrono>
#include <condition_variable>
#include <unordered_map>
Expand Down Expand Up @@ -84,6 +85,15 @@ namespace SIE
double totalMs = (double)duration_cast<std::chrono::milliseconds>(lastReset - lastReset).count();
};

struct ShaderCacheResult
{
ID3DBlob* blob;
ShaderCompilationTask::Status status;
system_clock::time_point compileTime = system_clock::now();
};

class UpdateListener;

class ShaderCache
{
public:
Expand Down Expand Up @@ -127,7 +137,16 @@ namespace SIE
void DeleteDiskCache();
void ValidateDiskCache();
void WriteDiskCacheInfo();
void StartFileWatcher();
/** @brief Whether the ShaderFile for RE::BSShader::Type has been modified since the timestamp.
@param a_type Case insensitive string for the type of shader. E.g., Lighting
@param a_current The current time in system_clock::time_point.
@return True if the shader for the type (i.e., Lighting.hlsl) has been modified since the timestamp
*/
bool ShaderModifiedSince(std::string a_type, system_clock::time_point a_current);

void Clear();
void Clear(RE::BSShader::Type a_type);

bool AddCompletedShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, ID3DBlob* a_blob);
ID3DBlob* GetCompletedShader(const std::string a_key);
Expand Down Expand Up @@ -264,7 +283,8 @@ namespace SIE

uint blockedKeyIndex = (uint)-1; // index in shaderMap; negative value indicates disabled
std::string blockedKey = "";
std::vector<uint32_t> blockedIDs; // more than one descriptor could be blocked based on shader hash
std::vector<uint32_t> blockedIDs; // more than one descriptor could be blocked based on shader hash
std::unordered_map<std::string, system_clock::time_point> modifiedShaderMap{}; // hashmap when a shader source file last modified

private:
ShaderCache();
Expand All @@ -290,7 +310,19 @@ namespace SIE
std::mutex vertexShadersMutex;
std::mutex pixelShadersMutex;
CompilationSet compilationSet;
std::unordered_map<std::string, std::pair<ID3DBlob*, ShaderCompilationTask::Status>> shaderMap{};
std::unordered_map<std::string, ShaderCacheResult> shaderMap{};
std::mutex mapMutex;

// efsw file watcher
efsw::FileWatcher* fileWatcher;
efsw::WatchID watchID;
UpdateListener* listener;
};

// Inherits from the abstract listener class, and implements the the file action handler
class UpdateListener : public efsw::FileWatchListener
{
public:
void handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string) override;
};
}
2 changes: 1 addition & 1 deletion src/XSEPlugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ void MessageHandler(SKSE::MessagingInterface::Message* message)
auto& shaderCache = SIE::ShaderCache::Instance();

shaderCache.ValidateDiskCache();

shaderCache.StartFileWatcher();
for (auto* feature : Feature::GetFeatureList()) {
if (feature->loaded) {
feature->PostPostLoad();
Expand Down
3 changes: 2 additions & 1 deletion vcpkg.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@
},
"eastl",
"clib-util",
"unordered-dense"
"unordered-dense",
"efsw"
],
"overrides": [
{
Expand Down

0 comments on commit 90ec85a

Please sign in to comment.