diff --git a/src/ShaderCache.cpp b/src/ShaderCache.cpp index c8da8bfef..eb779fadc 100644 --- a/src/ShaderCache.cpp +++ b/src/ShaderCache.cpp @@ -1861,8 +1861,8 @@ namespace SIE void ShaderCache::Clear() { - std::lock_guard lockGuardV(vertexShadersMutex); { + std::lock_guard lockGuardV(vertexShadersMutex); for (auto& shaders : vertexShaders) { for (auto& [id, shader] : shaders) { shader->shader->Release(); @@ -1870,8 +1870,8 @@ namespace SIE shaders.clear(); } } - std::lock_guard lockGuardP(pixelShadersMutex); { + std::lock_guard lockGuardP(pixelShadersMutex); for (auto& shaders : pixelShaders) { for (auto& [id, shader] : shaders) { shader->shader->Release(); @@ -1879,8 +1879,8 @@ namespace SIE shaders.clear(); } } - std::lock_guard lockGuardC(computeShadersMutex); { + std::lock_guard lockGuardC(computeShadersMutex); for (auto& shaders : computeShaders) { for (auto& [id, shader] : shaders) { shader->shader->Release(); @@ -1888,9 +1888,103 @@ namespace SIE shaders.clear(); } } + { + std::unique_lock lockM{ mapMutex }; + shaderMap.clear(); + } + { + std::unique_lock lockH{ hlslMapMutex }; + hlslToShaderMap.clear(); + } compilationSet.Clear(); - std::unique_lock lock{ mapMutex }; - shaderMap.clear(); + } + + template + void ReleaseShader(ShaderType& shaders, + MutexType& mutex, RE::BSShader::Type type, uint32_t descriptor) + { + std::lock_guard lockGuard(mutex); + + if (static_cast(type) < shaders.size()) { + auto& shaderMap = shaders[static_cast(type)]; + auto shaderIt = shaderMap.find(descriptor); + if (shaderIt != shaderMap.end()) { + auto& shaderPtr = shaderIt->second; + if (shaderPtr && shaderPtr->shader) { + shaderPtr->shader->Release(); + } + shaderMap.erase(shaderIt); + } + } + } + bool ShaderCache::Clear(const std::string& a_path) + { + std::string lowerFilePath = Util::FixFilePath(a_path); + + // Step 1: Lock hlslMapMutex to find and copy the relevant entries + std::set entries; + { + std::unique_lock lockH{ hlslMapMutex }; + auto it = hlslToShaderMap.find(lowerFilePath); + + if (it == hlslToShaderMap.end()) { + return false; + } + + entries = it->second; // Copy the entries + hlslToShaderMap.erase(it); + } + + // Step 2: Process the copied entries without holding hlslMapMutex + for (auto& entry : entries) { + // Remove shader key from shaderMap + { + std::unique_lock lockM{ mapMutex }; + shaderMap.erase(entry.key); + } + + // Handle vertex, pixel, and compute shaders (each will lock) + switch (entry.shaderClass) { + case SIE::ShaderClass::Vertex: + ReleaseShader(vertexShaders, vertexShadersMutex, entry.type, entry.descriptor); + break; + case SIE::ShaderClass::Pixel: + ReleaseShader(pixelShaders, pixelShadersMutex, entry.type, entry.descriptor); + break; + case SIE::ShaderClass::Compute: + ReleaseShader(computeShaders, computeShadersMutex, entry.type, entry.descriptor); + break; + default: + logger::warn("Unexpected shader class: {}", static_cast(entry.shaderClass)); + break; + } + + // Delete the associated file + const auto& filePath = entry.diskPath; + const auto& filePathString = Util::WStringToString(filePath); + { + std::scoped_lock lockD{ compilationSet.compilationMutex }; + try { + if (std::filesystem::exists(filePath)) { + std::filesystem::remove(filePath); + logger::debug("Deleted {}", filePathString); + } + } catch (const std::exception& e) { + logger::warn("Failed to delete file {}: {}", filePathString, e.what()); + } catch (...) { + logger::warn("An unknown error occurred while trying to delete file '{}'", filePathString); + } + } + + logger::debug("Marking recompile for shader: {}", entry.key); + } + + if (!entries.empty()) { + logger::debug("Marked {} entries for recompile due to change to {}", entries.size(), a_path); + compilationSet.Clear(); + } + + return true; } void ShaderCache::Clear(RE::BSShader::Type a_type) @@ -1924,20 +2018,57 @@ namespace SIE { auto key = SIE::SShaderCache::GetShaderString(shaderClass, shader, descriptor, true); auto status = a_blob ? ShaderCompilationTask::Status::Completed : ShaderCompilationTask::Status::Failed; - std::unique_lock lock{ mapMutex }; logger::debug("Adding {} shader to map: {}", magic_enum ::enum_name(status), key); - shaderMap.insert_or_assign(key, ShaderCacheResult{ a_blob, status, system_clock::now() }); - return (bool)a_blob; + { + std::unique_lock lockM{ mapMutex }; + shaderMap.insert_or_assign(key, ShaderCacheResult{ a_blob, status, system_clock::now() }); + } + const std::wstring path = SIE::SShaderCache::GetShaderPath( + shader.shaderType == RE::BSShader::Type::ImageSpace ? + static_cast(shader).originalShaderName : + shader.fxpFilename); + auto pathString = Util::WStringToString(path); + if (a_blob) { // only create hlsl record if successful + std::string lowerFilePath = Util::FixFilePath(pathString); + { + std::unique_lock lockH{ hlslMapMutex }; + auto it = hlslToShaderMap.find(lowerFilePath); + hlslRecord newRecord{ key, shader.shaderType.get(), descriptor, shaderClass, SIE::SShaderCache::GetDiskPath(shader.fxpFilename, descriptor, shaderClass) }; + + if (it != hlslToShaderMap.end()) { + auto& entries = it->second; + + // Find and remove existing record with the same key + auto existingRecord = std::find_if(entries.begin(), entries.end(), + [&](const hlslRecord& r) { return r.key == key; }); + + if (existingRecord != entries.end()) { + entries.erase(existingRecord); // Remove the old record + } + + // Insert the new or updated record + entries.insert(newRecord); + } else { + // Create a new entry in hlslToShaderMap for this file path + hlslToShaderMap.emplace(lowerFilePath, std::set{ newRecord }); + } + } + } + + return a_blob != nullptr; } ID3DBlob* ShaderCache::GetCompletedShader(const std::string& a_key) { std::string type = SIE::SShaderCache::GetTypeFromShaderString(a_key); - UpdateShaderModifiedTime(a_key); - std::scoped_lock lock{ mapMutex }; + UpdateShaderModifiedTime(type); + std::scoped_lock lockM{ mapMutex }; if (!shaderMap.empty() && shaderMap.contains(a_key)) { if (ShaderModifiedSince(type, shaderMap.at(a_key).compileTime)) { - logger::debug("Shader {} compiled {} before changes at {}", a_key, std::format("{:%Y%m%d%H%M}", shaderMap.at(a_key).compileTime), std::format("{:%Y%m%d%H%M}", GetModifiedShaderMapTime(type))); + logger::debug("Shader {} compiled {} before changes at {}", + a_key, + std::format("{:%H:%M:%S}", shaderMap.at(a_key).compileTime), + std::format("{:%H:%M:%S}", GetModifiedShaderMapTime(type))); return nullptr; } auto status = shaderMap.at(a_key).status; @@ -1962,7 +2093,7 @@ namespace SIE ShaderCompilationTask::Status ShaderCache::GetShaderStatus(const std::string& a_key) { - std::scoped_lock lock{ mapMutex }; + std::scoped_lock lockM{ mapMutex }; if (!shaderMap.empty() && shaderMap.contains(a_key)) { return shaderMap.at(a_key).status; } @@ -2135,17 +2266,32 @@ namespace SIE } } - bool ShaderCache::UpdateShaderModifiedTime(std::string a_type) + bool ShaderCache::UpdateShaderModifiedTime(const std::string& a_type, boolean a_forceUpdate) { if (!UseFileWatcher()) return false; - if (a_type.empty() || !magic_enum::enum_cast(a_type, magic_enum::case_insensitive).has_value()) // type is invalid - return false; - std::filesystem::path filePath{ SIE::SShaderCache::GetShaderPath(a_type) }; + // Validate the shader type + if (a_type.empty() || !magic_enum::enum_cast(a_type, magic_enum::case_insensitive).has_value()) { + return false; // Invalid type + } + std::lock_guard lockGuard(modifiedMapMutex); + + // Check for force update + if (a_forceUpdate) { + // Set an artificial timestamp far in the future (100 years) + auto futureTime = std::chrono::system_clock::now() + std::chrono::hours(24 * 365 * 100); + modifiedShaderMap.insert_or_assign(a_type, futureTime); + return true; + } + + // Otherwise, update with the actual file time + std::filesystem::path filePath{ SIE::SShaderCache::GetShaderPath(a_type) }; if (std::filesystem::exists(filePath)) { auto fileTime = std::chrono::clock_cast(std::filesystem::last_write_time(filePath)); - if (!modifiedShaderMap.contains(a_type) || modifiedShaderMap.at(a_type) != fileTime) { // insert if new or timestamp changed + + // Update only if timestamp has changed + if (!modifiedShaderMap.contains(a_type) || modifiedShaderMap.at(a_type) != fileTime) { modifiedShaderMap.insert_or_assign(a_type, fileTime); return true; } @@ -2153,15 +2299,19 @@ namespace SIE return false; } - bool ShaderCache::ShaderModifiedSince(std::string a_type, system_clock::time_point a_current) + bool ShaderCache::ShaderModifiedSince(const std::string& a_type, std::chrono::system_clock::time_point a_current) { if (!UseFileWatcher()) return false; - if (a_type.empty() || !magic_enum::enum_cast(a_type, magic_enum::case_insensitive).has_value()) // type is invalid - return false; + // Validate the shader type + if (a_type.empty() || !magic_enum::enum_cast(a_type, magic_enum::case_insensitive).has_value()) { + return false; // Invalid type + } + std::lock_guard lockGuard(modifiedMapMutex); - return !modifiedShaderMap.empty() && modifiedShaderMap.contains(a_type) // map has Type - && modifiedShaderMap.at(a_type) > a_current; //modification time is newer than a_current + + // Check if the shader type exists in the map and if its modification time is newer than a_current + return !modifiedShaderMap.empty() && modifiedShaderMap.contains(a_type) && modifiedShaderMap.at(a_type) > a_current; } RE::BSGraphics::VertexShader* ShaderCache::MakeAndAddVertexShader(const RE::BSShader& shader, @@ -2286,7 +2436,7 @@ namespace SIE return hideError; } - void ShaderCache::InsertModifiedShaderMap(std::string a_shader, std::chrono::time_point a_time) + void ShaderCache::InsertModifiedShaderMap(const std::string& a_shader, std::chrono::time_point a_time) { std::lock_guard lockGuard(modifiedMapMutex); modifiedShaderMap.insert_or_assign(a_shader, a_time); @@ -2305,7 +2455,7 @@ namespace SIE void ShaderCache::IterateShaderBlock(bool a_forward) { - std::scoped_lock lock{ mapMutex }; + std::scoped_lock lockM{ mapMutex }; auto targetIndex = a_forward ? 0 : shaderMap.size() - 1; // default start or last element if (blockedKeyIndex >= 0 && shaderMap.size() > blockedKeyIndex) { // grab next element targetIndex = (blockedKeyIndex + (a_forward ? 1 : -1)) % shaderMap.size(); @@ -2493,24 +2643,47 @@ namespace SIE void UpdateListener::UpdateCache(const std::filesystem::path& filePath, SIE::ShaderCache& cache, bool& clearCache, bool& fileDone) { - std::string extension = filePath.extension().string(); - std::string parentDir = filePath.parent_path().string(); - std::string shaderTypeString = filePath.stem().string(); + // Extract file components + const std::string extension = filePath.extension().string(); + const std::string shaderTypeString = filePath.stem().string(); std::chrono::time_point modifiedTime{}; auto shaderType = magic_enum::enum_cast(shaderTypeString, magic_enum::case_insensitive); fileDone = true; - if (std::filesystem::exists(filePath)) + // Check if the file exists and get its modified time + if (std::filesystem::exists(filePath)) { modifiedTime = std::chrono::clock_cast(std::filesystem::last_write_time(filePath)); - else // if file doesn't exist, don't do anything + } else { + fileDone = true; return; - if (!std::filesystem::is_directory(filePath) && extension.starts_with(".hlsl") && parentDir.ends_with("Shaders") && shaderType.has_value()) { // TODO: Case insensitive checks - // Shader types, so only invalidate specific shader type (e.g,. Lighting) + } + + // Ensure the file is not a directory and is a valid shader file (.hlsl) + std::string lowerExtension = extension; + std::transform(lowerExtension.begin(), lowerExtension.end(), lowerExtension.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + if (!std::filesystem::is_directory(filePath) && lowerExtension == ".hlsl") { + // Update cache with the modified shader cache.InsertModifiedShaderMap(shaderTypeString, modifiedTime); - cache.Clear(shaderType.value()); - } else if (!std::filesystem::is_directory(filePath) && extension.starts_with(".hlsl")) { // TODO: Case insensitive checks - // all other shaders, since we don't know what is using it, clear everything - clearCache = true; + + // Attempt to mark the shader for recompilation + bool foundPath = cache.Clear(filePath.string()); + + if (!foundPath) { + // File was not found in the the map so check its shader type + std::string parentDirName = filePath.parent_path().filename().string(); + std::transform(parentDirName.begin(), parentDirName.end(), parentDirName.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + + // Check if the parent directory name matches "shaders" in a case-insensitive way + if (lowerExtension == ".hlsl" && parentDirName == "shaders" && shaderType.has_value()) { + cache.Clear(shaderType.value()); + } else { + // If it's not specifically handled, clear all cache + clearCache = true; + } + } } + // Indicate that file processing is not yet complete fileDone = false; } diff --git a/src/ShaderCache.h b/src/ShaderCache.h index 4aee03df2..0e1d633da 100644 --- a/src/ShaderCache.h +++ b/src/ShaderCache.h @@ -301,20 +301,53 @@ namespace SIE void StartFileWatcher(); void StopFileWatcher(); - /** @brief Update the RE::BSShader::Type timestamp based on timestamp. - @param a_type Case insensitive string for the type of shader. E.g., Lighting - @return True if the shader for the type (i.e., Lighting.hlsl) timestamp was updated - */ - bool UpdateShaderModifiedTime(std::string a_type); - /** @brief Whether the ShaderFile for RE::BSShader::Type has been modified since the timestamp. - @param a_type Case insensitive string for the type of shader. E.g., Lighting - @param a_current The current time in system_clock::time_point. - @return True if the shader for the type (i.e., Lighting.hlsl) has been modified since the timestamp - */ - bool ShaderModifiedSince(std::string a_type, system_clock::time_point a_current); + /** + * @brief Updates the shader modification time for the given shader type. + * + * This function checks if the shader's file modification time has changed or + * forces an update based on the a_forceUpdate flag. If the file does not exist, + * or the shader type is invalid, the update is skipped. + * + * @param a_type The shader type as a string (case insensitive). + * @param a_forceUpdate If true, forces an update regardless of the actual file modification time. + * @return true if the shader modification time was updated, false otherwise. + */ + bool UpdateShaderModifiedTime(const std::string& a_type, boolean a_forceUpdate = false); + /** + * @brief Checks if the shader has been modified since the given time. + * + * This function compares the shader's last modification time against the provided + * time point to determine if it has been updated. + * + * @param a_type The shader type as a string (case insensitive). + * @param a_current The time point to compare against. + * @return true if the shader has been modified after the given time point, false otherwise. + */ + bool ShaderModifiedSince(const std::string& a_type, system_clock::time_point a_current); void Clear(); void Clear(RE::BSShader::Type a_type); + /** + * @brief Clears and marks shaders for recompilation based on the given path. + * + * This function looks up the provided `a_path` in the `hlslToShaderMap`. + * If the path exists in the map, it iterates through all the shader entries associated + * with that path, clears the shaders, and marks them for recompilation by updating their + * modified times, and logs the operation. + * + * @param a_path The file path associated with the shaders to be marked for recompilation. + * + * @returns bool whether a shader was found in the `hlslToShaderMap` + * + * @note The function assumes that `a_path` corresponds to shaders stored in `hlslToShaderMap`. + * If the path is not found in the map, the function does nothing. Also, only files compiled + * during session will be identified. Disk cached shaders will not be cleared and a further + * cache clear may be necessary. + * + * @threadsafe The function locks the internal map (`mapMutex`) to ensure thread safety when + * accessing or modifying shared shader map data. + */ + bool Clear(const std::string& a_path); bool AddCompletedShader(ShaderClass shaderClass, const RE::BSShader& shader, uint32_t descriptor, ID3DBlob* a_blob); ID3DBlob* GetCompletedShader(const std::string& a_key); @@ -348,7 +381,7 @@ namespace SIE void IterateShaderBlock(bool a_forward = true); bool IsHideErrors(); - void InsertModifiedShaderMap(std::string a_shader, std::chrono::time_point a_time); + void InsertModifiedShaderMap(const std::string& a_shader, std::chrono::time_point a_time); std::chrono::time_point GetModifiedShaderMapTime(const std::string& a_shader); int32_t compilationThreadCount = std::max({ static_cast(std::thread::hardware_concurrency()) - 4, static_cast(std::thread::hardware_concurrency()) * 3 / 4, 1 }); @@ -554,6 +587,19 @@ namespace SIE HANDLE managementThread = nullptr; private: + struct hlslRecord + { + std::string key; + RE::BSShader::Type type; + std::uint32_t descriptor; + SIE::ShaderClass shaderClass; + std::wstring diskPath; + + bool operator<(const hlslRecord& other) const + { + return key < other.key; + } + }; ShaderCache(); void ManageCompilationSet(std::stop_token stoken); void ProcessCompilationSet(std::stop_token stoken, SIE::ShaderCompilationTask task); @@ -583,9 +629,11 @@ namespace SIE std::mutex computeShadersMutex; CompilationSet compilationSet; std::unordered_map shaderMap{}; - std::mutex mapMutex; + std::mutex mapMutex; // guard for shaderMap std::unordered_map modifiedShaderMap{}; // hashmap when a shader source file last modified - std::mutex modifiedMapMutex; + std::mutex modifiedMapMutex; // guard for modifiedShaderMap + std::unordered_map> hlslToShaderMap{}; // hashmap linking specific hlsl files to shader keys in shaderMap + std::mutex hlslMapMutex; // guard for hlslToShaderMap // efsw file watcher efsw::FileWatcher* fileWatcher = nullptr; @@ -597,6 +645,23 @@ namespace SIE class UpdateListener : public efsw::FileWatchListener { public: + /** + * @brief Updates the shader cache for a specific file path and determines whether to clear the cache. + * + * This function checks if the given file exists and is a shader file (with the ".hlsl" extension). + * It then updates the cache with the modified time for the shader file and marks shaders for recompilation + * based on the given path. If a specific shader is not found in the cache, it may trigger a cache clear. + * + * @param filePath The path of the shader file to update. + * @param cache Reference to the shader cache to update. + * @param clearCache A boolean flag indicating whether the entire cache should be cleared. + * @param fileDone A boolean flag that signals whether the update process is done for the current file. + * + * @note The function only processes files with an ".hlsl" extension and ignores directories. + * It assumes case-insensitive handling for shader types and extensions. + * + * @return Void. Updates internal state and modifies `clearCache` and `fileDone` by reference. + */ void UpdateCache(const std::filesystem::path& filePath, SIE::ShaderCache& cache, bool& clearCache, bool& retFlag); void processQueue(); void handleFileAction(efsw::WatchID, const std::string& dir, const std::string& filename, efsw::Action action, std::string) override; diff --git a/src/Util.cpp b/src/Util.cpp index b327d3200..d0e96114d 100644 --- a/src/Util.cpp +++ b/src/Util.cpp @@ -238,6 +238,25 @@ namespace Util return result; } + std::string FixFilePath(const std::string& a_path) + { + std::string lowerFilePath = a_path; + + // Replace all backslashes with forward slashes + std::replace(lowerFilePath.begin(), lowerFilePath.end(), '\\', '/'); + + // Remove consecutive forward slashes + std::string::iterator newEnd = std::unique(lowerFilePath.begin(), lowerFilePath.end(), + [](char a, char b) { return a == '/' && b == '/'; }); + lowerFilePath.erase(newEnd, lowerFilePath.end()); + + // Convert all characters to lowercase + std::transform(lowerFilePath.begin(), lowerFilePath.end(), lowerFilePath.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + + return lowerFilePath; + } + std::string WStringToString(const std::wstring& wideString) { std::string result; diff --git a/src/Util.h b/src/Util.h index 270bbdf83..cabcd074c 100644 --- a/src/Util.h +++ b/src/Util.h @@ -24,6 +24,18 @@ namespace Util ID3D11DeviceChild* CompileShader(const wchar_t* FilePath, const std::vector>& Defines, const char* ProgramType, const char* Program = "main"); std::string DefinesToString(const std::vector>& defines); std::string DefinesToString(const std::vector& defines); + /** + * @brief Normalizes a file path by replacing backslashes with forward slashes, + * removing redundant slashes, and converting all characters to lowercase. + * + * This function ensures that the file path uses consistent forward slashes + * (`/`), eliminates consecutive slashes (`//`), and converts all characters + * in the path to lowercase for case-insensitive comparisons. + * + * @param a_path The original file path to be normalized. + * @return A normalized file path as a lowercase string with single forward slashes. + */ + std::string FixFilePath(const std::string& a_path); std::string WStringToString(const std::wstring& wideString); float4 TryGetWaterData(float offsetX, float offsetY);