Skip to content

Commit

Permalink
filesystem: got a Idea for initial faster times
Browse files Browse the repository at this point in the history
  • Loading branch information
@RaphaelIT7
RaphaelIT7 committed Nov 25, 2024
1 parent 9813481 commit f55c581
Showing 1 changed file with 176 additions and 7 deletions.
183 changes: 176 additions & 7 deletions source/modules/filesystem.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,14 @@ static ConVar holylib_filesystem_usesearchpathcache("holylib_filesystem_usesearc

static ConVar holylib_filesystem_precachehandle("holylib_filesystem_precachehandle", "1", 0,
"If enabled, it will try to predict which file it will open next and open the file to keep a handle ready to be opened.");
static ConVar holylib_filesystem_savesearchcache("holylib_filesystem_savesearchcache", "1", 0,
"If enabled, it will write the search cache into a file and restore it when starting, using it to improve performance.");


// Optimization Idea: When Gmod calls GetFileTime, we could try to get the filehandle in parallel to have it ready when gmod calls it.
// We could also cache every FULL searchpath to not have to look up a file every time.

IThreadPool* pFileSystemPool = NULL;
static IThreadPool* pFileSystemPool = NULL;

static void OnThreadsChange(IConVar* convar, const char* pOldValue, float flOldValue)
{
Expand Down Expand Up @@ -199,7 +201,7 @@ static void AddFileToSearchCache(const char* pFileName, int path, const char* pa
pathID = nullPath;

if (g_pFileSystemModule.InDebug())
Msg("holylib - AddFileToSearchCache: Added file %s to seach cache (%i, %s)\n", pFileName, path, pathID);
Msg("holylib - AddFileToSearchCache: Added file %s to seach cache (%i, %i, %s)\n", pFileName, path, pathID);

char* cFileName = new char[MAX_PATH];
V_strncpy(cFileName, pFileName, MAX_PATH);
Expand Down Expand Up @@ -249,6 +251,96 @@ static void NukeSearchCache() // NOTE: We actually never nuke it :D
m_SearchCache.clear(); // Now causes a memory leak :D (Should I try to solve it? naaaaaa :^)
}

struct SaveCacheEntry {
char pathID[24];
char path[128];
char absolutePath[255];
};

#define MaxSaveCacheEntries (1 << 14)
struct SaveCache {
unsigned int version = 1;
unsigned int usedPaths = 0;
SaveCacheEntry paths[MaxSaveCacheEntries]; // Support 16k entries for now
};

static void WriteSaveCache()
{
FileHandle_t handle = g_pFullFileSystem->Open("holylib_searchcache.dat", "wb", "MOD_WRITE");
if (handle)
{
SaveCache* savecache = new SaveCache;
for (auto& [strPath, cache] : m_SearchCache)
{
for (auto& [strEntry, storeID] : cache)
{
if (savecache->usedPaths >= MaxSaveCacheEntries)
break;

SaveCacheEntry& entry = savecache->paths[savecache->usedPaths++];
V_strcpy(entry.pathID, strPath.data());
V_strcpy(entry.path, strEntry.data());
g_pFullFileSystem->RelativePathToFullPath(strEntry.data(), strPath.data(), entry.absolutePath, sizeof(entry.absolutePath));
}
}

g_pFullFileSystem->Write(savecache, sizeof(SaveCache), handle);
g_pFullFileSystem->Close(handle);
Msg("holylib: successfully wrote searchcache file (%i)\n", savecache->usedPaths);
delete savecache;
}
else {
Warning("holylib: Failed to open searchcache file!\n");
}
}

static std::unordered_map<std::string_view, std::string_view> g_pAbsoluteSearchCache;
inline std::string_view* GetStringFromAbsoluteCache(std::string_view fileName)
{
auto it = g_pAbsoluteSearchCache.find(fileName);
if (it == g_pAbsoluteSearchCache.end())
return NULL;

return &it->second;
}

static void ReadSearchCache()
{
FileHandle_t handle = g_pFullFileSystem->Open("holylib_searchcache.dat", "rb", "MOD_WRITE");
if (handle)
{
SaveCache* savecache = new SaveCache;
g_pFullFileSystem->Read(savecache, sizeof(SaveCache), handle);
g_pFullFileSystem->Close(handle);
for (int i = 0; i < savecache->usedPaths; ++i)
{
SaveCacheEntry& pEntry = savecache->paths[i];

char* path = new char[128];
V_strcpy(path, pEntry.path);
std::string_view pathStr = path; // NOTE: We have to manually free it later

char* absolutePath = new char[255];
V_strcpy(absolutePath, pEntry.absolutePath);
std::string_view absolutePathStr = absolutePath; // NOTE: We have to manually free it later
g_pAbsoluteSearchCache[pathStr] = absolutePathStr;
}

for (auto& [key, val] : g_pAbsoluteSearchCache)
{
Msg("Key: %s\nValue: %s\n", key.data(), val.data());
}

//if (g_pFileSystemModule.InDebug())
Msg("holylib - filesystem: Loaded searchcache file (%i)\n", savecache->usedPaths);
delete savecache;
}
else {
if (g_pFileSystemModule.InDebug())
Msg("holylib - filesystem: Failed to find searchcache file\n");
}
}

static void DumpSearchcacheCmd(const CCommand &args)
{
Msg("---- Search cache ----\n");
Expand Down Expand Up @@ -312,6 +404,29 @@ static void ShowPredictionErrosCmd(const CCommand &args)
}
static ConCommand showpredictionerrors("holylib_filesystem_showpredictionerrors", ShowPredictionErrosCmd, "Shows all prediction errors that ocurred", 0);

static void WriteSaveCacheCMD(const CCommand& args)
{
WriteSaveCache();
}
static ConCommand writesearchcache("holylib_filesystem_writesearchcache", WriteSaveCacheCMD, "Writes the save cache into a file", 0);

static bool bInit = false;
static void InitFileSystem(IFileSystem* pFileSystem)
{
if (!pFileSystem)
return;

g_pFullFileSystem = pFileSystem;

if (holylib_filesystem_savesearchcache.GetBool())
{
ReadSearchCache();
}

if (g_pFileSystemModule.InDebug())
Msg("holylib - filesystem: Initialized filesystem\n");
}

inline void OnFileHandleOpen(FileHandle_t handle, const char* pFileMode)
{
if (pFileMode[0] == 'r') // I see a potential crash, but this should never happen.
Expand All @@ -327,7 +442,7 @@ static FileHandle_t hook_CBaseFileSystem_FindFileInSearchPath(void* filesystem,
return detour_CBaseFileSystem_FindFileInSearchPath.GetTrampoline<Symbols::CBaseFileSystem_FindFileInSearchPath>()(filesystem, openInfo);

if (!g_pFullFileSystem)
g_pFullFileSystem = (IFileSystem*)filesystem;
InitFileSystem((IFileSystem*)filesystem);

VPROF_BUDGET("HolyLib - CBaseFileSystem::FindFile", VPROF_BUDGETGROUP_OTHER_FILESYSTEM);

Expand Down Expand Up @@ -396,7 +511,7 @@ static long hook_CBaseFileSystem_FastFileTime(void* filesystem, const CSearchPat
return detour_CBaseFileSystem_FastFileTime.GetTrampoline<Symbols::CBaseFileSystem_FastFileTime>()(filesystem, path, pFileName);

if (!g_pFullFileSystem)
g_pFullFileSystem = (IFileSystem*)filesystem;
InitFileSystem((IFileSystem*)filesystem);

VPROF_BUDGET("HolyLib - CBaseFileSystem::FastFileTime", VPROF_BUDGETGROUP_OTHER_FILESYSTEM);

Expand Down Expand Up @@ -453,7 +568,7 @@ static bool hook_CBaseFileSystem_FixUpPath(IFileSystem* filesystem, const char *
VPROF_BUDGET("HolyLib - CBaseFileSystem::FixUpPath", VPROF_BUDGETGROUP_OTHER_FILESYSTEM);

if (!g_pFullFileSystem)
g_pFullFileSystem = (IFileSystem*)filesystem;
InitFileSystem((IFileSystem*)filesystem);

V_strncpy( pFixedUpFileName, pFileName, sizeFixedUpFileName );
V_FixSlashes( pFixedUpFileName, CORRECT_PATH_SEPARATOR );
Expand Down Expand Up @@ -610,6 +725,27 @@ static FileHandle_t hook_CBaseFileSystem_OpenForRead(CBaseFileSystem* filesystem

hook_CBaseFileSystem_FixUpPath(filesystem, pFileNameT, pFileNameBuff, sizeof(pFileNameBuff));

if (holylib_filesystem_savesearchcache.GetBool())
{
std::string_view* absoluteStr = GetStringFromAbsoluteCache(pFileName);
if (absoluteStr)
{
if (g_pFileSystemModule.InDebug())
Msg("holylib - OpenForRead: Found file in absolute path (%s, %s)\n", pFileName, absoluteStr->data());

FileHandle_t handle = detour_CBaseFileSystem_OpenForRead.GetTrampoline<Symbols::CBaseFileSystem_OpenForRead>()(filesystem, absoluteStr->data(), pOptions, flags, pathID, ppszResolvedFilename);
if (handle)
return handle;

if (g_pFileSystemModule.InDebug())
Msg("holylib - OpenForRead: Invalid absolute path! (%s, %s)\n", pFileName, absoluteStr->data());
}
else {
if (g_pFileSystemModule.InDebug())
Msg("holylib - OpenForRead: Failed to find file in absolute path (%s, %s)\n", pFileName, pFileNameT);
}
}

bool splitPath = false;
const char* origPath = pathID;
const char* newPath = GetOverridePath(pFileName, pathID);
Expand Down Expand Up @@ -889,6 +1025,27 @@ static long hook_CBaseFileSystem_GetFileTime(IFileSystem* filesystem, const char
if (!newPath && strFileName.rfind("gamemodes/terrortown") == 0)
newPath = "MOD_WRITE";

if (holylib_filesystem_savesearchcache.GetBool())
{
std::string_view* absoluteStr = GetStringFromAbsoluteCache(pFileName);
if (absoluteStr)
{
if (g_pFileSystemModule.InDebug())
Msg("holylib - GetFileTime: Found file in absolute path (%s, %s)\n", pFileName, absoluteStr->data());

// We pass it a absolute path which will be used in ::FastFileTime
long time = detour_CBaseFileSystem_GetFileTime.GetTrampoline<Symbols::CBaseFileSystem_GetFileTime>()(filesystem, absoluteStr->data(), pPathID);
if (time != 0L)
return time;

if (g_pFileSystemModule.InDebug())
Msg("holylib - GetFileTime: Invalid absolute path? (%s, %s)\n", pFileName, absoluteStr->data());
} else {
if (g_pFileSystemModule.InDebug())
Msg("holylib - GetFileTime: Failed to find file in absolute path (%s)\n", pFileName);
}
}

if (newPath)
{
long time = detour_CBaseFileSystem_GetFileTime.GetTrampoline<Symbols::CBaseFileSystem_GetFileTime>()(filesystem, pFileName, pPathID);
Expand Down Expand Up @@ -1263,6 +1420,9 @@ void CFileSystemModule::Init(CreateInterfaceFn* appfn, CreateInterfaceFn* gamefn
std::string workshopDir = pBaseDir;
workshopDir.append("garrysmod/workshop");

if (g_pFullFileSystem != NULL)
InitFileSystem(g_pFullFileSystem);

if (!DETOUR_ISVALID(detour_CBaseFileSystem_AddSearchPath))
{
Msg("holylib: CBaseFileSystem::AddSearchPath detour is invalid?\n");
Expand Down Expand Up @@ -1341,7 +1501,10 @@ inline const char* CPathIDInfo::GetPathIDString() const

inline const char* CSearchPath::GetPathIDString() const
{
return m_pPathIDInfo->GetPathIDString(); // When can we nuke it :>
if (m_pPathIDInfo)
return m_pPathIDInfo->GetPathIDString(); // When can we nuke it :>

return NULL;
}

static Symbols::CBaseFileSystem_CSearchPath_GetDebugString func_CBaseFileSystem_CSearchPath_GetDebugString;
Expand All @@ -1357,7 +1520,10 @@ inline const char* CBaseFileSystem::CPathIDInfo::GetPathIDString() const

inline const char* CBaseFileSystem::CSearchPath::GetPathIDString() const // Remove this duplicate later :<
{
return m_pPathIDInfo->GetPathIDString(); // When can we nuke it :>
if (m_pPathIDInfo)
return m_pPathIDInfo->GetPathIDString(); // When can we nuke it :>

return NULL;
}

inline const char* CBaseFileSystem::CSearchPath::GetPathString() const
Expand All @@ -1373,6 +1539,9 @@ void CFileSystemModule::InitDetour(bool bPreServer)
pFileSystemPool = V_CreateThreadPool();
Util::StartThreadPool(pFileSystemPool, holylib_filesystem_threads.GetInt());

if (g_pFullFileSystem != NULL)
InitFileSystem(g_pFullFileSystem);

// ToDo: Redo EVERY Hook so that we'll abuse the vtable instead of symbols.
// Use the ClassProxy or so which should also allow me to port this to windows.
SourceSDK::ModuleLoader dedicated_loader("dedicated");
Expand Down

0 comments on commit f55c581

Please sign in to comment.