Reliably save openmp data.

libHook-c/CMakeLists.txt

@@ -2,7 +2,7 @@ project(Hook VERSION 0.1.7)
 
 # Set global (Relative this folder until reset) compiler options
 
-set(SCALER_HOOK_COMPILATION_FLAGS "-O4" "-g" "-Werror")
+set(SCALER_HOOK_COMPILATION_FLAGS "-O4" "-g" )
 # https://json.nlohmann.me/integration/cmake/
 
 
@@ -36,18 +36,18 @@ include(${CMAKE_SOURCE_DIR}/proto/findThisGrpc.cmake)
 
 # Scaler hook library itself. Users need to manually install scalerhook through library functions.
 add_library(ScalerHook-HookManualAsm-C SHARED
-        ${HookSrc})
-target_include_directories(ScalerHook-HookManualAsm-C PUBLIC src/include lib/inireader)
+        ${HookSrc} src/libcProxy.cpp)
+target_include_directories(ScalerHook-HookManualAsm-C PUBLIC   src/include lib/inireader)
 target_link_libraries(ScalerHook-HookManualAsm-C PUBLIC pthread dl)
 target_compile_options(ScalerHook-HookManualAsm-C PRIVATE ${SCALER_HOOK_COMPILATION_FLAGS})
-target_compile_definitions(ScalerHook-HookManualAsm-C PRIVATE CMAKE_SCALERRUN_VERSION="${PROJECT_VERSION}")
+target_compile_definitions(ScalerHook-HookManualAsm-C PRIVATE MANUAL_INSTALL CMAKE_SCALERRUN_VERSION="${PROJECT_VERSION}")
 
 
 #message(" ${PROTO_SRCS}  ${GRPC_SRCS}")
 
 
 #ScalerHook with auto installation enabled. Scaler hook will run automatically before main.
-add_library(ScalerHook-HookAutoAsm-C SHARED src/libcProxy.cpp src/libcProxy.cpp ${HookSrc})# ${PROTO_SRCS} ${GRPC_SRCS})
+add_library(ScalerHook-HookAutoAsm-C SHARED src/libcProxy.cpp ${HookSrc})# ${PROTO_SRCS} ${GRPC_SRCS})
 target_include_directories(ScalerHook-HookAutoAsm-C PUBLIC src/include lib/inireader)# ${PROTO_SRC_DIR})
 target_link_libraries(ScalerHook-HookAutoAsm-C PUBLIC pthread dl)
 target_link_options(ScalerHook-HookAutoAsm-C PRIVATE -static-libgcc -static-libstdc++) #Todo; Statically link libc is not advised. Change to re-compile in o

libHook-c/src/HookContext.cpp

@@ -1,13 +1,20 @@
 #include <util/hook/HookContext.h>
 #include <util/tool/Timer.h>
 #include <util/tool/FileTool.h>
+#include <cxxabi.h>
 
 extern "C" {
 
 HookContext *constructContext(ssize_t libFileSize, ssize_t hookedSymbolSize) {
 
     HookContext *rlt = new HookContext();
     rlt->timingArr = new scaler::Array<uint64_t>(hookedSymbolSize);
+    rlt->threadDataSavingLock = new pthread_mutex_t();
+
+    pthread_mutexattr_t Attr;
+    pthread_mutexattr_init(&Attr);
+    pthread_mutexattr_settype(&Attr, PTHREAD_MUTEX_RECURSIVE);
+    pthread_mutex_init(rlt->threadDataSavingLock, &Attr);
     rlt->timingArr->setSize(hookedSymbolSize);
 //    memArrayHeap(1), timingArr(hookedSymbolSize),
 //            indexPosi(0)
@@ -29,6 +36,7 @@ HookContext *constructContext(ssize_t libFileSize, ssize_t hookedSymbolSize) {
     assert(scaler::ExtFuncCallHook::instance != nullptr);
     rlt->_this = scaler::ExtFuncCallHook::instance;
 
+    rlt->threadId = pthread_self();
     saverElem.initializeMe = 1; //Force initialize tls
 
     return rlt;
@@ -49,6 +57,7 @@ bool initTLS() {
         return false;
     }
 
+
     //RuntimeInfo newInfo;
 
     return true;
@@ -59,15 +68,19 @@ __thread HookContext *curContext __attribute((tls_model("initial-exec")));
 __thread uint8_t bypassCHooks __attribute((tls_model("initial-exec"))) = SCALER_FALSE; //Anything that is not SCALER_FALSE should be treated as SCALER_FALSE
 
 DataSaver::~DataSaver() {
-    saveData();
+    saveData(curContext);
 }
 
-void saveData() {
+void saveData(HookContext *curContextPtr, bool finalize) {
     bypassCHooks = SCALER_TRUE;
-    HookContext *curContextPtr = curContext;
+    if (!curContextPtr) {
+        curContextPtr = curContext;
+    }
+    pthread_mutex_lock(curContextPtr->threadDataSavingLock);
 
     if (curContextPtr->dataSaved) {
         DBG_LOG("Data already saved for this thread");
+        pthread_mutex_unlock(curContextPtr->threadDataSavingLock);
         return;
     }
     curContextPtr->dataSaved = true;
@@ -82,7 +95,7 @@ void saveData() {
         return;
     }
     std::stringstream ss;
-    ss << scaler::ExtFuncCallHook::instance->folderName << "/threadTiming_" << pthread_self() << ".bin";
+    ss << scaler::ExtFuncCallHook::instance->folderName << "/threadTiming_" << curContextPtr->threadId << ".bin";
     //INFO_LOGS("Saving timing data to %s", ss.str().c_str());
     FILE *threadDataSaver = fopen(ss.str().c_str(), "wb");
     if (!threadDataSaver) {
@@ -113,7 +126,7 @@ void saveData() {
     }
     INFO_LOGS("Saving data to %s, %lu", scaler::ExtFuncCallHook::instance->folderName.c_str(), pthread_self());
 
-    if (curContextPtr->isMainThread) {
+    if (curContextPtr->isMainThread || finalize) {
         ss.str("");
         ss << scaler::ExtFuncCallHook::instance->folderName << "/realFileId.bin";
         //The real id of each function is resolved in after hook, so I can only save it in datasaver
@@ -133,10 +146,24 @@ void saveData() {
         if (!scaler::fClose<size_t>(fd, realFileIdSizeInBytes, realFileIdMem)) {
             fatalError("Cannot close file");
         }
-    }
 
+        INFO_LOG("Save data of all existing threads");
+        for (int i = 0; i < threadContextMap.getSize(); ++i) {
+            HookContext *threadContext = threadContextMap[i];
+            if (!threadContext->dataSaved) {
+                pthread_mutex_lock(threadContext->threadDataSavingLock);
+                INFO_LOGS("Thread data not saved, save it %d/%zd", i, threadContextMap.getSize());
+                saveData(threadContext);
+                pthread_mutex_unlock(threadContext->threadDataSavingLock);
+            } else {
+                INFO_LOGS("Thread data already saved, skip %d/%zd", i, threadContextMap.getSize());
+            }
+        }
+    }
 
     fclose(threadDataSaver);
+    pthread_mutex_unlock(curContextPtr->threadDataSavingLock);
+
 }
 
 }

libHook-c/src/include/util/hook/HookContext.h

@@ -4,6 +4,7 @@
 #include <util/datastructure/FStack.h>
 #include <cstdio>
 #include <type/InvocationTree.h>
+#include <util/tool/Timer.h>
 #include "ExtFuncCallHook.h"
 
 extern "C" {
@@ -36,12 +37,12 @@ struct HookContext {
     uint8_t pad5 = 0;
     uint64_t startTImestamp;
     uint64_t endTImestamp;
-    uint64_t *realFileIdArr = nullptr; //Used to record real id
+    pthread_mutex_t *threadDataSavingLock = nullptr; //Used to record real id
     //New cacheline
     //Variables used to determine whether it's called by hook handler or not
     HookTuple hookTuple[MAX_CALL_DEPTH]; //8bytes aligned
+    pthread_t threadId;
 };
-
 const uint8_t SCALER_TRUE = 145;
 const uint8_t SCALER_FALSE = 167;
 
@@ -53,7 +54,9 @@ class DataSaver {
     ~DataSaver();
 };
 
-void saveData();
+
+void saveData(HookContext *context,bool finalize=false);
+
 
 static thread_local DataSaver saverElem;
 
@@ -63,7 +66,12 @@ extern __thread uint8_t bypassCHooks; //Anything that is not SCALER_FALSE should
 
 extern scaler::SymID pthreadCreateSymId;
 
+extern scaler::Vector<HookContext *> threadContextMap;
+
+extern pthread_mutex_t threadDataSavingLock;
 
 bool initTLS();
+
+
 }
 #endif

libHook-c/src/include/util/hook/proxy/libcProxy.h

@@ -4,12 +4,19 @@
 extern "C" {
 
 typedef int (*main_fn_t)(int, char **, char **);
+#ifndef MANUAL_INSTALL
 
 int __libc_start_main(main_fn_t, int, char **, void (*)(), void (*)(), void (*)(), void *)
 __attribute__((weak, alias("doubletake_libc_start_main")));
 
+
 int doubletake_libc_start_main(main_fn_t main_fn, int argc, char **argv, void (*init)(), void (*fini)(),
                                void (*rtld_fini)(), void *stack_end);
 
+typedef int (*exit_origt)(int __status) ;
+void exit(int __status) __attribute__ ((noreturn));
+
+
+#endif
 }
 #endif //SCALER_LIBCPROXY_H

libHook-c/src/include/util/tool/Timer.h

@@ -3,7 +3,7 @@
 //
 
 #ifndef SCALER_TIME_H
-#define SCALER_TIMER_H
+#define SCALER_TIME_H
 
 #include <inttypes.h>
 

libHook-c/src/libcProxy.cpp

@@ -10,10 +10,13 @@
 #include <util/hook/ExtFuncCallHook.h>
 #include <util/hook/HookContext.h>
 #include <util/hook/ExtFuncCallHook.h>
+#include <cxxabi.h>
 
 main_fn_t real_main;
 
 extern "C" {
+scaler::Vector<HookContext *> threadContextMap;
+#ifndef MANUAL_INSTALL
 
 int doubletake_main(int argc, char **argv, char **envp) {
     INFO_LOGS("libHook-c Ver %s", CMAKE_SCALERRUN_VERSION);
@@ -30,18 +33,24 @@ int doubletake_main(int argc, char **argv, char **envp) {
     scaler::ExtFuncCallHook::getInst(ss.str())->install();
     //Calculate the main application time
 
+
     HookContext *curContextPtr = curContext;
     curContextPtr->curFileId = 0;
     curContextPtr->endTImestamp = 0;
     curContextPtr->startTImestamp = getunixtimestampms();
     curContextPtr->isMainThread = true;
+
+    /**
+     * Register this thread with the main thread
+     */
+    threadContextMap.pushBack(curContextPtr);
+
     int ret = real_main(argc, argv, envp);
     curContextPtr->endTImestamp = getunixtimestampms();
-    saveData();
+    saveData(curContextPtr);
     return ret;
 }
 
-
 int doubletake_libc_start_main(main_fn_t main_fn, int argc, char **argv, void (*init)(), void (*fini)(),
                                void (*rtld_fini)(), void *stack_end) {
     using namespace scaler;
@@ -57,4 +66,14 @@ int doubletake_libc_start_main(main_fn_t main_fn, int argc, char **argv, void (*
 
     return real_libc_start_main(doubletake_main, argc, argv, init, fini, rtld_fini, stack_end);
 }
+
+void exit(int __status) {
+    auto realExit = (exit_origt) dlsym(RTLD_NEXT, "exit");
+    curContext->endTImestamp = getunixtimestampms();
+    saveData(curContext, true);
+    realExit(__status);
+}
+
+
+#endif
 }

libHook-c/src/pthreadProxy.cpp

@@ -43,13 +43,19 @@ void *dummy_thread_function(void *data) {
     assert(curContextPtr != NULL);
     curContextPtr->curFileId = curContextPtr->_this->pmParser.findExecNameByAddr(
             (void *) actualFuncPtr);
+
+    /**
+     * Register this thread with the main thread
+     */
+    threadContextMap.pushBack(curContextPtr);
+
     curContextPtr->startTImestamp = getunixtimestampms();
     actualFuncPtr(argData);
     /**
      * Perform required actions after each thread function completes
      */
     curContextPtr->endTImestamp = getunixtimestampms();
-    saveData();
+    saveData(curContextPtr);
     return nullptr;
 }