Merge branch 'master' into nacho/updateRuleset1.13.2

DataDog · Oct 31, 2024 · 7cf7e23 · 7cf7e23
2 parents c4edd27 + f9cf180
commit 7cf7e23
Show file tree

Hide file tree

Showing 104 changed files with 12,119 additions and 446 deletions.
diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml
@@ -2899,8 +2899,7 @@ stages:
         jobs: [Test, DockerTest]
 
     - job: Test
-      timeoutInMinutes: 60 #default value
-      # TODO: move this to GenerateVariables stage
+      timeoutInMinutes: 90 #default value
       strategy:
         matrix: $[stageDependencies.generate_variables.generate_variables_job.outputs['generate_variables_step.integration_tests_linux_arm64_matrix']]
       workspace:

diff --git a/.gitignore b/.gitignore
@@ -363,6 +363,7 @@ profiler/_build/
 obj/*
 obj_arm64/*
 obj_x86_64/*
+obj_arm64_x86_64/*
 
 # cmake build files
 .ionide/

diff --git a/Datadog.Trace.OSX.slnf b/Datadog.Trace.OSX.slnf
@@ -162,7 +162,6 @@
       "tracer\\test\\test-applications\\regression\\Sandbox.AutomaticInstrumentation\\Sandbox.AutomaticInstrumentation.csproj",
       "tracer\\test\\test-applications\\regression\\Sandbox.ManualTracing\\Sandbox.ManualTracing.csproj",
       "tracer\\test\\test-applications\\regression\\ServiceBus.Minimal.MassTransit\\ServiceBus.Minimal.MassTransit.csproj",
-      "tracer\\test\\test-applications\\regression\\ServiceBus.Minimal.NServiceBus\\ServiceBus.Minimal.NServiceBus.csproj",
       "tracer\\test\\test-applications\\regression\\ServiceBus.Minimal.Rebus\\ServiceBus.Minimal.Rebus.csproj",
       "tracer\\test\\test-applications\\regression\\StackExchange.Redis.AssemblyConflict.LegacyProject\\StackExchange.Redis.AssemblyConflict.LegacyProject.csproj",
       "tracer\\test\\test-applications\\regression\\StackExchange.Redis.AssemblyConflict.SdkProject\\StackExchange.Redis.AssemblyConflict.SdkProject.csproj",

diff --git a/build/cmake/FindGlibcCompat.cmake b/build/cmake/FindGlibcCompat.cmake
@@ -1,9 +1,13 @@
-add_library(glibc-compat OBJECT
-    ${DOTNET_TRACER_REPO_ROOT_PATH}/shared/src/glibc-compat/glibc-compat.c
-)
+if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
+    add_library(glibc-compat INTERFACE)
+else()
+    add_library(glibc-compat OBJECT
+            ${DOTNET_TRACER_REPO_ROOT_PATH}/shared/src/glibc-compat/glibc-compat.c
+    )
 
-set_target_properties(glibc-compat PROPERTIES POSITION_INDEPENDENT_CODE 1)
+    set_target_properties(glibc-compat PROPERTIES POSITION_INDEPENDENT_CODE 1)
 
-target_compile_options(glibc-compat PUBLIC
-    -std=c11
-)
+    target_compile_options(glibc-compat PUBLIC
+            -std=c11
+    )
+endif()
diff --git a/build/cmake/FindManagedLoader.cmake b/build/cmake/FindManagedLoader.cmake
@@ -5,8 +5,18 @@ SET(MANAGED_LOADER_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/tracer/src/bin/Profiler
 
 # Set specific custom commands to embed the loader
 if (ISMACOS)
-    SET(PDB_COMMAND touch stub.c && gcc -o stub.o -c stub.c -target ${OSX_ARCH}-apple-darwin${CMAKE_HOST_SYSTEM_VERSION} && ld -r -o Datadog.Trace.ClrProfiler.Managed.Loader.pdb.o -sectcreate binary pdb Datadog.Trace.ClrProfiler.Managed.Loader.pdb stub.o)
-    SET(DLL_COMMAND touch stub.c && gcc -o stub.o -c stub.c -target ${OSX_ARCH}-apple-darwin${CMAKE_HOST_SYSTEM_VERSION} && ld -r -o Datadog.Trace.ClrProfiler.Managed.Loader.dll.o -sectcreate binary dll Datadog.Trace.ClrProfiler.Managed.Loader.dll stub.o)
+    SET(PDB_COMMAND touch stub.c &&
+            gcc -o stub.arm64.o -c stub.c -target arm64-apple-darwin${CMAKE_HOST_SYSTEM_VERSION} &&
+            ld -r -o Datadog.Trace.ClrProfiler.Managed.Loader.pdb.arm64.o -sectcreate binary pdb Datadog.Trace.ClrProfiler.Managed.Loader.pdb stub.arm64.o &&
+            gcc -o stub.x86_64.o -c stub.c -target x86_64-apple-darwin${CMAKE_HOST_SYSTEM_VERSION} &&
+            ld -r -o Datadog.Trace.ClrProfiler.Managed.Loader.pdb.x86_64.o -sectcreate binary pdb Datadog.Trace.ClrProfiler.Managed.Loader.pdb stub.x86_64.o &&
+            lipo Datadog.Trace.ClrProfiler.Managed.Loader.pdb.arm64.o Datadog.Trace.ClrProfiler.Managed.Loader.pdb.x86_64.o -create -output Datadog.Trace.ClrProfiler.Managed.Loader.pdb.o)
+    SET(DLL_COMMAND touch stub.c &&
+            gcc -o stub.arm64.o -c stub.c -target arm64-apple-darwin${CMAKE_HOST_SYSTEM_VERSION} &&
+            ld -r -o Datadog.Trace.ClrProfiler.Managed.Loader.dll.arm64.o -sectcreate binary dll Datadog.Trace.ClrProfiler.Managed.Loader.dll stub.arm64.o &&
+            gcc -o stub.x86_64.o -c stub.c -target x86_64-apple-darwin${CMAKE_HOST_SYSTEM_VERSION} &&
+            ld -r -o Datadog.Trace.ClrProfiler.Managed.Loader.dll.x86_64.o -sectcreate binary dll Datadog.Trace.ClrProfiler.Managed.Loader.dll stub.x86_64.o &&
+            lipo Datadog.Trace.ClrProfiler.Managed.Loader.dll.arm64.o Datadog.Trace.ClrProfiler.Managed.Loader.dll.x86_64.o -create -output Datadog.Trace.ClrProfiler.Managed.Loader.dll.o)
 elseif(ISLINUX)
     SET(DLL_COMMAND ld -r -b binary -o Datadog.Trace.ClrProfiler.Managed.Loader.dll.o Datadog.Trace.ClrProfiler.Managed.Loader.dll)
     SET(PDB_COMMAND ld -r -b binary -o Datadog.Trace.ClrProfiler.Managed.Loader.pdb.o Datadog.Trace.ClrProfiler.Managed.Loader.pdb)

diff --git a/build/cmake/FindRe2.cmake b/build/cmake/FindRe2.cmake
@@ -5,14 +5,26 @@ SET(RE2_VERSION "2018-10-01")
 set (DOWNLOAD_COMMAND ${CMAKE_COMMAND} -DPROJECT_NAME=re2 -DPROJECT_REPOSITORY=https://github.com/google/re2.git -DPROJECT_BRANCH=${RE2_VERSION} -P ${CMAKE_SOURCE_DIR}/build/cmake/git-clone-quiet-once.cmake)
 
 if (ISMACOS)
+    SET (OSXRE2BUILDCOMMAND
+            echo "Building Re2 Arm64" &&
+            rm -f -r ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/obj &&
+            ${CMAKE_COMMAND} -E env LDFLAGS=-arch\ arm64 ARFLAGS=-r\ -s\ -c CXXFLAGS=-O3\ -g\ -fPIC\ -target\ arm64-apple-darwin${CMAKE_HOST_SYSTEM_VERSION}\ -Wno-unused-but-set-variable\ -D_GLIBCXX_USE_CXX11_ABI=0 $(MAKE) -j &&
+            mv ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/obj/libre2.a ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/libre2.arm64.a &&
+            echo "Building Re2 X86_64" &&
+            rm -f -r ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/obj &&
+            ${CMAKE_COMMAND} -E env LDFLAGS=-arch\ x86_64 ARFLAGS=-r\ -s\ -c CXXFLAGS=-O3\ -g\ -fPIC\ -target\ x86_64-apple-darwin${CMAKE_HOST_SYSTEM_VERSION}\ -Wno-unused-but-set-variable\ -D_GLIBCXX_USE_CXX11_ABI=0 $(MAKE) -j &&
+            mv ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/obj/libre2.a ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/libre2.x86_64.a &&
+            echo "Creating Re2 universal binary" &&
+            lipo ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/libre2.arm64.a ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/libre2.x86_64.a -create -output ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/obj/libre2.a
+    )
     ExternalProject_Add(re2
         DOWNLOAD_COMMAND ${DOWNLOAD_COMMAND}
         TIMEOUT 5
         INSTALL_COMMAND ""
         CONFIGURE_COMMAND ""
         UPDATE_COMMAND ""
         BUILD_IN_SOURCE TRUE
-        BUILD_COMMAND ${CMAKE_COMMAND} -E env LDFLAGS=-arch\ ${OSX_ARCH} ARFLAGS=-r\ -s\ -c CXXFLAGS=-O3\ -g\ -fPIC\ -target\ ${OSX_ARCH}-apple-darwin${CMAKE_HOST_SYSTEM_VERSION}\ -Wno-unused-but-set-variable\ -D_GLIBCXX_USE_CXX11_ABI=0 $(MAKE) -j
+        BUILD_COMMAND ${OSXRE2BUILDCOMMAND}
         BUILD_BYPRODUCTS ${CMAKE_CURRENT_BINARY_DIR}/re2-prefix/src/re2/obj/libre2.a
     )
 elseif(ISLINUX)

diff --git a/profiler/src/ProfilerEngine/Datadog.Profiler.Native.Windows/CrashReportingWindows.cpp b/profiler/src/ProfilerEngine/Datadog.Profiler.Native.Windows/CrashReportingWindows.cpp
@@ -22,6 +22,7 @@ CrashReporting* CrashReporting::Create(int32_t pid)
 CrashReportingWindows::CrashReportingWindows(int32_t pid)
     : CrashReporting(pid)
     , _process(NULL)
+    , _readMemory(nullptr)
 {
 }
 
@@ -40,6 +41,8 @@ int32_t CrashReportingWindows::Initialize()
             return 1;
         }
 
+        SetMemoryReader([this](uintptr_t address, SIZE_T size) { return ReadRemoteMemory(_process, address, size); });
+
         SymInitialize(_process, nullptr, TRUE);
 
         _modules = GetModules();
@@ -74,14 +77,14 @@ std::vector<std::pair<int32_t, std::string>> CrashReportingWindows::GetThreads()
                 {
                     std::string threadName;
                     PWSTR description;
-                    
+
                     if (SUCCEEDED(GetThreadDescription(thread, &description)))
                     {
                         threadName = shared::ToString(description);
                     }
 
                     threads.push_back({ threadEntry.th32ThreadID, std::move(threadName) });
-                }                
+                }
             }
         } while (Thread32Next(threadSnapshot, &threadEntry));
     }
@@ -109,7 +112,7 @@ std::vector<StackFrame> CrashReportingWindows::GetThreadFrames(int32_t tid, Reso
         {
             auto const& managedFrame = managedCallstack[i];
 
-            StackFrame stackFrame;
+            StackFrame stackFrame{};
             stackFrame.ip = managedFrame.ip;
             stackFrame.sp = managedFrame.sp;
             stackFrame.method = std::string(managedFrame.symbolName);
@@ -163,34 +166,45 @@ std::vector<StackFrame> CrashReportingWindows::GetThreadFrames(int32_t tid, Reso
             nullptr,
             SYM_STKWALK_DEFAULT))
         {
-            auto [moduleName, moduleAddress] = FindModule(nativeStackFrame.AddrPC.Offset);
+            auto module = FindModule(nativeStackFrame.AddrPC.Offset);
 
-            StackFrame stackFrame;
+            StackFrame stackFrame{};
             stackFrame.ip = nativeStackFrame.AddrPC.Offset;
             stackFrame.sp = nativeStackFrame.AddrStack.Offset;
             stackFrame.isSuspicious = false;
-            stackFrame.moduleAddress = moduleAddress;
             stackFrame.symbolAddress = nativeStackFrame.AddrPC.Offset;
 
-            std::ostringstream methodName;
-            methodName << moduleName << "!<unknown>+" << std::hex << (nativeStackFrame.AddrPC.Offset - moduleAddress);
-            stackFrame.method = methodName.str();
+            if (module != nullptr)
+            {
+                stackFrame.moduleAddress = module->startAddress;
+                stackFrame.hasPdbInfo = module->hasPdbInfo;
+                stackFrame.pdbAge = module->pdbAge;
+                stackFrame.pdbSig = module->pdbSig;
 
-            fs::path modulePath(moduleName);
+                std::ostringstream methodName;
+                methodName << module->path << "!<unknown>+" << std::hex << (nativeStackFrame.AddrPC.Offset - module->startAddress);
+                stackFrame.method = methodName.str();
 
-            if (modulePath.has_filename())
-            {
-                const auto moduleFilename = modulePath.stem().string();
+                fs::path modulePath(module->path);
 
-                if (moduleFilename.rfind("Datadog", 0) == 0
-                    || moduleFilename == "libdatadog"
-                    || moduleFilename == "datadog"
-                    || moduleFilename == "libddwaf"
-                    || moduleFilename == "ddwaf")
+                if (modulePath.has_filename())
                 {
-                    stackFrame.isSuspicious = true;
+                    const auto moduleFilename = modulePath.stem().string();
+
+                    if (moduleFilename.rfind("Datadog", 0) == 0
+                        || moduleFilename == "libdatadog"
+                        || moduleFilename == "datadog"
+                        || moduleFilename == "libddwaf"
+                        || moduleFilename == "ddwaf")
+                    {
+                        stackFrame.isSuspicious = true;
+                    }
                 }
             }
+            else
+            {
+                stackFrame.method = "<unknown>";
+            }
 
             frames.push_back(std::move(stackFrame));
         }
@@ -225,23 +239,165 @@ std::vector<ModuleInfo> CrashReportingWindows::GetModules()
                     resolvedModuleName = moduleName;
                 }
 
-                modules.push_back({ (uintptr_t)moduleInfo.lpBaseOfDll, (uintptr_t)moduleInfo.lpBaseOfDll + moduleInfo.SizeOfImage, std::move(resolvedModuleName) });
+                ModuleInfo module{ (uintptr_t)moduleInfo.lpBaseOfDll, (uintptr_t)moduleInfo.lpBaseOfDll + moduleInfo.SizeOfImage, std::move(resolvedModuleName), false, 0, 0 };
+
+                FillPdbInfo((uintptr_t)moduleInfo.lpBaseOfDll, module);
+
+                modules.push_back(std::move(module));
             }
         }
     }
 
     return modules;
 }
 
-std::pair<std::string_view, uintptr_t> CrashReportingWindows::FindModule(uintptr_t ip)
+const ModuleInfo* CrashReportingWindows::FindModule(uintptr_t ip)
 {
     for (auto const& module : _modules)
     {
         if (ip >= module.startAddress && ip < module.endAddress)
         {
-            return std::make_pair(module.path, module.startAddress);
+            return &module;
+        }
+    }
+
+    return nullptr;
+}
+
+std::vector<BYTE> CrashReportingWindows::ReadRemoteMemory(HANDLE process, uintptr_t address, SIZE_T size)
+{
+    std::vector<BYTE> buffer(size);
+    SIZE_T bytesRead = 0;
+
+    if (ReadProcessMemory(process, reinterpret_cast<LPCVOID>(address), buffer.data(), size, &bytesRead) && bytesRead == size)
+    {
+        return buffer;
+    }
+
+    return {};
+}
+
+bool CrashReportingWindows::FillPdbInfo(uintptr_t baseAddress, ModuleInfo& moduleInfo)
+{
+    // Read the DOS header
+    auto dosHeaderBuffer = _readMemory(baseAddress, sizeof(IMAGE_DOS_HEADER));
+    if (dosHeaderBuffer.empty())
+    {
+        return false;
+    }
+
+    auto dosHeader = reinterpret_cast<PIMAGE_DOS_HEADER>(dosHeaderBuffer.data());
+
+    if (dosHeader->e_magic != IMAGE_DOS_SIGNATURE)
+    {
+        return false;
+    }
+
+    // Read the NT headers
+    uintptr_t ntHeadersAddress = baseAddress + dosHeader->e_lfanew;
+    auto ntHeadersBuffer = _readMemory(ntHeadersAddress, sizeof(IMAGE_NT_HEADERS_GENERIC));
+    if (ntHeadersBuffer.empty())
+    {
+        return false;
+    }
+
+    auto ntHeaders = reinterpret_cast<IMAGE_NT_HEADERS_GENERIC*>(ntHeadersBuffer.data());
+
+    if (ntHeaders->Signature != IMAGE_NT_SIGNATURE)
+    {
+        return false;
+    }
+
+    // Check the PE type
+    bool isPE32 = (ntHeaders->Magic == IMAGE_NT_OPTIONAL_HDR32_MAGIC);
+    bool isPE64 = (ntHeaders->Magic == IMAGE_NT_OPTIONAL_HDR64_MAGIC);
+
+    if (!isPE32 && !isPE64)
+    {
+        return false;
+    }
+
+    // Read the debug directory according to the PE type
+    IMAGE_DATA_DIRECTORY debugDataDir;
+
+    if (isPE32)
+    {
+        auto header32Buffer = _readMemory(ntHeadersAddress, sizeof(IMAGE_NT_HEADERS32));
+        if (header32Buffer.empty())
+        {
+            return false;
+        }
+
+        auto header32 = reinterpret_cast<IMAGE_NT_HEADERS32*>(header32Buffer.data());
+        debugDataDir = header32->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_DEBUG];
+    }
+    else
+    {
+        auto header64Buffer = _readMemory(ntHeadersAddress, sizeof(IMAGE_NT_HEADERS64));
+        if (header64Buffer.empty())
+        {
+            return false;
+        }
+
+        auto header64 = reinterpret_cast<IMAGE_NT_HEADERS64*>(header64Buffer.data());
+        debugDataDir = header64->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_DEBUG];
+    }
+
+    uintptr_t debugDirectoryAddress = baseAddress + debugDataDir.VirtualAddress;
+    if (debugDirectoryAddress == 0)
+    {
+        return false;
+    }
+
+    auto debugDirectoryBuffer = _readMemory(debugDirectoryAddress, debugDataDir.Size);
+    if (debugDirectoryBuffer.empty())
+    {
+        return false;
+    }
+
+    auto debugDirectory = reinterpret_cast<PIMAGE_DEBUG_DIRECTORY>(debugDirectoryBuffer.data());
+
+    // Iterate over the debug directory entries and look for IMAGE_DEBUG_TYPE_CODEVIEW entries
+    for (size_t i = 0; i < debugDataDir.Size / sizeof(IMAGE_DEBUG_DIRECTORY); i++)
+    {
+        if (debugDirectory[i].Type == IMAGE_DEBUG_TYPE_CODEVIEW)
+        {
+            struct CV_INFO_PDB70
+            {
+                DWORD Signature;
+                GUID Guid;
+                DWORD Age;
+                char PdbFileName[];
+            };
+
+            // Extract the PDB info from the codeview entry
+            auto pdbInfoAddress = baseAddress + debugDirectory[i].AddressOfRawData;
+            auto pdbInfoBuffer = _readMemory(pdbInfoAddress, sizeof(CV_INFO_PDB70));
+
+            if (pdbInfoBuffer.empty())
+            {
+                return false;
+            }
+
+            auto pdbInfo = reinterpret_cast<CV_INFO_PDB70*>(pdbInfoBuffer.data());
+
+            constexpr DWORD PDB70_SIGNATURE = 0x53445352; // "SDSR"
+
+            if (pdbInfo->Signature == PDB70_SIGNATURE)
+            {
+                moduleInfo.pdbAge = pdbInfo->Age;
+                moduleInfo.pdbSig = pdbInfo->Guid;
+                moduleInfo.hasPdbInfo = true;
+
+                return true;
+            }
         }
     }
 
-    return std::make_pair("", 0);
+    return false;
+}
+
+void CrashReportingWindows::SetMemoryReader(std::function<std::vector<BYTE>(uintptr_t, SIZE_T)> readMemory)
+{
+    _readMemory = readMemory;
 }