Merge branch 'master' into nicholas.hulston/eventbridge-inject-trace-…

…context
DataDog · Oct 15, 2024 · c01b3a5 · c01b3a5
2 parents 9f09e40 + aa5cedc
commit c01b3a5
Show file tree

Hide file tree

Showing 168 changed files with 1,539 additions and 161 deletions.
diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml
@@ -504,7 +504,7 @@ stages:
         build: true
         target: builder
         baseImage: "universal"
-        useNativeSdkVersion: true
+        useNativeSdkVersion: false
         command: "Clean BuildNativeLoader BuildNativeWrapper ExtractDebugInfoLinux"
         retryCountForRunCommand: 1
 

diff --git a/Datadog.Trace.sln b/Datadog.Trace.sln
@@ -591,6 +591,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Generated", "Generated", "{
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Samples.AWS.EventBridge", "tracer\test\test-applications\integrations\Samples.AWS.EventBridge\Samples.AWS.EventBridge.csproj", "{D6155F26-8245-4B66-8944-79C3DF9F9DA3}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AssemblyLoadContextResolve", "tracer\test\test-applications\regression\AssemblyLoadContextResolve\AssemblyLoadContextResolve.csproj", "{8B1AF6A7-DD41-4347-B637-90C23D69B50E}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1415,6 +1417,10 @@ Global
 		{D6155F26-8245-4B66-8944-79C3DF9F9DA3}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{D6155F26-8245-4B66-8944-79C3DF9F9DA3}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{D6155F26-8245-4B66-8944-79C3DF9F9DA3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8B1AF6A7-DD41-4347-B637-90C23D69B50E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8B1AF6A7-DD41-4347-B637-90C23D69B50E}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8B1AF6A7-DD41-4347-B637-90C23D69B50E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{8B1AF6A7-DD41-4347-B637-90C23D69B50E}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1644,6 +1650,7 @@ Global
 		{7131FE5A-6B27-4BBC-B0CF-09780F6D2DFE} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A}
 		{2CA0D70C-DFC1-458A-871B-328AB6E87E3A} = {BAF8F246-3645-42AD-B1D0-0F7EAFBAB34A}
 		{E1B0F72C-991A-409D-9266-DE5ED1BD940E} = {A0C5FBBB-CFB2-4FB9-B8F0-55676E9DCF06}
+		{8B1AF6A7-DD41-4347-B637-90C23D69B50E} = {498A300E-D036-49B7-A43D-821D1CAF11A5}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {160A1D00-1F5B-40F8-A155-621B4459D78F}
@@ -1723,6 +1730,7 @@ Global
 		tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{83290961-40bf-48cb-b925-fbbe48e629f3}*SharedItemsImports = 5
 		tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{87d57940-9a6e-473c-a4d6-777e3bafd5f9}*SharedItemsImports = 5
 		tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{887ac8ba-35a6-4646-bf9a-59357155805e}*SharedItemsImports = 5
+		tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{8b1af6a7-dd41-4347-b637-90c23d69b50e}*SharedItemsImports = 5
 		tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{8b457e8f-8716-4f29-bbe2-dd6c7bc4ac37}*SharedItemsImports = 5
 		tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{8bdf1de0-e6de-48ad-aaa3-ce09cb544e2c}*SharedItemsImports = 5
 		tracer\test\test-applications\Samples.Shared\Samples.Shared.projitems*{8dfe1168-b1cc-43d1-b256-b5708badd47b}*SharedItemsImports = 5

diff --git a/shared/src/Datadog.Trace.ClrProfiler.Native/dllmain.cpp b/shared/src/Datadog.Trace.ClrProfiler.Native/dllmain.cpp
@@ -80,7 +80,7 @@ EXTERN_C BOOL STDMETHODCALLTYPE DllMain(HMODULE hModule, DWORD ul_reason_for_cal
         bool telemetry_enabled = true;
         shared::TryParseBooleanEnvironmentValue(shared::GetEnvironmentValue(L"DD_INSTRUMENTATION_TELEMETRY_ENABLED"), telemetry_enabled);
 
-        bool crashtracking_enabled = true;
+        bool crashtracking_enabled = false;
         shared::TryParseBooleanEnvironmentValue(shared::GetEnvironmentValue(L"DD_CRASHTRACKING_ENABLED"), crashtracking_enabled);
 
         if (telemetry_enabled && crashtracking_enabled)

diff --git a/shared/src/Datadog.Trace.ClrProfiler.Native/dynamic_dispatcher.cpp b/shared/src/Datadog.Trace.ClrProfiler.Native/dynamic_dispatcher.cpp
@@ -80,13 +80,6 @@ namespace datadog::shared::nativeloader
         fs::path configFolder = fs::path(configFilePath).remove_filename();
         Log::Debug("DynamicDispatcherImpl::LoadConfiguration: Config Folder: ", configFolder);
 
-        // Get the current path
-        fs::path oldCurrentPath = fs::current_path();
-        Log::Debug("DynamicDispatcherImpl::LoadConfiguration: Current Path: ", oldCurrentPath);
-
-        // Set the current path to the configuration folder (to allow relative paths)
-        fs::current_path(configFolder);
-
         const auto isRunningOnAlpine = IsRunningOnAlpine();
         const auto currentOsArch = GetCurrentOsArch(isRunningOnAlpine);
 
@@ -128,9 +121,11 @@ namespace datadog::shared::nativeloader
                     {
                         // Convert possible relative paths to absolute paths using the configuration file folder as base
                         // (current_path)
-                        std::string absoluteFilepathValue = fs::absolute(filepathValue).string();
+                        std::string absoluteFilepathValue = (configFolder / filepathValue).string();
                         Log::Debug("DynamicDispatcherImpl::LoadConfiguration: [", type, "] Loading: ", filepathValue, " [AbsolutePath=", absoluteFilepathValue,"] (", currentOsArch, ")" );
-                        if (fs::exists(absoluteFilepathValue))
+
+                        ec.clear();
+                        if (fs::exists(absoluteFilepathValue, ec))
                         {
                             Log::Debug("[", type, "] Creating a new DynamicInstance object");
 
@@ -162,8 +157,17 @@ namespace datadog::shared::nativeloader
                         }
                         else
                         {
-                            Log::Warn("DynamicDispatcherImpl::LoadConfiguration: [", type, "] Dynamic library for '", absoluteFilepathValue,
-                                 "' cannot be loaded, file doesn't exist.");
+                            if (ec)
+                            {
+                                Log::Warn("DynamicDispatcherImpl::LoadConfiguration: [", type, "] Dynamic library for '", absoluteFilepathValue,
+                                    "' cannot be loaded, error code: ", ec.value(), ", message: ", ec.message());
+                            }
+                            else
+                            {
+
+                                Log::Warn("DynamicDispatcherImpl::LoadConfiguration: [", type, "] Dynamic library for '", absoluteFilepathValue,
+                                    "' cannot be loaded, file doesn't exist.");
+                            }
                         }
                     }
                     else
@@ -182,9 +186,6 @@ namespace datadog::shared::nativeloader
             }
         }
         t.close();
-
-        // Set the current path to the original one
-        fs::current_path(oldCurrentPath);
     }
 
     HRESULT DynamicDispatcherImpl::LoadClassFactory(REFIID riid)

diff --git a/tracer/build/_build/Build.Steps.cs b/tracer/build/_build/Build.Steps.cs
@@ -62,7 +62,7 @@ partial class Build
 
     AbsolutePath NativeBuildDirectory => RootDirectory / "obj";
 
-    const string LibDdwafVersion = "1.19.1";
+    const string LibDdwafVersion = "1.20.0";
 
     string[] OlderLibDdwafVersions = { "1.3.0", "1.10.0", "1.14.0", "1.16.0" };
 

diff --git a/tracer/build/_build/_build.csproj b/tracer/build/_build/_build.csproj
@@ -2,10 +2,10 @@
 
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>net7.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
+    <TargetFramework Condition="$(USE_NATIVE_SDK_VERSION) == 'true'">net7.0</TargetFramework>
     <TargetFramework Condition="$(OS.StartsWith('Windows')) AND '$(NUKE_NOTIFY)' != ''">$(TargetFramework)-windows10.0.19041</TargetFramework>
     <DefineConstants Condition="$(OS.StartsWith('Windows')) AND '$(NUKE_NOTIFY)' != ''">$(DefineConstants);NUKE_NOTIFY</DefineConstants>
-    <RollForward>LatestMajor</RollForward>
     <RootNamespace></RootNamespace>
     <!-- NU* are workaround for Rider bug: https://youtrack.jetbrains.com/issue/RIDER-103207/Cannot-suppress-vulnerable-package-errors -->
     <NoWarn>CS0649;CS0169;SA1652;NU1901;NU1902;NU1903;NU1904</NoWarn>

diff --git a/tracer/build/_build/docker/centos7.dockerfile b/tracer/build/_build/docker/centos7.dockerfile
@@ -78,6 +78,8 @@ ENV \
 
 FROM base as builder
 
+ENV USE_NATIVE_SDK_VERSION=true
+
 # Copy the build project in and build it
 COPY *.csproj *.props *.targets /build/
 RUN dotnet restore /build
@@ -104,6 +106,8 @@ RUN if [ "$(uname -m)" = "x86_64" ]; \
     && rm dotnet-install.sh
 
 
+ENV USE_NATIVE_SDK_VERSION=true
+
 # Copy the build project in and build it
 COPY *.csproj *.props *.targets /build/
 RUN dotnet restore /build

diff --git a/tracer/build/_build/docker/universal.dockerfile b/tracer/build/_build/docker/universal.dockerfile
@@ -55,6 +55,7 @@ RUN ln -s `which clang-16` /usr/bin/clang  && \
 
 ENV \
     DOTNET_ROLL_FORWARD_TO_PRERELEASE=1 \
+    USE_NATIVE_SDK_VERSION=true \
     CXX=clang++ \
     CC=clang
 

diff --git a/tracer/build/smoke_test_snapshots/smoke_test_snapshots.json b/tracer/build/smoke_test_snapshots/smoke_test_snapshots.json
@@ -38,7 +38,7 @@
       "parent_id": 1,
       "type": "web",
       "meta": {
-        "_dd.appsec.waf.version": "1.19.1",
+        "_dd.appsec.waf.version": "1.20.0",
         "_dd.runtime_family": "dotnet",
         "_dd.appsec.s.req.params": "H4sIAAAAAAAAA4uuVkrOzyspys/JSS1Ssoq2iNVRSkwuyczPA3NqYwH+CR9jIQAAAA==",
         "_dd.appsec.s.res.body": "H4sIAAAAAAAAA4u2iAUA8YntnQMAAAA=",
@@ -61,7 +61,7 @@
       "metrics": {
         "_dd.appsec.enabled": 1.0,
         "_dd.appsec.event_rules.error_count": 0.0,
-        "_dd.appsec.event_rules.loaded": 153.0,
+        "_dd.appsec.event_rules.loaded": 158.0,
         "_dd.top_level": 1.0,
         "_dd.tracer_kr": 0.0,
         "_sampling_priority_v1": 2.0,

diff --git a/tracer/build/smoke_test_snapshots/smoke_test_snapshots_2_1.json b/tracer/build/smoke_test_snapshots/smoke_test_snapshots_2_1.json
@@ -38,7 +38,7 @@
       "parent_id": 1,
       "type": "web",
       "meta": {
-        "_dd.appsec.waf.version": "1.19.1",
+        "_dd.appsec.waf.version": "1.20.0",
         "_dd.runtime_family": "dotnet",
         "_dd.appsec.s.res.body": "H4sIAAAAAAAAA4u2iAUA8YntnQMAAAA=",
         "_dd.appsec.s.req.headers": "H4sIAAAAAAAAA4WOMQrAIBDA/uKsQ7fiVw6Ho4oVrIp3Q4v491JcLc4JJNAEVzwcMbITGgB2Y2QT0SWht27kwAWrSzzlt7LIaLNXjJ4WCuFVYkhelRpyDfws/NFVwa7S3+SfdmaarXfzAg6PMlH9AAAA",
@@ -59,7 +59,7 @@
       "metrics": {
         "_dd.appsec.enabled": 1.0,
         "_dd.appsec.event_rules.error_count": 0.0,
-        "_dd.appsec.event_rules.loaded": 153.0,
+        "_dd.appsec.event_rules.loaded": 158.0,
         "_dd.top_level": 1.0,
         "_dd.tracer_kr": 0.0,
         "_sampling_priority_v1": 2.0,

diff --git a/tracer/missing-nullability-files.csv b/tracer/missing-nullability-files.csv
@@ -60,7 +60,6 @@ src/Datadog.Trace/Agent/TracesTransportType.cs
 src/Datadog.Trace/AppSec/AddressesConstants.cs
 src/Datadog.Trace/AppSec/AppSecRateLimiter.cs
 src/Datadog.Trace/AppSec/BlockingAction.cs
-src/Datadog.Trace/AppSec/CoreHttpContextStore.cs
 src/Datadog.Trace/AppSec/EventTrackingSdk.cs
 src/Datadog.Trace/AppSec/IDatadogSecurity.cs
 src/Datadog.Trace/AppSec/IEvent.cs

diff --git a/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs b/tracer/src/Datadog.Trace.ClrProfiler.Managed.Loader/Startup.cs
@@ -59,6 +59,17 @@ static Startup()
                     StartupLogger.Log(ex, "Unable to register a callback to the CurrentDomain.AssemblyResolve event.");
                 }
 
+#if NETCOREAPP
+                try
+                {
+                    System.Runtime.Loader.AssemblyLoadContext.Default.Resolving += (_, assemblyName) => ResolveAssembly(assemblyName.Name);
+                }
+                catch (Exception ex)
+                {
+                    StartupLogger.Log(ex, "Unable to register a callback to the AssemblyLoadContext.Default.Resolving event.");
+                }
+#endif
+
                 var runInAas = ReadBooleanEnvironmentVariable(AzureAppServicesKey, false);
                 if (runInAas)
                 {

diff --git a/tracer/src/Datadog.Trace/AppSec/AttackerFingerprint/AttackerFingerprintHelper.cs b/tracer/src/Datadog.Trace/AppSec/AttackerFingerprint/AttackerFingerprintHelper.cs
@@ -24,10 +24,15 @@ public static void AddSpanTags(Span span, IResult result)
             return;
         }
 
-        var securityCoordinator = new SecurityCoordinator(Security.Instance, span);
+        var securityCoordinator = SecurityCoordinator.TryGet(Security.Instance, span);
+
+        if (securityCoordinator is null)
+        {
+            return;
+        }
 
         // We need a context
-        if (!securityCoordinator.HasContext() || securityCoordinator.IsAdditiveContextDisposed())
+        if (securityCoordinator.Value.IsAdditiveContextDisposed())
         {
             return;
         }

diff --git a/tracer/src/Datadog.Trace/AppSec/ControllerContextExtensions.Framework.cs b/tracer/src/Datadog.Trace/AppSec/ControllerContextExtensions.Framework.cs
@@ -9,6 +9,7 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Web;
+using Datadog.Trace.AppSec.Coordinator;
 using Datadog.Trace.AspNet;
 using Datadog.Trace.ClrProfiler.AutoInstrumentation.AspNet;
 using Datadog.Trace.Iast;
@@ -78,7 +79,7 @@ internal static void MonitorBodyAndPathParams(this IControllerContext controller
 
             if (security.Enabled)
             {
-                var securityTransport = new Coordinator.SecurityCoordinator(security, scope.Span!);
+                var securityTransport = SecurityCoordinator.Get(security, scope.Span!, context);
                 if (!securityTransport.IsBlocked)
                 {
                     var inputData = new Dictionary<string, object>();

diff --git a/tracer/src/Datadog.Trace/AppSec/Coordinator/HttpTransportBase.cs b/tracer/src/Datadog.Trace/AppSec/Coordinator/HttpTransportBase.cs
@@ -4,6 +4,7 @@
 // </copyright>
 
 #nullable enable
+using System;
 using System.Collections.Generic;
 using Datadog.Trace.AppSec.Waf;
 using Datadog.Trace.Headers;

diff --git a/tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Core.cs b/tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Core.cs
@@ -20,15 +20,31 @@ namespace Datadog.Trace.AppSec.Coordinator;
 
 internal readonly partial struct SecurityCoordinator
 {
-    internal SecurityCoordinator(Security security, Span span, HttpTransport? transport = null)
+    private SecurityCoordinator(Security security, Span span, HttpTransport transport)
     {
         _security = security;
         _localRootSpan = TryGetRoot(span);
-        _httpTransport = transport ?? new HttpTransport(CoreHttpContextStore.Instance.Get());
+        _httpTransport = transport;
     }
 
     private static bool CanAccessHeaders => true;
 
+    internal static SecurityCoordinator? TryGet(Security security, Span span)
+    {
+        var context = CoreHttpContextStore.Instance.Get();
+        if (context is null)
+        {
+            Log.Warning("Can't instantiate SecurityCoordinator.Core as no transport has been provided and CoreHttpContextStore.Instance.Get() returned null, make sure HttpContext is available");
+            return null;
+        }
+
+        return new SecurityCoordinator(security, span, new(context));
+    }
+
+    internal static SecurityCoordinator Get(Security security, Span span, HttpContext context) => new(security, span, new HttpTransport(context));
+
+    internal static SecurityCoordinator Get(Security security, Span span, HttpTransport transport) => new(security, span, transport);
+
     public static Dictionary<string, object> ExtractHeadersFromRequest(IHeaderDictionary headers)
     {
         var headersDic = new Dictionary<string, object>(headers.Keys.Count);
@@ -162,7 +178,7 @@ internal override bool IsBlocked
             {
                 if (Context.Items.TryGetValue(BlockingAction.BlockDefaultActionName, out var value))
                 {
-                    return value is bool boolValue && boolValue;
+                    return value is true;
                 }
 
                 return false;
@@ -175,8 +191,16 @@ internal override bool IsBlocked
 
         internal override bool ReportedExternalWafsRequestHeaders
         {
-            get => Context.Items["ReportedExternalWafsRequestHeaders"] is true;
-            set => Context.Items["ReportedExternalWafsRequestHeaders"] = value;
+            get
+            {
+                if (Context.Items.TryGetValue(ReportedExternalWafsRequestHeadersStr, out var value))
+                {
+                    return value is bool boolValue && boolValue;
+                }
+
+                return false;
+            }
+            set => Context.Items[ReportedExternalWafsRequestHeadersStr] = value;
         }
 
         internal override void MarkBlocked() => Context.Items[BlockingAction.BlockDefaultActionName] = true;

diff --git a/tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Framework.cs b/tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Framework.cs
@@ -45,15 +45,32 @@ static SecurityCoordinator()
         }
     }
 
-    internal SecurityCoordinator(Security security, Span span, HttpTransport? transport = null)
+    private SecurityCoordinator(Security security, Span span, HttpTransport transport)
     {
         _security = security;
         _localRootSpan = TryGetRoot(span);
-        _httpTransport = transport ?? new HttpTransport(HttpContext.Current);
+        _httpTransport = transport;
     }
 
     private bool CanAccessHeaders => UsingIntegratedPipeline is true or null;
 
+    internal static SecurityCoordinator? TryGet(Security security, Span span)
+    {
+        if (HttpContext.Current is not { } current)
+        {
+            Log.Warning("Can't instantiate SecurityCoordinator.Framework as no transport has been provided and HttpContext.Current null, make sure HttpContext is available");
+            return null;
+        }
+
+        var transport = new HttpTransport(current);
+
+        return new SecurityCoordinator(security, span, transport);
+    }
+
+    internal static SecurityCoordinator Get(Security security, Span span, HttpContext context) => new(security, span, new HttpTransport(context));
+
+    internal static SecurityCoordinator Get(Security security, Span span, HttpTransport transport) => new(security, span, transport);
+
     private static Action<IResult, HttpStatusCode, string, string>? CreateThrowHttpResponseExceptionDynMeth()
     {
         try
@@ -541,8 +558,8 @@ public HttpTransport(HttpContext context)
 
         internal override bool ReportedExternalWafsRequestHeaders
         {
-            get => Context.Items["ReportedExternalWafsRequestHeaders"] is true;
-            set => Context.Items["ReportedExternalWafsRequestHeaders"] = value;
+            get => Context.Items[ReportedExternalWafsRequestHeadersStr] is true;
+            set => Context.Items[ReportedExternalWafsRequestHeadersStr] = value;
         }
 
         internal override void MarkBlocked() => Context.Items[BlockingAction.BlockDefaultActionName] = true;

diff --git a/tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Reporter.cs b/tracer/src/Datadog.Trace/AppSec/Coordinator/SecurityCoordinator.Reporter.cs
@@ -114,6 +114,8 @@ internal void TryReport(IResult result, bool blocked, int? status = null)
             _httpTransport.ReportedExternalWafsRequestHeaders = true;
         }
 
+        AttackerFingerprintHelper.AddSpanTags(_localRootSpan, result);
+
         if (result.ShouldReportSecurityResult)
         {
             _localRootSpan.SetTag(Tags.AppSecEvent, "true");
@@ -132,8 +134,6 @@ internal void TryReport(IResult result, bool blocked, int? status = null)
                 traceContext.AppSecRequestContext.AddWafSecurityEvents(result.Data);
             }
 
-            AttackerFingerprintHelper.AddSpanTags(_localRootSpan, result);
-
             var clientIp = _localRootSpan.GetTag(Tags.HttpClientIp);
             if (!string.IsNullOrEmpty(clientIp))
             {