Guard from out of memory during probe processing

tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembers.cs

@@ -9,32 +9,17 @@
 
 namespace Datadog.Trace.Debugger.Expressions;
 
-internal class MethodScopeMembers
+internal record struct MethodScopeMembersParameters(int NumberOfLocals, int NumberOfArguments);
+internal class MethodScopeMembers : IPoolable<MethodScopeMembersParameters>
 {
-    private readonly int _initialSize;
     private int _index;
 
-    internal MethodScopeMembers(int numberOfLocals, int numberOfArguments)
-    {
-        _initialSize = numberOfLocals + numberOfArguments;
-        if (_initialSize == 0)
-        {
-            _initialSize = 1;
-        }
-
-        Members = ArrayPool<ScopeMember>.Shared.Rent(_initialSize);
-        Array.Clear(Members, 0, Members.Length);
-        Exception = null;
-        Return = default;
-        InvocationTarget = default;
-    }
-
     internal ScopeMember[] Members { get; private set; }
 
     internal Exception Exception { get; set; }
 
     // food for thought:
-    // we can save Return and InvocationTarget as T if we will change the native side so we will have MethodDebuggerState<T, TReturn> instead MethodDebuggerState.
+    // we can save Return and InvocationTarget as T if we will change the native side, so we will have MethodDebuggerState<T, TReturn> instead MethodDebuggerState.
     internal ScopeMember Return { get; set; }
 
     internal ScopeMember InvocationTarget { get; set; }
@@ -59,4 +44,26 @@ internal void Dispose()
             ArrayPool<ScopeMember>.Shared.Return(Members);
         }
     }
+
+    public void Set(MethodScopeMembersParameters parameters)
+    {
+        var initialSize = parameters.NumberOfLocals + parameters.NumberOfArguments;
+        if (initialSize == 0)
+        {
+            initialSize = 1;
+        }
+
+        Members = ArrayPool<ScopeMember>.Shared.Rent(initialSize * 2);
+        Array.Clear(Members, 0, Members.Length);
+        Exception = null;
+        Duration = default;
+        Return = default;
+        InvocationTarget = default;
+        _index = 0;
+    }
+
+    public void Reset()
+    {
+        Dispose();
+    }
 }

tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs

@@ -160,7 +160,7 @@ private ProbeExpressionEvaluator GetOrCreateEvaluator()
 
         public bool ShouldProcess(in ProbeData probeData)
         {
-            return HasCondition() || probeData.Sampler.Sample();
+            return HasCondition() || (probeData.Sampler.Sample() && GlobalMemoryCircuitBreaker.Instance.CanAllocate(1024));
         }
 
         public bool Process<TCapture>(ref CaptureInfo<TCapture> info, IDebuggerSnapshotCreator inSnapshotCreator, in ProbeData probeData)

tracer/src/Datadog.Trace/Debugger/Helpers/IPoolable.cs

@@ -0,0 +1,14 @@
+// <copyright file="IPoolable.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+namespace Datadog.Trace.Debugger.Helpers
+{
+    internal interface IPoolable<TSetParameters>
+    {
+        void Set(TSetParameters parameters);
+
+        void Reset();
+    }
+}

tracer/src/Datadog.Trace/Debugger/Helpers/MemoryInfoRetriever.cs

@@ -0,0 +1,230 @@
+// <copyright file="MemoryInfoRetriever.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+#nullable enable
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+
+namespace Datadog.Trace.Debugger.Helpers
+{
+    internal static class MemoryInfoRetriever
+    {
+        private const long DEFAULT_MEMORY_THRESHOLD = 1024L * 1024 * 1024; // 1 GB
+        private const double DEFAULT_PERCENTAGE = 0.7;
+
+        public static long GetTotalPhysicalMemory()
+        {
+            if (IsRunningInContainer())
+            {
+                return GetContainerMemoryLimit();
+            }
+            else if (IsWindows())
+            {
+                return GetWindowsTotalPhysicalMemory();
+            }
+            else if (IsLinux())
+            {
+                return GetLinuxTotalPhysicalMemory();
+            }
+            else if (IsMacOS())
+            {
+                return GetMacOSTotalPhysicalMemory();
+            }
+
+            return 0; // Indicates failure to retrieve memory info
+        }
+
+        internal static long GetDynamicMemoryThreshold(double percentageOfPhysicalMemory = DEFAULT_PERCENTAGE)
+        {
+            long totalPhysicalMemory = GetTotalPhysicalMemory();
+
+            if (totalPhysicalMemory > 0)
+            {
+                return (long)(totalPhysicalMemory * percentageOfPhysicalMemory);
+            }
+            else
+            {
+                // Fallback to default values based on the environment
+                if (IsWindows())
+                {
+                    return 2L * 1024 * 1024 * 1024; // 2 GB for Windows
+                }
+                else if (IsLinux())
+                {
+                    return 1024L * 1024 * 1024; // 1 GB for Linux
+                }
+                else if (IsMacOS())
+                {
+                    return 2L * 1024 * 1024 * 1024; // 2 GB for macOS
+                }
+                else
+                {
+                    return DEFAULT_MEMORY_THRESHOLD;
+                }
+            }
+        }
+
+        private static bool IsRunningInContainer()
+        {
+            return File.Exists("/.dockerenv") || File.Exists("/run/.containerenv");
+        }
+
+        private static bool IsWindows()
+        {
+#if NETCOREAPP3_0_OR_GREATER
+            return RuntimeInformation.IsOSPlatform(OSPlatform.Windows)
+#else
+            return Environment.OSVersion.Platform == PlatformID.Win32NT;
+#endif
+        }
+
+        private static bool IsLinux()
+        {
+#if NETCOREAPP3_0_OR_GREATER
+            return RuntimeInformation.IsOSPlatform(OSPlatform.Linux);
+#else
+            return Environment.OSVersion.Platform == PlatformID.Unix && !IsMacOS();
+#endif
+        }
+
+        private static bool IsMacOS()
+        {
+#if NETCOREAPP3_0_OR_GREATER
+            return RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
+#else
+            return Environment.OSVersion.Platform == PlatformID.Unix &&
+                   Directory.Exists("/System/Library/CoreServices");
+#endif
+        }
+
+        private static long GetContainerMemoryLimit()
+        {
+            try
+            {
+                string cgroupMemLimitPath = "/sys/fs/cgroup/memory/memory.limit_in_bytes";
+                if (File.Exists(cgroupMemLimitPath))
+                {
+                    string memLimitStr = File.ReadAllText(cgroupMemLimitPath).Trim();
+                    if (long.TryParse(memLimitStr, out long memLimit))
+                    {
+                        return memLimit;
+                    }
+                }
+            }
+            catch
+            {
+                // Silently handle any exceptions
+            }
+
+            return 0; // Indicates failure to retrieve container memory limit
+        }
+
+        private static long GetWindowsTotalPhysicalMemory()
+        {
+            try
+            {
+                var memStatus = new MEMORYSTATUSEX();
+                if (GlobalMemoryStatusEx(memStatus))
+                {
+                    return (long)memStatus.ullTotalPhys;
+                }
+            }
+            catch
+            {
+                // Silently handle any exceptions
+            }
+
+            return 0;
+        }
+
+        private static long GetLinuxTotalPhysicalMemory()
+        {
+            try
+            {
+                string[] lines = File.ReadAllLines("/proc/meminfo");
+                foreach (string line in lines)
+                {
+                    if (line.StartsWith("MemTotal:"))
+                    {
+                        string[] parts = line.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries);
+                        if (parts.Length >= 2 && long.TryParse(parts[1], out long memKb))
+                        {
+                            return memKb * 1024; // Convert KB to bytes
+                        }
+                    }
+                }
+            }
+            catch
+            {
+                // Silently handle any exceptions
+            }
+
+            return 0;
+        }
+
+        private static long GetMacOSTotalPhysicalMemory()
+        {
+            try
+            {
+                var output = ExecuteCommand("sysctl", "-n hw.memsize");
+                if (long.TryParse(output, out long memSize))
+                {
+                    return memSize;
+                }
+            }
+            catch
+            {
+                // Silently handle any exceptions
+            }
+
+            return 0;
+        }
+
+        private static string ExecuteCommand(string command, string arguments)
+        {
+            var process = new System.Diagnostics.Process()
+            {
+                StartInfo = new System.Diagnostics.ProcessStartInfo
+                {
+                    FileName = command,
+                    Arguments = arguments,
+                    RedirectStandardOutput = true,
+                    UseShellExecute = false,
+                    CreateNoWindow = true,
+                }
+            };
+            process.Start();
+            string result = process.StandardOutput.ReadToEnd();
+            process.WaitForExit();
+            return result.Trim();
+        }
+
+        // Windows-specific structures and methods
+        [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)]
+        private class MEMORYSTATUSEX
+        {
+            public uint dwLength;
+#pragma warning disable CS0649 // Field is never assigned to, and will always have its default value
+            public uint dwMemoryLoad;
+            public ulong ullTotalPhys;
+            public ulong ullAvailPhys;
+            public ulong ullTotalPageFile;
+            public ulong ullAvailPageFile;
+            public ulong ullTotalVirtual;
+            public ulong ullAvailVirtual;
+            public ulong ullAvailExtendedVirtual;
+#pragma warning restore CS0649 // Field is never assigned to, and will always have its default value
+
+            public MEMORYSTATUSEX()
+            {
+                dwLength = (uint)Marshal.SizeOf(typeof(MEMORYSTATUSEX));
+            }
+        }
+
+        [return: MarshalAs(UnmanagedType.Bool)]
+        [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)]
+        static extern bool GlobalMemoryStatusEx([In, Out] MEMORYSTATUSEX lpBuffer);
+    }
+}

tracer/src/Datadog.Trace/Debugger/Helpers/ObjectPool.cs

@@ -0,0 +1,46 @@
+// <copyright file="ObjectPool.cs" company="Datadog">
+// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License.
+// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc.
+// </copyright>
+
+#nullable enable
+using System;
+using System.Collections.Concurrent;
+
+namespace Datadog.Trace.Debugger.Helpers
+{
+    internal class ObjectPool<T, TSetParameters>
+        where T : class, IPoolable<TSetParameters>, new()
+    {
+        private readonly ConcurrentBag<T> _objects;
+        private readonly Func<T> _objectFactory;
+        private readonly int _maxSize;
+
+        public ObjectPool(Func<T>? objectFactory = null, int maxSize = 100)
+        {
+            _objectFactory = objectFactory ?? (() => new T());
+            _maxSize = maxSize;
+            _objects = new ConcurrentBag<T>();
+        }
+
+        public T Get() => _objects.TryTake(out T item) ? item : _objectFactory();
+
+        public T? Get(TSetParameters parameters)
+        {
+            var item = _objects.TryTake(out var obj) ? obj : _objectFactory();
+            item?.Set(parameters);
+            return item;
+        }
+
+        public void Return(T? item)
+        {
+            item?.Reset();
+            if (item != null && _objects.Count < _maxSize)
+            {
+                _objects.Add(item);
+            }
+        }
+
+        public int Count => _objects.Count;
+    }
+}