From ff396c56741968c7d7caa3b0f0e3380e36e9f2ed Mon Sep 17 00:00:00 2001 From: dudik Date: Thu, 31 Oct 2024 17:37:31 +0100 Subject: [PATCH] removing the memory guard from this PR --- .../TrackedStackFrameNode.cs | 4 +- .../Expressions/MethodScopeMembers.cs | 10 +- .../MethodScopeMembersParameters.cs | 8 + .../Debugger/Expressions/ProbeProcessor.cs | 2 +- .../Debugger/Helpers/IPoolable.cs | 2 +- .../Debugger/Helpers/MemoryInfoRetriever.cs | 230 ------------------ .../Debugger/Helpers/ObjectPool.cs | 19 +- .../GlobalMemoryCircuitBreaker.cs | 40 --- .../RateLimiting/MemoryCircuitBreaker.cs | 67 ----- 9 files changed, 27 insertions(+), 355 deletions(-) create mode 100644 tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembersParameters.cs delete mode 100644 tracer/src/Datadog.Trace/Debugger/Helpers/MemoryInfoRetriever.cs delete mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/GlobalMemoryCircuitBreaker.cs delete mode 100644 tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryCircuitBreaker.cs diff --git a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs index 8dc8cef7fd8e..7afccd2d91ca 100644 --- a/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs +++ b/tracer/src/Datadog.Trace/Debugger/ExceptionAutoInstrumentation/TrackedStackFrameNode.cs @@ -124,7 +124,7 @@ private string CreateSnapshot() if (members == null) { - members = new MethodScopeMembers(0, 0); + members = new MethodScopeMembers(new MethodScopeMembersParameters(0, 0)); } var limitInfo = new CaptureLimitInfo( @@ -164,7 +164,7 @@ internal void AddScopeMember(string name, Type type, T value, ScopeMemberKind { if (Members == null) { - Members = new MethodScopeMembers(0, 0); + Members = new MethodScopeMembers(new MethodScopeMembersParameters(0, 0)); } type = (type.IsGenericTypeDefinition ? value?.GetType() : type) ?? type; diff --git a/tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembers.cs b/tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembers.cs index 3aaef1c89805..78bc087d88ae 100644 --- a/tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembers.cs +++ b/tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembers.cs @@ -9,11 +9,19 @@ namespace Datadog.Trace.Debugger.Expressions; -internal record struct MethodScopeMembersParameters(int NumberOfLocals, int NumberOfArguments); internal class MethodScopeMembers : IPoolable { private int _index; + public MethodScopeMembers() + { + } + + internal MethodScopeMembers(MethodScopeMembersParameters parameters) + { + Set(parameters); + } + internal ScopeMember[] Members { get; private set; } internal Exception Exception { get; set; } diff --git a/tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembersParameters.cs b/tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembersParameters.cs new file mode 100644 index 000000000000..f2f957eab874 --- /dev/null +++ b/tracer/src/Datadog.Trace/Debugger/Expressions/MethodScopeMembersParameters.cs @@ -0,0 +1,8 @@ +// +// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. +// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. +// + +namespace Datadog.Trace.Debugger.Expressions; + +internal record struct MethodScopeMembersParameters(int NumberOfLocals, int NumberOfArguments); diff --git a/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs b/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs index daf77c0253d7..75512d8180ed 100644 --- a/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs +++ b/tracer/src/Datadog.Trace/Debugger/Expressions/ProbeProcessor.cs @@ -160,7 +160,7 @@ private ProbeExpressionEvaluator GetOrCreateEvaluator() public bool ShouldProcess(in ProbeData probeData) { - return HasCondition() || (probeData.Sampler.Sample() && GlobalMemoryCircuitBreaker.Instance.CanAllocate(1024)); + return HasCondition() || (probeData.Sampler.Sample()); } public bool Process(ref CaptureInfo info, IDebuggerSnapshotCreator inSnapshotCreator, in ProbeData probeData) diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/IPoolable.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/IPoolable.cs index 98617031b150..eead6dac7ef9 100644 --- a/tracer/src/Datadog.Trace/Debugger/Helpers/IPoolable.cs +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/IPoolable.cs @@ -5,7 +5,7 @@ namespace Datadog.Trace.Debugger.Helpers { - internal interface IPoolable + internal interface IPoolable { void Set(TSetParameters parameters); diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/MemoryInfoRetriever.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/MemoryInfoRetriever.cs deleted file mode 100644 index c80de2bcb2df..000000000000 --- a/tracer/src/Datadog.Trace/Debugger/Helpers/MemoryInfoRetriever.cs +++ /dev/null @@ -1,230 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// -#nullable enable -using System; -using System.IO; -using System.Runtime.InteropServices; - -namespace Datadog.Trace.Debugger.Helpers -{ - internal static class MemoryInfoRetriever - { - private const long DEFAULT_MEMORY_THRESHOLD = 1024L * 1024 * 1024; // 1 GB - private const double DEFAULT_PERCENTAGE = 0.7; - - public static long GetTotalPhysicalMemory() - { - if (IsRunningInContainer()) - { - return GetContainerMemoryLimit(); - } - else if (IsWindows()) - { - return GetWindowsTotalPhysicalMemory(); - } - else if (IsLinux()) - { - return GetLinuxTotalPhysicalMemory(); - } - else if (IsMacOS()) - { - return GetMacOSTotalPhysicalMemory(); - } - - return 0; // Indicates failure to retrieve memory info - } - - internal static long GetDynamicMemoryThreshold(double percentageOfPhysicalMemory = DEFAULT_PERCENTAGE) - { - long totalPhysicalMemory = GetTotalPhysicalMemory(); - - if (totalPhysicalMemory > 0) - { - return (long)(totalPhysicalMemory * percentageOfPhysicalMemory); - } - else - { - // Fallback to default values based on the environment - if (IsWindows()) - { - return 2L * 1024 * 1024 * 1024; // 2 GB for Windows - } - else if (IsLinux()) - { - return 1024L * 1024 * 1024; // 1 GB for Linux - } - else if (IsMacOS()) - { - return 2L * 1024 * 1024 * 1024; // 2 GB for macOS - } - else - { - return DEFAULT_MEMORY_THRESHOLD; - } - } - } - - private static bool IsRunningInContainer() - { - return File.Exists("/.dockerenv") || File.Exists("/run/.containerenv"); - } - - private static bool IsWindows() - { -#if NETCOREAPP3_0_OR_GREATER - return RuntimeInformation.IsOSPlatform(OSPlatform.Windows) -#else - return Environment.OSVersion.Platform == PlatformID.Win32NT; -#endif - } - - private static bool IsLinux() - { -#if NETCOREAPP3_0_OR_GREATER - return RuntimeInformation.IsOSPlatform(OSPlatform.Linux); -#else - return Environment.OSVersion.Platform == PlatformID.Unix && !IsMacOS(); -#endif - } - - private static bool IsMacOS() - { -#if NETCOREAPP3_0_OR_GREATER - return RuntimeInformation.IsOSPlatform(OSPlatform.OSX); -#else - return Environment.OSVersion.Platform == PlatformID.Unix && - Directory.Exists("/System/Library/CoreServices"); -#endif - } - - private static long GetContainerMemoryLimit() - { - try - { - string cgroupMemLimitPath = "/sys/fs/cgroup/memory/memory.limit_in_bytes"; - if (File.Exists(cgroupMemLimitPath)) - { - string memLimitStr = File.ReadAllText(cgroupMemLimitPath).Trim(); - if (long.TryParse(memLimitStr, out long memLimit)) - { - return memLimit; - } - } - } - catch - { - // Silently handle any exceptions - } - - return 0; // Indicates failure to retrieve container memory limit - } - - private static long GetWindowsTotalPhysicalMemory() - { - try - { - var memStatus = new MEMORYSTATUSEX(); - if (GlobalMemoryStatusEx(memStatus)) - { - return (long)memStatus.ullTotalPhys; - } - } - catch - { - // Silently handle any exceptions - } - - return 0; - } - - private static long GetLinuxTotalPhysicalMemory() - { - try - { - string[] lines = File.ReadAllLines("/proc/meminfo"); - foreach (string line in lines) - { - if (line.StartsWith("MemTotal:")) - { - string[] parts = line.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); - if (parts.Length >= 2 && long.TryParse(parts[1], out long memKb)) - { - return memKb * 1024; // Convert KB to bytes - } - } - } - } - catch - { - // Silently handle any exceptions - } - - return 0; - } - - private static long GetMacOSTotalPhysicalMemory() - { - try - { - var output = ExecuteCommand("sysctl", "-n hw.memsize"); - if (long.TryParse(output, out long memSize)) - { - return memSize; - } - } - catch - { - // Silently handle any exceptions - } - - return 0; - } - - private static string ExecuteCommand(string command, string arguments) - { - var process = new System.Diagnostics.Process() - { - StartInfo = new System.Diagnostics.ProcessStartInfo - { - FileName = command, - Arguments = arguments, - RedirectStandardOutput = true, - UseShellExecute = false, - CreateNoWindow = true, - } - }; - process.Start(); - string result = process.StandardOutput.ReadToEnd(); - process.WaitForExit(); - return result.Trim(); - } - - // Windows-specific structures and methods - [StructLayout(LayoutKind.Sequential, CharSet = CharSet.Auto)] - private class MEMORYSTATUSEX - { - public uint dwLength; -#pragma warning disable CS0649 // Field is never assigned to, and will always have its default value - public uint dwMemoryLoad; - public ulong ullTotalPhys; - public ulong ullAvailPhys; - public ulong ullTotalPageFile; - public ulong ullAvailPageFile; - public ulong ullTotalVirtual; - public ulong ullAvailVirtual; - public ulong ullAvailExtendedVirtual; -#pragma warning restore CS0649 // Field is never assigned to, and will always have its default value - - public MEMORYSTATUSEX() - { - dwLength = (uint)Marshal.SizeOf(typeof(MEMORYSTATUSEX)); - } - } - - [return: MarshalAs(UnmanagedType.Bool)] - [DllImport("kernel32.dll", CharSet = CharSet.Auto, SetLastError = true)] - static extern bool GlobalMemoryStatusEx([In, Out] MEMORYSTATUSEX lpBuffer); - } -} diff --git a/tracer/src/Datadog.Trace/Debugger/Helpers/ObjectPool.cs b/tracer/src/Datadog.Trace/Debugger/Helpers/ObjectPool.cs index c18fa37c805b..49f46fafd9ff 100644 --- a/tracer/src/Datadog.Trace/Debugger/Helpers/ObjectPool.cs +++ b/tracer/src/Datadog.Trace/Debugger/Helpers/ObjectPool.cs @@ -9,21 +9,16 @@ namespace Datadog.Trace.Debugger.Helpers { - internal class ObjectPool + internal class ObjectPool(Func? objectFactory = null, int maxSize = 100) where T : class, IPoolable, new() { - private readonly ConcurrentBag _objects; - private readonly Func _objectFactory; - private readonly int _maxSize; + private readonly ConcurrentBag _objects = new(); + private readonly Func _objectFactory = objectFactory ?? (() => new T()); + private readonly int _maxSize = maxSize; - public ObjectPool(Func? objectFactory = null, int maxSize = 100) - { - _objectFactory = objectFactory ?? (() => new T()); - _maxSize = maxSize; - _objects = new ConcurrentBag(); - } + public int Count => _objects.Count; - public T Get() => _objects.TryTake(out T item) ? item : _objectFactory(); + public T? Get() => _objects.TryTake(out var item) ? item : _objectFactory(); public T? Get(TSetParameters parameters) { @@ -40,7 +35,5 @@ public void Return(T? item) _objects.Add(item); } } - - public int Count => _objects.Count; } } diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/GlobalMemoryCircuitBreaker.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/GlobalMemoryCircuitBreaker.cs deleted file mode 100644 index 8b13c64d5e39..000000000000 --- a/tracer/src/Datadog.Trace/Debugger/RateLimiting/GlobalMemoryCircuitBreaker.cs +++ /dev/null @@ -1,40 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// -#nullable enable -using System; -using Datadog.Trace.Debugger.Helpers; - -namespace Datadog.Trace.Debugger.RateLimiting -{ - internal class GlobalMemoryCircuitBreaker - { - private static readonly Lazy _lazyInstance = - new Lazy(() => new GlobalMemoryCircuitBreaker( - memoryThreshold: MemoryInfoRetriever.GetDynamicMemoryThreshold(0.5), // 50% of physical memory - cooldownPeriod: TimeSpan.FromSeconds(10), - allocationThresholdPercentage: 0.1 // 10% default - )); - - internal static GlobalMemoryCircuitBreaker Instance => _lazyInstance.Value; - - private readonly MemoryCircuitBreaker _circuitBreaker; - - private GlobalMemoryCircuitBreaker(long memoryThreshold, TimeSpan cooldownPeriod, double allocationThresholdPercentage) - { - _circuitBreaker = new MemoryCircuitBreaker(memoryThreshold, cooldownPeriod, allocationThresholdPercentage); - } - - internal bool CanAllocate(long estimatedAllocationSize = 0) - { - return _circuitBreaker.CanAllocate(estimatedAllocationSize); - } - - internal long GetCurrentMemoryUsage() => _circuitBreaker.GetCurrentMemoryUsage(); - - internal long GetMemoryThreshold() => _circuitBreaker.GetMemoryThreshold(); - - internal double GetAvailableMemoryPercentage() => _circuitBreaker.GetAvailableMemoryPercentage(); - } -} diff --git a/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryCircuitBreaker.cs b/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryCircuitBreaker.cs deleted file mode 100644 index 2f8877aa0a29..000000000000 --- a/tracer/src/Datadog.Trace/Debugger/RateLimiting/MemoryCircuitBreaker.cs +++ /dev/null @@ -1,67 +0,0 @@ -// -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache 2 License. -// This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2017 Datadog, Inc. -// -#nullable enable -using System; -using System.Threading; - -namespace Datadog.Trace.Debugger.RateLimiting -{ - internal class MemoryCircuitBreaker - { - private readonly long _memoryThreshold; - private readonly TimeSpan _cooldownPeriod; - private readonly double _allocationThresholdPercentage; - private long _lastTrippedTicks = 0; - private long _currentMemoryUsage = 0; - - internal MemoryCircuitBreaker( - long memoryThreshold = 1024L * 1024 * 1024, // 1 GB default - TimeSpan? cooldownPeriod = null, - double allocationThresholdPercentage = 0.05) // 5% default - { - _memoryThreshold = memoryThreshold; - _cooldownPeriod = cooldownPeriod ?? TimeSpan.FromSeconds(10); - _allocationThresholdPercentage = allocationThresholdPercentage; - } - - public bool CanAllocate(long estimatedAllocationSize = 0) - { - long currentTicks = DateTime.UtcNow.Ticks; - long lastTrippedTicks = Interlocked.Read(ref _lastTrippedTicks); - - if ((new TimeSpan(currentTicks - lastTrippedTicks)) < _cooldownPeriod) - { - return false; - } - - _currentMemoryUsage = GC.GetTotalMemory(false); - - if (_currentMemoryUsage > _memoryThreshold) - { - Interlocked.Exchange(ref _lastTrippedTicks, currentTicks); - return false; - } - - if (estimatedAllocationSize > 0) - { - long availableMemory = _memoryThreshold - _currentMemoryUsage; - if (estimatedAllocationSize > availableMemory * _allocationThresholdPercentage) - { - Interlocked.Exchange(ref _lastTrippedTicks, currentTicks); - return false; - } - } - - return true; - } - - internal long GetCurrentMemoryUsage() => _currentMemoryUsage; - - internal long GetMemoryThreshold() => _memoryThreshold; - - internal double GetAvailableMemoryPercentage() => - (double)(_memoryThreshold - _currentMemoryUsage) / _memoryThreshold * 100; - } -}