From 94159d369a3ba960bae07da1b4b87b7972b8037b Mon Sep 17 00:00:00 2001 From: Dmitry Anoshin Date: Fri, 28 Jun 2024 11:32:15 -0700 Subject: [PATCH] Set Go garbage collection target percentage to 400% After removal of memory_ballast extension in v0.97.0, the Go garbage collection is running more aggressively, which increased CPU usage and leads to reduced throughput of the collector. This change reduces the frequency of garbage collection cycles to improves performance of the collector for typical workloads. As a result, the collector will report higher memory usage, but it will be bound to the same configured limits. If you want to revert to the previous behavior, set the `GOGC` environment variable to `100`. --- CHANGELOG.md | 9 +++++++++ internal/settings/settings.go | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42574d8fd9..f6867762a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +### 💡 Enhancements 💡 + +- (Splunk) Set Go garbage collection target percentage to 400% ([#5034](https://github.com/signalfx/splunk-otel-collector/pull/5034)) + After removal of memory_ballast extension in v0.97.0, the Go garbage collection is running more aggressively, which + increased CPU usage and leads to reduced throughput of the collector. This change reduces the frequency of garbage + collection cycles to improves performance of the collector for typical workloads. As a result, the collector will + report higher memory usage, but it will be bound to the same configured limits. If you want to revert to the previous + behavior, set the `GOGC` environment variable to `100`. + ### 🧰 Bug fixes 🧰 - (Splunk) `receiver/discovery`: Do not emit entity events for discovered endpoints that are not evaluated yet diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 7350abea3b..d402ac8103 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -43,6 +43,7 @@ const ( HecLogIngestURLEnvVar = "SPLUNK_HEC_URL" ListenInterfaceEnvVar = "SPLUNK_LISTEN_INTERFACE" GoMemLimitEnvVar = "GOMEMLIMIT" + GoGCEnvVar = "GOGC" // nolint:gosec HecTokenEnvVar = "SPLUNK_HEC_TOKEN" // this isn't a hardcoded token IngestURLEnvVar = "SPLUNK_INGEST_URL" @@ -59,6 +60,7 @@ const ( DefaultMemoryLimitPercentage = 90 DefaultMemoryTotalMiB = 512 + DefaultGoGC = 400 DefaultListenInterface = "0.0.0.0" DefaultAgentConfigLinux = "/etc/otel/collector/agent_config.yaml" featureGates = "feature-gates" @@ -371,6 +373,11 @@ func checkRuntimeParams(settings *Settings) error { setSoftMemoryLimit(memTotalSize) } + if _, ok := os.LookupEnv(GoGCEnvVar); !ok { + debug.SetGCPercent(DefaultGoGC) + log.Printf("Set garbage collection target percentage (GOGC) to %d", DefaultGoGC) + } + return nil }