From d3aba5df6fa23df1954890cd912296fb6e1f448c Mon Sep 17 00:00:00 2001 From: Dmitry Anoshin Date: Fri, 28 Jun 2024 11:32:15 -0700 Subject: [PATCH] Set Go garbage collection target percentage to 400% After removal of memory_ballast extension in v0.97.0, the Go garbage collection is running more aggressively, which increased CPU usage and leads to reduced throughput of the collector. This change reduces the frequency of garbage collection cycles to improves performance of the collector for typical workloads. As a result, the collector will report higher memory usage, but it will be bound to the same configured limits. If you want to revert to the previous behavior, set the `GOGC` environment variable to `100`. --- CHANGELOG.md | 9 +++++++++ internal/settings/settings.go | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42574d8fd9..f6867762a4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,15 @@ ## Unreleased +### 💡 Enhancements 💡 + +- (Splunk) Set Go garbage collection target percentage to 400% ([#5034](https://github.com/signalfx/splunk-otel-collector/pull/5034)) + After removal of memory_ballast extension in v0.97.0, the Go garbage collection is running more aggressively, which + increased CPU usage and leads to reduced throughput of the collector. This change reduces the frequency of garbage + collection cycles to improves performance of the collector for typical workloads. As a result, the collector will + report higher memory usage, but it will be bound to the same configured limits. If you want to revert to the previous + behavior, set the `GOGC` environment variable to `100`. + ### 🧰 Bug fixes 🧰 - (Splunk) `receiver/discovery`: Do not emit entity events for discovered endpoints that are not evaluated yet diff --git a/internal/settings/settings.go b/internal/settings/settings.go index 7350abea3b..047308ff63 100644 --- a/internal/settings/settings.go +++ b/internal/settings/settings.go @@ -43,6 +43,7 @@ const ( HecLogIngestURLEnvVar = "SPLUNK_HEC_URL" ListenInterfaceEnvVar = "SPLUNK_LISTEN_INTERFACE" GoMemLimitEnvVar = "GOMEMLIMIT" + GoGCEnvVar = "GOGC" // nolint:gosec HecTokenEnvVar = "SPLUNK_HEC_TOKEN" // this isn't a hardcoded token IngestURLEnvVar = "SPLUNK_INGEST_URL" @@ -59,6 +60,7 @@ const ( DefaultMemoryLimitPercentage = 90 DefaultMemoryTotalMiB = 512 + DefaultGoGC = 400 DefaultListenInterface = "0.0.0.0" DefaultAgentConfigLinux = "/etc/otel/collector/agent_config.yaml" featureGates = "feature-gates" @@ -371,6 +373,11 @@ func checkRuntimeParams(settings *Settings) error { setSoftMemoryLimit(memTotalSize) } + if _, ok := os.LookupEnv(GoGCEnvVar); !ok { + debug.SetGCPercent(DefaultGoGC) + log.Printf("GOGC is set to %d by default", DefaultGoGC) + } + return nil }