Set Go garbage collection target percentage to 400%

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 ## Unreleased
 
+### 💡 Enhancements 💡
+
+- (Splunk) Set Go garbage collection target percentage to 400% ([#5034](https://github.com/signalfx/splunk-otel-collector/pull/5034))
+  After removal of memory_ballast extension in v0.97.0, the Go garbage collection is running more aggressively, which 
+  increased CPU usage and leads to reduced throughput of the collector. This change reduces the frequency of garbage 
+  collection cycles to improves performance of the collector for typical workloads. As a result, the collector will
+  report higher memory usage, but it will be bound to the same configured limits. If you want to revert to the previous
+  behavior, set the `GOGC` environment variable to `100`.
+
 ### 🧰 Bug fixes 🧰
 
 - (Splunk) `receiver/discovery`: Do not emit entity events for discovered endpoints that are not evaluated yet

internal/settings/settings.go

@@ -43,6 +43,7 @@ const (
 	HecLogIngestURLEnvVar     = "SPLUNK_HEC_URL"
 	ListenInterfaceEnvVar     = "SPLUNK_LISTEN_INTERFACE"
 	GoMemLimitEnvVar          = "GOMEMLIMIT"
+	GoGCEnvVar                = "GOGC"
 	// nolint:gosec
 	HecTokenEnvVar    = "SPLUNK_HEC_TOKEN" // this isn't a hardcoded token
 	IngestURLEnvVar   = "SPLUNK_INGEST_URL"
@@ -59,6 +60,7 @@ const (
 
 	DefaultMemoryLimitPercentage = 90
 	DefaultMemoryTotalMiB        = 512
+	DefaultGoGC                  = 400
 	DefaultListenInterface       = "0.0.0.0"
 	DefaultAgentConfigLinux      = "/etc/otel/collector/agent_config.yaml"
 	featureGates                 = "feature-gates"
@@ -371,6 +373,11 @@ func checkRuntimeParams(settings *Settings) error {
 		setSoftMemoryLimit(memTotalSize)
 	}
 
+	if _, ok := os.LookupEnv(GoGCEnvVar); !ok {
+		debug.SetGCPercent(DefaultGoGC)
+		log.Printf("Set garbage collection target percentage (GOGC) to %d", DefaultGoGC)
+	}
+
 	return nil
 }