[12.0-stable] Configure Go GC based on pillar memory limit or global config

pkg/pillar/cmd/zedmanager/zedmanager.go

@@ -1437,6 +1437,17 @@ func quantifyChanges(config types.AppInstanceConfig, oldConfig types.AppInstance
 	return needPurge, needRestart, purgeReason, restartReason
 }
 
+func configureGOGC(gcp *types.ConfigItemValueMap) {
+	lim := gcp.GlobalValueInt(types.GOGCMemoryLimitInBytes)
+	per := gcp.GlobalValueInt(types.GOGCPercent)
+	plim, pper, err := types.ConfigureGOGC(int64(lim), int(per))
+	if err != nil {
+		log.Warningf("configureGOGC: failed '%v'", err)
+	} else {
+		log.Functionf("configureGOGC: memory limit set to '%v' (previous '%v'), GC percent set to '%v' (previous '%v')", lim, plim, per, pper)
+	}
+}
+
 func handleGlobalConfigCreate(ctxArg interface{}, key string,
 	statusArg interface{}) {
 	handleGlobalConfigImpl(ctxArg, key, statusArg)
@@ -1462,6 +1473,7 @@ func handleGlobalConfigImpl(ctxArg interface{}, key string,
 		ctx.globalConfig = gcp
 		ctx.GCInitialized = true
 	}
+	configureGOGC(gcp)
 	log.Functionf("handleGlobalConfigImpl done for %s", key)
 }
 

pkg/pillar/types/global.go

@@ -229,6 +229,10 @@ const (
 	EveMemoryLimitInBytes GlobalSettingKey = "memory.eve.limit.bytes"
 	// How much memory overhead is allowed for VMM needs
 	VmmMemoryLimitInMiB GlobalSettingKey = "memory.vmm.limit.MiB"
+	// GOGCMemoryLimitInBytes global setting key
+	GOGCMemoryLimitInBytes GlobalSettingKey = "gogc.memory.limit.bytes"
+	// GOGCPercent global setting key
+	GOGCPercent GlobalSettingKey = "gogc.percent"
 	// IgnoreMemoryCheckForApps global setting key
 	IgnoreMemoryCheckForApps GlobalSettingKey = "memory.apps.ignore.check"
 	// IgnoreDiskCheckForApps global setting key
@@ -872,7 +876,10 @@ func NewConfigItemSpecMap() ConfigItemSpecMap {
 		100*1024*1024, 0xFFFFFFFF)
 	configItemSpecMap.AddIntItem(StorageZfsReserved, 20, 1, 99)
 	configItemSpecMap.AddIntItem(ForceFallbackCounter, 0, 0, 0xFFFFFFFF)
-
+	// Default GOGC memory limit is 0
+	configItemSpecMap.AddIntItem(GOGCMemoryLimitInBytes, 0, 0, 0xFFFFFFFF)
+	// Default GOGC target percentage is 100, 0 means disable GC
+	configItemSpecMap.AddIntItem(GOGCPercent, 100, 0, 500)
 	configItemSpecMap.AddIntItem(EveMemoryLimitInBytes, uint32(eveMemoryLimitInBytes),
 		uint32(eveMemoryLimitInBytes), 0xFFFFFFFF)
 	// Limit manual vmm overhead override to 1 PiB

pkg/pillar/types/global_test.go

@@ -181,6 +181,8 @@ func TestNewConfigItemSpecMap(t *testing.T) {
 		VgaAccess,
 		ConsoleAccess,
 		AllowAppVnc,
+		GOGCMemoryLimitInBytes,
+		GOGCPercent,
 		EveMemoryLimitInBytes,
 		VmmMemoryLimitInMiB,
 		IgnoreMemoryCheckForApps,

pkg/pillar/types/locationconsts.go

@@ -98,6 +98,8 @@ const (
 	NewlogUploadAppDir = NewlogDir + "/appUpload"
 	// NewlogKeepSentQueueDir - a circular queue of gzip files already been sent
 	NewlogKeepSentQueueDir = NewlogDir + "/keepSentQueue"
+	// PillarHardMemoryLimitFile - hard memory reserved for pillar
+	PillarHardMemoryLimitFile = "/hostfs/sys/fs/cgroup/memory/eve/services/pillar/memory.limit_in_bytes"
 	// EveMemoryLimitFile - stores memory reserved for eve
 	EveMemoryLimitFile = "/hostfs/sys/fs/cgroup/memory/eve/memory.soft_limit_in_bytes"
 	// EveMemoryUsageFile - current usage

pkg/pillar/types/memory.go

@@ -4,11 +4,19 @@
 package types
 
 import (
+	"fmt"
 	"os"
+	"runtime/debug"
 	"strconv"
 	"strings"
 )
 
+// GetPillarHardMemoryLimitInBytes returns hard memory limit
+// reserved for pillar in bytes
+func GetPillarHardMemoryLimitInBytes() (uint64, error) {
+	return readUint64File(PillarHardMemoryLimitFile)
+}
+
 // GetEveMemoryLimitInBytes returns memory limit
 // reserved for eve in bytes
 func GetEveMemoryLimitInBytes() (uint64, error) {
@@ -42,3 +50,36 @@ func readUint64File(filename string) (uint64, error) {
 	dataUint64, err := strconv.ParseUint(dataString, 10, 64)
 	return dataUint64, err
 }
+
+// ConfigureGOGC sets two main configuration parameters for the
+// garbage collector (GOGC): memory limit and percentage (see
+// explanation here: https://tip.golang.org/doc/gc-guide).
+// If limit is 0, create GOGC limit from the pillar cgroups hard
+// memory limit.
+func ConfigureGOGC(limit int64, percent int) (int64, int, error) {
+	if limit == 0 {
+		// Fallback to value from cgroups if no limit in the configuration
+		ulimit, err := GetPillarHardMemoryLimitInBytes()
+		if err != nil {
+			err := fmt.Errorf("can't receive pillar memory hard limit: '%w'", err)
+			return -1, -1, err
+		}
+		// Reduce actual memory limit to 0.6 of cgroup limit. The logic behind
+		// the constant is simple: cgroup limit is a hard limit for the whole
+		// pillar cgroup, meaning when reached, we are killed by OOM. In turn
+		// GOGC memory limit is a soft limit, so the difference must be
+		// significant to ensure that after the soft limit is reached, there
+		// will be enough memory for the GOGC to do its job and, fortunately,
+		// not to hit the hard limit.
+		limit = int64(ulimit) * 600 / 1000
+	}
+	if percent == 0 {
+		// Disable GC
+		percent = -1
+	}
+	// Set new and retrieve previous values
+	limit = debug.SetMemoryLimit(limit)
+	percent = debug.SetGCPercent(percent)
+
+	return limit, percent, nil
+}