pillar: log increased memory watermark for an application

Every time metrics are collected, check the cgroup's
memory.max_usage_in_bytes has been increased since last time
checked. If yes, print the new value to the logs.

This will allow us to understand the overhead brought by qemu and
containerd in the real use cases.

Signed-off-by: Yuri Volchkov <yuri@zededa.com>

pkg/pillar/cmd/domainmgr/metric.go

@@ -41,6 +41,33 @@ func metricsTimerTask(ctx *domainContext, hyper hypervisor.Hypervisor) {
 	}
 }
 
+func logWatermarks(ctx *domainContext, status *types.DomainStatus, dm *types.DomainMetric) {
+	if status == nil {
+		return
+	}
+
+	config := lookupDomainConfig(ctx, status.Key())
+	if config == nil {
+		return
+	}
+
+	var CurrMaxUsedMemory uint32
+	st, _ := ctx.pubDomainMetric.Get(dm.Key())
+	if st != nil {
+		previousMetric := st.(types.DomainMetric)
+		CurrMaxUsedMemory = previousMetric.MaxUsedMemory
+	}
+
+	if CurrMaxUsedMemory < dm.MaxUsedMemory && config.Memory != 0 {
+		usedPercents := dm.MaxUsedMemory * 100 * 1024 / uint32(config.Memory)
+		log.Noticef("Memory watermark for %s increased: %d MiB,"+
+			" app-memory %d MiB (%d%%), %.2f%% of cgroup limit",
+			status.DomainName,
+			dm.MaxUsedMemory, config.Memory>>10,
+			usedPercents, dm.UsedMemoryPercent)
+	}
+}
+
 func getAndPublishMetrics(ctx *domainContext, hyper hypervisor.Hypervisor) {
 	dmList, _ := hyper.GetDomsCPUMem()
 	now := time.Now()
@@ -70,8 +97,12 @@ func getAndPublishMetrics(ctx *domainContext, hyper hypervisor.Hypervisor) {
 			// We clear the memory so it doesn't accidentally get
 			// reported.  We keep the CPUTotal and AvailableMemory
 			dm.UsedMemory = 0
+			dm.MaxUsedMemory = 0
 			dm.UsedMemoryPercent = 0
 		}
+
+		logWatermarks(ctx, status, &dm)
+
 		dm.LastHeard = now
 		ctx.pubDomainMetric.Publish(dm.Key(), dm)
 	}

pkg/pillar/hypervisor/containerd.go

@@ -244,7 +244,7 @@ func (ctx ctrdContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) {
 	}
 
 	for _, id := range ids {
-		var usedMem, availMem, totalMem uint32
+		var usedMem, maxUsedMem, availMem, totalMem uint32
 		var usedMemPerc float64
 		var cpuTotal uint64
 
@@ -253,6 +253,7 @@ func (ctx ctrdContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) {
 				logrus.Errorf("GetDomsCPUMem nil returned in metric.Memory: %v", metric)
 			} else {
 				usedMem = uint32(roundFromBytesToMbytes(metric.Memory.Usage.Usage))
+				maxUsedMem = uint32(roundFromBytesToMbytes(metric.Memory.Usage.Max))
 				totalMem = uint32(roundFromBytesToMbytes(metric.Memory.HierarchicalMemoryLimit))
 				availMem = 0
 				if totalMem > usedMem {
@@ -277,6 +278,7 @@ func (ctx ctrdContext) GetDomsCPUMem() (map[string]types.DomainMetric, error) {
 			UUIDandVersion:    types.UUIDandVersion{},
 			CPUTotal:          cpuTotal,
 			UsedMemory:        usedMem,
+			MaxUsedMemory:     maxUsedMem,
 			AvailableMemory:   availMem,
 			UsedMemoryPercent: usedMemPerc,
 		}

pkg/pillar/types/domainmgrtypes.go

@@ -393,6 +393,7 @@ type DomainMetric struct {
 	UUIDandVersion    UUIDandVersion
 	CPUTotal          uint64 // Seconds since Domain boot
 	UsedMemory        uint32
+	MaxUsedMemory     uint32
 	AvailableMemory   uint32
 	UsedMemoryPercent float64
 	LastHeard         time.Time