From 453227293141cb041a073df04b4d16cc1d33d05c Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 1 Apr 2021 11:50:17 -0400 Subject: [PATCH 1/4] drivers/exec: Account for cgroup-v2 memory stats If the host is running with cgroup-v2, RSS and Max Usage doesn't get reported anymore. --- drivers/shared/executor/executor_linux.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go index 0952d332e2f4..ef302caf3f74 100644 --- a/drivers/shared/executor/executor_linux.go +++ b/drivers/shared/executor/executor_linux.go @@ -39,8 +39,11 @@ const ( ) var ( - // ExecutorCgroupMeasuredMemStats is the list of memory stats captured by the executor - ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage", "Kernel Usage", "Kernel Max Usage"} + // ExecutorCgroupV1MeasuredMemStats is the list of memory stats captured by the executor with cgroup-v1 + ExecutorCgroupV1MeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage", "Kernel Usage", "Kernel Max Usage"} + + // ExecutorCgroupV2MeasuredMemStats is the list of memory stats captured by the executor with cgroup-v2. cgroup-v2 exposes different memory stats and no longer reports rss or max usage. + ExecutorCgroupV2MeasuredMemStats = []string{"Cache", "Swap", "Usage"} // ExecutorCgroupMeasuredCpuStats is the list of CPU stats captures by the executor ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"} @@ -342,6 +345,12 @@ func (l *LibcontainerExecutor) Stats(ctx context.Context, interval time.Duration func (l *LibcontainerExecutor) handleStats(ch chan *cstructs.TaskResourceUsage, ctx context.Context, interval time.Duration) { defer close(ch) timer := time.NewTimer(0) + + measuredMemStats := ExecutorCgroupV1MeasuredMemStats + if cgroups.IsCgroup2UnifiedMode() { + measuredMemStats = ExecutorCgroupV2MeasuredMemStats + } + for { select { case <-ctx.Done(): @@ -379,7 +388,7 @@ func (l *LibcontainerExecutor) handleStats(ch chan *cstructs.TaskResourceUsage, MaxUsage: maxUsage, KernelUsage: stats.MemoryStats.KernelUsage.Usage, KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage, - Measured: ExecutorCgroupMeasuredMemStats, + Measured: measuredMemStats, } // CPU Related Stats From f7b7b1cf3f64b46e85b33805eea569333f2b0a84 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 1 Apr 2021 11:51:29 -0400 Subject: [PATCH 2/4] drivers/docker: account for cgroup-v2 memory stats If the docker engine is running on cgroup-v2 host, then RSS and Max Usage doesn't get reported. Using a heauristic here to avoid adding more API calls to the Docker Engine to infer cgroups version. Also, opted to avoid coordinating stats collection with fingerprinting, which adds concurrency complexities. --- drivers/docker/util/stats_posix.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/drivers/docker/util/stats_posix.go b/drivers/docker/util/stats_posix.go index f27bd2d9a002..60219cf076f5 100644 --- a/drivers/docker/util/stats_posix.go +++ b/drivers/docker/util/stats_posix.go @@ -12,17 +12,28 @@ import ( var ( DockerMeasuredCPUStats = []string{"Throttled Periods", "Throttled Time", "Percent"} - DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage"} + + // cgroup-v2 only exposes a subset of memory stats + DockerCgroupV1MeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage"} + DockerCgroupV2MeasuredMemStats = []string{"Cache", "Swap", "Usage"} ) func DockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage { + measuredMems := DockerCgroupV1MeasuredMemStats + + // use a simple heauristic to check if cgroup-v2 is used. + // go-dockerclient doesn't distinguish between 0 and not-present value + if s.MemoryStats.Stats.Rss == 0 && s.MemoryStats.MaxUsage == 0 && s.MemoryStats.Usage != 0 { + measuredMems = DockerCgroupV2MeasuredMemStats + } + ms := &cstructs.MemoryStats{ RSS: s.MemoryStats.Stats.Rss, Cache: s.MemoryStats.Stats.Cache, Swap: s.MemoryStats.Stats.Swap, Usage: s.MemoryStats.Usage, MaxUsage: s.MemoryStats.MaxUsage, - Measured: DockerMeasuredMemStats, + Measured: measuredMems, } cs := &cstructs.CpuStats{ From f14921d41dc6ea9a368f3fcc10fe74bfa5e18a52 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Thu, 1 Apr 2021 11:56:23 -0400 Subject: [PATCH 3/4] cli: Show memory usage instead of RSS If a task doesn't report RSS, let's use memory usage. --- command/alloc_status.go | 8 +++++++- command/helpers.go | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/command/alloc_status.go b/command/alloc_status.go index 4e21d18adff2..b9d2b46be5cb 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -586,7 +586,13 @@ func (c *AllocStatusCommand) outputTaskResources(alloc *api.Allocation, task str cpuUsage = fmt.Sprintf("%v/%v", math.Floor(cs.TotalTicks), cpuUsage) } if ms := ru.ResourceUsage.MemoryStats; ms != nil { - memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(ms.RSS), memUsage) + // Nomad uses RSS as the top-level metric to report, for historical reasons, + // but it's not always measured (e.g. with cgroup-v2) + usage := ms.RSS + if usage == 0 && !stringsContain(ms.Measured, "RSS") { + usage = ms.Usage + } + memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(usage), memUsage) } deviceStats = ru.ResourceUsage.DeviceStats } diff --git a/command/helpers.go b/command/helpers.go index 74b221bba406..177cf48c17c6 100644 --- a/command/helpers.go +++ b/command/helpers.go @@ -542,3 +542,14 @@ func (w *uiErrorWriter) Close() error { } return nil } + +// stringsContains returns true if s is present in the vs string slice +func stringsContain(vs []string, s string) bool { + for _, v := range vs { + if v == s { + return true + } + } + + return false +} From 13a9e4bc3188a94d8d9e836994f89c9cf7352e01 Mon Sep 17 00:00:00 2001 From: Mahmood Ali Date: Fri, 2 Apr 2021 11:56:27 -0400 Subject: [PATCH 4/4] reuse existing function and typo fix --- command/alloc_status.go | 3 ++- command/helpers.go | 11 ----------- drivers/docker/util/stats_posix.go | 2 +- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/command/alloc_status.go b/command/alloc_status.go index b9d2b46be5cb..d8a5e841b655 100644 --- a/command/alloc_status.go +++ b/command/alloc_status.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/api" "github.com/hashicorp/nomad/api/contexts" "github.com/hashicorp/nomad/client/allocrunner/taskrunner/restarts" + "github.com/hashicorp/nomad/helper" "github.com/hashicorp/nomad/nomad/structs" "github.com/posener/complete" ) @@ -589,7 +590,7 @@ func (c *AllocStatusCommand) outputTaskResources(alloc *api.Allocation, task str // Nomad uses RSS as the top-level metric to report, for historical reasons, // but it's not always measured (e.g. with cgroup-v2) usage := ms.RSS - if usage == 0 && !stringsContain(ms.Measured, "RSS") { + if usage == 0 && !helper.SliceStringContains(ms.Measured, "RSS") { usage = ms.Usage } memUsage = fmt.Sprintf("%v/%v", humanize.IBytes(usage), memUsage) diff --git a/command/helpers.go b/command/helpers.go index 177cf48c17c6..74b221bba406 100644 --- a/command/helpers.go +++ b/command/helpers.go @@ -542,14 +542,3 @@ func (w *uiErrorWriter) Close() error { } return nil } - -// stringsContains returns true if s is present in the vs string slice -func stringsContain(vs []string, s string) bool { - for _, v := range vs { - if v == s { - return true - } - } - - return false -} diff --git a/drivers/docker/util/stats_posix.go b/drivers/docker/util/stats_posix.go index 60219cf076f5..356096395763 100644 --- a/drivers/docker/util/stats_posix.go +++ b/drivers/docker/util/stats_posix.go @@ -21,7 +21,7 @@ var ( func DockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage { measuredMems := DockerCgroupV1MeasuredMemStats - // use a simple heauristic to check if cgroup-v2 is used. + // use a simple heuristic to check if cgroup-v2 is used. // go-dockerclient doesn't distinguish between 0 and not-present value if s.MemoryStats.Stats.Rss == 0 && s.MemoryStats.MaxUsage == 0 && s.MemoryStats.Usage != 0 { measuredMems = DockerCgroupV2MeasuredMemStats