Merge pull request #5190 from hashicorp/f-memory-usage

Track Basic Memory Usage as reported by cgroups
hashicorp · Jan 18, 2019 · 9f76193 · 9f76193
2 parents 256ac52 + b5c20aa
commit 9f76193
Show file tree

Hide file tree

Showing 19 changed files with 338 additions and 308 deletions.
diff --git a/api/tasks.go b/api/tasks.go
@@ -16,6 +16,7 @@ type MemoryStats struct {
 	RSS            uint64
 	Cache          uint64
 	Swap           uint64
+	Usage          uint64
 	MaxUsage       uint64
 	KernelUsage    uint64
 	KernelMaxUsage uint64

diff --git a/client/allocrunner/taskrunner/task_runner.go b/client/allocrunner/taskrunner/task_runner.go
@@ -1118,14 +1118,14 @@ func (tr *TaskRunner) UpdateStats(ru *cstructs.TaskResourceUsage) {
 //TODO Remove Backwardscompat or use tr.Alloc()?
 func (tr *TaskRunner) setGaugeForMemory(ru *cstructs.TaskResourceUsage) {
 	if !tr.clientConfig.DisableTaggedMetrics {
-		metrics.SetGaugeWithLabels([]string{"client", "allocs", "memory", "rss"},
-			float32(ru.ResourceUsage.MemoryStats.RSS), tr.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocs", "memory", "rss"},
 			float32(ru.ResourceUsage.MemoryStats.RSS), tr.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocs", "memory", "cache"},
 			float32(ru.ResourceUsage.MemoryStats.Cache), tr.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocs", "memory", "swap"},
 			float32(ru.ResourceUsage.MemoryStats.Swap), tr.baseLabels)
+		metrics.SetGaugeWithLabels([]string{"client", "allocs", "memory", "usage"},
+			float32(ru.ResourceUsage.MemoryStats.Usage), tr.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocs", "memory", "max_usage"},
 			float32(ru.ResourceUsage.MemoryStats.MaxUsage), tr.baseLabels)
 		metrics.SetGaugeWithLabels([]string{"client", "allocs", "memory", "kernel_usage"},
@@ -1138,6 +1138,7 @@ func (tr *TaskRunner) setGaugeForMemory(ru *cstructs.TaskResourceUsage) {
 		metrics.SetGauge([]string{"client", "allocs", tr.alloc.Job.Name, tr.alloc.TaskGroup, tr.allocID, tr.taskName, "memory", "rss"}, float32(ru.ResourceUsage.MemoryStats.RSS))
 		metrics.SetGauge([]string{"client", "allocs", tr.alloc.Job.Name, tr.alloc.TaskGroup, tr.allocID, tr.taskName, "memory", "cache"}, float32(ru.ResourceUsage.MemoryStats.Cache))
 		metrics.SetGauge([]string{"client", "allocs", tr.alloc.Job.Name, tr.alloc.TaskGroup, tr.allocID, tr.taskName, "memory", "swap"}, float32(ru.ResourceUsage.MemoryStats.Swap))
+		metrics.SetGauge([]string{"client", "allocs", tr.alloc.Job.Name, tr.alloc.TaskGroup, tr.allocID, tr.taskName, "memory", "usage"}, float32(ru.ResourceUsage.MemoryStats.Usage))
 		metrics.SetGauge([]string{"client", "allocs", tr.alloc.Job.Name, tr.alloc.TaskGroup, tr.allocID, tr.taskName, "memory", "max_usage"}, float32(ru.ResourceUsage.MemoryStats.MaxUsage))
 		metrics.SetGauge([]string{"client", "allocs", tr.alloc.Job.Name, tr.alloc.TaskGroup, tr.allocID, tr.taskName, "memory", "kernel_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelUsage))
 		metrics.SetGauge([]string{"client", "allocs", tr.alloc.Job.Name, tr.alloc.TaskGroup, tr.allocID, tr.taskName, "memory", "kernel_max_usage"}, float32(ru.ResourceUsage.MemoryStats.KernelMaxUsage))

diff --git a/client/logmon/proto/logmon.pb.go b/client/logmon/proto/logmon.pb.go
diff --git a/client/structs/structs.go b/client/structs/structs.go
@@ -168,6 +168,7 @@ type MemoryStats struct {
 	RSS            uint64
 	Cache          uint64
 	Swap           uint64
+	Usage          uint64
 	MaxUsage       uint64
 	KernelUsage    uint64
 	KernelMaxUsage uint64
@@ -184,6 +185,7 @@ func (ms *MemoryStats) Add(other *MemoryStats) {
 	ms.RSS += other.RSS
 	ms.Cache += other.Cache
 	ms.Swap += other.Swap
+	ms.Usage += other.Usage
 	ms.MaxUsage += other.MaxUsage
 	ms.KernelUsage += other.KernelUsage
 	ms.KernelMaxUsage += other.KernelMaxUsage

diff --git a/command/alloc_status.go b/command/alloc_status.go
@@ -557,6 +557,8 @@ func (c *AllocStatusCommand) outputVerboseResourceUsage(task string, resourceUsa
 				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Cache))
 			case "Swap":
 				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Swap))
+			case "Usage":
+				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.Usage))
 			case "Max Usage":
 				measuredStats = append(measuredStats, humanize.IBytes(memoryStats.MaxUsage))
 			case "Kernel Usage":

diff --git a/drivers/docker/docklog/proto/docker_logger.pb.go b/drivers/docker/docklog/proto/docker_logger.pb.go
diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go
@@ -41,7 +41,7 @@ var (
 	waitClient *docker.Client
 
 	// The statistics the Docker driver exposes
-	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage"}
+	DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage"}
 	DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"}
 
 	// recoverableErrTimeouts returns a recoverable error if the error was due

diff --git a/drivers/docker/stats.go b/drivers/docker/stats.go
@@ -134,6 +134,7 @@ func dockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage
 		RSS:      s.MemoryStats.Stats.Rss,
 		Cache:    s.MemoryStats.Stats.Cache,
 		Swap:     s.MemoryStats.Stats.Swap,
+		Usage:    s.MemoryStats.Usage,
 		MaxUsage: s.MemoryStats.MaxUsage,
 		Measured: DockerMeasuredMemStats,
 	}

diff --git a/drivers/docker/stats_test.go b/drivers/docker/stats_test.go
@@ -23,6 +23,7 @@ func TestDriver_DockerStatsCollector(t *testing.T) {
 	stats.MemoryStats.Stats.Rss = 6537216
 	stats.MemoryStats.Stats.Cache = 1234
 	stats.MemoryStats.Stats.Swap = 0
+	stats.MemoryStats.Usage = 5651904
 	stats.MemoryStats.MaxUsage = 6651904
 
 	go dockerStatsCollector(dst, src, time.Second)
@@ -38,6 +39,7 @@ func TestDriver_DockerStatsCollector(t *testing.T) {
 		require.Equal(stats.MemoryStats.Stats.Rss, ru.ResourceUsage.MemoryStats.RSS)
 		require.Equal(stats.MemoryStats.Stats.Cache, ru.ResourceUsage.MemoryStats.Cache)
 		require.Equal(stats.MemoryStats.Stats.Swap, ru.ResourceUsage.MemoryStats.Swap)
+		require.Equal(stats.MemoryStats.Usage, ru.ResourceUsage.MemoryStats.Usage)
 		require.Equal(stats.MemoryStats.MaxUsage, ru.ResourceUsage.MemoryStats.MaxUsage)
 		require.Equal(stats.CPUStats.ThrottlingData.ThrottledPeriods, ru.ResourceUsage.CpuStats.ThrottledPeriods)
 		require.Equal(stats.CPUStats.ThrottlingData.ThrottledTime, ru.ResourceUsage.CpuStats.ThrottledTime)

diff --git a/drivers/shared/executor/executor_linux.go b/drivers/shared/executor/executor_linux.go
@@ -39,7 +39,7 @@ const (
 
 var (
 	// ExecutorCgroupMeasuredMemStats is the list of memory stats captured by the executor
-	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
+	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
 
 	// ExecutorCgroupMeasuredCpuStats is the list of CPU stats captures by the executor
 	ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
@@ -400,6 +400,7 @@ func (l *LibcontainerExecutor) handleStats(ch chan *cstructs.TaskResourceUsage,
 			RSS:            rss,
 			Cache:          cache,
 			Swap:           swap.Usage,
+			Usage:          stats.MemoryStats.Usage.Usage,
 			MaxUsage:       maxUsage,
 			KernelUsage:    stats.MemoryStats.KernelUsage.Usage,
 			KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage,

diff --git a/plugins/base/proto/base.pb.go b/plugins/base/proto/base.pb.go