From 6624d3667b02186831b7f64a0689bf2f0f9e00f0 Mon Sep 17 00:00:00 2001 From: Danielle Tomlinson Date: Fri, 22 Feb 2019 13:22:02 +0100 Subject: [PATCH] docker: Support stats on Windows --- drivers/docker/driver.go | 4 -- drivers/docker/stats.go | 53 ++------------------------ drivers/docker/stats_test.go | 27 ++++++++++---- drivers/docker/util/stats_posix.go | 53 ++++++++++++++++++++++++++ drivers/docker/util/stats_windows.go | 56 ++++++++++++++++++++++++++++ drivers/docker/util/util.go | 11 ++++++ 6 files changed, 143 insertions(+), 61 deletions(-) create mode 100644 drivers/docker/util/stats_posix.go create mode 100644 drivers/docker/util/stats_windows.go create mode 100644 drivers/docker/util/util.go diff --git a/drivers/docker/driver.go b/drivers/docker/driver.go index e38bf5898aed..de8fbfa0100e 100644 --- a/drivers/docker/driver.go +++ b/drivers/docker/driver.go @@ -42,10 +42,6 @@ var ( // running operations such as waiting on containers and collect stats waitClient *docker.Client - // The statistics the Docker driver exposes - DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage"} - DockerMeasuredCpuStats = []string{"Throttled Periods", "Throttled Time", "Percent"} - // recoverableErrTimeouts returns a recoverable error if the error was due // to timeouts recoverableErrTimeouts = func(err error) error { diff --git a/drivers/docker/stats.go b/drivers/docker/stats.go index c0a208a9bce1..3a08273ce63a 100644 --- a/drivers/docker/stats.go +++ b/drivers/docker/stats.go @@ -4,12 +4,11 @@ import ( "context" "fmt" "io" - "runtime" "time" docker "github.com/fsouza/go-dockerclient" cstructs "github.com/hashicorp/nomad/client/structs" - "github.com/hashicorp/nomad/helper/stats" + "github.com/hashicorp/nomad/drivers/docker/util" nstructs "github.com/hashicorp/nomad/nomad/structs" ) @@ -86,6 +85,7 @@ func (h *taskHandle) collectStats(ctx context.Context, ch chan *cstructs.TaskRes return } } + func dockerStatsCollector(destCh chan *cstructs.TaskResourceUsage, statsCh <-chan *docker.Stats, interval time.Duration) { var resourceUsage *cstructs.TaskResourceUsage @@ -117,7 +117,7 @@ func dockerStatsCollector(destCh chan *cstructs.TaskResourceUsage, statsCh <-cha } // s should always be set, but check and skip just in case if s != nil { - resourceUsage = dockerStatsToTaskResourceUsage(s) + resourceUsage = util.DockerStatsToTaskResourceUsage(s) // send stats next interation if this is the first time received // from docker if !hasSentInitialStats { @@ -128,50 +128,3 @@ func dockerStatsCollector(destCh chan *cstructs.TaskResourceUsage, statsCh <-cha } } } - -func dockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage { - ms := &cstructs.MemoryStats{ - RSS: s.MemoryStats.Stats.Rss, - Cache: s.MemoryStats.Stats.Cache, - Swap: s.MemoryStats.Stats.Swap, - Usage: s.MemoryStats.Usage, - MaxUsage: s.MemoryStats.MaxUsage, - Measured: DockerMeasuredMemStats, - } - - cs := &cstructs.CpuStats{ - ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods, - ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime, - Measured: DockerMeasuredCpuStats, - } - - // Calculate percentage - cs.Percent = calculatePercent( - s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, - s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, runtime.NumCPU()) - cs.SystemMode = calculatePercent( - s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode, - s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU()) - cs.UserMode = calculatePercent( - s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode, - s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU()) - cs.TotalTicks = (cs.Percent / 100) * stats.TotalTicksAvailable() / float64(runtime.NumCPU()) - - return &cstructs.TaskResourceUsage{ - ResourceUsage: &cstructs.ResourceUsage{ - MemoryStats: ms, - CpuStats: cs, - }, - Timestamp: s.Read.UTC().UnixNano(), - } -} - -func calculatePercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 { - numerator := newSample - oldSample - denom := newTotal - oldTotal - if numerator <= 0 || denom <= 0 { - return 0.0 - } - - return (float64(numerator) / float64(denom)) * float64(cores) * 100.0 -} diff --git a/drivers/docker/stats_test.go b/drivers/docker/stats_test.go index 66cd42306e13..78e8b987229c 100644 --- a/drivers/docker/stats_test.go +++ b/drivers/docker/stats_test.go @@ -1,6 +1,7 @@ package docker import ( + "runtime" "testing" "time" @@ -25,6 +26,9 @@ func TestDriver_DockerStatsCollector(t *testing.T) { stats.MemoryStats.Stats.Swap = 0 stats.MemoryStats.Usage = 5651904 stats.MemoryStats.MaxUsage = 6651904 + stats.MemoryStats.Commit = 123231 + stats.MemoryStats.CommitPeak = 321323 + stats.MemoryStats.PrivateWorkingSet = 62222 go dockerStatsCollector(dst, src, time.Second) @@ -36,13 +40,22 @@ func TestDriver_DockerStatsCollector(t *testing.T) { select { case ru := <-dst: - require.Equal(stats.MemoryStats.Stats.Rss, ru.ResourceUsage.MemoryStats.RSS) - require.Equal(stats.MemoryStats.Stats.Cache, ru.ResourceUsage.MemoryStats.Cache) - require.Equal(stats.MemoryStats.Stats.Swap, ru.ResourceUsage.MemoryStats.Swap) - require.Equal(stats.MemoryStats.Usage, ru.ResourceUsage.MemoryStats.Usage) - require.Equal(stats.MemoryStats.MaxUsage, ru.ResourceUsage.MemoryStats.MaxUsage) - require.Equal(stats.CPUStats.ThrottlingData.ThrottledPeriods, ru.ResourceUsage.CpuStats.ThrottledPeriods) - require.Equal(stats.CPUStats.ThrottlingData.ThrottledTime, ru.ResourceUsage.CpuStats.ThrottledTime) + if runtime.GOOS != "windows" { + require.Equal(stats.MemoryStats.Stats.Rss, ru.ResourceUsage.MemoryStats.RSS) + require.Equal(stats.MemoryStats.Stats.Cache, ru.ResourceUsage.MemoryStats.Cache) + require.Equal(stats.MemoryStats.Stats.Swap, ru.ResourceUsage.MemoryStats.Swap) + require.Equal(stats.MemoryStats.Usage, ru.ResourceUsage.MemoryStats.Usage) + require.Equal(stats.MemoryStats.MaxUsage, ru.ResourceUsage.MemoryStats.MaxUsage) + require.Equal(stats.CPUStats.ThrottlingData.ThrottledPeriods, ru.ResourceUsage.CpuStats.ThrottledPeriods) + require.Equal(stats.CPUStats.ThrottlingData.ThrottledTime, ru.ResourceUsage.CpuStats.ThrottledTime) + } else { + require.Equal(stats.MemoryStats.PrivateWorkingSet, ru.ResourceUsage.MemoryStats.RSS) + require.Equal(stats.MemoryStats.Commit, ru.ResourceUsage.MemoryStats.Usage) + require.Equal(stats.MemoryStats.CommitPeak, ru.ResourceUsage.MemoryStats.MaxUsage) + require.Equal(stats.CPUStats.ThrottlingData.ThrottledPeriods, ru.ResourceUsage.CpuStats.ThrottledPeriods) + require.Equal(stats.CPUStats.ThrottlingData.ThrottledTime, ru.ResourceUsage.CpuStats.ThrottledTime) + + } case <-time.After(time.Second): require.Fail("receiving stats should not block here") } diff --git a/drivers/docker/util/stats_posix.go b/drivers/docker/util/stats_posix.go new file mode 100644 index 000000000000..f27bd2d9a002 --- /dev/null +++ b/drivers/docker/util/stats_posix.go @@ -0,0 +1,53 @@ +// +build !windows + +package util + +import ( + "runtime" + + docker "github.com/fsouza/go-dockerclient" + cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/helper/stats" +) + +var ( + DockerMeasuredCPUStats = []string{"Throttled Periods", "Throttled Time", "Percent"} + DockerMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Usage", "Max Usage"} +) + +func DockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage { + ms := &cstructs.MemoryStats{ + RSS: s.MemoryStats.Stats.Rss, + Cache: s.MemoryStats.Stats.Cache, + Swap: s.MemoryStats.Stats.Swap, + Usage: s.MemoryStats.Usage, + MaxUsage: s.MemoryStats.MaxUsage, + Measured: DockerMeasuredMemStats, + } + + cs := &cstructs.CpuStats{ + ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods, + ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime, + Measured: DockerMeasuredCPUStats, + } + + // Calculate percentage + cs.Percent = CalculateCPUPercent( + s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, + s.CPUStats.SystemCPUUsage, s.PreCPUStats.SystemCPUUsage, runtime.NumCPU()) + cs.SystemMode = CalculateCPUPercent( + s.CPUStats.CPUUsage.UsageInKernelmode, s.PreCPUStats.CPUUsage.UsageInKernelmode, + s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU()) + cs.UserMode = CalculateCPUPercent( + s.CPUStats.CPUUsage.UsageInUsermode, s.PreCPUStats.CPUUsage.UsageInUsermode, + s.CPUStats.CPUUsage.TotalUsage, s.PreCPUStats.CPUUsage.TotalUsage, runtime.NumCPU()) + cs.TotalTicks = (cs.Percent / 100) * stats.TotalTicksAvailable() / float64(runtime.NumCPU()) + + return &cstructs.TaskResourceUsage{ + ResourceUsage: &cstructs.ResourceUsage{ + MemoryStats: ms, + CpuStats: cs, + }, + Timestamp: s.Read.UTC().UnixNano(), + } +} diff --git a/drivers/docker/util/stats_windows.go b/drivers/docker/util/stats_windows.go new file mode 100644 index 000000000000..057edd399dc1 --- /dev/null +++ b/drivers/docker/util/stats_windows.go @@ -0,0 +1,56 @@ +package util + +import ( + "runtime" + + docker "github.com/fsouza/go-dockerclient" + cstructs "github.com/hashicorp/nomad/client/structs" + "github.com/hashicorp/nomad/helper/stats" +) + +var ( + // The statistics the Docker driver exposes + DockerMeasuredCPUStats = []string{"Throttled Periods", "Throttled Time", "Percent"} + DockerMeasuredMemStats = []string{"RSS", "Usage", "Max Usage"} +) + +func DockerStatsToTaskResourceUsage(s *docker.Stats) *cstructs.TaskResourceUsage { + ms := &cstructs.MemoryStats{ + RSS: s.MemoryStats.PrivateWorkingSet, + Usage: s.MemoryStats.Commit, + MaxUsage: s.MemoryStats.CommitPeak, + Measured: DockerMeasuredMemStats, + } + + cpuPercent := 0.0 + + // https://github.com/moby/moby/blob/cbb885b07af59225eef12a8159e70d1485616d57/integration-cli/docker_api_stats_test.go#L47-L58 + // Max number of 100ns intervals between the previous time read and now + possIntervals := uint64(s.Read.Sub(s.PreRead).Nanoseconds()) // Start with number of ns intervals + possIntervals /= 100 // Convert to number of 100ns intervals + possIntervals *= uint64(s.NumProcs) // Multiple by the number of processors + + // Intervals used + intervalsUsed := s.CPUStats.CPUUsage.TotalUsage - s.PreCPUStats.CPUUsage.TotalUsage + + // Percentage avoiding divide-by-zero + if possIntervals > 0 { + cpuPercent = float64(intervalsUsed) / float64(possIntervals) * 100.0 + } + + cs := &cstructs.CpuStats{ + ThrottledPeriods: s.CPUStats.ThrottlingData.ThrottledPeriods, + ThrottledTime: s.CPUStats.ThrottlingData.ThrottledTime, + Percent: cpuPercent, + TotalTicks: (cpuPercent / 100) * stats.TotalTicksAvailable() / float64(runtime.NumCPU()), + Measured: DockerMeasuredCPUStats, + } + + return &cstructs.TaskResourceUsage{ + ResourceUsage: &cstructs.ResourceUsage{ + MemoryStats: ms, + CpuStats: cs, + }, + Timestamp: s.Read.UTC().UnixNano(), + } +} diff --git a/drivers/docker/util/util.go b/drivers/docker/util/util.go new file mode 100644 index 000000000000..e5d10ce0b680 --- /dev/null +++ b/drivers/docker/util/util.go @@ -0,0 +1,11 @@ +package util + +func CalculateCPUPercent(newSample, oldSample, newTotal, oldTotal uint64, cores int) float64 { + numerator := newSample - oldSample + denom := newTotal - oldTotal + if numerator <= 0 || denom <= 0 { + return 0.0 + } + + return (float64(numerator) / float64(denom)) * float64(cores) * 100.0 +}