Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add stats to stdout storage #2612

Merged
merged 1 commit into from
Sep 11, 2020
Merged

Add stats to stdout storage #2612

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 135 additions & 27 deletions cmd/internal/storage/stdout/stdout.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package stdout
import (
"bytes"
"fmt"
"strconv"
"time"

info "github.com/google/cadvisor/info/v1"
Expand All @@ -32,27 +33,66 @@ type stdoutStorage struct {
}

const (
colTimestamp = "timestamp"
// CPU Uasge
colCpuCumulativeUsage = "cpu_cumulative_usage"
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm fine with the rename, but just in-case someone is using these metrics, lets keep the old name around as well, with a link to an issue to remove it in a few releases (v0.39.0)

serTimestamp string = "timestamp"
// Cumulative CPU usage
// To be deprecated in 0.39
// https://github.com/google/cadvisor/issues/2637
colCpuCumulativeUsage string = "cpu_cumulative_usage"
// Cumulative CPU usage
serCpuUsageTotal string = "cpu_usage_total"
serCpuUsageSystem string = "cpu_usage_system"
serCpuUsageUser string = "cpu_usage_user"
serCpuUsagePerCpu string = "cpu_usage_per_cpu"
// Smoothed average of number of runnable threads x 1000.
serLoadAverage string = "load_average"
// Memory Usage
colMemoryUsage = "memory_usage"
serMemoryUsage string = "memory_usage"
// Maximum memory usage recorded
serMemoryMaxUsage string = "memory_max_usage"
// Number of bytes of page cache memory
serMemoryCache string = "memory_cache"
// Size of RSS
serMemoryRss string = "memory_rss"
// Container swap usage
serMemorySwap string = "memory_swap"
// Size of memory mapped files in bytes
serMemoryMappedFile string = "memory_mapped_file"
// Working set size
colMemoryWorkingSet = "memory_working_set"
serMemoryWorkingSet string = "memory_working_set"
// Number of memory usage hits limits
serMemoryFailcnt string = "memory_failcnt"
// Cumulative count of memory allocation failures
serMemoryFailure string = "memory_failure"
// Cumulative count of bytes received.
colRxBytes = "rx_bytes"
serRxBytes string = "rx_bytes"
// Cumulative count of receive errors encountered.
colRxErrors = "rx_errors"
serRxErrors string = "rx_errors"
// Cumulative count of bytes transmitted.
colTxBytes = "tx_bytes"
serTxBytes string = "tx_bytes"
// Cumulative count of transmit errors encountered.
colTxErrors = "tx_errors"
serTxErrors string = "tx_errors"
// Filesystem summary
colFsSummary = "fs_summary"
serFsSummary string = "fs_summary"
// Filesystem limit.
colFsLimit = "fs_limit"
serFsLimit string = "fs_limit"
// Filesystem usage.
colFsUsage = "fs_usage"
serFsUsage string = "fs_usage"
// Hugetlb stat - current res_counter usage for hugetlb
setHugetlbUsage string = "hugetlb_usage"
// Hugetlb stat - maximum usage ever recorded
setHugetlbMaxUsage string = "hugetlb_max_usage"
// Hugetlb stat - number of times hugetlb usage allocation failure
setHugetlbFailcnt string = "hugetlb_failcnt"
// Perf statistics
serPerfStat string = "perf_stat"
// Referenced memory
serReferencedMemory string = "referenced_memory"
// Resctrl - Total memory bandwidth
serResctrlMemoryBandwidthTotal string = "resctrl_memory_bandwidth_total"
// Resctrl - Local memory bandwidth
serResctrlMemoryBandwidthLocal string = "resctrl_memory_bandwidth_local"
// Resctrl - Last level cache usage
serResctrlLLCOccupancy string = "resctrl_llc_occupancy"
)

func new() (storage.StorageDriver, error) {
Expand All @@ -63,38 +103,102 @@ func (driver *stdoutStorage) containerStatsToValues(stats *info.ContainerStats)
series = make(map[string]uint64)

// Unix Timestamp
series[colTimestamp] = uint64(time.Now().UnixNano())
series[serTimestamp] = uint64(time.Now().UnixNano())

// Cumulative Cpu Usage
series[colCpuCumulativeUsage] = stats.Cpu.Usage.Total
// Total usage in nanoseconds
series[serCpuUsageTotal] = stats.Cpu.Usage.Total

// Memory Usage
series[colMemoryUsage] = stats.Memory.Usage
// To be deprecated in 0.39
series[colCpuCumulativeUsage] = series[serCpuUsageTotal]

// Working set size
series[colMemoryWorkingSet] = stats.Memory.WorkingSet
// CPU usage: Time spend in system space (in nanoseconds)
series[serCpuUsageSystem] = stats.Cpu.Usage.System

// CPU usage: Time spent in user space (in nanoseconds)
series[serCpuUsageUser] = stats.Cpu.Usage.User

// CPU usage per CPU
for i := 0; i < len(stats.Cpu.Usage.PerCpu); i++ {
series[serCpuUsagePerCpu+"."+strconv.Itoa(i)] = stats.Cpu.Usage.PerCpu[i]
}

// Load Average
series[serLoadAverage] = uint64(stats.Cpu.LoadAverage)

// Network stats.
series[colRxBytes] = stats.Network.RxBytes
series[colRxErrors] = stats.Network.RxErrors
series[colTxBytes] = stats.Network.TxBytes
series[colTxErrors] = stats.Network.TxErrors
series[serRxBytes] = stats.Network.RxBytes
series[serRxErrors] = stats.Network.RxErrors
series[serTxBytes] = stats.Network.TxBytes
series[serTxErrors] = stats.Network.TxErrors

// Referenced Memory
series[serReferencedMemory] = stats.ReferencedMemory

return series
}

func (driver *stdoutStorage) containerFsStatsToValues(series *map[string]uint64, stats *info.ContainerStats) {
for _, fsStat := range stats.Filesystem {
// Summary stats.
(*series)[colFsSummary+"."+colFsLimit] += fsStat.Limit
(*series)[colFsSummary+"."+colFsUsage] += fsStat.Usage
(*series)[serFsSummary+"."+serFsLimit] += fsStat.Limit
(*series)[serFsSummary+"."+serFsUsage] += fsStat.Usage

// Per device stats.
(*series)[fsStat.Device+"."+colFsLimit] = fsStat.Limit
(*series)[fsStat.Device+"."+colFsUsage] = fsStat.Usage
(*series)[fsStat.Device+"."+serFsLimit] = fsStat.Limit
(*series)[fsStat.Device+"."+serFsUsage] = fsStat.Usage
}
}

func (driver *stdoutStorage) memoryStatsToValues(series *map[string]uint64, stats *info.ContainerStats) {
// Memory Usage
(*series)[serMemoryUsage] = stats.Memory.Usage
// Maximum memory usage recorded
(*series)[serMemoryMaxUsage] = stats.Memory.MaxUsage
//Number of bytes of page cache memory
(*series)[serMemoryCache] = stats.Memory.Cache
// Size of RSS
(*series)[serMemoryRss] = stats.Memory.RSS
// Container swap usage
(*series)[serMemorySwap] = stats.Memory.Swap
// Size of memory mapped files in bytes
(*series)[serMemoryMappedFile] = stats.Memory.MappedFile
// Working Set Size
(*series)[serMemoryWorkingSet] = stats.Memory.WorkingSet
// Number of memory usage hits limits
(*series)[serMemoryFailcnt] = stats.Memory.Failcnt

// Cumulative count of memory allocation failures
(*series)[serMemoryFailure+".container.pgfault"] = stats.Memory.ContainerData.Pgfault
(*series)[serMemoryFailure+".container.pgmajfault"] = stats.Memory.ContainerData.Pgmajfault
(*series)[serMemoryFailure+".hierarchical.pgfault"] = stats.Memory.HierarchicalData.Pgfault
(*series)[serMemoryFailure+".hierarchical.pgmajfault"] = stats.Memory.HierarchicalData.Pgmajfault
}

func (driver *stdoutStorage) hugetlbStatsToValues(series *map[string]uint64, stats *info.ContainerStats) {
for pageSize, hugetlbStat := range stats.Hugetlb {
(*series)[setHugetlbUsage+"."+pageSize] = hugetlbStat.Usage
(*series)[setHugetlbMaxUsage+"."+pageSize] = hugetlbStat.MaxUsage
(*series)[setHugetlbFailcnt+"."+pageSize] = hugetlbStat.Failcnt
}
}

func (driver *stdoutStorage) perfStatsToValues(series *map[string]uint64, stats *info.ContainerStats) {
for _, perfStat := range stats.PerfStats {
(*series)[serPerfStat+"."+perfStat.Name+"."+strconv.Itoa(perfStat.Cpu)] = perfStat.Value
}
}

func (driver *stdoutStorage) resctrlStatsToValues(series *map[string]uint64, stats *info.ContainerStats) {
for nodeID, rdtMemoryBandwidth := range stats.Resctrl.MemoryBandwidth {
(*series)[serResctrlMemoryBandwidthTotal+"."+strconv.Itoa(nodeID)] = rdtMemoryBandwidth.TotalBytes
(*series)[serResctrlMemoryBandwidthLocal+"."+strconv.Itoa(nodeID)] = rdtMemoryBandwidth.LocalBytes
}
for nodeID, rdtCache := range stats.Resctrl.Cache {
(*series)[serResctrlLLCOccupancy+"."+strconv.Itoa(nodeID)] = rdtCache.LLCOccupancy
}

}

func (driver *stdoutStorage) AddStats(cInfo *info.ContainerInfo, stats *info.ContainerStats) error {
if stats == nil {
return nil
Expand All @@ -110,6 +214,10 @@ func (driver *stdoutStorage) AddStats(cInfo *info.ContainerInfo, stats *info.Con

series := driver.containerStatsToValues(stats)
driver.containerFsStatsToValues(&series, stats)
driver.memoryStatsToValues(&series, stats)
driver.hugetlbStatsToValues(&series, stats)
driver.perfStatsToValues(&series, stats)
driver.resctrlStatsToValues(&series, stats)
for key, value := range series {
buffer.WriteString(fmt.Sprintf(" %s=%v", key, value))
}
Expand Down