Aggregate perf metrics

Signed-off-by: Katarzyna Kujawa <katarzyna.kujawa@intel.com>
google · Jul 7, 2020 · ab35e42 · ab35e42
1 parent 0587e3d
commit ab35e42
Show file tree

Hide file tree

Showing 2 changed files with 207 additions and 18 deletions.
diff --git a/metrics/prometheus.go b/metrics/prometheus.go
@@ -15,6 +15,7 @@
 package metrics
 
 import (
+	"flag"
 	"fmt"
 	"regexp"
 	"strconv"
@@ -29,6 +30,8 @@ import (
 	"k8s.io/utils/clock"
 )
 
+var perfAggregateFlag = flag.Bool("perf_aggregate", false, "Enable core perf metrics aggregation by 'event' and 'id'")
+
 // asFloat64 converts a uint64 into a float64.
 func asFloat64(v uint64) float64 { return float64(v) }
 
@@ -1556,15 +1559,7 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 				valueType:   prometheus.CounterValue,
 				extraLabels: []string{"cpu", "event"},
 				getValues: func(s *info.ContainerStats) metricValues {
-					values := make(metricValues, 0, len(s.PerfStats))
-					for _, metric := range s.PerfStats {
-						values = append(values, metricValue{
-							value:     float64(metric.Value),
-							labels:    []string{strconv.Itoa(metric.Cpu), metric.Name},
-							timestamp: s.Timestamp,
-						})
-					}
-					return values
+					return getCorePerfEvents(s)
 				},
 			},
 			{
@@ -1573,15 +1568,7 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
 				valueType:   prometheus.GaugeValue,
 				extraLabels: []string{"cpu", "event"},
 				getValues: func(s *info.ContainerStats) metricValues {
-					values := make(metricValues, 0, len(s.PerfStats))
-					for _, metric := range s.PerfStats {
-						values = append(values, metricValue{
-							value:     metric.ScalingRatio,
-							labels:    []string{strconv.Itoa(metric.Cpu), metric.Name},
-							timestamp: s.Timestamp,
-						})
-					}
-					return values
+					return getCoreScalingRatio(s)
 				},
 			},
 			{
@@ -1903,3 +1890,65 @@ var invalidNameCharRE = regexp.MustCompile(`[^a-zA-Z0-9_]`)
 func sanitizeLabelName(name string) string {
 	return invalidNameCharRE.ReplaceAllString(name, "_")
 }
+
+func getCorePerfEvents(s *info.ContainerStats) metricValues {
+	values := make(metricValues, 0)
+	if *perfAggregateFlag {
+		perfEventStatAgg := make(map[string]uint64)
+		// aggregate by event
+		for _, perfStat := range s.PerfStats {
+			perfEventStatAgg[perfStat.Name] += perfStat.Value
+		}
+		// create aggregated metrics
+		for perfEvent, perfValue := range perfEventStatAgg {
+			values = append(values, metricValue{
+				value:     float64(perfValue),
+				labels:    []string{"", perfEvent},
+				timestamp: s.Timestamp,
+			})
+		}
+
+	} else {
+		for _, metric := range s.PerfStats {
+			values = append(values, metricValue{
+				value:     float64(metric.Value),
+				labels:    []string{strconv.Itoa(metric.Cpu), metric.Name},
+				timestamp: s.Timestamp,
+			})
+		}
+	}
+	return values
+}
+
+func getCoreScalingRatio(s *info.ContainerStats) metricValues {
+	values := make(metricValues, 0)
+	if *perfAggregateFlag {
+		perfEventStatAgg := make(map[string][]float64)
+		// collect scaling ratios for event
+		for _, perfStat := range s.PerfStats {
+			perfEventStatAgg[perfStat.Name] = append(perfEventStatAgg[perfStat.Name], perfStat.ScalingRatio)
+		}
+		// calculate average scaling ratio
+		for perfEvent, perfScalingRatio := range perfEventStatAgg {
+			sumScalingRatio := 0.0
+			for _, scalingRatio := range perfScalingRatio {
+				sumScalingRatio += scalingRatio
+			}
+
+			values = append(values, metricValue{
+				value:     sumScalingRatio / float64(len(perfScalingRatio)),
+				labels:    []string{"", perfEvent},
+				timestamp: s.Timestamp,
+			})
+		}
+	} else {
+		for _, metric := range s.PerfStats {
+			values = append(values, metricValue{
+				value:     metric.ScalingRatio,
+				labels:    []string{strconv.Itoa(metric.Cpu), metric.Name},
+				timestamp: s.Timestamp,
+			})
+		}
+	}
+	return values
+}
diff --git a/metrics/prometheus_test.go b/metrics/prometheus_test.go
@@ -122,3 +122,143 @@ func (m *mockInfoProvider) GetMachineInfo() (*info.MachineInfo, error) {
 func mockLabelFunc(*info.ContainerInfo) map[string]string {
 	return map[string]string{}
 }
+
+func TestGetCorePerfEvents(t *testing.T) {
+	containerStats := &info.ContainerStats{
+		Timestamp: time.Unix(1395066367, 0),
+		PerfStats: []info.PerfStat{
+			{
+				ScalingRatio: 1.0,
+				Value:        123,
+				Name:         "instructions",
+				Cpu:          0,
+			},
+			{
+				ScalingRatio: 0.5,
+				Value:        456,
+				Name:         "instructions",
+				Cpu:          1,
+			},
+			{
+				ScalingRatio: 0.7,
+				Value:        321,
+				Name:         "instructions_retired",
+				Cpu:          0,
+			},
+			{
+				ScalingRatio: 0.3,
+				Value:        789,
+				Name:         "instructions_retired",
+				Cpu:          1,
+			},
+		},
+	}
+	metricVals := getCorePerfEvents(containerStats)
+	assert.Equal(t, 4, len(metricVals))
+}
+
+func TestGetCorePerfEventsAggregated(t *testing.T) {
+	*perfAggregateFlag = true
+	containerStats := &info.ContainerStats{
+		Timestamp: time.Unix(1395066367, 0),
+		PerfStats: []info.PerfStat{
+			{
+				ScalingRatio: 1.0,
+				Value:        123,
+				Name:         "instructions",
+				Cpu:          0,
+			},
+			{
+				ScalingRatio: 0.5,
+				Value:        456,
+				Name:         "instructions",
+				Cpu:          1,
+			},
+			{
+				ScalingRatio: 0.7,
+				Value:        321,
+				Name:         "instructions_retired",
+				Cpu:          0,
+			},
+			{
+				ScalingRatio: 0.3,
+				Value:        789,
+				Name:         "instructions_retired",
+				Cpu:          1,
+			},
+		},
+	}
+	metricVals := getCorePerfEvents(containerStats)
+	assert.Equal(t, 2, len(metricVals))
+	*perfAggregateFlag = false
+}
+
+func TestGetCoreScalingRatio(t *testing.T) {
+	containerStats := &info.ContainerStats{
+		Timestamp: time.Unix(1395066367, 0),
+		PerfStats: []info.PerfStat{
+			{
+				ScalingRatio: 1.0,
+				Value:        123,
+				Name:         "instructions",
+				Cpu:          0,
+			},
+			{
+				ScalingRatio: 0.5,
+				Value:        456,
+				Name:         "instructions",
+				Cpu:          1,
+			},
+			{
+				ScalingRatio: 0.7,
+				Value:        321,
+				Name:         "instructions_retired",
+				Cpu:          0,
+			},
+			{
+				ScalingRatio: 0.3,
+				Value:        789,
+				Name:         "instructions_retired",
+				Cpu:          1,
+			},
+		},
+	}
+	metricVals := getCoreScalingRatio(containerStats)
+	assert.Equal(t, 4, len(metricVals))
+}
+
+func TestGetCoreScalingRatioAverage(t *testing.T) {
+	*perfAggregateFlag = true
+	containerStats := &info.ContainerStats{
+		Timestamp: time.Unix(1395066367, 0),
+		PerfStats: []info.PerfStat{
+			{
+				ScalingRatio: 1.0,
+				Value:        123,
+				Name:         "instructions",
+				Cpu:          0,
+			},
+			{
+				ScalingRatio: 0.5,
+				Value:        456,
+				Name:         "instructions",
+				Cpu:          1,
+			},
+			{
+				ScalingRatio: 0.7,
+				Value:        321,
+				Name:         "instructions_retired",
+				Cpu:          0,
+			},
+			{
+				ScalingRatio: 0.3,
+				Value:        789,
+				Name:         "instructions_retired",
+				Cpu:          1,
+			},
+		},
+	}
+	metricVals := getCoreScalingRatio(containerStats)
+	assert.Equal(t, 2, len(metricVals))
+	*perfAggregateFlag = false
+}