[receiver/kubeletstats] Add k8s.pod.cpu.node.utilization metric (#33390)

**Description:** <Describe what has changed.>
<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
This PR adds the `k8s.pod.cpu.node.utilization` metric.
Follow up from
#32295 (comment)
(cc @TylerHelmuth) .

**Link to tracking Issue:** <Issue number if applicable>
Related to
#27885.

**Testing:** <Describe what testing was performed and which tests were
added.> Adjusted the respective unit test to cover this metric as well.

**Documentation:** <Describe the documentation added.> Added

Tested with a single container Pod:


![podCpu](https://github.com/open-telemetry/opentelemetry-collector-contrib/assets/11754898/9a0069c2-7077-4944-93b6-2dde00979bf3)

---------

Signed-off-by: ChrsMark <chrismarkou92@gmail.com>
Co-authored-by: Tiffany Hrabusa <30397949+tiffany76@users.noreply.github.com>
Co-authored-by: Tyler Helmuth <12352919+TylerHelmuth@users.noreply.github.com>

.chloggen/add_k8s_pod_utilization.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: kubeletstats
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: "Add k8s.pod.cpu.node.utilization metric"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [33390]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [user]

receiver/kubeletstatsreceiver/README.md

@@ -218,10 +218,10 @@ receivers:
       - pod
 ```
 
-### Collect k8s.container.cpu.node.utilization as ratio of total node's capacity
+### Collect k8s.container.cpu.node.utilization, `k8s.pod.cpu.node.utilization` as ratio of total node's capacity
 
-In order to calculate the `k8s.container.cpu.node.utilization` metric, the information of the node's capacity
-must be retrieved from the k8s API. In this, the `k8s_api_config` needs to be set.
+In order to calculate the `k8s.container.cpu.node.utilization` or `k8s.pod.cpu.node.utilization` metrics, the
+information of the node's capacity must be retrieved from the k8s API. In this, the `k8s_api_config` needs to be set.
 In addition, the node name must be identified properly. The `K8S_NODE_NAME` env var can be set using the
 downward API inside the collector pod spec as follows:
 
@@ -246,6 +246,8 @@ receivers:
       metrics:
         k8s.container.cpu.node.utilization:
           enabled: true
+        k8s.pod.cpu.node.utilization:
+          enabled: true
 ```
 
 ### Optional parameters

receiver/kubeletstatsreceiver/config.go

@@ -122,6 +122,8 @@ func (cfg *Config) Unmarshal(componentParser *confmap.Conf) error {
 func (cfg *Config) Validate() error {
 	if cfg.Metrics.K8sContainerCPUNodeUtilization.Enabled && cfg.NodeName == "" {
 		return errors.New("for k8s.container.cpu.node.utilization node setting is required. Check the readme on how to set the required setting")
+	} else if cfg.Metrics.K8sPodCPUNodeUtilization.Enabled && cfg.NodeName == "" {
+		return errors.New("for k8s.pod.cpu.node.utilization node setting is required. Check the readme on how to set the required setting")
 	}
 	return nil
 }

receiver/kubeletstatsreceiver/config_test.go

@@ -201,6 +201,34 @@ func TestLoadConfig(t *testing.T) {
 			},
 			expectedValidationErr: "for k8s.container.cpu.node.utilization node setting is required. Check the readme on how to set the required setting",
 		},
+		{
+			id: component.NewIDWithName(metadata.Type, "pod_cpu_node_utilization"),
+			expected: &Config{
+				ControllerConfig: scraperhelper.ControllerConfig{
+					CollectionInterval: duration,
+					InitialDelay:       time.Second,
+				},
+				ClientConfig: kube.ClientConfig{
+					APIConfig: k8sconfig.APIConfig{
+						AuthType: "tls",
+					},
+				},
+				MetricGroupsToCollect: []kubelet.MetricGroup{
+					kubelet.ContainerMetricGroup,
+					kubelet.PodMetricGroup,
+					kubelet.NodeMetricGroup,
+				},
+				MetricsBuilderConfig: metadata.MetricsBuilderConfig{
+					Metrics: metadata.MetricsConfig{
+						K8sPodCPUNodeUtilization: metadata.MetricConfig{
+							Enabled: true,
+						},
+					},
+					ResourceAttributes: metadata.DefaultResourceAttributesConfig(),
+				},
+			},
+			expectedValidationErr: "for k8s.pod.cpu.node.utilization node setting is required. Check the readme on how to set the required setting",
+		},
 	}
 
 	for _, tt := range tests {

receiver/kubeletstatsreceiver/documentation.md

@@ -458,6 +458,14 @@ The time since the node started
 | ---- | ----------- | ---------- | ----------------------- | --------- |
 | s | Sum | Int | Cumulative | true |
 
+### k8s.pod.cpu.node.utilization
+
+Pod cpu utilization as a ratio of the node's capacity
+
+| Unit | Metric Type | Value Type |
+| ---- | ----------- | ---------- |
+| 1 | Gauge | Double |
+
 ### k8s.pod.cpu.usage
 
 Total CPU usage (sum of all cores per second) averaged over the sample window

receiver/kubeletstatsreceiver/internal/kubelet/accumulator.go

@@ -76,7 +76,7 @@ func (a *metricDataAccumulator) podStats(s stats.PodStats) {
 
 	currentTime := pcommon.NewTimestampFromTime(a.time)
 	addUptimeMetric(a.mbs.PodMetricsBuilder, metadata.PodUptimeMetrics.Uptime, s.StartTime, currentTime)
-	addCPUMetrics(a.mbs.PodMetricsBuilder, metadata.PodCPUMetrics, s.CPU, currentTime, a.metadata.podResources[s.PodRef.UID], 0)
+	addCPUMetrics(a.mbs.PodMetricsBuilder, metadata.PodCPUMetrics, s.CPU, currentTime, a.metadata.podResources[s.PodRef.UID], a.metadata.cpuNodeLimit)
 	addMemoryMetrics(a.mbs.PodMetricsBuilder, metadata.PodMemoryMetrics, s.Memory, currentTime, a.metadata.podResources[s.PodRef.UID])
 	addFilesystemMetrics(a.mbs.PodMetricsBuilder, metadata.PodFilesystemMetrics, s.EphemeralStorage, currentTime)
 	addNetworkMetrics(a.mbs.PodMetricsBuilder, metadata.PodNetworkMetrics, s.Network, currentTime)

receiver/kubeletstatsreceiver/internal/metadata/metrics.go

@@ -37,6 +37,7 @@ var PodCPUMetrics = CPUMetrics{
 	Time:               (*MetricsBuilder).RecordK8sPodCPUTimeDataPoint,
 	Usage:              (*MetricsBuilder).RecordK8sPodCPUUsageDataPoint,
 	Utilization:        (*MetricsBuilder).RecordK8sPodCPUUtilizationDataPoint,
+	NodeUtilization:    (*MetricsBuilder).RecordK8sPodCPUNodeUtilizationDataPoint,
 	LimitUtilization:   (*MetricsBuilder).RecordK8sPodCPULimitUtilizationDataPoint,
 	RequestUtilization: (*MetricsBuilder).RecordK8sPodCPURequestUtilizationDataPoint,
 }

receiver/kubeletstatsreceiver/internal/metadata/testdata/config.yaml

@@ -67,6 +67,8 @@ all_set:
       enabled: true
     k8s.node.uptime:
       enabled: true
+    k8s.pod.cpu.node.utilization:
+      enabled: true
     k8s.pod.cpu.time:
       enabled: true
     k8s.pod.cpu.usage:
@@ -214,6 +216,8 @@ none_set:
       enabled: false
     k8s.node.uptime:
       enabled: false
+    k8s.pod.cpu.node.utilization:
+      enabled: false
     k8s.pod.cpu.time:
       enabled: false
     k8s.pod.cpu.usage:

receiver/kubeletstatsreceiver/metadata.yaml

@@ -236,6 +236,13 @@ metrics:
     gauge:
       value_type: int
     attributes: []
+  k8s.pod.cpu.node.utilization:
+    enabled: false
+    description: "Pod cpu utilization as a ratio of the node's capacity"
+    unit: 1
+    gauge:
+      value_type: double
+    attributes: [ ]
   k8s.pod.cpu_limit_utilization:
     enabled: false
     description: "Pod cpu utilization as a ratio of the pod's total container limits. If any container is missing a limit the metric is not emitted."

receiver/kubeletstatsreceiver/scraper.go

@@ -83,7 +83,8 @@ func newKubletScraper(
 		nodeLimits: &kubelet.NodeLimits{},
 	}
 
-	if metricsConfig.Metrics.K8sContainerCPUNodeUtilization.Enabled {
+	if metricsConfig.Metrics.K8sContainerCPUNodeUtilization.Enabled ||
+		metricsConfig.Metrics.K8sPodCPUNodeUtilization.Enabled {
 		ks.nodeInformer = k8sconfig.NewNodeSharedInformer(rOptions.k8sAPIClient, nodeName, 5*time.Minute)
 	}
 

receiver/kubeletstatsreceiver/scraper_test.go

@@ -99,6 +99,7 @@ func TestScraperWithNodeUtilization(t *testing.T) {
 	options := &scraperOptions{
 		metricGroupsToCollect: map[kubelet.MetricGroup]bool{
 			kubelet.ContainerMetricGroup: true,
+			kubelet.PodMetricGroup:       true,
 		},
 		k8sAPIClient: client,
 	}
@@ -111,6 +112,9 @@ func TestScraperWithNodeUtilization(t *testing.T) {
 				K8sContainerCPUNodeUtilization: metadata.MetricConfig{
 					Enabled: true,
 				},
+				K8sPodCPUNodeUtilization: metadata.MetricConfig{
+					Enabled: true,
+				},
 			},
 			ResourceAttributes: metadata.DefaultResourceAttributesConfig(),
 		},
@@ -132,7 +136,7 @@ func TestScraperWithNodeUtilization(t *testing.T) {
 
 	md, err := r.Scrape(context.Background())
 	require.NoError(t, err)
-	require.Equal(t, numContainers, md.DataPointCount())
+	require.Equal(t, numContainers+numPods, md.DataPointCount())
 	expectedFile := filepath.Join("testdata", "scraper", "test_scraper_cpu_util_nodelimit_expected.yaml")
 
 	// Uncomment to regenerate '*_expected.yaml' files

receiver/kubeletstatsreceiver/testdata/config.yaml

@@ -34,3 +34,9 @@ kubeletstats/container_cpu_node_utilization:
   metrics:
     k8s.container.cpu.node.utilization:
       enabled: true
+kubeletstats/pod_cpu_node_utilization:
+  collection_interval: 10s
+  metric_groups: [ container, pod, node ]
+  metrics:
+    k8s.pod.cpu.node.utilization:
+      enabled: true