Merge pull request #952 from tydra-wang/fix-kubelet-metrics-1.26

Cherry pick #949 fix kubelet stats summary collector panic
kubernetes-sigs · Jan 15, 2024 · 303a494 · 303a494
2 parents 51f49b7 + 33afa88
commit 303a494
Show file tree

Hide file tree

Showing 2 changed files with 18 additions and 8 deletions.
diff --git a/pkg/metric/collector.go b/pkg/metric/collector.go
@@ -2,6 +2,7 @@ package metric
 
 import (
 	"errors"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -98,6 +99,11 @@ func (csi CSICollector) Collect(ch chan<- prometheus.Metric) {
 }
 
 func execute(name string, c Collector, ch chan<- prometheus.Metric) {
+	defer func() {
+		if err := recover(); err != nil {
+			logrus.Errorf("Collecotor panic occurred: %v. stacktrace: %s", err, string(debug.Stack()))
+		}
+	}()
 	begin := time.Now()
 	err := c.Update(ch)
 	duration := time.Since(begin)

diff --git a/pkg/metric/kubelet_stats_summary_collector.go b/pkg/metric/kubelet_stats_summary_collector.go
@@ -97,18 +97,22 @@ func (c *kubeletStatsSummaryCollector) Update(ch chan<- prometheus.Metric) error
 		return err
 	}
 	for _, pod := range summary.Pods {
-		for _, m := range ephemeralStorageMetrics {
-			metric := m.Metric(pod.EphemeralStorage, pod.PodRef.Namespace, pod.PodRef.Name, pod.PodRef.UID)
-			if metric != nil {
-				ch <- metric
+		if pod.EphemeralStorage != nil {
+			for _, m := range ephemeralStorageMetrics {
+				metric := m.Metric(pod.EphemeralStorage, pod.PodRef.Namespace, pod.PodRef.Name, pod.PodRef.UID)
+				if metric != nil {
+					ch <- metric
+				}
 			}
 		}
 
 		for _, container := range pod.Containers {
-			for _, m := range rootfsMetrics {
-				metric := m.Metric(container.Rootfs, pod.PodRef.Namespace, pod.PodRef.Name, pod.PodRef.UID, container.Name)
-				if metric != nil {
-					ch <- metric
+			if container.Rootfs != nil {
+				for _, m := range rootfsMetrics {
+					metric := m.Metric(container.Rootfs, pod.PodRef.Namespace, pod.PodRef.Name, pod.PodRef.UID, container.Name)
+					if metric != nil {
+						ch <- metric
+					}
 				}
 			}
 		}