perf(metrics): recover from single collector panic

kubernetes-sigs · Jan 15, 2024 · 5156ac2 · 5156ac2
1 parent a6a9cea
commit 5156ac2
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/pkg/metric/collector.go b/pkg/metric/collector.go
@@ -2,6 +2,7 @@ package metric
 
 import (
 	"errors"
+	"runtime/debug"
 	"sync"
 	"time"
 
@@ -98,6 +99,11 @@ func (csi CSICollector) Collect(ch chan<- prometheus.Metric) {
 }
 
 func execute(name string, c Collector, ch chan<- prometheus.Metric) {
+	defer func() {
+		if err := recover(); err != nil {
+			logrus.Errorf("Collecotor panic occurred: %v. stacktrace: %s", err, string(debug.Stack()))
+		}
+	}()
 	begin := time.Now()
 	err := c.Update(ch)
 	duration := time.Since(begin)