Merge pull request #1125 from alvaroaleman/moar-metrics

✨ Add metrics for total workers and active workers

pkg/internal/controller/controller.go

@@ -169,6 +169,7 @@ func (c *Controller) Start(stop <-chan struct{}) error {
 
 		// Launch workers to process resources
 		c.Log.Info("Starting workers", "worker count", c.MaxConcurrentReconciles)
+		ctrlmetrics.WorkerCount.WithLabelValues(c.Name).Set(float64(c.MaxConcurrentReconciles))
 		for i := 0; i < c.MaxConcurrentReconciles; i++ {
 			// Process work items
 			go wait.Until(c.worker, c.JitterPeriod, stop)
@@ -210,6 +211,9 @@ func (c *Controller) processNextWorkItem() bool {
 	// period.
 	defer c.Queue.Done(obj)
 
+	ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(1)
+	defer ctrlmetrics.ActiveWorkers.WithLabelValues(c.Name).Add(-1)
+
 	return c.reconcileHandler(obj)
 }
 

pkg/internal/controller/metrics/metrics.go

@@ -44,13 +44,29 @@ var (
 		Name: "controller_runtime_reconcile_time_seconds",
 		Help: "Length of time per reconciliation per controller",
 	}, []string{"controller"})
+
+	// WorkerCount is a prometheus metric which holds the number of
+	// concurrent reconciles per controller
+	WorkerCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "controller_runtime_max_concurrent_reconciles",
+		Help: "Maximum number of concurrent reconciles per controller",
+	}, []string{"controller"})
+
+	// ActiveWorkers is a prometheus metric which holds the number
+	// of active workers per controller
+	ActiveWorkers = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: "controller_runtime_active_workers",
+		Help: "Number of currently used workers per controller",
+	}, []string{"controller"})
 )
 
 func init() {
 	metrics.Registry.MustRegister(
 		ReconcileTotal,
 		ReconcileErrors,
 		ReconcileTime,
+		WorkerCount,
+		ActiveWorkers,
 		// expose process metrics like CPU, Memory, file descriptor usage etc.
 		prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}),
 		// expose Go runtime metrics like GC stats, memory stats etc.