Add reconcile errors counter

kubernetes-sigs · Sep 17, 2018 · fefc0f7 · fefc0f7
1 parent 49b5825
commit fefc0f7
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 1 deletion.
diff --git a/pkg/internal/controller/controller.go b/pkg/internal/controller/controller.go
@@ -211,6 +211,7 @@ func (c *Controller) processNextWorkItem() bool {
 	if result, err := c.Do.Reconcile(req); err != nil {
 		c.Queue.AddRateLimited(req)
 		log.Error(err, "Reconciler error", "Controller", c.Name, "Request", req)
+		ctrlmetrics.ReconcileErrors.WithLabelValues(c.Name).Inc()
 
 		return false
 	} else if result.RequeueAfter > 0 {

diff --git a/pkg/internal/controller/controller_test.go b/pkg/internal/controller/controller_test.go
@@ -414,6 +414,10 @@ var _ = Describe("controller", func() {
 					Name: "controller_runtime_reconcile_queue_length",
 					Help: "Length of reconcile queue per controller",
 				}, []string{"controller"})
+				ctrlmetrics.ReconcileErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
+					Name: "controller_runtime_reconcile_errors_total",
+					Help: "Total number of reconcile errors per controller",
+				}, []string{"controller"})
 
 				fakeReconcile.Err = fmt.Errorf("expected error: reconcile")
 				go func() {
@@ -427,14 +431,21 @@ var _ = Describe("controller", func() {
 
 				By("Invoking Reconciler which will give an error")
 				Expect(<-reconciled).To(Equal(request))
-				var queueLength dto.Metric
+				var queueLength, reconcileErrs dto.Metric
 				Eventually(func() error {
 					ctrlmetrics.QueueLength.WithLabelValues(ctrl.Name).Write(&queueLength)
 					if queueLength.GetGauge().GetValue() != 1.0 {
 						return fmt.Errorf("metrics not updated")
 					}
 					return nil
 				}, 2.0).Should(Succeed())
+				Eventually(func() error {
+					ctrlmetrics.ReconcileErrors.WithLabelValues(ctrl.Name).Write(&reconcileErrs)
+					if reconcileErrs.GetCounter().GetValue() != 1.0 {
+						return fmt.Errorf("metrics not updated")
+					}
+					return nil
+				}, 2.0).Should(Succeed())
 
 				By("Invoking Reconciler a second time without error")
 				fakeReconcile.Err = nil

diff --git a/pkg/internal/controller/metrics/metrics.go b/pkg/internal/controller/metrics/metrics.go
@@ -26,8 +26,14 @@ var (
 		Name: "controller_runtime_reconcile_queue_length",
 		Help: "Length of reconcile queue per controller",
 	}, []string{"controller"})
+
+	ReconcileErrors = prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "controller_runtime_reconcile_errors_total",
+		Help: "Total number of reconcile errors per controller",
+	}, []string{"controller"})
 )
 
 func init() {
 	metrics.Registry.MustRegister(QueueLength)
+	metrics.Registry.MustRegister(ReconcileErrors)
 }