feat: add deprovisioning metrics for eligible machines (#331)

* feat: add metrics for deprovisioning for each deprovisioner * add string functions and populate metric call * rebase * remove naming * fix names * comments * revert to histogram * revert label * inc * add expiration
kubernetes-sigs · May 10, 2023 · 4b3fcb8 · 4b3fcb8
1 parent 6a6e5fe
commit 4b3fcb8
Show file tree

Hide file tree

Showing 9 changed files with 54 additions and 31 deletions.
diff --git a/pkg/controllers/deprovisioning/consolidation.go b/pkg/controllers/deprovisioning/consolidation.go
@@ -104,7 +104,6 @@ func (c *consolidation) ShouldDeprovision(_ context.Context, cn *Candidate) bool
 //
 // nolint:gocyclo
 func (c *consolidation) computeConsolidation(ctx context.Context, candidates ...*Candidate) (Command, error) {
-	defer metrics.Measure(deprovisioningDurationHistogram.WithLabelValues("Replace/Delete"))()
 	// Run scheduling simulation to compute consolidation option
 	results, err := simulateScheduling(ctx, c.kubeClient, c.cluster, c.provisioner, candidates...)
 	if err != nil {

diff --git a/pkg/controllers/deprovisioning/controller.go b/pkg/controllers/deprovisioning/controller.go
@@ -144,6 +144,7 @@ func (c *Controller) Reconcile(ctx context.Context, _ reconcile.Request) (reconc
 }
 
 func (c *Controller) deprovision(ctx context.Context, deprovisioner Deprovisioner) (bool, error) {
+	defer metrics.Measure(deprovisioningDurationHistogram.WithLabelValues(deprovisioner.String()))()
 	candidates, err := GetCandidates(ctx, c.cluster, c.kubeClient, c.recorder, c.clock, c.cloudProvider, deprovisioner.ShouldDeprovision)
 	if err != nil {
 		return false, fmt.Errorf("determining candidates, %w", err)
@@ -171,8 +172,12 @@ func (c *Controller) deprovision(ctx context.Context, deprovisioner Deprovisione
 }
 
 func (c *Controller) executeCommand(ctx context.Context, d Deprovisioner, command Command) error {
-	deprovisioningActionsPerformedCounter.With(prometheus.Labels{"action": fmt.Sprintf("%s/%s", d, command.Action())}).Inc()
-	logging.FromContext(ctx).Infof("deprovisioning via %s %s", d, command.Action())
+	deprovisioningActionsPerformedCounter.With(map[string]string{
+		// TODO: make this just command.Action() since we've added the deprovisioner as its own label.
+		actionLabel:        fmt.Sprintf("%s/%s", d, command.Action()),
+		deprovisionerLabel: d.String(),
+	}).Inc()
+	logging.FromContext(ctx).Infof("deprovisioning via %s %s", d, command)
 
 	reason := fmt.Sprintf("%s/%s", d, command.Action())
 	if command.Action() == ReplaceAction {

diff --git a/pkg/controllers/deprovisioning/drift.go b/pkg/controllers/deprovisioning/drift.go
@@ -64,6 +64,7 @@ func (d *Drift) ComputeCommand(ctx context.Context, nodes ...*Candidate) (Comman
 	if err != nil {
 		return Command{}, fmt.Errorf("filtering candidates, %w", err)
 	}
+	deprovisioningEligibleMachinesGauge.WithLabelValues(d.String()).Set(float64(len(candidates)))
 
 	// Deprovision all empty drifted nodes, as they require no scheduling simulations.
 	if empty := lo.Filter(candidates, func(c *Candidate, _ int) bool {

diff --git a/pkg/controllers/deprovisioning/emptiness.go b/pkg/controllers/deprovisioning/emptiness.go
@@ -66,6 +66,7 @@ func (e *Emptiness) ComputeCommand(_ context.Context, candidates ...*Candidate)
 	emptyCandidates := lo.Filter(candidates, func(cn *Candidate, _ int) bool {
 		return cn.Machine.DeletionTimestamp.IsZero() && len(cn.pods) == 0
 	})
+	deprovisioningEligibleMachinesGauge.WithLabelValues(e.String()).Set(float64(len(candidates)))
 
 	return Command{
 		candidates: emptyCandidates,

diff --git a/pkg/controllers/deprovisioning/emptymachineconsolidation.go b/pkg/controllers/deprovisioning/emptymachineconsolidation.go
@@ -49,6 +49,7 @@ func (c *EmptyMachineConsolidation) ComputeCommand(ctx context.Context, candidat
 	if err != nil {
 		return Command{}, fmt.Errorf("sorting candidates, %w", err)
 	}
+	deprovisioningEligibleMachinesGauge.WithLabelValues(c.String()).Set(float64(len(candidates)))
 
 	// select the entirely empty nodes
 	emptyCandidates := lo.Filter(candidates, func(n *Candidate, _ int) bool { return len(n.pods) == 0 })

diff --git a/pkg/controllers/deprovisioning/expiration.go b/pkg/controllers/deprovisioning/expiration.go
@@ -85,6 +85,7 @@ func (e *Expiration) ComputeCommand(ctx context.Context, nodes ...*Candidate) (C
 	if err != nil {
 		return Command{}, fmt.Errorf("filtering candidates, %w", err)
 	}
+	deprovisioningEligibleMachinesGauge.WithLabelValues(e.String()).Set(float64(len(candidates)))
 
 	// Deprovision all empty expired nodes, as they require no scheduling simulations.
 	if empty := lo.Filter(candidates, func(c *Candidate, _ int) bool {

diff --git a/pkg/controllers/deprovisioning/metrics.go b/pkg/controllers/deprovisioning/metrics.go
@@ -25,36 +25,49 @@ func init() {
 	crmetrics.Registry.MustRegister(deprovisioningDurationHistogram)
 	crmetrics.Registry.MustRegister(deprovisioningReplacementNodeInitializedHistogram)
 	crmetrics.Registry.MustRegister(deprovisioningActionsPerformedCounter)
+	crmetrics.Registry.MustRegister(deprovisioningEligibleMachinesGauge)
 }
 
-const deprovisioningSubsystem = "deprovisioning"
-
-var deprovisioningDurationHistogram = prometheus.NewHistogramVec(
-	prometheus.HistogramOpts{
-		Namespace: metrics.Namespace,
-		Subsystem: deprovisioningSubsystem,
-		Name:      "evaluation_duration_seconds",
-		Help:      "Duration of the deprovisioning evaluation process in seconds.",
-		Buckets:   metrics.DurationBuckets(),
-	},
-	[]string{"method"},
+const (
+	deprovisioningSubsystem = "deprovisioning"
+	deprovisionerLabel      = "deprovisioner"
+	actionLabel             = "action"
 )
 
-var deprovisioningReplacementNodeInitializedHistogram = prometheus.NewHistogram(
-	prometheus.HistogramOpts{
-		Namespace: metrics.Namespace,
-		Subsystem: deprovisioningSubsystem,
-		Name:      "replacement_machine_initialized_seconds",
-		Help:      "Amount of time required for a replacement machine to become initialized.",
-		Buckets:   metrics.DurationBuckets(),
-	})
-
-var deprovisioningActionsPerformedCounter = prometheus.NewCounterVec(
-	prometheus.CounterOpts{
-		Namespace: metrics.Namespace,
-		Subsystem: deprovisioningSubsystem,
-		Name:      "actions_performed",
-		Help:      "Number of deprovisioning actions performed. Labeled by action.",
-	},
-	[]string{"action"},
+var (
+	deprovisioningDurationHistogram = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: metrics.Namespace,
+			Subsystem: deprovisioningSubsystem,
+			Name:      "evaluation_duration_seconds",
+			Help:      "Duration of the deprovisioning evaluation process in seconds.",
+			Buckets:   metrics.DurationBuckets(),
+		},
+		[]string{"method"})
+	deprovisioningReplacementNodeInitializedHistogram = prometheus.NewHistogram(
+		prometheus.HistogramOpts{
+			Namespace: metrics.Namespace,
+			Subsystem: deprovisioningSubsystem,
+			Name:      "replacement_machine_initialized_seconds",
+			Help:      "Amount of time required for a replacement machine to become initialized.",
+			Buckets:   metrics.DurationBuckets(),
+		})
+	deprovisioningActionsPerformedCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: metrics.Namespace,
+			Subsystem: deprovisioningSubsystem,
+			Name:      "actions_performed",
+			Help:      "Number of deprovisioning actions performed. Labeled by deprovisioner.",
+		},
+		[]string{actionLabel, deprovisionerLabel},
+	)
+	deprovisioningEligibleMachinesGauge = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: metrics.Namespace,
+			Subsystem: deprovisioningSubsystem,
+			Name:      "eligible_machines",
+			Help:      "Number of machines eligible for deprovisioning by Karpenter. Labeled by deprovisioner",
+		},
+		[]string{deprovisionerLabel},
+	)
 )
diff --git a/pkg/controllers/deprovisioning/multimachineconsolidation.go b/pkg/controllers/deprovisioning/multimachineconsolidation.go
@@ -47,6 +47,7 @@ func (m *MultiMachineConsolidation) ComputeCommand(ctx context.Context, candidat
 	if err != nil {
 		return Command{}, fmt.Errorf("sorting candidates, %w", err)
 	}
+	deprovisioningEligibleMachinesGauge.WithLabelValues(m.String()).Set(float64(len(candidates)))
 
 	// For now, we will consider up to every machine in the cluster, might be configurable in the future.
 	maxParallel := len(candidates)

diff --git a/pkg/controllers/deprovisioning/singlemachineconsolidation.go b/pkg/controllers/deprovisioning/singlemachineconsolidation.go
@@ -48,6 +48,7 @@ func (c *SingleMachineConsolidation) ComputeCommand(ctx context.Context, candida
 	if err != nil {
 		return Command{}, fmt.Errorf("sorting candidates, %w", err)
 	}
+	deprovisioningEligibleMachinesGauge.WithLabelValues(c.String()).Set(float64(len(candidates)))
 
 	v := NewValidation(consolidationTTL, c.clock, c.cluster, c.kubeClient, c.provisioner, c.cloudProvider, c.recorder)
 	for _, candidate := range candidates {