feat: Manifest duration metrics

internal/declarative/v2/reconciler.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"time"
 
 	apicorev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -92,6 +93,8 @@ func newResourcesCondition(obj Object) apimetav1.Condition {
 
 //nolint:funlen,cyclop,gocognit // Declarative pkg will be removed soon
 func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	startTime := time.Now()
+	defer r.recordReconciliationDuration(startTime, req.Name)
 	obj, ok := r.prototype.DeepCopyObject().(Object)
 	if !ok {
 		r.Metrics.RecordRequeueReason(metrics.ManifestTypeCast, queue.UnexpectedRequeue)
@@ -124,6 +127,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	spec, err := r.Spec(ctx, obj)
 	if err != nil {
 		if !obj.GetDeletionTimestamp().IsZero() {
+			r.Metrics.RemoveManifestDuration(req.Name)
 			return r.removeFinalizers(ctx, obj, []string{r.Finalizer}, metrics.ManifestRemoveFinalizerWhenParseSpec)
 		}
 		return r.ssaStatus(ctx, obj, metrics.ManifestParseSpec)
@@ -137,6 +141,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	clnt, err := r.getTargetClient(ctx, obj)
 	if err != nil {
 		if !obj.GetDeletionTimestamp().IsZero() && errors.Is(err, ErrAccessSecretNotFound) {
+			r.Metrics.RemoveManifestDuration(req.Name)
 			return r.removeFinalizers(ctx, obj, obj.GetFinalizers(), metrics.ManifestRemoveFinalizerWhenSecretGone)
 		}
 
@@ -190,6 +195,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
 	}
 
 	if !obj.GetDeletionTimestamp().IsZero() {
+		r.Metrics.RemoveManifestDuration(req.Name)
 		return r.removeFinalizers(ctx, obj, []string{r.Finalizer}, metrics.ManifestRemoveFinalizerInDeleting)
 	}
 	return ctrl.Result{RequeueAfter: r.Success}, nil
@@ -654,3 +660,12 @@ func (r *Reconciler) updateObject(ctx context.Context, obj client.Object,
 	}
 	return ctrl.Result{Requeue: true}, nil
 }
+
+func (r *Reconciler) recordReconciliationDuration(startTime time.Time, name string) {
+	duration := time.Since(startTime)
+	if duration >= 1*time.Minute {
+		r.Metrics.RecordManifestDuration(name, duration)
+	} else {
+		r.Metrics.RemoveManifestDuration(name)
+	}
+}

internal/pkg/metrics/manifest.go

@@ -1,10 +1,19 @@
 package metrics
 
-import "github.com/kyma-project/lifecycle-manager/pkg/queue"
+import (
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+	ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
+
+	"github.com/kyma-project/lifecycle-manager/pkg/queue"
+)
 
 type ManifestRequeueReason string
 
 const (
+	MetricManifestDuration                                      = "reconcile_duration_seconds"
+	ManifestNameLabel                                           = "manifest_name"
 	ManifestTypeCast                      ManifestRequeueReason = "manifest_type_cast"
 	ManifestRetrieval                     ManifestRequeueReason = "manifest_retrieval"
 	ManifestInit                          ManifestRequeueReason = "manifest_initialize"
@@ -28,12 +37,32 @@ const (
 
 type ManifestMetrics struct {
 	*SharedMetrics
+	ManifestDurationGauge *prometheus.GaugeVec
 }
 
 func NewManifestMetrics(sharedMetrics *SharedMetrics) *ManifestMetrics {
-	return &ManifestMetrics{SharedMetrics: sharedMetrics}
+	metrics := &ManifestMetrics{
+		SharedMetrics: sharedMetrics,
+		ManifestDurationGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: MetricManifestDuration,
+			Help: "Indicates the duration for manifest reconciliation in seconds",
+		}, []string{ManifestNameLabel}),
+	}
+
+	ctrlmetrics.Registry.MustRegister(metrics.ManifestDurationGauge)
+	return metrics
 }
 
 func (k *ManifestMetrics) RecordRequeueReason(requeueReason ManifestRequeueReason, requeueType queue.RequeueType) {
 	k.requeueReasonCounter.WithLabelValues(string(requeueReason), string(requeueType)).Inc()
 }
+
+func (k *ManifestMetrics) RecordManifestDuration(manifestName string, duration time.Duration) {
+	k.ManifestDurationGauge.WithLabelValues(manifestName).Set(duration.Seconds())
+}
+
+func (k *ManifestMetrics) RemoveManifestDuration(manifestName string) {
+	k.ManifestDurationGauge.DeletePartialMatch(prometheus.Labels{
+		ManifestNameLabel: manifestName,
+	})
+}

internal/pkg/metrics/metrics_test.go

@@ -44,6 +44,11 @@ func Test_ConstantMetricNames(t *testing.T) {
 			constValue:    SelfSignedCertNotRenewMetrics,
 			expectedValue: "lifecycle_mgr_self_signed_cert_not_renew",
 		},
+		{
+			constName:     "MetricManifestDuration",
+			constValue:    MetricManifestDuration,
+			expectedValue: "reconcile_duration_seconds",
+		},
 	}
 	for _, testcase := range tests {
 		testcase := testcase