feat(metrics-operator): expose analysis results as Prometheus Metric

metrics-operator/controllers/analysis/controller.go

@@ -26,6 +26,7 @@ import (
 	"github.com/keptn/lifecycle-toolkit/metrics-operator/api/v1alpha3"
 	metricsapi "github.com/keptn/lifecycle-toolkit/metrics-operator/api/v1alpha3"
 	common "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/analysis"
+	evalType "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/analysis/types"
 	"golang.org/x/exp/maps"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -44,6 +45,7 @@ type AnalysisReconciler struct {
 	MaxWorkers int //maybe 2 or 4 as def
 	NewWorkersPoolFactory
 	common.IAnalysisEvaluator
+	analysisResults chan evalType.AnalysisCompletion
 }
 
 //+kubebuilder:rbac:groups=metrics.keptn.sh,resources=analyses,verbs=get;list;watch;create;update;patch;delete
@@ -105,15 +107,15 @@ func (a *AnalysisReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c
 	}
 
 	maps.Copy(res, done)
-	a.evaluateObjectives(ctx, res, analysisDef, analysis)
+	a.evaluateObjectives(res, analysisDef, analysis)
 	if err := a.updateStatus(ctx, analysis); err != nil {
 		return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, err
 	}
 
 	return ctrl.Result{}, nil
 }
 
-func (a *AnalysisReconciler) evaluateObjectives(ctx context.Context, res map[string]metricsapi.ProviderResult, analysisDef *metricsapi.AnalysisDefinition, analysis *metricsapi.Analysis) {
+func (a *AnalysisReconciler) evaluateObjectives(res map[string]metricsapi.ProviderResult, analysisDef *metricsapi.AnalysisDefinition, analysis *metricsapi.Analysis) {
 	eval := a.Evaluate(res, analysisDef)
 	analysisResultJSON, err := json.Marshal(eval)
 	if err != nil {
@@ -126,6 +128,22 @@ func (a *AnalysisReconciler) evaluateObjectives(ctx context.Context, res map[str
 	analysis.Status.State = metricsapi.StateCompleted
 	// if evaluation was successful remove the stored values
 	analysis.Status.StoredValues = nil
+	go a.reportAnalysisResult(eval, *analysis)
+}
+
+func (a *AnalysisReconciler) reportAnalysisResult(eval evalType.AnalysisResult, analysis metricsapi.Analysis) {
+	if a.analysisResults == nil {
+		return
+	}
+
+	a.analysisResults <- evalType.AnalysisCompletion{
+		Result:   eval,
+		Analysis: analysis,
+	}
+}
+
+func (a *AnalysisReconciler) SetAnalysisResultsChannel(c chan evalType.AnalysisCompletion) {
+	a.analysisResults = c
 }
 
 func (a *AnalysisReconciler) updateStatus(ctx context.Context, analysis *metricsapi.Analysis) error {

metrics-operator/controllers/analysis/controller_test.go

@@ -22,6 +22,56 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+func TestAnalysisReconciler_SendResultToChannel(t *testing.T) {
+	analysis, analysisDef, template, _ := getTestCRDs()
+	fakeclient := fake2.NewClient(&analysis, &analysisDef, &template)
+	res := metricstypes.AnalysisResult{
+		Pass: true,
+		ObjectiveResults: []metricstypes.ObjectiveResult{
+			{
+				Objective: analysisDef.Spec.Objectives[0],
+			},
+		},
+	}
+
+	req := controllerruntime.Request{
+		NamespacedName: types.NamespacedName{Namespace: "default", Name: "my-analysis"},
+	}
+	mockFactory := func(ctx context.Context, analysisMoqParam *metricsapi.Analysis, obj []metricsapi.Objective, numWorkers int, c client.Client, log logr.Logger, namespace string) (context.Context, IAnalysisPool) {
+		mymock := fake.IAnalysisPoolMock{
+			DispatchAndCollectFunc: func(ctx context.Context) (map[string]metricsapi.ProviderResult, error) {
+				return map[string]metricsapi.ProviderResult{}, nil
+			},
+		}
+		return ctx, &mymock
+	}
+
+	a := &AnalysisReconciler{
+		Client:                fakeclient,
+		Scheme:                fakeclient.Scheme(),
+		Log:                   testr.New(t),
+		MaxWorkers:            2,
+		NewWorkersPoolFactory: mockFactory,
+		IAnalysisEvaluator: &fakeEvaluator.IAnalysisEvaluatorMock{
+			EvaluateFunc: func(values map[string]metricsapi.ProviderResult, ad *metricsapi.AnalysisDefinition) metricstypes.AnalysisResult {
+				return res
+			}},
+	}
+
+	resChan := make(chan metricstypes.AnalysisCompletion)
+	a.SetAnalysisResultsChannel(resChan)
+
+	_, err := a.Reconcile(context.TODO(), req)
+	require.Nil(t, err)
+
+	select {
+	case <-time.After(5 * time.Second):
+		t.Error("timed out waiting for the analysis result to be reported")
+	case analysisResult := <-resChan:
+		require.Equal(t, "my-analysis", analysisResult.Analysis.Name)
+	}
+}
+
 func TestAnalysisReconciler_Reconcile_BasicControlLoop(t *testing.T) {
 
 	analysis, analysisDef, template, _ := getTestCRDs()

metrics-operator/controllers/common/analysis/objective_evaluator.go

@@ -20,8 +20,9 @@ func NewObjectiveEvaluator(t ITargetEvaluator) ObjectiveEvaluator {
 
 func (oe *ObjectiveEvaluator) Evaluate(values map[string]v1alpha3.ProviderResult, obj *v1alpha3.Objective) types.ObjectiveResult {
 	result := types.ObjectiveResult{
-		Score: 0.0,
-		Value: 0.0,
+		Score:     0.0,
+		Value:     0.0,
+		Objective: *obj,
 	}
 
 	// get the value

metrics-operator/controllers/common/analysis/objective_evaluator_test.go

@@ -30,6 +30,11 @@ func TestObjectiveEvaluator_Evaluate(t *testing.T) {
 			want: types.ObjectiveResult{
 				Score: 0.0,
 				Error: fmt.Errorf("required value 'name' not available"),
+				Objective: v1alpha3.Objective{
+					AnalysisValueTemplateRef: v1alpha3.ObjectReference{
+						Name: "name",
+					},
+				},
 			},
 		},
 		{
@@ -57,6 +62,12 @@ func TestObjectiveEvaluator_Evaluate(t *testing.T) {
 				Result: types.TargetResult{
 					Pass: true,
 				},
+				Objective: v1alpha3.Objective{
+					AnalysisValueTemplateRef: v1alpha3.ObjectReference{
+						Name: "name",
+					},
+					Weight: 2,
+				},
 			},
 		},
 		{
@@ -86,6 +97,12 @@ func TestObjectiveEvaluator_Evaluate(t *testing.T) {
 					Pass:    false,
 					Warning: true,
 				},
+				Objective: v1alpha3.Objective{
+					AnalysisValueTemplateRef: v1alpha3.ObjectReference{
+						Name: "name",
+					},
+					Weight: 2,
+				},
 			},
 		},
 		{
@@ -115,6 +132,12 @@ func TestObjectiveEvaluator_Evaluate(t *testing.T) {
 					Pass:    false,
 					Warning: false,
 				},
+				Objective: v1alpha3.Objective{
+					AnalysisValueTemplateRef: v1alpha3.ObjectReference{
+						Name: "name",
+					},
+					Weight: 2,
+				},
 			},
 		},
 	}

metrics-operator/controllers/common/analysis/types/types.go

@@ -31,10 +31,11 @@ type OperatorResult struct {
 }
 
 type ObjectiveResult struct {
-	Result TargetResult `json:"result"`
-	Value  float64      `json:"value"`
-	Score  float64      `json:"score"`
-	Error  error        `json:"error,omitempty"`
+	Result    TargetResult       `json:"result"`
+	Objective v1alpha3.Objective `json:"objective"`
+	Value     float64            `json:"value"`
+	Score     float64            `json:"score"`
+	Error     error              `json:"error,omitempty"`
 }
 
 func (o *ObjectiveResult) IsFail() bool {
@@ -66,3 +67,10 @@ func (a *AnalysisResult) GetAchievedPercentage() float64 {
 	}
 	return (a.TotalScore / a.MaximumScore) * 100.0
 }
+
+// AnalysisCompletion consolidates an analysis definition and its result into one struct, which is needed to communicate
+// both objects via a channel
+type AnalysisCompletion struct {
+	Result   AnalysisResult
+	Analysis v1alpha3.Analysis
+}

metrics-operator/main.go

@@ -36,10 +36,12 @@ import (
 	"github.com/keptn/lifecycle-toolkit/metrics-operator/cmd/metrics/adapter"
 	analysiscontroller "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/analysis"
 	"github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/analysis"
+	analysistypes "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/analysis/types"
 	"github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/providers"
 	metricscontroller "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/metrics"
 	"github.com/keptn/lifecycle-toolkit/metrics-operator/converter"
 	keptnserver "github.com/keptn/lifecycle-toolkit/metrics-operator/pkg/metrics"
+	analysismetrics "github.com/keptn/lifecycle-toolkit/metrics-operator/pkg/metrics/analysis"
 	"github.com/open-feature/go-sdk/pkg/openfeature"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -200,23 +202,29 @@ func main() {
 	}
 
 	if env.EnableKeptnAnalysis {
-
 		analysisLogger := ctrl.Log.WithName("KeptnAnalysis Controller")
 		targetEval := analysis.NewTargetEvaluator(&analysis.OperatorEvaluator{})
 		objEval := analysis.NewObjectiveEvaluator(&targetEval)
 		analysisEval := analysis.NewAnalysisEvaluator(&objEval)
 
-		if err = (&analysiscontroller.AnalysisReconciler{
+		ac := &analysiscontroller.AnalysisReconciler{
 			Client:                mgr.GetClient(),
 			Scheme:                mgr.GetScheme(),
 			Log:                   analysisLogger.V(env.AnalysisControllerLogLevel),
 			MaxWorkers:            2,
 			NewWorkersPoolFactory: analysiscontroller.NewWorkersPool,
 			IAnalysisEvaluator:    &analysisEval,
-		}).SetupWithManager(mgr); err != nil {
+		}
+		if err = ac.SetupWithManager(mgr); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "KeptnMetric")
 			os.Exit(1)
 		}
+
+		res := make(chan analysistypes.AnalysisCompletion)
+
+		ac.SetAnalysisResultsChannel(res)
+
+		_ = analysismetrics.GetResultsReporter(ctx, res)
 	}
 	// +kubebuilder:scaffold:builder
 

metrics-operator/pkg/metrics/analysis/results.go

@@ -0,0 +1,118 @@
+package analysis
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	analysistypes "github.com/keptn/lifecycle-toolkit/metrics-operator/controllers/common/analysis/types"
+	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/klog/v2"
+)
+
+const analysisResultMetricName = "keptn_analysis_result"
+const objectiveResultMetricName = "keptn_objective_result"
+
+type Metrics struct {
+	AnalysisResult  *prometheus.GaugeVec
+	ObjectiveResult *prometheus.GaugeVec
+}
+
+// use singleton pattern here to avoid registering the same metrics on Prometheus multiple times
+var instance *resultsReporter
+var once sync.Once
+
+func GetResultsReporter(ctx context.Context, res chan analysistypes.AnalysisCompletion) *resultsReporter {
+	once.Do(func() {
+		instance = &resultsReporter{}
+		instance.initialize(ctx, res)
+	})
+
+	return instance
+}
+
+type resultsReporter struct {
+	metrics Metrics
+	mtx     sync.Mutex
+}
+
+func (r *resultsReporter) initialize(ctx context.Context, res chan analysistypes.AnalysisCompletion) {
+	labelNamesAnalysis := []string{"name", "namespace", "from", "to"}
+	a := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: analysisResultMetricName,
+		Help: "Result of Analysis",
+	}, labelNamesAnalysis)
+	err := prometheus.Register(a)
+
+	if err != nil {
+		klog.Errorf("Could not register Analysis results as Prometheus metric: %v", err)
+	}
+
+	labelNames := []string{"name", "namespace", "analysis_name", "analysis_namespace", "key_objective", "weight", "from", "to"}
+	o := prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Name: objectiveResultMetricName,
+		Help: "Result of the Analysis Objective",
+	}, labelNames)
+	err = prometheus.Register(o)
+	if err != nil {
+		klog.Errorf("Could not register Analysis Objective results as Prometheus metric: %v", err)
+	}
+
+	r.metrics = Metrics{
+		AnalysisResult:  a,
+		ObjectiveResult: o,
+	}
+
+	go r.watchForResults(ctx, res)
+}
+
+func (r *resultsReporter) watchForResults(ctx context.Context, res chan analysistypes.AnalysisCompletion) {
+	for {
+		select {
+		case <-ctx.Done():
+			klog.Info("Exiting due to termination of context")
+			return
+		case finishedAnalysis := <-res:
+			r.reportResult(finishedAnalysis)
+		}
+	}
+}
+
+func (r *resultsReporter) reportResult(finishedAnalysis analysistypes.AnalysisCompletion) {
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	f := finishedAnalysis.Analysis.Spec.From.String()
+	t := finishedAnalysis.Analysis.Spec.To.String()
+	labelsAnalysis := prometheus.Labels{
+		"name":      finishedAnalysis.Analysis.Name,
+		"namespace": finishedAnalysis.Analysis.Namespace,
+		"from":      f,
+		"to":        t,
+	}
+	if m, err := r.metrics.AnalysisResult.GetMetricWith(labelsAnalysis); err == nil {
+		m.Set(finishedAnalysis.Result.GetAchievedPercentage())
+	} else {
+		klog.Errorf("unable to set value for analysis result metric: %v", err)
+	}
+	// expose also the individual objectives
+	for _, o := range finishedAnalysis.Result.ObjectiveResults {
+		name := o.Objective.AnalysisValueTemplateRef.Name
+		ns := o.Objective.AnalysisValueTemplateRef.Namespace
+		labelsObjective := prometheus.Labels{
+			"name":               name,
+			"namespace":          ns,
+			"analysis_name":      finishedAnalysis.Analysis.Name,
+			"analysis_namespace": finishedAnalysis.Analysis.Namespace,
+			"key_objective":      fmt.Sprintf("%v", o.Objective.KeyObjective),
+			"weight":             fmt.Sprintf("%v", o.Objective.Weight),
+			"from":               f,
+			"to":                 t,
+		}
+		if m, err := r.metrics.ObjectiveResult.GetMetricWith(labelsObjective); err == nil {
+			m.Set(o.Value)
+		} else {
+			klog.Errorf("unable to set value for objective result metric: %v", err)
+		}
+	}
+}