metrics-exporter: configure Prometheus operator

Deploy the needed configuration to make the prometheus
operator to find and scrape the sriov-network-metrics-exporter
endpoints, including the ServiceMonitor, Role and RoleBinding

Signed-off-by: Andrea Panattoni <apanatto@redhat.com>

bindata/manifests/metrics-exporter/metrics-service.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: {{.Namespace}}
   annotations:
     prometheus.io/target: "true"
-    {{- if eq .ClusterType "openshift" }}
+    {{ if .IsOpenshift }}
     service.beta.openshift.io/serving-cert-secret-name: {{ .MetricsExporterSecretName }}
     {{- end }}
   labels:
@@ -18,6 +18,7 @@ spec:
       name: sriov-network-metrics
       port: {{ .MetricsExporterPort }}
       targetPort: {{ .MetricsExporterPort }}
+{{ if .IsOpenshift }}
 ---
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
@@ -72,3 +73,4 @@ rules:
   - get
   - list
   - watch
+{{ end }}

controllers/sriovoperatorconfig_controller.go

@@ -46,6 +46,7 @@ import (
 	snolog "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/log"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/platforms"
 	render "github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/render"
+	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/utils"
 	"github.com/k8snetworkplumbingwg/sriov-network-operator/pkg/vars"
 )
 
@@ -228,7 +229,7 @@ func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context,
 	data.Data["MetricsExporterSecretName"] = os.Getenv("METRICS_EXPORTER_SECRET_NAME")
 	data.Data["MetricsExporterPort"] = os.Getenv("METRICS_EXPORTER_PORT")
 	data.Data["KubeRbacProxyImage"] = os.Getenv("KUBE_RBAC_PROXY_IMAGE")
-	data.Data["ClusterType"] = vars.ClusterType
+	data.Data["IsOpenshift"] = r.PlatformHelper.IsOpenshiftCluster()
 
 	objs, err := render.RenderDir(consts.MetricsExporterPath, &data)
 	if err != nil {
@@ -237,24 +238,25 @@ func (r *SriovOperatorConfigReconciler) syncMetricsExporter(ctx context.Context,
 	}
 
 	deployMetricsExporter, ok := dc.Spec.FeatureGates[consts.MetricsExporterFeatureGate]
-	if ok && deployMetricsExporter {
-		for _, obj := range objs {
-			err = updateDaemonsetNodeSelector(obj, dc.Spec.ConfigDaemonNodeSelector)
-			if err != nil {
-				return err
-			}
+	if !ok || !deployMetricsExporter {
+		return r.deleteK8sResources(ctx, objs)
+	}
 
-			err = r.syncK8sResource(ctx, dc, obj)
-			if err != nil {
-				logger.Error(err, "Couldn't sync metrics exporter objects")
-				return err
-			}
+	for _, obj := range objs {
+		err = updateDaemonsetNodeSelector(obj, dc.Spec.ConfigDaemonNodeSelector)
+		if err != nil {
+			return err
+		}
+
+		err = r.syncK8sResource(ctx, dc, obj)
+		if err != nil {
+			logger.Error(err, "Couldn't sync metrics exporter objects")
+			return err
 		}
-		return nil
 	}
 
-	for _, obj := range objs {
-		err = r.deleteK8sResource(ctx, obj)
+	if r.PlatformHelper.IsOpenshiftCluster() {
+		err = utils.AddLabelToNamespace(ctx, vars.Namespace, "openshift.io/cluster-monitoring", "true", r.Client)
 		if err != nil {
 			return err
 		}
@@ -353,6 +355,16 @@ func (r *SriovOperatorConfigReconciler) deleteK8sResource(ctx context.Context, i
 	return nil
 }
 
+func (r *SriovOperatorConfigReconciler) deleteK8sResources(ctx context.Context, objs []*uns.Unstructured) error {
+	for _, obj := range objs {
+		err := r.deleteK8sResource(ctx, obj)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (r *SriovOperatorConfigReconciler) syncK8sResource(ctx context.Context, cr *sriovnetworkv1.SriovOperatorConfig, in *uns.Unstructured) error {
 	switch in.GetKind() {
 	case clusterRoleResourceName, clusterRoleBindingResourceName, mutatingWebhookConfigurationCRDName, validatingWebhookConfigurationCRDName, machineConfigCRDName:

controllers/sriovoperatorconfig_controller_test.go

@@ -9,8 +9,11 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/golang/mock/gomock"
 	. "github.com/onsi/ginkgo/v2"
@@ -28,6 +31,7 @@ import (
 var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 	var cancel context.CancelFunc
 	var ctx context.Context
+	var reconciler *SriovOperatorConfigReconciler
 
 	BeforeAll(func() {
 		By("Create SriovOperatorConfig controller k8s objs")
@@ -73,12 +77,13 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 		platformHelper.EXPECT().IsOpenshiftCluster().Return(false).AnyTimes()
 		platformHelper.EXPECT().IsHypershift().Return(false).AnyTimes()
 
-		err = (&SriovOperatorConfigReconciler{
+		reconciler = &SriovOperatorConfigReconciler{
 			Client:         k8sManager.GetClient(),
 			Scheme:         k8sManager.GetScheme(),
 			PlatformHelper: platformHelper,
 			FeatureGate:    featuregate.New(),
-		}).SetupWithManager(k8sManager)
+		}
+		err = reconciler.SetupWithManager(k8sManager)
 		Expect(err).ToNot(HaveOccurred())
 
 		ctx, cancel = context.WithCancel(context.Background())
@@ -100,21 +105,21 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 		})
 	})
 
-	Context("When is up", func() {
-		JustBeforeEach(func() {
-			config := &sriovnetworkv1.SriovOperatorConfig{}
-			err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout)
-			Expect(err).NotTo(HaveOccurred())
-			config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{
-				EnableInjector:        true,
-				EnableOperatorWebhook: true,
-				LogLevel:              2,
-				FeatureGates:          map[string]bool{},
-			}
-			err = k8sClient.Update(ctx, config)
-			Expect(err).NotTo(HaveOccurred())
-		})
+	BeforeEach(func() {
+		config := &sriovnetworkv1.SriovOperatorConfig{}
+		err := util.WaitForNamespacedObject(config, k8sClient, testNamespace, "default", util.RetryInterval, util.APITimeout)
+		Expect(err).NotTo(HaveOccurred())
+		config.Spec = sriovnetworkv1.SriovOperatorConfigSpec{
+			EnableInjector:        true,
+			EnableOperatorWebhook: true,
+			LogLevel:              2,
+			FeatureGates:          map[string]bool{},
+		}
+		err = k8sClient.Update(ctx, config)
+		Expect(err).NotTo(HaveOccurred())
+	})
 
+	Context("When is up", func() {
 		It("should have webhook enable", func() {
 			mutateCfg := &admv1.MutatingWebhookConfiguration{}
 			err := util.WaitForNamespacedObject(mutateCfg, k8sClient, testNamespace, "sriov-operator-webhook-config", util.RetryInterval, util.APITimeout*3)
@@ -328,6 +333,7 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 			})
 			Expect(err).ToNot(HaveOccurred())
 		})
+
 		It("should deploy the metrics-exporter when the feature gate is enabled", func() {
 			config := &sriovnetworkv1.SriovOperatorConfig{}
 			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())
@@ -356,6 +362,83 @@ var _ = Describe("SriovOperatorConfig controller", Ordered, func() {
 			err = util.WaitForNamespacedObject(&v1.Service{}, k8sClient, testNamespace, "sriov-network-metrics-exporter-service", util.RetryInterval, util.APITimeout)
 			Expect(err).ToNot(HaveOccurred())
 		})
+	})
+
+	Context("on OpenShift clusters", func() {
+
+		BeforeAll(func() {
+			t := GinkgoT()
+			mockCtrl := gomock.NewController(t)
+			openshiftPlatformHelper := mock_platforms.NewMockInterface(mockCtrl)
+			openshiftPlatformHelper.EXPECT().GetFlavor().Return(openshift.OpenshiftFlavorDefault).AnyTimes()
+			openshiftPlatformHelper.EXPECT().IsHypershift().Return(false).AnyTimes()
+
+			openshiftPlatformHelper.EXPECT().IsOpenshiftCluster().Return(true).AnyTimes()
+
+			previousPlatformHelper := reconciler.PlatformHelper
+			DeferCleanup(func() {
+				reconciler.PlatformHelper = previousPlatformHelper
+			})
+
+			reconciler.PlatformHelper = openshiftPlatformHelper
+		})
+
+		It("should deploy specific featureswhen the metrics-exporter feature is enabled", func() {
+			config := &sriovnetworkv1.SriovOperatorConfig{}
+			Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())
+
+			config.Spec.FeatureGates = map[string]bool{constants.MetricsExporterFeatureGate: true}
+			err := k8sClient.Update(ctx, config)
+			Expect(err).NotTo(HaveOccurred())
 
+			DeferCleanup(func() {
+				Expect(k8sClient.Get(ctx, types.NamespacedName{Namespace: testNamespace, Name: "default"}, config)).NotTo(HaveOccurred())
+				config.Spec.FeatureGates = map[string]bool{}
+				err = k8sClient.Update(ctx, config)
+				Expect(err).NotTo(HaveOccurred())
+			})
+
+			By("check the `openshift.io/cluster-monitoring` label has been addedd to the operator's namespace")
+
+			Eventually(func(g Gomega) {
+				ns := &v1.Namespace{}
+				err := k8sClient.Get(ctx, types.NamespacedName{Name: testNamespace}, ns)
+				g.Expect(err).NotTo(HaveOccurred())
+				g.Expect(ns.Labels).To(HaveKeyWithValue("openshift.io/cluster-monitoring", "true"))
+			}).Should(Succeed())
+
+			By("check prometheus related configuration has been deployed")
+			assertCRIsPresent(
+				schema.GroupVersionKind{
+					Group:   "monitoring.coreos.com",
+					Kind:    "ServiceMonitor",
+					Version: "v1",
+				},
+				client.ObjectKey{Namespace: testNamespace, Name: "sriov-network-metrics-exporter"})
+
+			assertCRIsPresent(
+				schema.GroupVersionKind{
+					Group:   "rbac.authorization.k8s.io",
+					Kind:    "Role",
+					Version: "v1",
+				},
+				client.ObjectKey{Namespace: testNamespace, Name: "prometheus-k8s"})
+
+			assertCRIsPresent(
+				schema.GroupVersionKind{
+					Group:   "rbac.authorization.k8s.io",
+					Kind:    "RoleBinding",
+					Version: "v1",
+				},
+				client.ObjectKey{Namespace: testNamespace, Name: "prometheus-k8s"})
+
+		})
 	})
 })
+
+func assertCRIsPresent(gvk schema.GroupVersionKind, key client.ObjectKey) {
+	u := &unstructured.Unstructured{}
+	u.SetGroupVersionKind(gvk)
+	err := k8sClient.Get(context.Background(), key, u)
+	Expect(err).NotTo(HaveOccurred())
+}

pkg/utils/cluster.go

@@ -161,3 +161,29 @@ func AnnotateNode(ctx context.Context, nodeName string, key, value string, c cli
 
 	return AnnotateObject(ctx, node, key, value, c)
 }
+
+
+func AddLabelToNamespace(ctx context.Context, namespaceName, key, value string, c client.Client) error {
+	ns := &corev1.Namespace{}
+	err := c.Get(ctx, client.ObjectKey{Name: namespaceName}, ns)
+	if err != nil {
+		return fmt.Errorf("failed to get namespace [%s]: %v", namespaceName, err)
+	}
+
+	if ns.Labels == nil {
+		ns.Labels = make(map[string]string)
+	}
+
+	if ns.Labels[key] == value {
+		return nil
+	}
+
+	ns.Labels[key] = value
+
+	err = c.Update(ctx, ns)
+	if err != nil {
+		return fmt.Errorf("failed to update namespace [%s] with label [%s: %s]: %v", namespaceName, key, value, err)
+	}
+
+	return nil
+}