Added operator metrics ssl and auth configuration

bundle/manifests/netobserv-metrics-monitor_monitoring.coreos.com_v1_servicemonitor.yaml

@@ -0,0 +1,19 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: netobserv-metrics-monitor
+spec:
+  endpoints:
+  - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+    interval: 30s
+    path: /metrics
+    port: https
+    scheme: https
+    tlsConfig:
+      caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+      serverName: netobserv-metrics-service.netobserv.svc
+  selector:
+    matchLabels:
+      control-plane: controller-manager

bundle/manifests/netobserv-metrics-service_v1_service.yaml

@@ -1,6 +1,8 @@
 apiVersion: v1
 kind: Service
 metadata:
+  annotations:
+    service.beta.openshift.io/serving-cert-secret-name: manager-metrics-tls
   creationTimestamp: null
   labels:
     control-plane: controller-manager

bundle/manifests/netobserv-operator.clusterserviceversion.yaml

@@ -432,6 +432,10 @@ spec:
     spec:
       clusterPermissions:
       - rules:
+        - nonResourceURLs:
+          - /metrics
+          verbs:
+          - get
         - apiGroups:
           - apiregistration.k8s.io
           resources:
@@ -700,7 +704,8 @@ spec:
                 - --upstream=http://127.0.0.1:8080/
                 - --logtostderr=true
                 - --v=10
-                - --ignore-paths=/metrics
+                - --tls-cert-file=/etc/tls/private/tls.crt
+                - --tls-private-key-file=/etc/tls/private/tls.key
                 image: gcr.io/kubebuilder/kube-rbac-proxy:v0.8.0
                 name: kube-rbac-proxy
                 ports:
@@ -714,6 +719,10 @@ spec:
                   requests:
                     cpu: 5m
                     memory: 64Mi
+                volumeMounts:
+                - mountPath: /etc/tls/private
+                  name: manager-metric-tls
+                  readOnly: true
               securityContext:
                 runAsNonRoot: true
               serviceAccountName: netobserv-controller-manager
@@ -723,6 +732,10 @@ spec:
                 secret:
                   defaultMode: 420
                   secretName: webhook-server-cert
+              - name: manager-metric-tls
+                secret:
+                  defaultMode: 420
+                  secretName: manager-metrics-tls
       permissions:
       - rules:
         - apiGroups:

config/default/manager_auth_proxy_patch.yaml

@@ -23,7 +23,6 @@ spec:
           - "--secure-listen-address=0.0.0.0:8443"
           - "--upstream=http://127.0.0.1:8080/"
           - "--logtostderr=true"
-          - "--ignore-paths=/metrics"
           - "--v=10"
         ports:
           - containerPort: 8443

config/manager/manager.yaml

@@ -67,7 +67,6 @@ spec:
           - "--upstream=http://127.0.0.1:8080/"
           - "--logtostderr=true"
           - "--v=10"
-          - "--ignore-paths=/metrics"
         ports:
           - containerPort: 8443
             protocol: TCP

config/openshift-olm/kustomization.yaml

@@ -8,6 +8,14 @@ namespace: netobserv
 # field above.
 namePrefix: netobserv-
 
+resources:
+- monitor.yaml
+
+patches:
+  - path: ./monitoring-patch.yaml
+    target:
+      kind: Deployment
+      name: controller-manager
 # Labels to add to all resources and selectors.
 #commonLabels:
 #  someName: someValue

config/prometheus/monitor.yaml → config/openshift-olm/monitor.yaml

@@ -1,4 +1,3 @@
-
 # Prometheus Monitor Service (Metrics)
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
@@ -9,11 +8,14 @@ metadata:
   namespace: system
 spec:
   endpoints:
-    - path: /metrics
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      path: /metrics
+      interval: 30s
       port: https
       scheme: https
       tlsConfig:
-        insecureSkipVerify: true
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: netobserv-metrics-service.netobserv.svc
   selector:
     matchLabels:
       control-plane: controller-manager

config/openshift-olm/monitoring-patch.yaml

@@ -0,0 +1,27 @@
+# Check that the 0 container is the expected one
+- op: test
+  path: /spec/template/spec/containers/1/name
+  value: kube-rbac-proxy
+
+- op: add
+  path: "/spec/template/spec/volumes/-"
+  value:
+    name: manager-metric-tls
+    secret:
+      secretName: manager-metrics-tls
+      defaultMode: 420
+
+- op: add
+  path: "/spec/template/spec/containers/1/volumeMounts"
+  value:
+    - name: manager-metric-tls
+      readOnly: true
+      mountPath: /etc/tls/private
+
+- op: add
+  path: "/spec/template/spec/containers/1/args/-"
+  value: '--tls-cert-file=/etc/tls/private/tls.crt'
+
+- op: add
+  path: "/spec/template/spec/containers/1/args/-"
+  value: '--tls-private-key-file=/etc/tls/private/tls.key'

config/openshift/kustomization.yaml

@@ -8,16 +8,19 @@ namespace: netobserv
 # field above.
 namePrefix: netobserv-
 
-# Labels to add to all resources and selectors.
-#commonLabels:
-#  someName: someValue
+resources:
+- monitor.yaml
 
+patches:
+  - path: ./monitoring-patch.yaml
+    target:
+      kind: Deployment
+      name: controller-manager
 bases:
 - ../crd
 - ../rbac
 - ../manager
 - ../webhook
-- ../prometheus
 - namespace.yaml
 patchesStrategicMerge:
 - patch.yaml

config/openshift/monitor.yaml

@@ -0,0 +1,21 @@
+# Prometheus Monitor Service (Metrics)
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: metrics-monitor
+  namespace: system
+spec:
+  endpoints:
+    - bearerTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token
+      path: /metrics
+      interval: 30s
+      port: https
+      scheme: https
+      tlsConfig:
+        caFile: /etc/prometheus/configmaps/serving-certs-ca-bundle/service-ca.crt
+        serverName: netobserv-metrics-service.netobserv.svc
+  selector:
+    matchLabels:
+      control-plane: controller-manager

config/openshift/monitoring-patch.yaml

@@ -0,0 +1,27 @@
+# Check that the 0 container is the expected one
+- op: test
+  path: /spec/template/spec/containers/1/name
+  value: kube-rbac-proxy
+
+- op: add
+  path: "/spec/template/spec/volumes/-"
+  value:
+    name: manager-metric-tls
+    secret:
+      secretName: manager-metrics-tls
+      defaultMode: 420
+
+- op: add
+  path: "/spec/template/spec/containers/1/volumeMounts"
+  value:
+    - name: manager-metric-tls
+      readOnly: true
+      mountPath: /etc/tls/private
+
+- op: add
+  path: "/spec/template/spec/containers/1/args/-"
+  value: '--tls-cert-file=/etc/tls/private/tls.crt'
+
+- op: add
+  path: "/spec/template/spec/containers/1/args/-"
+  value: '--tls-private-key-file=/etc/tls/private/tls.key'

config/rbac/auth_proxy_service.yaml

@@ -3,6 +3,8 @@ kind: Service
 metadata:
   labels:
     control-plane: controller-manager
+  annotations:
+        service.beta.openshift.io/serving-cert-secret-name: manager-metrics-tls
   name: metrics-service
   namespace: system
 spec:

config/rbac/kustomization.yaml

@@ -15,4 +15,3 @@ resources:
 - auth_proxy_service.yaml
 - auth_proxy_role.yaml
 - auth_proxy_role_binding.yaml
-- auth_proxy_client_clusterrole.yaml

config/rbac/role.yaml

@@ -6,6 +6,10 @@ metadata:
   creationTimestamp: null
   name: manager-role
 rules:
+- nonResourceURLs:
+  - /metrics
+  verbs:
+  - get
 - apiGroups:
   - apiregistration.k8s.io
   resources:

controllers/flowcollector_controller.go

@@ -75,6 +75,7 @@ func NewFlowCollectorReconciler(client client.Client, scheme *runtime.Scheme, co
 //+kubebuilder:rbac:groups=security.openshift.io,resources=securitycontextconstraints,verbs=list;create;update;watch
 //+kubebuilder:rbac:groups=apiregistration.k8s.io,resources=apiservices,verbs=list;get;watch
 //+kubebuilder:rbac:groups=monitoring.coreos.com,resources=servicemonitors;prometheusrules,verbs=get;create;delete;update;patch;list;watch
+//+kubebuilder:rbac:urls="/metrics",verbs=get
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
 // move the current state of the cluster closer to the desired state.

controllers/flowcollector_objects.go

@@ -34,10 +34,15 @@ func buildRoleMonitoringReader(ns string) *rbacv1.Role {
 			Name:      constants.OperatorName + roleSuffix,
 			Namespace: ns,
 		},
-		Rules: []rbacv1.PolicyRule{{APIGroups: []string{""},
-			Verbs:     []string{"get", "list", "watch"},
-			Resources: []string{"pods", "services", "endpoints"},
-		},
+		Rules: []rbacv1.PolicyRule{
+			{APIGroups: []string{""},
+				Verbs:     []string{"get", "list", "watch"},
+				Resources: []string{"pods", "services", "endpoints"},
+			},
+			{
+				NonResourceURLs: []string{"/metrics"},
+				Verbs:           []string{"get"},
+			},
 		},
 	}
 	return &cr

monitor.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cluster-monitoring-config
+  namespace: openshift-monitoring
+data:
+  config.yaml: |
+    enableUserWorkload: true