Update magnum k8s monitoring infra

* Prometheus-server now runs only on master nodes. * Update prometheus-operator helm chart and tag. * Update prometheus-adapter version. * Deprecation notice for prometheus_monitoring component. Since v1.20 (changes are gradually introduced in next versions) k8s components are exposing (by default) metrics on 127.0.0.1 and are changing expose port. * Fix metrics address and ports post 1.20 * Extended memory limits to 64M * Changed expose addr to 127.0.0.1 * Changed expose ports task: 41569 story: 2006765 Signed-off-by: Diogo Guerra <diogo.filipe.tomas.guerra@cern.ch> Co-authored-by: Grzegorz Bialas <grzegorz@stackhpc.com> Change-Id: I05e8c2be4e4c8e66a166b485ec7851875dca8b1c (cherry picked from commit c92f605) (cherry picked from commit 32b22e5) (cherry picked from commit 5b56182) (cherry picked from commit 25e3bdc) (cherry picked from commit d0e1aa2)
stackhpc · Jul 1, 2024 · 903f7f6 · 903f7f6
1 parent a7a698c
commit 903f7f6
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 29 deletions.
diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh
@@ -271,7 +271,7 @@ CERT_DIR=/etc/kubernetes/certs
 
 # kube-proxy config
 PROXY_KUBECONFIG=/etc/kubernetes/proxy-kubeconfig.yaml
-KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME}"
+KUBE_PROXY_ARGS="--kubeconfig=${PROXY_KUBECONFIG} --cluster-cidr=${PODS_NETWORK_CIDR} --hostname-override=${INSTANCE_NAME} --metrics-bind-address=0.0.0.0"
 cat > /etc/kubernetes/proxy << EOF
 KUBE_PROXY_ARGS="${KUBE_PROXY_ARGS} ${KUBEPROXY_OPTIONS}"
 EOF
@@ -406,6 +406,8 @@ KUBE_CONTROLLER_MANAGER_ARGS="--leader-elect=true --kubeconfig=/etc/kubernetes/a
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --cluster-name=${CLUSTER_UUID}"
 KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --allocate-node-cidrs=true"
 KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --cluster-cidr=${PODS_NETWORK_CIDR}"
+KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --secure-port=10257"
+KUBE_CONTROLLER_MANAGER_ARGS="${KUBE_CONTROLLER_MANAGER_ARGS} --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics"
 KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS $KUBECONTROLLER_OPTIONS"
 if [ -n "${ADMISSION_CONTROL_LIST}" ] && [ "${TLS_DISABLED}" == "False" ]; then
     KUBE_CONTROLLER_MANAGER_ARGS="$KUBE_CONTROLLER_MANAGER_ARGS --service-account-private-key-file=$CERT_DIR/service_account_private.key --root-ca-file=$CERT_DIR/ca.crt"
@@ -428,7 +430,7 @@ sed -i '
     /^KUBE_CONTROLLER_MANAGER_ARGS=/ s#\(KUBE_CONTROLLER_MANAGER_ARGS\).*#\1="'"${KUBE_CONTROLLER_MANAGER_ARGS}"'"#
 ' /etc/kubernetes/controller-manager
 
-sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf"#' /etc/kubernetes/scheduler
+sed -i '/^KUBE_SCHEDULER_ARGS=/ s#=.*#="--leader-elect=true --kubeconfig=/etc/kubernetes/admin.conf --authorization-always-allow-paths=/healthz,/readyz,/livez,/metrics "#' /etc/kubernetes/scheduler
 
 $ssh_cmd mkdir -p /etc/kubernetes/manifests
 KUBELET_ARGS="--register-node=true --pod-manifest-path=/etc/kubernetes/manifests --hostname-override=${INSTANCE_NAME}"

diff --git a/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh b/magnum/drivers/common/templates/kubernetes/helm/prometheus-operator.sh
@@ -180,18 +180,10 @@ ${APP_INGRESS_ANNOTATIONS}
     endpoints: ${KUBE_MASTERS_PRIVATE}
     ## If using kubeControllerManager.endpoints only the port and targetPort are used
     service:
-      port: 10252
-      targetPort: 10252
+      port: 10257
+      targetPort: 10257
       # selector:
       #   component: kube-controller-manager
-    serviceMonitor:
-      ## Enable scraping kube-controller-manager over https.
-      ## Requires proper certs (not self-signed) and delegated authentication/authorization checks
-      https: "True"
-      # Skip TLS certificate validation when scraping
-      insecureSkipVerify: "True"
-      # Name of the server to use when validating TLS certificate
-      serverName: null
 
   coreDns:
     enabled: true
@@ -224,28 +216,17 @@ ${APP_INGRESS_ANNOTATIONS}
     endpoints: ${KUBE_MASTERS_PRIVATE}
     ## If using kubeScheduler.endpoints only the port and targetPort are used
     service:
-      port: 10251
-      targetPort: 10251
+      port: 10259
+      targetPort: 10259
       # selector:
       #   component: kube-scheduler
     serviceMonitor:
-      ## Enable scraping kube-scheduler over https.
-      ## Requires proper certs (not self-signed) and delegated authentication/authorization checks
-      https: "True"
-      ## Skip TLS certificate validation when scraping
-      insecureSkipVerify: "True"
-      ## Name of the server to use when validating TLS certificate
-      serverName: null
+      scheme: https
+      insecureSkipVerify: true
 
   kubeProxy:
     ## If your kube proxy is not deployed as a pod, specify IPs it can be found on
     endpoints: ${KUBE_MASTERS_PRIVATE} # masters + minions
-    serviceMonitor:
-      ## Enable scraping kube-proxy over https.
-      ## Requires proper certs (not self-signed) and delegated authentication/authorization checks
-      https: "True"
-      ## Skip TLS certificate validation when scraping
-      insecureSkipVerify: "True"
 
   kube-state-metrics:
     priorityClassName: "system-cluster-critical"
@@ -272,7 +253,7 @@ ${APP_INGRESS_ANNOTATIONS}
           requests:
             cpu: 2m
           limits:
-            memory: 30M
+            memory: 64M
     # clusterDomain: ${CLUSTER_ROOT_DOMAIN_NAME}
     priorityClassName: "system-cluster-critical"
     logFormat: json
@@ -281,7 +262,7 @@ ${APP_INGRESS_ANNOTATIONS}
       requests:
         cpu: 2m
       limits:
-        memory: 32M
+        memory: 64M
     image:
       repository: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus-operator/}prometheus-operator
     prometheusDefaultBaseImage: ${CONTAINER_INFRA_PREFIX:-quay.io/prometheus/}prometheus

diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml
@@ -1121,6 +1121,18 @@ resources:
         - protocol: tcp
           port_range_min: 9090
           port_range_max: 9090
+        - protocol: tcp
+          port_range_min: 10259
+          port_range_max: 10259
+        - protocol: tcp
+          port_range_min: 10257
+          port_range_max: 10257
+        - protocol: tcp
+          port_range_min: 10249
+          port_range_max: 10249
+        - protocol: tcp
+          port_range_min: 9153
+          port_range_max: 9153
 
   secgroup_kube_minion:
     condition: create_cluster_resources