eks-prow-build-cluster: create simple monitoring stack

infra/aws/terraform/prow-build-cluster/resources/README.md

@@ -0,0 +1 @@
+[Installing AWS Node Termination Handler](https://github.com/aws/aws-node-termination-handler/tree/main/config/helm/aws-node-termination-handler#installing-the-chart)

infra/aws/terraform/prow-build-cluster/resources/monitoring/README.md

@@ -0,0 +1,51 @@
+# EKS monitoring
+
+## Setting up monitoring
+
+```bash
+# Create monitoring namespace:
+kubectl apply -f ../namespaces.yaml
+
+# Install CRDs for Prometheus Operator:
+# (server side is required due to long annotations)
+kubectl apply --server-side -f ./prometheus-operator-crds
+
+# Install Prometheus Operator:
+kubectl apply  -f ./prometheus-operator
+
+# Install kube-state-metrics
+kubectl apply -f ./kube-state-metrics
+
+# Install node-exporter
+kubectl apply -f ./node-exporter
+
+# Install cadvisor
+kubectl apply -f ./cadvisor
+
+# Install dashboards for Grafana
+kubectl apply --server-side -f ./grafana/dashboards
+
+# Install Grafana
+kubectl apply -f ./grafana
+```
+
+[Prometheus Operator CRDs](https://github.com/prometheus-operator/prometheus-operator/tree/v0.63.0/example/prometheus-operator-crd-full)
+
+## Local access
+
+```bash
+# Prometheus
+kubectl --namespace monitoring port-forward svc/prometheus-operated 9090
+
+# Grafana
+kubectl --namespace monitoring port-forward svc/grafana 3000
+```
+
+## Debugging
+
+- [Troubleshooting Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/troubleshooting.md)
+
+Checking Prometheus configuration:
+```
+kubectl -n monitoring get secret prometheus-main -ojson | jq -r '.data["prometheus.yaml.gz"]' | base64 -d | gunzip | less
+```

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/cluster-role-binding.yaml

@@ -0,0 +1,12 @@
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cadvisor
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cadvisor
+subjects:
+  - kind: ServiceAccount
+    name: cadvisor
+    namespace: monitoring

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/cluster-role.yaml

@@ -0,0 +1,10 @@
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cadvisor
+rules:
+  - apiGroups: ['policy']
+    resources: ['podsecuritypolicies']
+    verbs: ['use']
+    resourceNames:
+      - cadvisor

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/ds.yaml

@@ -0,0 +1,63 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cadvisor
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      name: cadvisor
+  template:
+    metadata:
+      labels:
+        name: cadvisor
+    spec:
+      serviceAccountName: cadvisor
+      containers:
+        - name: cadvisor
+          image: gcr.io/cadvisor/cadvisor:v0.45.0
+          resources:
+            requests:
+              memory: 400Mi
+              cpu: 400m
+            limits:
+              memory: 2000Mi
+              cpu: 800m
+          volumeMounts:
+            - name: rootfs
+              mountPath: /rootfs
+              readOnly: true
+            - name: var-run
+              mountPath: /var/run
+              readOnly: true
+            - name: sys
+              mountPath: /sys
+              readOnly: true
+            - name: docker
+              mountPath: /var/lib/docker
+              readOnly: true
+            - name: disk
+              mountPath: /dev/disk
+              readOnly: true
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 30
+      volumes:
+        - name: rootfs
+          hostPath:
+            path: /
+        - name: var-run
+          hostPath:
+            path: /var/run
+        - name: sys
+          hostPath:
+            path: /sys
+        - name: docker
+          hostPath:
+            path: /var/lib/docker
+        - name: disk
+          hostPath:
+            path: /dev/disk

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cadvisor
+  namespace: monitoring

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/service-monitor.yaml

@@ -0,0 +1,13 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: cadvisor
+  namespace: monitoring
+  labels:
+    prometheus: main
+spec:
+  selector:
+    matchLabels:
+      name: cadvisor
+  endpoints:
+    - port: http

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: cadvisor
+  namespace: monitoring
+  labels:
+    name: cadvisor
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8080
+      protocol: TCP
+      name: http
+  selector:
+    name: cadvisor

infra/aws/terraform/prow-build-cluster/resources/monitoring/grafana/config.yaml

@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: monitoring
+  name: grafana
+data:
+  grafana.ini: |-
+    [analytics]
+    reporting_enabled = false
+    check_for_updates = true
+
+    [grafana_net]
+    url = https://grafana.net
+
+    [log]
+    mode = console
+
+    [auth.anonymous]
+    enabled = true
+
+    [paths]
+    data = /var/lib/grafana/data
+    logs = /var/log/grafana
+    plugins = /var/lib/grafana/plugins
+    provisioning = /etc/grafana/provisioning
+
+    [server]
+    root_url = "https://my-grafana.example.pvt"
+    [auth.anonymous]
+    enabled = false
+    [metrics]
+    enabled = true
+    disable_total_stats = false

infra/aws/terraform/prow-build-cluster/resources/monitoring/grafana/dashboards.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: monitoring
+  name: dashboards
+data:
+  dashboardproviders.yaml: |-
+    apiVersion: 1
+    providers:
+    - disableDeletion: false
+      editable: false
+      folder: Kubernetes
+      name: kubernetes
+      options:
+        path: /var/lib/grafana/dashboards/kubernetes
+      orgId: 1
+      type: file