eks-prow-build-cluster: create simple monitoring stack

infra/aws/terraform/prow-build-cluster/resources/monitoring/README.md

@@ -0,0 +1,51 @@
+# EKS monitoring
+
+## Setting up monitoring
+
+```bash
+# Create monitoring namespace:
+kubectl apply -f ../namespaces.yaml
+
+# Install CRDs for Prometheus Operator:
+# (server side is required due to long annotations)
+kubectl apply --server-side -f ./prometheus-operator-crds
+
+# Install Prometheus Operator:
+kubectl apply  -f ./prometheus-operator
+
+# Install kube-state-metrics
+kubectl apply -f ./kube-state-metrics
+
+# Install node-exporter
+kubectl apply -f ./node-exporter
+
+# Install cadvisor
+kubectl apply -f ./cadvisor
+
+# Install dashboards for Grafana
+kubectl apply --server-side -f ./grafana/dashboards
+
+# Install Grafana
+kubectl apply -f ./grafana
+```
+
+[Prometheus Operator CRDs](https://github.com/prometheus-operator/prometheus-operator/tree/v0.63.0/example/prometheus-operator-crd-full)
+
+## Local access
+
+```bash
+# Prometheus
+kubectl --namespace monitoring port-forward svc/prometheus-operated 9090
+
+# Grafana
+kubectl --namespace monitoring port-forward svc/grafana 3000
+```
+
+## Debugging
+
+- [Troubleshooting Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/troubleshooting.md)
+
+Checking Prometheus configuration:
+```
+kubectl -n monitoring get secret prometheus-main -ojson | jq -r '.data["prometheus.yaml.gz"]' | base64 -d | gunzip | less
+```

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/ds.yaml

@@ -0,0 +1,57 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cadvisor
+  namespace: monitoring
+spec:
+  selector:
+    matchLabels:
+      name: cadvisor
+  template:
+    metadata:
+      labels:
+        name: cadvisor
+    spec:
+      serviceAccountName: cadvisor
+      containers:
+        - name: cadvisor
+          image: gcr.io/cadvisor/cadvisor:v0.45.0
+          resources:
+            requests:
+              memory: 400Mi
+              cpu: 400m
+            limits:
+              memory: 2000Mi
+              cpu: 800m
+          volumeMounts:
+            - name: rootfs
+              mountPath: /rootfs
+              readOnly: true
+            - name: var-run
+              mountPath: /var/run
+              readOnly: true
+            - name: sys
+              mountPath: /sys
+              readOnly: true
+            - name: disk
+              mountPath: /dev/disk
+              readOnly: true
+          ports:
+            - name: http
+              containerPort: 8080
+              protocol: TCP
+      automountServiceAccountToken: false
+      terminationGracePeriodSeconds: 30
+      volumes:
+        - name: rootfs
+          hostPath:
+            path: /
+        - name: var-run
+          hostPath:
+            path: /var/run
+        - name: sys
+          hostPath:
+            path: /sys
+        - name: disk
+          hostPath:
+            path: /dev/disk

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/sa.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: cadvisor
+  namespace: monitoring

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/service-monitor.yaml

@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: cadvisor
+  namespace: monitoring
+  labels:
+    prometheus: main
+spec:
+  selector:
+    matchLabels:
+      name: cadvisor
+  endpoints:
+    - port: http
+      honorLabels: true

infra/aws/terraform/prow-build-cluster/resources/monitoring/cadvisor/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: cadvisor
+  namespace: monitoring
+  labels:
+    name: cadvisor
+spec:
+  type: ClusterIP
+  ports:
+    - port: 8080
+      protocol: TCP
+      name: http
+  selector:
+    name: cadvisor

infra/aws/terraform/prow-build-cluster/resources/monitoring/grafana/config.yaml

@@ -0,0 +1,39 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: monitoring
+  name: grafana
+data:
+  grafana.ini: |-
+    [analytics]
+    reporting_enabled = false
+    check_for_updates = true
+
+    [log]
+    mode = console
+
+    [auth.basic]
+    enabled = false
+
+    [auth.anonymous]
+    enabled = true
+    org_role = Viewer
+
+    [auth]
+    disable_login_form = true
+
+    [paths]
+    data = /var/lib/grafana/data
+    logs = /var/log/grafana
+    plugins = /var/lib/grafana/plugins
+    provisioning = /etc/grafana/provisioning
+
+    [server]
+    http_port = 3001
+
+    [security]
+    disable_gravatar = true
+
+    [metrics]
+    enabled = true
+    disable_total_stats = false

infra/aws/terraform/prow-build-cluster/resources/monitoring/grafana/dashboards.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  namespace: monitoring
+  name: dashboards
+data:
+  dashboardproviders.yaml: |-
+    apiVersion: 1
+    providers:
+    - disableDeletion: false
+      editable: false
+      folder: Kubernetes
+      name: kubernetes
+      options:
+        path: /var/lib/grafana/dashboards/kubernetes
+      orgId: 1
+      type: file