feat: mdraid monitoring

Obmondo · Nov 27, 2024 · 4f6bcb1 · 4f6bcb1
1 parent ef3de66
commit 4f6bcb1
Show file tree

Hide file tree

Showing 2 changed files with 29 additions and 1 deletion.
diff --git a/argocd-helm-charts/prometheus-linuxaid/rules/mdraid.yaml b/argocd-helm-charts/prometheus-linuxaid/rules/mdraid.yaml
@@ -23,7 +23,7 @@ groups:
             Array '{{ .Labels.device }}' needs attention and possibly a disk swap.
           summary: Failed device in RAID array
         expr: |
-          node_md_disks{state="failed"} > 0
+          (node_md_disks{state="failed"} > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}
           and on(certname) obmondo_monitoring{alert_id="monitor::raid::mdraid::failed"} > 0
         labels:
           severity: warning

diff --git a/build/kube-prometheus/mixins/mdraid/mixin.libsonnet b/build/kube-prometheus/mixins/mdraid/mixin.libsonnet
@@ -0,0 +1,28 @@
+{
+  _config+:: {
+    selector: '',
+  },
+
+  prometheusAlerts+:: {
+    groups+: [
+      {
+        name: 'mdraid-status',
+        rules: [
+          {
+            alert: 'HostRaidDiskFailure',
+            expr: '(node_md_disks{state="failed"} > 0) * on(instance) group_left (nodename) node_uname_info{nodename=~".+"}',
+            'for': '30m',
+            labels: {
+              severity: 'critical',
+              alert_id: 'HostRaidDiskFailure',
+            },
+            annotations: {
+              summary: 'Host RAID disk failure (instance {{ $labels.instance }})',
+              description: 'At least one device in RAID array on {{ $labels.instance }} failed. Array {{ $labels.md_device }} needs attention and possibly a disk swap\n  VALUE = {{ $value }}\n  LABELS = {{ $labels }}',
+            },
+          },
+        ],
+      },
+    ],
+  },
+}