Improve promtail alerts to retain the namespace label (#2976)

The `PromtailFileMissing` should actually work in this simplified form. Please let me know if I'm missing something.
grafana · Nov 24, 2020 · e14463b · e14463b
1 parent 14a5e22
commit e14463b
Show file tree

Hide file tree

Showing 2 changed files with 4 additions and 4 deletions.
diff --git a/production/promtail-mixin/alerts.libsonnet b/production/promtail-mixin/alerts.libsonnet
@@ -25,7 +25,7 @@
           {
             alert: 'PromtailRequestLatency',
             expr: |||
-              job_status_code:promtail_request_duration_seconds:99quantile > 1
+              job_status_code_namespace:promtail_request_duration_seconds:99quantile > 1
             |||,
             'for': '15m',
             labels: {
@@ -55,7 +55,7 @@
           {
             alert: 'PromtailFileMissing',
             expr: |||
-              count by (path,instance,job) (promtail_file_bytes_total) unless count by (path,instance,job) (promtail_read_bytes_total)
+              promtail_file_bytes_total unless promtail_read_bytes_total
             |||,
             'for': '15m',
             labels: {

diff --git a/production/promtail-mixin/recording_rules.libsonnet b/production/promtail-mixin/recording_rules.libsonnet
@@ -5,8 +5,8 @@ local utils = import 'mixin-utils/utils.libsonnet';
     groups+: [{
       name: 'promtail_rules',
       rules:
-        utils.histogramRules('promtail_request_duration_seconds', ['job']) +
-        utils.histogramRules('promtail_request_duration_seconds', ['job', 'status_code']),
+        utils.histogramRules('promtail_request_duration_seconds', ['job', 'namespace']) +
+        utils.histogramRules('promtail_request_duration_seconds', ['job', 'status_code', 'namespace']),
     }],
   },
 }