From d87cafa9806be389fea266bc1702854c25323071 Mon Sep 17 00:00:00 2001 From: beorn7 Date: Mon, 23 Nov 2020 12:44:50 +0100 Subject: [PATCH] Improve promtail alerts to retain the namespace label The `PromtailFileMissing` should actually work in this simplified form. Please let me know if I'm missing something. --- production/promtail-mixin/alerts.libsonnet | 4 ++-- production/promtail-mixin/recording_rules.libsonnet | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/production/promtail-mixin/alerts.libsonnet b/production/promtail-mixin/alerts.libsonnet index 1f3776b863df..f6ef5da60100 100644 --- a/production/promtail-mixin/alerts.libsonnet +++ b/production/promtail-mixin/alerts.libsonnet @@ -25,7 +25,7 @@ { alert: 'PromtailRequestLatency', expr: ||| - job_status_code:promtail_request_duration_seconds:99quantile > 1 + job_status_code_namespace:promtail_request_duration_seconds:99quantile > 1 |||, 'for': '15m', labels: { @@ -55,7 +55,7 @@ { alert: 'PromtailFileMissing', expr: ||| - count by (path,instance,job) (promtail_file_bytes_total) unless count by (path,instance,job) (promtail_read_bytes_total) + promtail_file_bytes_total unless promtail_read_bytes_total |||, 'for': '15m', labels: { diff --git a/production/promtail-mixin/recording_rules.libsonnet b/production/promtail-mixin/recording_rules.libsonnet index a93c19a7ec40..80e41dce888e 100644 --- a/production/promtail-mixin/recording_rules.libsonnet +++ b/production/promtail-mixin/recording_rules.libsonnet @@ -5,8 +5,8 @@ local utils = import 'mixin-utils/utils.libsonnet'; groups+: [{ name: 'promtail_rules', rules: - utils.histogramRules('promtail_request_duration_seconds', ['job']) + - utils.histogramRules('promtail_request_duration_seconds', ['job', 'status_code']), + utils.histogramRules('promtail_request_duration_seconds', ['job', 'namespace']) + + utils.histogramRules('promtail_request_duration_seconds', ['job', 'status_code', 'namespace']), }], }, }