From 2c4d2009108e409bf9c0c03ef0db978c19c74695 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Thu, 15 Apr 2021 13:48:57 +0200 Subject: [PATCH] mixin: Downgrade severity of forward failures We had this alert fire during a rollout, however, it is expected that during rollouts some forward requests fail. As long as ingestion as a whole is still succeeding with quorum, there's no need to alert on anything. Signed-off-by: Frederic Branczyk --- examples/alerts/alerts.md | 2 +- examples/alerts/alerts.yaml | 2 +- mixin/alerts/receive.libsonnet | 2 +- mixin/runbook.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/alerts/alerts.md b/examples/alerts/alerts.md index 291c97ae97..98bfe8a6c5 100644 --- a/examples/alerts/alerts.md +++ b/examples/alerts/alerts.md @@ -521,7 +521,7 @@ rules: ) * 100 > 20 for: 5m labels: - severity: warning + severity: info - alert: ThanosReceiveHighHashringFileRefreshFailures annotations: description: Thanos Receive {{$labels.job}} is failing to refresh hashring file, diff --git a/examples/alerts/alerts.yaml b/examples/alerts/alerts.yaml index f4c6e8d237..46c3cb0727 100644 --- a/examples/alerts/alerts.yaml +++ b/examples/alerts/alerts.yaml @@ -241,7 +241,7 @@ groups: ) * 100 > 20 for: 5m labels: - severity: warning + severity: info - alert: ThanosReceiveHighHashringFileRefreshFailures annotations: description: Thanos Receive {{$labels.job}} is failing to refresh hashring file, diff --git a/mixin/alerts/receive.libsonnet b/mixin/alerts/receive.libsonnet index 1cda9d7ae9..d491be7536 100644 --- a/mixin/alerts/receive.libsonnet +++ b/mixin/alerts/receive.libsonnet @@ -93,7 +93,7 @@ ||| % thanos.receive, 'for': '5m', labels: { - severity: 'warning', + severity: 'info', }, }, { diff --git a/mixin/runbook.md b/mixin/runbook.md index 37d2a4ff7d..d87c7bc2ca 100755 --- a/mixin/runbook.md +++ b/mixin/runbook.md @@ -59,7 +59,7 @@ |ThanosReceiveHttpRequestErrorRateHigh|Thanos Receive is failing to handle requests.|Thanos Receive {{$labels.job}} is failing to handle {{$value humanize}}% of requests.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehttprequesterrorratehigh](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehttprequesterrorratehigh)| |ThanosReceiveHttpRequestLatencyHigh|Thanos Receive has high HTTP requests latency.|Thanos Receive {{$labels.job}} has a 99th percentile latency of {{ $value }} seconds for requests.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehttprequestlatencyhigh](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehttprequestlatencyhigh)| |ThanosReceiveHighReplicationFailures|Thanos Receive is having high number of replication failures.|Thanos Receive {{$labels.job}} is failing to replicate {{$value humanize}}% of requests.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighreplicationfailures](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighreplicationfailures)| -|ThanosReceiveHighForwardRequestFailures|Thanos Receive is failing to forward requests.|Thanos Receive {{$labels.job}} is failing to forward {{$value humanize}}% of requests.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighforwardrequestfailures](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighforwardrequestfailures)| +|ThanosReceiveHighForwardRequestFailures|Thanos Receive is failing to forward requests.|Thanos Receive {{$labels.job}} is failing to forward {{$value humanize}}% of requests.|info|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighforwardrequestfailures](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighforwardrequestfailures)| |ThanosReceiveHighHashringFileRefreshFailures|Thanos Receive is failing to refresh hasring file.|Thanos Receive {{$labels.job}} is failing to refresh hashring file, {{$value humanize}} of attempts failed.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighhashringfilerefreshfailures](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivehighhashringfilerefreshfailures)| |ThanosReceiveConfigReloadFailure|Thanos Receive has not been able to reload configuration.|Thanos Receive {{$labels.job}} has not been able to reload hashring configurations.|warning|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceiveconfigreloadfailure](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceiveconfigreloadfailure)| |ThanosReceiveNoUpload|Thanos Receive has not uploaded latest data to object storage.|Thanos Receive {{$labels.instance}} has not uploaded latest data to object storage.|critical|[https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivenoupload](https://github.com/thanos-io/thanos/tree/main/mixin/runbook.md#alert-name-thanosreceivenoupload)|