From 79446ea677da3fe6505eba0913d84b56b20d0fe9 Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Wed, 15 Nov 2017 14:33:52 +0100 Subject: [PATCH 1/2] Documentation/op-guide: Adapt alerting rules to new gRPC metrics --- Documentation/op-guide/etcd3_alert.rules | 10 +++++----- Documentation/op-guide/etcd3_alert.rules.yml | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Documentation/op-guide/etcd3_alert.rules b/Documentation/op-guide/etcd3_alert.rules index e25cbc4c2b2..1001926654b 100644 --- a/Documentation/op-guide/etcd3_alert.rules +++ b/Documentation/op-guide/etcd3_alert.rules @@ -69,7 +69,7 @@ ANNOTATIONS { # alert if the 99th percentile of gRPC method calls take more than 150ms ALERT GRPCRequestsSlow -IF histogram_quantile(0.99, rate(etcd_grpc_unary_requests_duration_seconds_bucket[5m])) > 0.15 +IF histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15 FOR 10m LABELS { severity = "critical" @@ -84,8 +84,8 @@ ANNOTATIONS { # alert if more than 1% of requests to an HTTP endpoint have failed within the last 5 minutes ALERT HighNumberOfFailedHTTPRequests -IF sum by(method) (rate(etcd_http_failed_total{job="etcd"}[5m])) - / sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.01 +IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method) + / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01 FOR 10m LABELS { severity = "warning" @@ -97,8 +97,8 @@ ANNOTATIONS { # alert if more than 5% of requests to an HTTP endpoint have failed within the last 5 minutes ALERT HighNumberOfFailedHTTPRequests -IF sum by(method) (rate(etcd_http_failed_total{job="etcd"}[5m])) - / sum by(method) (rate(etcd_http_received_total{job="etcd"}[5m])) > 0.05 +IF sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method) + / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05 FOR 5m LABELS { severity = "critical" diff --git a/Documentation/op-guide/etcd3_alert.rules.yml b/Documentation/op-guide/etcd3_alert.rules.yml index 47a24773ec8..e68f1a42e66 100644 --- a/Documentation/op-guide/etcd3_alert.rules.yml +++ b/Documentation/op-guide/etcd3_alert.rules.yml @@ -26,8 +26,8 @@ groups: changes within the last hour summary: a high number of leader changes within the etcd cluster are happening - alert: HighNumberOfFailedGRPCRequests - expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method) - / sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.01 + expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method) + / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.01 for: 10m labels: severity: warning @@ -36,8 +36,8 @@ groups: on etcd instance {{ $labels.instance }}' summary: a high number of gRPC requests are failing - alert: HighNumberOfFailedGRPCRequests - expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method) - / sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.05 + expr: sum(rate(grpc_server_handled_total{grpc_code!="OK",job="etcd"}[5m])) BY (grpc_service, grpc_method) + / sum(rate(grpc_server_handled_total{job="etcd"}[5m])) BY (grpc_service, grpc_method) > 0.05 for: 5m labels: severity: critical @@ -46,7 +46,7 @@ groups: on etcd instance {{ $labels.instance }}' summary: a high number of gRPC requests are failing - alert: GRPCRequestsSlow - expr: histogram_quantile(0.99, rate(etcd_grpc_unary_requests_duration_seconds_bucket[5m])) + expr: histogram_quantile(0.99, sum(rate(grpc_server_handling_seconds_bucket{job="etcd",grpc_type="unary"}[5m])) by (grpc_service, grpc_method, le)) > 0.15 for: 10m labels: From 092b2706978ab70ac7242e37c82812fc48c0a8ec Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Wed, 15 Nov 2017 14:34:55 +0100 Subject: [PATCH 2/2] Documentation/op-guide: Fix link to Prometheus 2.0 alerting rules --- Documentation/op-guide/monitoring.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/op-guide/monitoring.md b/Documentation/op-guide/monitoring.md index e2f439adf9c..04e717df016 100644 --- a/Documentation/op-guide/monitoring.md +++ b/Documentation/op-guide/monitoring.md @@ -100,7 +100,7 @@ Now Prometheus will scrape etcd metrics every 10 seconds. ### Alerting -There is a set of default alerts for etcd v3 clusters for [Prometheus 1.x](./etcd3_alert.rules) as well as [Prometheus 2.x](./etcd3_alert.rules). +There is a set of default alerts for etcd v3 clusters for [Prometheus 1.x](./etcd3_alert.rules) as well as [Prometheus 2.x](./etcd3_alert.rules.yml). > Note: `job` labels may need to be adjusted to fit a particular need. The rules were written to apply to a single cluster so it is recommended to choose labels unique to a cluster.