From e72d49b97cf52d2b91f30d40361e55f34d020628 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 6 Mar 2024 13:57:34 +0800 Subject: [PATCH] metrics: update tso metrics in micro service (#7873) close tikv/pd#5839 Signed-off-by: lhy1024 Co-authored-by: Hu# --- metrics/grafana/pd.json | 67 +++++++++++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index e70c1eb89ef..2ead4b1e249 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -1809,7 +1809,7 @@ "steppedLine": false, "targets": [ { - "expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}[30s])", + "expr": "irate(process_cpu_seconds_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}[30s])", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1819,7 +1819,7 @@ }, { "exemplar": true, - "expr": "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "pd_service_maxprocs{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "hide": false, "interval": "", "legendFormat": "{{job}}-{{instance}}-limit", @@ -1917,7 +1917,7 @@ "steppedLine": false, "targets": [ { - "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "process_resident_memory_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "format": "time_series", "hide": false, "interval": "", @@ -1927,42 +1927,42 @@ "step": 4 }, { - "expr": "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "go_memstats_heap_sys_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "hide": true, "interval": "", "legendFormat": "HeapSys-{{job}}-{{instance}}", "refId": "B" }, { - "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "go_memstats_heap_inuse_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "hide": false, "interval": "", "legendFormat": "HeapInuse-{{job}}-{{instance}}", "refId": "C" }, { - "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "go_memstats_heap_alloc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "hide": true, "interval": "", "legendFormat": "HeapAlloc-{{job}}-{{instance}}", "refId": "D" }, { - "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "go_memstats_heap_idle_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "hide": true, "interval": "", "legendFormat": "HeapIdle-{{job}}-{{instance}}", "refId": "E" }, { - "expr": "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "go_memstats_heap_released_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "hide": true, "interval": "", "legendFormat": "HeapReleased-{{job}}-{{instance}}", "refId": "F" }, { - "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "go_memstats_next_gc_bytes{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "hide": true, "interval": "", "legendFormat": "GCTrigger-{{job}}-{{instance}}", @@ -2059,7 +2059,7 @@ "steppedLine": false, "targets": [ { - "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"})", + "expr": "(time() - process_start_time_seconds{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"})", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2158,7 +2158,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*pd.*\"}", + "expr": "go_goroutines{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\",job=~\".*(pd|tso|scheduling).*\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -10618,6 +10618,40 @@ "intervalFactor": 1, "legendFormat": "99.999% tso", "refId": "D" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "90% tso", + "refId": "E", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "99% tso", + "refId": "F", + "step": 2 + }, + { + "expr": "histogram_quantile(0.999, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "99.9% tso", + "refId": "G", + "step": 2 + }, + { + "expr": "histogram_quantile(0.99999, sum(rate(tso_server_handle_tso_duration_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[30s])) by (type, le))", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "99.999% tso", + "refId": "H" } ], "thresholds": [], @@ -10830,6 +10864,15 @@ "hide": true, "refId": "B", "step": 2 + }, + { + "expr": "sum(rate(tso_server_handle_tso_duration_seconds_count{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (type)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "tso request/secs", + "refId": "C", + "step": 2 } ], "thresholds": [], @@ -13590,4 +13633,4 @@ "title": "Test-Cluster-PD", "uid": "Q6RuHYIWk", "version": 1 -} +} \ No newline at end of file