From f3c1fafab5f3ffd65fa8a7f9d66a3a2722869e06 Mon Sep 17 00:00:00 2001 From: Shiranka Miskin Date: Tue, 21 Mar 2023 15:59:41 +0000 Subject: [PATCH] fixes --- pkg/ccl/changefeedccl/helpers_test.go | 8 +- pkg/ccl/changefeedccl/parallel_io.go | 45 +- ...hangefeed-roachtest-grafana-dashboard.json | 3532 ++++------------- pkg/cmd/roachtest/tests/cdc.go | 2 +- 4 files changed, 773 insertions(+), 2814 deletions(-) diff --git a/pkg/ccl/changefeedccl/helpers_test.go b/pkg/ccl/changefeedccl/helpers_test.go index 1235e2b920e8..90b7575c48a7 100644 --- a/pkg/ccl/changefeedccl/helpers_test.go +++ b/pkg/ccl/changefeedccl/helpers_test.go @@ -830,11 +830,11 @@ func randomSinkType(opts ...feedTestOption) string { func randomSinkTypeWithOptions(options feedTestOptions) string { sinkWeights := map[string]int{ - "kafka": 3, - "enterprise": 1, + "kafka": 0, + "enterprise": 0, "webhook": 1, - "pubsub": 1, - "sinkless": 2, + "pubsub": 0, + "sinkless": 0, "cloudstorage": 0, } if options.externalIODir != "" { diff --git a/pkg/ccl/changefeedccl/parallel_io.go b/pkg/ccl/changefeedccl/parallel_io.go index 3c2b47c9519f..186dd6d7b25d 100644 --- a/pkg/ccl/changefeedccl/parallel_io.go +++ b/pkg/ccl/changefeedccl/parallel_io.go @@ -154,25 +154,17 @@ func (pe *parallelIO) runWorkers(ctx context.Context, numEmitWorkers int) error // Clear out the completed keys to check for newly valid pending requests inflight.DifferenceWith(req.Keys()) - var stillPending = pending[:0] // Reuse underlying space - for _, pendingReq := range pending { - // If no intersection, nothing changed for this request's validity - if !req.Keys().Intersects(pendingReq.Keys()) { - stillPending = append(stillPending, pendingReq) - continue - } - - // If it is now free to send, send it - if !inflight.Intersects(pendingReq.Keys()) { + pendingKeys := intsets.Fast{} + for i, pendingReq := range pending { + if !inflight.Intersects(pendingReq.Keys()) && !pendingKeys.Intersects(pendingReq.Keys()) { + inflight.UnionWith(pendingReq.Keys()) + pending = append(pending[:i], pending[i+1:]...) sendToWorker(ctx, pendingReq) - } else { - stillPending = append(stillPending, pendingReq) + break } - // Re-add whatever keys in the pending request that were removed - inflight.UnionWith(pendingReq.Keys()) + pendingKeys.UnionWith(pendingReq.Keys()) } - pending = stillPending select { case <-ctx.Done(): @@ -181,6 +173,18 @@ func (pe *parallelIO) runWorkers(ctx context.Context, numEmitWorkers int) error } } + keysInFlight := func(keys intsets.Fast) bool { + if inflight.Intersects(keys) { + return true + } + for _, pendingReq := range pending { + if pendingReq.Keys().Intersects(keys) { + return true + } + } + return false + } + for { // Results read from sendToWorker need to be first added to a pendingResults // list and then handled separately here rather than calling handleResult @@ -195,16 +199,11 @@ func (pe *parallelIO) runWorkers(ctx context.Context, numEmitWorkers int) error select { case req := <-pe.requestCh: - if !inflight.Intersects(req.Keys()) { - inflight.UnionWith(req.Keys()) - sendToWorker(ctx, req) + if keysInFlight(req.Keys()) { + pending = append(pending, req) } else { - // Even if the request isn't going to be immediately sent out, it must - // still be considered "inflight" as future incoming events overlapping - // its keys must not be sent until this event is removed from the queue - // and successfully emitted. inflight.UnionWith(req.Keys()) - pending = append(pending, req) + sendToWorker(ctx, req) } case res := <-emitSuccessCh: handleSuccess(res) diff --git a/pkg/cmd/roachtest/grafana/configs/changefeed-roachtest-grafana-dashboard.json b/pkg/cmd/roachtest/grafana/configs/changefeed-roachtest-grafana-dashboard.json index b458d8f406c0..8b6d460720d4 100644 --- a/pkg/cmd/roachtest/grafana/configs/changefeed-roachtest-grafana-dashboard.json +++ b/pkg/cmd/roachtest/grafana/configs/changefeed-roachtest-grafana-dashboard.json @@ -4,11 +4,10 @@ { "builtIn": 1, "datasource": { - "type": "prometheus", - "uid": "localprom" + "type": "grafana", + "uid": "-- Grafana --" }, "enable": true, - "expr": "", "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", @@ -19,31 +18,15 @@ "type": "dashboard" }, "type": "dashboard" - }, - { - "datasource": { - "type": "datasource", - "uid": "grafana" - }, - "enable": true, - "iconColor": "red", - "name": "Annotation", - "target": { - "limit": 100, - "matchAny": false, - "tags": [], - "type": "dashboard" - } } ] }, - "description": "", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": 1, "links": [], - "liveNow": true, + "liveNow": false, "panels": [ { "collapsed": false, @@ -53,9 +36,9 @@ "x": 0, "y": 0 }, - "id": 174, + "id": 67, "panels": [], - "title": "Elastic CPU limiter", + "title": "General", "type": "row" }, { @@ -111,18 +94,17 @@ "value": 80 } ] - }, - "unit": "percentunit" + } }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 5, + "w": 3, "x": 0, "y": 1 }, - "id": 162, + "id": 49, "options": { "legend": { "calcs": [], @@ -141,14 +123,14 @@ "type": "prometheus", "uid": "localprom" }, - "editorMode": "builder", - "expr": "admission_elastic_cpu_utilization_limit", + "editorMode": "code", + "expr": "sum(changefeed_running{scope=\"\"})", "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Elastic CPU utilization limit", + "title": "Running Changefeeds", "type": "timeseries" }, { @@ -156,7 +138,6 @@ "type": "prometheus", "uid": "localprom" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -165,7 +146,7 @@ "custom": { "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "Latency", + "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -180,8 +161,7 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 10, - "type": "log" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -190,11 +170,10 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "line" + "mode": "off" } }, "mappings": [], - "min": 50000, "thresholds": { "mode": "absolute", "steps": [ @@ -204,21 +183,20 @@ }, { "color": "red", - "value": 1000000 + "value": 80 } ] - }, - "unit": "ns" + } }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, + "h": 5, + "w": 3, + "x": 3, "y": 1 }, - "id": 2, + "id": 53, "options": { "legend": { "calcs": [], @@ -238,16 +216,13 @@ "uid": "localprom" }, "editorMode": "code", - "exemplar": false, - "expr": "histogram_quantile(0.99, sum by(le, node) (rate(go_scheduler_latency_bucket{node=~\"$node\"}[1m])))", - "format": "time_series", - "hide": false, - "instant": false, + "expr": "sum by (node) (changefeed_backfill_pending_ranges{scope=\"\"})", + "legendFormat": "node={{node}}", "range": true, - "refId": "B" + "refId": "A" } ], - "title": "Go scheduler latency (p99)", + "title": "Backfill Pending Ranges", "type": "timeseries" }, { @@ -291,7 +266,6 @@ } }, "mappings": [], - "min": 0, "thresholds": { "mode": "absolute", "steps": [ @@ -305,17 +279,17 @@ } ] }, - "unit": "percentunit" + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 + "h": 5, + "w": 3, + "x": 6, + "y": 1 }, - "id": 166, + "id": 56, "options": { "legend": { "calcs": [], @@ -335,49 +309,132 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(admission_elastic_cpu_acquired_nanos[2m])", - "hide": true, - "legendFormat": "{{label_name}}", + "expr": "sum(rate(changefeed_emitted_bytes{scope=\"\"}[$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" + } + ], + "title": "Processed Bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.3", + "targets": [ { "datasource": { "type": "prometheus", "uid": "localprom" }, "editorMode": "code", - "expr": "rate(admission_elastic_cpu_returned_nanos[2m])", - "hide": true, - "legendFormat": "__auto", + "expr": "histogram_quantile(0.50, sum(rate(changefeed_commit_latency_bucket[5m])) by (le))", + "hide": false, + "legendFormat": "p50 {{label_name}}", "range": true, - "refId": "B" + "refId": "C" }, { "datasource": { - "name": "Expression", - "type": "__expr__", - "uid": "__expr__" + "type": "prometheus", + "uid": "localprom" }, - "expression": "($A-$B)/$C", + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum(rate(changefeed_commit_latency_bucket[5m])) by (le))", "hide": false, - "refId": "Measured", - "type": "math" + "legendFormat": "p75 {{label_name}}", + "range": true, + "refId": "D" }, { "datasource": { "type": "prometheus", "uid": "localprom" }, - "editorMode": "builder", - "expr": "admission_elastic_cpu_max_available_nanos", - "hide": true, - "legendFormat": "__auto", + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(changefeed_commit_latency_bucket[5m])) by (le))", + "hide": false, + "legendFormat": "p95 {{label_name}}", "range": true, - "refId": "C" + "refId": "E" } ], - "title": "Elastic CPU utilization (smoothed; 2m)", + "title": "Commit Latencies", "type": "timeseries" }, { @@ -408,8 +465,7 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 10, - "type": "log" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -418,7 +474,7 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "line" + "mode": "off" } }, "mappings": [], @@ -430,22 +486,21 @@ "value": null }, { - "color": "orange", - "value": 15000000 + "color": "red", + "value": 80 } ] - }, - "unit": "ns" + } }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 5, + "w": 3, "x": 12, - "y": 9 + "y": 1 }, - "id": 168, + "id": 74, "options": { "legend": { "calcs": [], @@ -464,13 +519,14 @@ "type": "prometheus", "uid": "localprom" }, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(sql_service_latency_bucket[$__rate_interval])))", + "editorMode": "code", + "expr": "sum by (scope) (rate(changefeed_emitted_messages{scope!=\"\"}[$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Service latency (p99)", + "title": "Emitted Messages", "type": "timeseries" }, { @@ -514,8 +570,6 @@ } }, "mappings": [], - "max": 1, - "min": 0.4, "thresholds": { "mode": "absolute", "steps": [ @@ -528,18 +582,17 @@ "value": 80 } ] - }, - "unit": "percentunit" + } }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 + "h": 5, + "w": 3, + "x": 15, + "y": 1 }, - "id": 170, + "id": 43, "options": { "legend": { "calcs": [], @@ -558,10 +611,9 @@ "type": "prometheus", "uid": "localprom" }, - "editorMode": "builder", - "expr": "rate(admission_elastic_cpu_acquired_nanos[2m]) / 8000000000", - "hide": true, - "legendFormat": "{{label_name}}", + "editorMode": "code", + "expr": "sum by(scope) (changefeed_error_retries{scope!=\"\"})", + "legendFormat": "retries[scope={{scope}}]", "range": true, "refId": "A" }, @@ -570,10 +622,10 @@ "type": "prometheus", "uid": "localprom" }, - "editorMode": "builder", - "expr": "rate(admission_elastic_cpu_returned_nanos[2m]) / 8000000000", - "hide": true, - "legendFormat": "__auto", + "editorMode": "code", + "expr": "sum by (scope) (changefeed_error_failures{scope!=\"\"})", + "hide": false, + "legendFormat": "failures[scope={{scope}}]", "range": true, "refId": "B" }, @@ -583,27 +635,41 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "avg_over_time(sys_cpu_combined_percent_normalized[2m])", - "hide": true, - "legendFormat": "__auto", + "expr": "sum by (scope) (changefeed_replan_count{scope!=\"\"})", + "hide": false, + "legendFormat": "replans[scope={{scope}}]", "range": true, "refId": "C" }, { "datasource": { - "name": "Expression", - "type": "__expr__", - "uid": "__expr__" + "type": "prometheus", + "uid": "localprom" }, - "expression": "$C-(($A-$B))", + "editorMode": "builder", + "expr": "", "hide": false, - "refId": "D", - "type": "math" + "legendFormat": "__auto", + "range": true, + "refId": "D" } ], - "title": "Non-elastic CPU utilization (smoothed; 2m)", + "title": "Failures/Retries/Replans", "type": "timeseries" }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 47, + "panels": [], + "title": "Traffic", + "type": "row" + }, { "datasource": { "type": "prometheus", @@ -645,8 +711,6 @@ } }, "mappings": [], - "max": 1, - "min": 0.4, "thresholds": { "mode": "absolute", "steps": [ @@ -660,17 +724,17 @@ } ] }, - "unit": "percentunit" + "unit": "cps" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 + "h": 10, + "w": 8, + "x": 0, + "y": 7 }, - "id": 172, + "id": 2, "options": { "legend": { "calcs": [], @@ -690,14 +754,26 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "avg_over_time(sys_cpu_combined_percent_normalized[2m])", + "expr": "sum(rate(changefeed_emitted_messages{scope=\"\"}[$__rate_interval]))", "hide": false, - "legendFormat": "__auto", + "legendFormat": "total", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "expr": "sum by(node) (rate(changefeed_emitted_messages{scope!=\"\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "node={{node}}", + "range": true, + "refId": "B" } ], - "title": "Aggregate CPU utilization (smoothed; 2m)", + "title": "Changefeed Emitted Messages", "type": "timeseries" }, { @@ -754,17 +830,17 @@ } ] }, - "unit": "goroutines/cpu" + "unit": "Bps" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 + "h": 10, + "w": 8, + "x": 8, + "y": 7 }, - "id": 16, + "id": 20, "options": { "legend": { "calcs": [], @@ -784,12 +860,28 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "max by(node) (sys_runnable_goroutines_per_cpu{node=~\"$node\"})", + "expr": "sum(rate(changefeed_flushed_bytes{scope!=\"\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "total", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by(node) (rate(changefeed_flushed_bytes{scope!=\"\"}[$__rate_interval]))", + "hide": false, + "instant": false, + "legendFormat": "node={{node}}", + "range": true, + "refId": "D" } ], - "title": "Average runnable goroutines per CPU", + "title": "Flushed Bytes", "type": "timeseries" }, { @@ -820,8 +912,7 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 10, - "type": "log" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -843,21 +934,21 @@ }, { "color": "red", - "value": 0.2 + "value": 80 } ] }, - "unit": "ns" + "unit": "cpm" }, "overrides": [] }, "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 25 + "h": 5, + "w": 5, + "x": 16, + "y": 7 }, - "id": 12, + "id": 29, "options": { "legend": { "calcs": [], @@ -877,2109 +968,118 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(go_scheduler_latency_sum{node=~\"$node\"}[1m])", + "expr": "rate(changefeed_frontier_updates[$__rate_interval])*60", + "legendFormat": "node={{node}}", "range": true, "refId": "A" } ], - "title": "Aggregate Go scheduler latency", + "title": "Frontier Resolved Updates", "type": "timeseries" }, { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 34 + "datasource": { + "type": "prometheus", + "uid": "localprom" }, - "id": 10, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 10, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "max": 500000000, - "min": 5000000, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - } - ] - }, - "unit": "ns" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 4, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(sql_service_latency_bucket{node=~\"$node\"}[$__rate_interval])))", - "hide": false, - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.999, sum by(le) (rate(sql_service_latency_bucket{node=~\"$node\"}[$__rate_interval])))", - "hide": false, - "range": true, - "refId": "B" - } - ], - "title": "Service latency (p99, p99.9)", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(sql_insert_count{node=~\"$node\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "inserts", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(sql_delete_count{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "interval": "", - "legendFormat": "deletes", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(sql_select_count{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "legendFormat": "selects", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(sql_update_count{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "legendFormat": "updates", - "range": true, - "refId": "D" - } - ], - "title": "SQL statements", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 152, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "changefeed_running", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Changefeeds running", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 119, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "replicas{node=~\"$node\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Replicas", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 17 - }, - "id": 154, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "changefeed_backfill_pending_ranges", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Ranges pending changefeed backfill", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "percentunit" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 17 - }, - "id": 20, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sys_cpu_combined_percent_normalized{node=~\"$node\"}", - "range": true, - "refId": "A" - } - ], - "title": "CPU utilization", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 25 - }, - "id": 160, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "rate(changefeed_failures[$__rate_interval])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Changefeed failures", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 25 - }, - "id": 121, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "replicas_leaseholders{node=~\"$node\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Leaseholders", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 33 - }, - "id": 156, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "rate(changefeed_emitted_bytes[$__rate_interval])", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "C" - } - ], - "title": "Changefeed emitted bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 33 - }, - "id": 102, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(rpc_batches_recv{node=~\"$node\"}[$__rate_interval])", - "range": true, - "refId": "A" - } - ], - "title": "# of batch requests", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 41 - }, - "id": 158, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "rate(changefeed_emitted_messages[$__rate_interval])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Changefeed emitted messages", - "type": "timeseries" - } - ], - "title": "CRDB", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 35 - }, - "id": 144, - "panels": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "flushes/sec" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 139, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "rate(rocksdb_flushes{node=~\"$node\"}[$__rate_interval])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Memtable flushes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 142, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "rate(rocksdb_flushed_bytes{node=~\"$node\"}[$__rate_interval])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Memtable flushed bytes", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 10 - }, - "id": 132, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "rocksdb_read_amplification{node=~\"$node\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Read amplification", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "compactions/sec" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 10 - }, - "id": 141, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(rocksdb_compactions{node=~\"$node\"}[$__rate_interval])", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "# of compactions", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 128, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "capacity_used{node=~\"$node\"}", - "legendFormat": "__auto", - "range": true, - "refId": "A" - } - ], - "title": "Used disk space", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 134, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(storage_l6_bytes_ingested{node=~\"$node\"}[$__rate_interval]))", - "legendFormat": "__auto", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(storage_l5_bytes_ingested{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(storage_l4_bytes_ingested{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(storage_l3_bytes_ingested{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(storage_l2_bytes_ingested{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "E" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(storage_l1_bytes_ingested{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum(rate(storage_l0_bytes_ingested{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "legendFormat": "__auto", - "range": true, - "refId": "G" - } - ], - "title": "LSM ingestions by level", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 26 - }, - "id": 46, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "sum by(node) (rate(node_disk_written_bytes_total{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "range": true, - "refId": "B" - } - ], - "title": "Write bandwidth", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 26 - }, - "id": 24, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum by(node) (rate(node_disk_read_bytes_total{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "range": true, - "refId": "B" - } - ], - "title": "Read bandwidth", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + } }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 10, - "type": "log" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "line" - } - }, - "mappings": [], - "min": 100000, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 6000000 - } - ] - }, - "unit": "ns" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 34 - }, - "id": 30, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum by(le, node) (rate(raft_process_logcommit_latency_bucket{node=~\"$node\"}[$__rate_interval])))", - "hide": false, - "range": true, - "refId": "A" - } - ], - "title": "Log commit latency (p99)", - "type": "timeseries" + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 5, + "x": 16, + "y": 12 + }, + "id": 41, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { "type": "prometheus", "uid": "localprom" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "Bps" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 34 - }, - "id": 117, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(rocksdb_compacted_bytes_written{node=~\"$node\"}[$__rate_interval])", - "hide": true, - "legendFormat": "written-n{{node}}s{{store}}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(rocksdb_compacted_bytes_read{node=~\"$node\"}[$__rate_interval])", - "hide": true, - "interval": "", - "legendFormat": "read-n{{node}}s{{store}}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "name": "Expression", - "type": "__expr__", - "uid": "__expr__" - }, - "expression": "$A+$B", - "hide": false, - "refId": "read+write", - "type": "math" - } - ], - "title": "Compaction bandwidth", - "type": "timeseries" - }, + "editorMode": "code", + "expr": "rate(changefeed_forwarded_resolved_messages[$__rate_interval])", + "legendFormat": "node={{node}}", + "range": true, + "refId": "A" + } + ], + "title": "Aggregator Resolved Updates", + "type": "timeseries" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 62, + "panels": [ { "datasource": { "type": "prometheus", @@ -3033,17 +1133,17 @@ } ] }, - "unit": "iops" + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 7, + "w": 6, "x": 0, - "y": 42 + "y": 24 }, - "id": 32, + "id": 18, "options": { "legend": { "calcs": [], @@ -3056,6 +1156,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -3063,12 +1164,38 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "sum by(node) (rate(node_disk_writes_completed_total{node=~\"$node\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.50, sum(rate(changefeed_admit_latency_bucket[1m])) by (le))", + "hide": false, + "legendFormat": "p50 {{label_name}}", "range": true, - "refId": "A" + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum(rate(changefeed_admit_latency_bucket[1m])) by (le))", + "hide": false, + "legendFormat": "p75 {{label_name}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(changefeed_admit_latency_bucket[1m])) by (le))", + "hide": false, + "legendFormat": "p95 {{label_name}}", + "range": true, + "refId": "E" } ], - "title": "Write IOPS", + "title": "Admit Latencies", "type": "timeseries" }, { @@ -3076,6 +1203,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3124,17 +1252,17 @@ } ] }, - "unit": "iops" + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 42 + "h": 7, + "w": 6, + "x": 6, + "y": 24 }, - "id": 28, + "id": 57, "options": { "legend": { "calcs": [], @@ -3154,12 +1282,13 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(sys_host_disk_read_count{node=~\"$node\"}[$__rate_interval])", + "expr": "sum by (node) (irate(kv_rangefeed_catchup_scan_nanos[$__rate_interval]))", + "legendFormat": "node={{node}}", "range": true, "refId": "A" } ], - "title": "Read IOPS", + "title": "Rangefeed Catchup Scan Time", "type": "timeseries" }, { @@ -3167,6 +1296,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3215,17 +1345,17 @@ } ] }, - "unit": "s" + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 50 + "h": 7, + "w": 5, + "x": 12, + "y": 24 }, - "id": 42, + "id": 72, "options": { "legend": { "calcs": [], @@ -3245,14 +1375,29 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "sum by(node) (rate(node_disk_io_time_weighted_seconds_total{node=~\"$node\"}[$__rate_interval]))", + "expr": "sum(rate(changefeed_table_metadata_nanos[$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" } ], - "title": "Weighted IO time", + "title": "SchemaFeed Metadata Wait Time", "type": "timeseries" - }, + } + ], + "title": "Ingest", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 69, + "panels": [ { "datasource": { "type": "prometheus", @@ -3277,7 +1422,7 @@ "tooltip": false, "viz": false }, - "lineInterpolation": "smooth", + "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { @@ -3306,17 +1451,17 @@ } ] }, - "unit": "ios" + "unit": "decbytes" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 50 + "h": 7, + "w": 6, + "x": 0, + "y": 27 }, - "id": 44, + "id": 35, "options": { "legend": { "calcs": [], @@ -3336,12 +1481,25 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "sum by(node) (node_disk_io_now{node=~\"$node\"})", + "expr": "changefeed_buffer_entries_mem_acquired - changefeed_buffer_entries_mem_released", + "legendFormat": "node={{node}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "expr": "sum(changefeed_buffer_entries_mem_acquired - changefeed_buffer_entries_mem_released)", + "hide": false, + "legendFormat": "Total", + "range": true, + "refId": "B" } ], - "title": "IOPS in progress", + "title": "KVEvent Bytes Buffered", "type": "timeseries" }, { @@ -3349,6 +1507,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "Time events spent waiting in the kv_feed blocking buffer", "fieldConfig": { "defaults": { "color": { @@ -3397,17 +1556,17 @@ } ] }, - "unit": "decbytes" + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 58 + "h": 7, + "w": 6, + "x": 6, + "y": 27 }, - "id": 150, + "id": 33, "options": { "legend": { "calcs": [], @@ -3427,13 +1586,13 @@ "uid": "localprom" }, "editorMode": "builder", - "expr": "rocksdb_estimated_pending_compaction{node=~\"$node\"}", - "legendFormat": "__auto", + "expr": "rate(changefeed_buffer_pushback_nanos[$__rate_interval])", + "legendFormat": "node={{node}}", "range": true, "refId": "A" } ], - "title": "Estimated pending compaction", + "title": "Blocking Buffer Wait Time", "type": "timeseries" }, { @@ -3441,6 +1600,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "changefeed_queue_time_nanos", "fieldConfig": { "defaults": { "color": { @@ -3489,17 +1649,17 @@ } ] }, - "unit": "decbytes" + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 7, + "w": 6, "x": 12, - "y": 58 + "y": 27 }, - "id": 146, + "id": 39, "options": { "legend": { "calcs": [], @@ -3519,17 +1679,17 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rocksdb_block_cache_usage{node=~\"$node\"}", - "legendFormat": "__auto", + "expr": "irate(changefeed_queue_time_nanos[$__rate_interval])", + "legendFormat": "node={{node}}", "range": true, "refId": "A" } ], - "title": "Block cache size", + "title": "KVEvent Total Queue Time", "type": "timeseries" } ], - "title": "Pebble", + "title": "KV Feed", "type": "row" }, { @@ -3538,9 +1698,9 @@ "h": 1, "w": 24, "x": 0, - "y": 36 + "y": 19 }, - "id": 49, + "id": 71, "panels": [ { "datasource": { @@ -3597,113 +1757,17 @@ } ] }, - "unit": "slots" + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 6, + "w": 4, "x": 0, - "y": 4 - }, - "id": 55, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "admission_granter_total_slots_kv{node=~\"$node\"}", - "hide": false, - "interval": "", - "legendFormat": "KV {node=\"{{node}}\"}", - "range": true, - "refId": "D" - } - ], - "title": "Granter total slots", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "slots" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 4 + "y": 20 }, - "id": 54, + "id": 63, "options": { "legend": { "calcs": [], @@ -3716,6 +1780,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -3723,12 +1788,11 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "admission_granter_used_slots_kv{node=~\"$node\"}", + "expr": "histogram_quantile(0.50, sum(rate(changefeed_nprocs_consume_event_nanos_bucket[$__rate_interval])) by (le))", "hide": false, - "interval": "", - "legendFormat": "KV {node=\"{{node}}\"}", + "legendFormat": "p50 {{label_name}}", "range": true, - "refId": "B" + "refId": "C" }, { "datasource": { @@ -3736,10 +1800,9 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "admission_granter_used_slots_sql_root_start{node=~\"$node\"}", + "expr": "histogram_quantile(0.75, sum(rate(changefeed_nprocs_consume_event_nanos_bucket[$__rate_interval])) by (le))", "hide": false, - "interval": "", - "legendFormat": "SQL root start {node=\"{{node}}\"}", + "legendFormat": "p75 {{label_name}}", "range": true, "refId": "A" }, @@ -3749,15 +1812,14 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "admission_granter_used_slots_sql_leaf_start{node=~\"$node\"}", + "expr": "histogram_quantile(0.95, sum(rate(changefeed_nprocs_consume_event_nanos_bucket[$__rate_interval])) by (le))", "hide": false, - "interval": "", - "legendFormat": "SQL leaf start {node=\"{{node}}\"}", + "legendFormat": "p95 {{label_name}}", "range": true, - "refId": "C" + "refId": "B" } ], - "title": "Granter used slots", + "title": "Event Consumer Ingest Wait Time", "type": "timeseries" }, { @@ -3765,6 +1827,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3814,17 +1877,17 @@ } ] }, - "unit": "reqps" + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 12 + "h": 6, + "w": 4, + "x": 4, + "y": 20 }, - "id": 60, + "id": 64, "options": { "legend": { "calcs": [], @@ -3837,6 +1900,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -3844,10 +1908,11 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "sum by(node) (rate(admission_requested_kv{node=~\"$node\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.50, sum(rate(changefeed_nprocs_flush_nanos_bucket[$__rate_interval])) by (le))", "hide": false, + "legendFormat": "p50 {{label_name}}", "range": true, - "refId": "B" + "refId": "C" }, { "datasource": { @@ -3855,13 +1920,26 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "sum by() (rate(admission_requested_kv{node=~\"$node\"}[$__rate_interval]))", + "expr": "histogram_quantile(0.75, sum(rate(changefeed_nprocs_flush_nanos_bucket[$__rate_interval])) by (le))", "hide": false, + "legendFormat": "p75 {{label_name}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(changefeed_nprocs_flush_nanos_bucket[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p95 {{label_name}}", + "range": true, + "refId": "B" } ], - "title": "KV admission requests", + "title": "Event Consumer Flush Wait Time", "type": "timeseries" }, { @@ -3869,6 +1947,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -3918,17 +1997,17 @@ } ] }, - "unit": "reqps" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 12 + "h": 6, + "w": 4, + "x": 8, + "y": 20 }, - "id": 51, + "id": 65, "options": { "legend": { "calcs": [], @@ -3941,6 +2020,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -3948,23 +2028,14 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "sum by(node) (rate(admission_admitted_kv{node=~\"$node\"}[$__rate_interval]))", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum by() (rate(admission_admitted_kv{node=~\"$node\"}[$__rate_interval]))", + "expr": "sum(changefeed_nprocs_in_flight_count)", "hide": false, + "legendFormat": "__auto", "range": true, - "refId": "B" + "refId": "C" } ], - "title": "KV requests admitted", + "title": "Event Consumer Inflight Events", "type": "timeseries" }, { @@ -3972,6 +2043,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -4021,17 +2093,17 @@ } ] }, - "unit": "reqps" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, + "h": 6, + "w": 4, + "x": 12, "y": 20 }, - "id": 59, + "id": 75, "options": { "legend": { "calcs": [], @@ -4044,6 +2116,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -4051,25 +2124,14 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(admission_requested_kv_stores{node=~\"$node\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{node=\"{{node}}\"}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "sum by() (rate(admission_requested_kv_stores{node=~\"$node\"}[$__rate_interval]))", + "expr": "sum(changefeed_nprocs_buffered_sink_events_count)", "hide": false, + "legendFormat": "__auto", "range": true, - "refId": "B" + "refId": "C" } ], - "title": "KV stores requests", + "title": "Event Consumer Buffered Sink Events", "type": "timeseries" }, { @@ -4126,17 +2188,17 @@ } ] }, - "unit": "reqps" + "unit": "none" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, + "h": 6, + "w": 4, + "x": 16, "y": 20 }, - "id": 52, + "id": 19, "options": { "legend": { "calcs": [], @@ -4156,34 +2218,35 @@ "uid": "localprom" }, "editorMode": "code", - "exemplar": false, - "expr": "sum by(node) (rate(admission_admitted_kv_stores{node=~\"$node\"}[$__rate_interval]))", - "hide": false, - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "exemplar": false, - "expr": "sum by() (rate(admission_admitted_kv_stores{node=~\"$node\"}[$__rate_interval]))", + "expr": "sum by (scope) (changefeed_emitted_bytes{scope!=\"\"}) / sum by (scope) (changefeed_flushed_bytes{scope!=\"\"})", "hide": false, + "legendFormat": "label={{scope}}", "range": true, - "refId": "A" + "refId": "D" } ], - "title": "KV stores requests admitted", + "title": "Compression Ratio", "type": "timeseries" - }, + } + ], + "title": "Encoding", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 20 + }, + "id": 14, + "panels": [ { "datasource": { "type": "prometheus", "uid": "localprom" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4233,17 +2296,17 @@ } ] }, - "unit": "µs" + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, + "h": 6, + "w": 6, "x": 0, - "y": 28 + "y": 21 }, - "id": 53, + "id": 54, "options": { "legend": { "calcs": [], @@ -4256,6 +2319,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -4263,13 +2327,38 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(admission_granter_io_tokens_exhausted_duration_kv{node=~\"$node\"}[$__rate_interval])", + "expr": "histogram_quantile(0.50, sum(rate(changefeed_sink_batch_hist_nanos_bucket[$__rate_interval])) by (le))", "hide": false, + "legendFormat": "p50 {{label_name}}", "range": true, "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, sum(rate(changefeed_sink_batch_hist_nanos_bucket[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p75 {{label_name}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(changefeed_sink_batch_hist_nanos_bucket[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p95 {{label_name}}", + "range": true, + "refId": "E" } ], - "title": "IO token exhaustion duration", + "title": "Sink Buffer Time Batched", "type": "timeseries" }, { @@ -4277,7 +2366,6 @@ "type": "prometheus", "uid": "localprom" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4326,17 +2414,18 @@ "value": 80 } ] - } + }, + "unit": "ns" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 28 + "h": 6, + "w": 5, + "x": 6, + "y": 21 }, - "id": 62, + "id": 55, "options": { "legend": { "calcs": [], @@ -4349,6 +2438,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -4356,11 +2446,11 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "admission_wait_queue_length_kv_stores{node=~\"$node\"}", - "interval": "", - "legendFormat": "KV stores {node=\"{{node}}\"}", + "expr": "histogram_quantile(0.50, sum(rate(changefeed_flush_hist_nanos_bucket[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p50 {{label_name}}", "range": true, - "refId": "A" + "refId": "C" }, { "datasource": { @@ -4368,15 +2458,26 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "admission_wait_queue_length_kv{node=~\"$node\"}", + "expr": "histogram_quantile(0.75, sum(rate(changefeed_flush_hist_nanos_bucket[$__rate_interval])) by (le))", "hide": false, - "interval": "", - "legendFormat": "KV {node=\"{{node}}\"}", + "legendFormat": "p75 {{label_name}}", "range": true, - "refId": "B" + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(changefeed_flush_hist_nanos_bucket[$__rate_interval])) by (le))", + "hide": false, + "legendFormat": "p95 {{label_name}}", + "range": true, + "refId": "E" } ], - "title": "KV wait queue length", + "title": "Sink Flush Time", "type": "timeseries" }, { @@ -4384,7 +2485,6 @@ "type": "prometheus", "uid": "localprom" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4408,8 +2508,7 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 10, - "type": "log" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -4435,21 +2534,21 @@ } ] }, - "unit": "µs" + "unit": "cps" }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 36 + "h": 6, + "w": 5, + "x": 11, + "y": 21 }, - "id": 64, + "id": 60, "options": { "legend": { "calcs": [], - "displayMode": "table", + "displayMode": "list", "placement": "bottom", "showLegend": true }, @@ -4458,6 +2557,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -4465,61 +2565,9 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(admission_wait_sum_kv_stores{node=~\"$node\"}[$__rate_interval])", - "interval": "", - "legendFormat": "KV stores {node=\"{{node}}\"}", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(admission_wait_sum_kv{node=~\"$node\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "KV {node=\"{{node}}\"}", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(admission_wait_sum_sql_kv_response{node=~\"$node\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "SQL-KV {node=\"{{node}}\"}", - "range": true, - "refId": "C" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(admission_wait_sum_sql_sql_response{node=~\"$node\"}[$__rate_interval])", + "expr": "sum(rate(changefeed_flushes[$__rate_interval]))", "hide": false, - "interval": "", - "legendFormat": "SQL-SQL {node=\"{{node}}\"}", - "range": true, - "refId": "D" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(admission_wait_sum_sql_leaf_start{node=~\"$node\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "SQL leaf start {node=\"{{node}}\"}", + "legendFormat": "total_flushes", "range": true, "refId": "E" }, @@ -4529,28 +2577,14 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(admission_wait_sum_sql_root_start{node=~\"$node\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "SQL root start {node=\"{{node}}\"}", - "range": true, - "refId": "F" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "builder", - "expr": "rate(admission_wait_sum_elastic_cpu{node=~\"$node\"}[$__rate_interval])", + "expr": "sum(rate(changefeed_size_based_flushes[$__rate_interval]))", "hide": false, - "interval": "", - "legendFormat": "Elastic CPU {node=\"{{node}}\"}", + "legendFormat": "size_based_flushes", "range": true, - "refId": "G" + "refId": "A" } ], - "title": "Aggregate wait time", + "title": "Sink Flush Rate", "type": "timeseries" }, { @@ -4558,7 +2592,6 @@ "type": "prometheus", "uid": "localprom" }, - "description": "", "fieldConfig": { "defaults": { "color": { @@ -4582,8 +2615,7 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 10, - "type": "log" + "type": "linear" }, "showPoints": "auto", "spanNulls": false, @@ -4614,12 +2646,12 @@ "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 36 + "h": 6, + "w": 5, + "x": 16, + "y": 21 }, - "id": 65, + "id": 25, "options": { "legend": { "calcs": [], @@ -4632,6 +2664,7 @@ "sort": "none" } }, + "pluginVersion": "9.2.3", "targets": [ { "datasource": { @@ -4639,62 +2672,9 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(admission_wait_durations_kv_bucket{node=~\"$node\"}[$__rate_interval])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "KV", - "range": true, - "refId": "G" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(admission_wait_durations_kv_stores_bucket{node=~\"$node\"}[$__rate_interval])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "KV Stores", - "range": true, - "refId": "H" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(admission_wait_durations_sql_kv_response_bucket{node=~\"$node\"}[$__rate_interval])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "SQL-KV", - "range": true, - "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(admission_wait_durations_sql_sql_response_bucket{node=~\"$node\"}[$__rate_interval])) by (le))", - "hide": false, - "interval": "", - "legendFormat": "SQL-SQL", - "range": true, - "refId": "B" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(admission_wait_durations_sql_leaf_start_bucket{node=~\"$node\"}[$__rate_interval])) by (le)) ", + "expr": "histogram_quantile(0.50, sum(rate(changefeed_checkpoint_hist_nanos_bucket[5m])) by (le))", "hide": false, - "interval": "", - "legendFormat": "SQL leaf start", + "legendFormat": "p50 {{label_name}}", "range": true, "refId": "C" }, @@ -4704,10 +2684,9 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(admission_wait_durations_sql_root_start_bucket{node=~\"$node\"}[$__rate_interval])) by (le)) ", + "expr": "histogram_quantile(0.75, sum(rate(changefeed_checkpoint_hist_nanos_bucket[5m])) by (le))", "hide": false, - "interval": "", - "legendFormat": "SQL root start", + "legendFormat": "p75 {{label_name}}", "range": true, "refId": "D" }, @@ -4716,24 +2695,37 @@ "type": "prometheus", "uid": "localprom" }, - "editorMode": "builder", - "expr": "histogram_quantile(0.99, sum by(le) (rate(admission_wait_durations_elastic_cpu_bucket{node=~\"$node\"}[$__rate_interval])))", + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum(rate(changefeed_checkpoint_hist_nanos_bucket[5m])) by (le))", "hide": false, - "interval": "", - "legendFormat": "Elastic CPU", + "legendFormat": "p95 {{label_name}}", "range": true, "refId": "E" } ], - "title": "Queue wait durations (p99)", + "title": "Time Spent Checkpointing", "type": "timeseries" - }, + } + ], + "title": "Sink", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 21 + }, + "id": 16, + "panels": [ { "datasource": { "type": "prometheus", "uid": "localprom" }, - "description": "at 1, IO admission control kicks in", + "description": "sys_cpu_combined_percent_normalized", "fieldConfig": { "defaults": { "color": { @@ -4744,7 +2736,6 @@ "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", - "axisSoftMax": 1.3, "barAlignment": 0, "drawStyle": "line", "fillOpacity": 0, @@ -4767,7 +2758,7 @@ "mode": "none" }, "thresholdsStyle": { - "mode": "area" + "mode": "off" } }, "mappings": [], @@ -4780,20 +2771,21 @@ }, { "color": "red", - "value": 1 + "value": 80 } ] - } + }, + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 44 + "y": 22 }, - "id": 67, + "id": 22, "options": { "legend": { "calcs": [], @@ -4813,12 +2805,13 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "admission_io_overload{node=~\"$node\"}", + "expr": "sys_cpu_combined_percent_normalized", + "legendFormat": "node={{node}}", "range": true, "refId": "A" } ], - "title": "IO Admission Threshold Fraction", + "title": "Node CPU Usage", "type": "timeseries" }, { @@ -4826,6 +2819,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -4875,17 +2869,17 @@ } ] }, - "unit": "Bps" + "unit": "percentunit" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 44 + "w": 10, + "x": 8, + "y": 22 }, - "id": 71, + "id": 24, "options": { "legend": { "calcs": [], @@ -4905,26 +2899,13 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(storage_l0_bytes_ingested{node=~\"$node\"}[$__rate_interval])", - "interval": "", - "legendFormat": "ingested {node=\"{{node}}\"}", + "expr": "sum by (cpu, node) (rate(node_cpu_seconds_total{mode != \"idle\"}[$__rate_interval]))", + "legendFormat": "__auto", "range": true, "refId": "A" - }, - { - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "editorMode": "code", - "expr": "rate(storage_l0_bytes_flushed{node=~\"$node\"}[$__rate_interval])", - "hide": false, - "legendFormat": "flushed {node=\"{{node}}\"}", - "range": true, - "refId": "B" } ], - "title": "L0 flushed + ingested bytes", + "title": "CPU Usage (Per processor)", "type": "timeseries" }, { @@ -4932,6 +2913,7 @@ "type": "prometheus", "uid": "localprom" }, + "description": "sys_cpu_combined_percent_normalized", "fieldConfig": { "defaults": { "color": { @@ -4987,11 +2969,11 @@ }, "gridPos": { "h": 8, - "w": 12, + "w": 8, "x": 0, - "y": 52 + "y": 30 }, - "id": 148, + "id": 23, "options": { "legend": { "calcs": [], @@ -5011,29 +2993,16 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "admission_elastic_cpu_utilization_limit{node=~\"$node\"}", - "legendFormat": "__auto", + "expr": "node_memory_Active_bytes/node_memory_MemTotal_bytes", + "hide": false, + "legendFormat": "node={{node}}", "range": true, - "refId": "A" + "refId": "B" } ], - "title": "Elastic CPU utilization limit", + "title": "Node Memory Usage", "type": "timeseries" - } - ], - "title": "Admission Control", - "type": "row" - }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 37 - }, - "id": 130, - "panels": [ + }, { "datasource": { "type": "prometheus", @@ -5088,17 +3057,17 @@ } ] }, - "unit": "s" + "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 0, - "y": 5 + "w": 5, + "x": 8, + "y": 30 }, - "id": 123, + "id": 50, "options": { "legend": { "calcs": [], @@ -5118,13 +3087,13 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(node_pressure_io_stalled_seconds_total{node=~\"$node\"}[$__rate_interval])", - "legendFormat": "n{{node}}", + "expr": "sum by (node) (rate(node_network_transmit_bytes_total[$__rate_interval]))", + "legendFormat": "transmit[node={{node}}]", "range": true, "refId": "A" } ], - "title": "IO Stalled PSI", + "title": "Node Network Bytes Transmit", "type": "timeseries" }, { @@ -5181,17 +3150,17 @@ } ] }, - "unit": "s" + "unit": "Bps" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, - "x": 12, - "y": 5 + "w": 5, + "x": 13, + "y": 30 }, - "id": 124, + "id": 51, "options": { "legend": { "calcs": [], @@ -5211,15 +3180,43 @@ "uid": "localprom" }, "editorMode": "code", - "expr": "rate(node_pressure_cpu_waiting_seconds_total{node=~\"$node\"}[$__rate_interval])", - "legendFormat": "cpu-psi-n{{node}}", + "expr": "sum by (node) (rate(node_network_receive_bytes_total[$__rate_interval]))", + "hide": false, + "legendFormat": "receive[node={{node}}]", "range": true, - "refId": "A" + "refId": "B" } ], - "title": "CPU PSI", + "title": "Node Network Bytes Receive", "type": "timeseries" - }, + } + ], + "title": "Resources", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 59, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 27, + "panels": [ { "datasource": { "type": "prometheus", @@ -5273,18 +3270,17 @@ "value": 80 } ] - }, - "unit": "s" + } }, "overrides": [] }, "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 13 + "h": 7, + "w": 8, + "x": 0, + "y": 24 }, - "id": 126, + "id": 31, "options": { "legend": { "calcs": [], @@ -5303,84 +3299,48 @@ "type": "prometheus", "uid": "localprom" }, - "editorMode": "code", - "expr": "rate(node_pressure_io_waiting_seconds_total{node=~\"$node\"}[$__rate_interval])", - "legendFormat": "n{{node}}", + "editorMode": "builder", + "expr": "rate(changefeed_buffer_entries_in[$__rate_interval])", + "legendFormat": "entries_in node={{node}}", "range": true, "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "localprom" + }, + "editorMode": "builder", + "expr": "rate(changefeed_buffer_entries_out[$__rate_interval])", + "hide": false, + "legendFormat": "entries_out node={{node}}", + "range": true, + "refId": "B" } ], - "title": "IO Waiting PSI", + "title": "KVFeed Buffer Entries In/Out", "type": "timeseries" } ], - "title": "PSI", + "title": "Other", "type": "row" } ], - "refresh": false, + "refresh": "5s", "schemaVersion": 37, "style": "dark", "tags": [], "templating": { - "list": [ - { - "allValue": "everything", - "current": { - "selected": true, - "text": [ - "1", - "2", - "3" - ], - "value": [ - "1", - "2", - "3" - ] - }, - "datasource": { - "type": "prometheus", - "uid": "localprom" - }, - "definition": "replicas", - "hide": 0, - "includeAll": false, - "label": "Node", - "multi": true, - "name": "node", - "options": [], - "query": { - "query": "replicas", - "refId": "StandardVariableQuery" - }, - "refresh": 1, - "regex": "/.*node=\"([^\"]*).*/", - "skipUrlSync": false, - "sort": 3, - "type": "query" - } - ] + "list": [] }, "time": { - "from": "now-3h", + "from": "now-15m", "to": "now" }, - "timepicker": { - "hidden": false, - "refresh_intervals": [ - "10s", - "1m", - "15m", - "30m", - "1h", - "2h", - "1d" - ] - }, - "timezone": "utc", - "title": "Changefeed Admission Control", - "uid": "ngnKs_j7z", - "version": 8, + "timepicker": {}, + "timezone": "", + "title": "CDC", + "uid": "928XNlN4k", + "version": 2, "weekStart": "" } diff --git a/pkg/cmd/roachtest/tests/cdc.go b/pkg/cmd/roachtest/tests/cdc.go index e8235c020127..5e83fa0afcff 100644 --- a/pkg/cmd/roachtest/tests/cdc.go +++ b/pkg/cmd/roachtest/tests/cdc.go @@ -552,7 +552,7 @@ func (ct *cdcTester) startGrafana() { ct.promCfg = cfg - if !ct.t.SkipInit() { + if true || !ct.t.SkipInit() { err := ct.cluster.StartGrafana(ct.ctx, ct.t.L(), cfg) if err != nil { ct.t.Errorf("error starting prometheus/grafana: %s", err)