From 5fd42d71516420b7f500a5c768288023959e1923 Mon Sep 17 00:00:00 2001 From: Koenraad Verheyden Date: Tue, 14 Sep 2021 15:49:54 +0200 Subject: [PATCH] tempo-mixin: regenerate yamls/ --- operations/tempo-mixin/yamls/alerts.yaml | 46 +- .../tempo-mixin/yamls/tempo-operational.json | 6290 ++++++++++------- 2 files changed, 3591 insertions(+), 2745 deletions(-) diff --git a/operations/tempo-mixin/yamls/alerts.yaml b/operations/tempo-mixin/yamls/alerts.yaml index a865152a51a..603d937da5e 100644 --- a/operations/tempo-mixin/yamls/alerts.yaml +++ b/operations/tempo-mixin/yamls/alerts.yaml @@ -47,14 +47,52 @@ "message": "Greater than 2 compactions have failed in the past hour." "runbook_url": "https://github.com/grafana/tempo/tree/main/operations/tempo-mixin/runbook.md#TempoCompactionsFailing" "expr": | - sum by (cluster, namespace) (increase(tempodb_compaction_errors_total{}[1h])) > 2 + sum by (cluster, namespace) (increase(tempodb_compaction_errors_total{}[1h])) > 2 and + sum by (cluster, namespace) (increase(tempodb_compaction_errors_total{}[5m])) > 0 "labels": "severity": "critical" - - "alert": "TempoFlushesFailing" + - "alert": "TempoIngesterFlushesFailing" "annotations": "message": "Greater than 2 flushes have failed in the past hour." - "runbook_url": "https://github.com/grafana/tempo/tree/main/operations/tempo-mixin/runbook.md#TempoFlushesFailing" + "runbook_url": "https://github.com/grafana/tempo/tree/main/operations/tempo-mixin/runbook.md#TempoIngesterFlushesFailing" "expr": | - sum by (cluster, namespace) (increase(tempo_ingester_failed_flushes_total{}[1h])) > 2 + sum by (cluster, namespace) (increase(tempo_ingester_failed_flushes_total{}[1h])) > 2 and + sum by (cluster, namespace) (increase(tempo_ingester_failed_flushes_total{}[5m])) > 0 + "labels": + "severity": "critical" + - "alert": "TempoPollsFailing" + "annotations": + "message": "Greater than 2 polls have failed in the past hour." + "runbook_url": "https://github.com/grafana/tempo/tree/main/operations/tempo-mixin/runbook.md#TempoPollsFailing" + "expr": | + sum by (cluster, namespace) (increase(tempodb_blocklist_poll_errors_total{}[1h])) > 2 and + sum by (cluster, namespace) (increase(tempodb_blocklist_poll_errors_total{}[5m])) > 0 + "labels": + "severity": "critical" + - "alert": "TempoTenantIndexFailures" + "annotations": + "message": "Greater than 2 tenant index failures in the past hour." + "runbook_url": "https://github.com/grafana/tempo/tree/main/operations/tempo-mixin/runbook.md#TempoTenantIndexFailures" + "expr": | + sum by (cluster, namespace) (increase(tempodb_blocklist_tenant_index_errors_total{}[1h])) > 2 and + sum by (cluster, namespace) (increase(tempodb_blocklist_tenant_index_errors_total{}[5m])) > 0 + "labels": + "severity": "critical" + - "alert": "TempoNoTenantIndexBuilders" + "annotations": + "message": "No tenant index builders. Tenant index is out of date." + "runbook_url": "https://github.com/grafana/tempo/tree/main/operations/tempo-mixin/runbook.md#TempoNoTenantIndexBuilders" + "expr": | + sum by (cluster, namespace) (tempodb_blocklist_tenant_index_builder{}) == 0 + "for": "5m" + "labels": + "severity": "critical" + - "alert": "TempoTenantIndexTooOld" + "annotations": + "message": "Tenant index age is 600 seconds old." + "runbook_url": "https://github.com/grafana/tempo/tree/main/operations/tempo-mixin/runbook.md#TempoTenantIndexTooOld" + "expr": | + max by (cluster, namespace) (tempodb_blocklist_tenant_index_age_seconds{}) > 600 + "for": "5m" "labels": "severity": "critical" diff --git a/operations/tempo-mixin/yamls/tempo-operational.json b/operations/tempo-mixin/yamls/tempo-operational.json index 8af85047cbe..10e43565ad4 100644 --- a/operations/tempo-mixin/yamls/tempo-operational.json +++ b/operations/tempo-mixin/yamls/tempo-operational.json @@ -8,6 +8,14 @@ "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [ + + ], + "type": "dashboard" + }, "type": "dashboard" }, { @@ -27,8 +35,7 @@ "editable": true, "gnetId": null, "graphTooltip": 1, - "id": 152, - "iteration": 1625468849630, + "iteration": 1630589498735, "links": [ { "asDropdown": true, @@ -43,18 +50,11 @@ "type": "dashboards" } ], + "liveNow": false, "panels": [ { - "collapsed": false, + "collapsed": true, "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, "gridPos": { "h": 1, "w": 24, @@ -63,1150 +63,1573 @@ }, "id": 38, "panels": [ + { + "aliasColors": { - ], - "title": "General", - "type": "row" - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 1 - }, - "hiddenSeries": false, - "id": 24, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 1 + }, + "hiddenSeries": false, + "id": 24, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_gc_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "gcs", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "gcs", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 1 - }, - "hiddenSeries": false, - "id": 25, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 1 + }, + "hiddenSeries": false, + "id": 25, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "go heap", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "go heap", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 1 - }, - "hiddenSeries": false, - "id": 23, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 1 + }, + "hiddenSeries": false, + "id": 23, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "goroutines", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "goroutines", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 9, - "y": 1 - }, - "hiddenSeries": false, - "id": 42, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 1 + }, + "hiddenSeries": false, + "id": 42, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}[$__rate_interval])", - "interval": "", - "intervalFactor": 5, - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}[$__rate_interval])", + "interval": "", + "intervalFactor": 5, + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "cpu", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "cpu", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 12, - "y": 1 - }, - "hiddenSeries": false, - "id": 43, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 1 + }, + "hiddenSeries": false, + "id": 43, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}", - "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\", container!=\"POD\"}", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "working set", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "working set", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 15, - "y": 1 - }, - "hiddenSeries": false, - "id": 44, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 1 + }, + "hiddenSeries": false, + "id": 44, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "rx-{{pod}}", - "refId": "A" - }, - { - "expr": "rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])", - "hide": false, - "interval": "", - "legendFormat": "tx-{{pod}}", - "refId": "B" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "rx-{{pod}}", + "refId": "A" + }, + { + "expr": "rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$component.*\"}[$__rate_interval])", + "hide": false, + "interval": "", + "legendFormat": "tx-{{pod}}", + "refId": "B" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "tx/rx", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "tx/rx", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 0, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 18, - "y": 1 - }, - "hiddenSeries": false, - "id": 45, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 1 + }, + "hiddenSeries": false, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=~\"$component.*\"}", - "legendFormat": "{{persistentvolumeclaim}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "kubelet_volume_stats_available_bytes{cluster=\"$cluster\", namespace=\"$namespace\", persistentvolumeclaim=~\"$component.*\"}", + "legendFormat": "{{persistentvolumeclaim}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "data volume free", - "tooltip": { - "shared": true, - "sort": 1, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "data volume free", + "tooltip": { + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "decbytes", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 21, + "y": 1 + }, + "hiddenSeries": false, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", app=~\"$component.*\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{exported_pod}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "bad words", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "General", + "type": "row" }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, + "collapsed": true, + "datasource": null, "gridPos": { - "h": 5, - "w": 3, - "x": 21, + "h": 1, + "w": 24, + "x": 0, "y": 1 }, - "hiddenSeries": false, - "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", app=~\"$component.*\"}[$__rate_interval])", - "interval": "", - "legendFormat": "{{exported_pod}}", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeRegions": [ - - ], - "timeShift": null, - "title": "bad words", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "collapsed": false, - "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 6 - }, "id": 49, "panels": [ + { + "aliasColors": { - ], - "title": "Maintenance", - "type": "row" - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 7 - }, - "hiddenSeries": false, - "id": 33, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 2 + }, + "hiddenSeries": false, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "tempodb_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"} / tempodb_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", - "legendFormat": "{{instance}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "tempodb_work_queue_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"} / tempodb_work_queue_max{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}", + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "%age total work queue", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "%age total work queue", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "decimals": null, - "format": "percentunit", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "decimals": null, + "format": "percentunit", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 3, - "y": 7 - }, - "hiddenSeries": false, - "id": 32, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 2 + }, + "hiddenSeries": false, + "id": 32, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(tempodb_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "legendFormat": "compaction_err", + "refId": "B" + }, + { + "expr": "sum(increase(tempodb_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "legendFormat": "retention_err", + "refId": "C" + }, + { + "expr": "sum(increase(tempodb_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", + "legendFormat": "blocklist_err", + "refId": "D" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "maintenance errors", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 2 + }, + "hiddenSeries": false, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Blocklist Poll Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 2 + }, + "hiddenSeries": false, + "id": 47, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "exemplar": true, + "expr": "avg(tempodb_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}) by (tenant)", + "instant": false, + "interval": "", + "legendFormat": "{{tenant}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Blocklist Length", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 2 + }, + "hiddenSeries": false, + "id": 51, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "retention duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 2 + }, + "hiddenSeries": false, + "id": 53, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(tempodb_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "deleted", + "refId": "A" + }, + { + "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "marked_for_deletion", + "refId": "B" + } + ], + "thresholds": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(increase(tempodb_compaction_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", - "legendFormat": "compaction_err", - "refId": "B" - }, - { - "expr": "sum(increase(tempodb_retention_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", - "legendFormat": "retention_err", - "refId": "C" + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "retention", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "expr": "sum(increase(tempodb_blocklist_poll_errors_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (job)", - "legendFormat": "blocklist_err", - "refId": "D" - } - ], - "thresholds": [ + "aliasColors": { - ], - "timeFrom": null, - "timeRegions": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 18, + "y": 2 + }, + "hiddenSeries": false, + "id": 70, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "timeShift": null, - "title": "maintenance errors", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0", + "hide": false, + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Container Restarts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Maintenance", + "type": "row" }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, + "collapsed": false, + "datasource": null, "gridPos": { - "h": 5, - "w": 3, - "x": 6, - "y": 7 - }, - "hiddenSeries": false, - "id": 35, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", - "legendFormat": ".9", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(tempodb_blocklist_poll_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeRegions": [ - - ], - "timeShift": null, - "title": "Blocklist Poll Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" + "h": 1, + "w": 24, + "x": 0, + "y": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + "id": 21, + "panels": [ - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Read", + "type": "row" }, { "aliasColors": { @@ -1220,18 +1643,18 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 3, - "x": 9, - "y": 7 + "w": 4, + "x": 0, + "y": 3 }, "hiddenSeries": false, - "id": 47, + "id": 34, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -1242,7 +1665,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1250,15 +1673,14 @@ ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "exemplar": true, - "expr": "avg(tempodb_blocklist_length{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}) by (tenant)", - "instant": false, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\"}[$__rate_interval])) by (status_code)", + "hide": false, "interval": "", - "legendFormat": "{{tenant}}", + "legendFormat": "{{status_code}}", "refId": "A" } ], @@ -1270,7 +1692,7 @@ ], "timeShift": null, - "title": "Blocklist Length", + "title": "Queries/Sec (cortex-gw)", "tooltip": { "shared": true, "sort": 0, @@ -1321,31 +1743,31 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 3, - "x": 12, - "y": 7 + "w": 6, + "x": 4, + "y": 3 }, "hiddenSeries": false, - "id": 51, + "id": 78, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, - "lines": false, + "lines": true, "linewidth": 1, "nullPointMode": "null", "options": { "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, - "points": true, + "points": false, "renderer": "flot", "seriesOverrides": [ @@ -1355,19 +1777,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempodb_retention_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1381,7 +1803,7 @@ ], "timeShift": null, - "title": "retention duration", + "title": "Query Latency (Gateway)", "tooltip": { "shared": true, "sort": 0, @@ -1432,18 +1854,18 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 3, - "x": 15, - "y": 7 + "w": 4, + "x": 12, + "y": 3 }, "hiddenSeries": false, - "id": 53, + "id": 97, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -1454,7 +1876,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1466,16 +1888,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tempodb_retention_deleted_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "deleted", - "refId": "A" - }, - { - "expr": "sum(increase(tempodb_retention_marked_for_deletion_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval]))", + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\"}[$__rate_interval])) by (status_code)", + "hide": false, "interval": "", - "legendFormat": "marked_for_deletion", - "refId": "B" + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" } ], "thresholds": [ @@ -1486,10 +1905,10 @@ ], "timeShift": null, - "title": "retention", + "title": "Search Queries/Sec (cortex-gw)", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -1537,18 +1956,18 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 3, - "x": 18, - "y": 7 + "w": 6, + "x": 16, + "y": 3 }, "hiddenSeries": false, - "id": 70, + "id": 93, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -1559,7 +1978,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1571,11 +1990,30 @@ "steppedLine": false, "targets": [ { - "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0", + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "hide": false, "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", + "hide": false, + "interval": "", + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" } ], "thresholds": [ @@ -1586,7 +2024,7 @@ ], "timeShift": null, - "title": "Container Restarts", + "title": "Search Query Latency (Gateway)", "tooltip": { "shared": true, "sort": 0, @@ -1604,7 +2042,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -1625,30 +2063,6 @@ "alignLevel": null } }, - { - "collapsed": false, - "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 12 - }, - "id": 21, - "panels": [ - - ], - "title": "Read", - "type": "row" - }, { "aliasColors": { @@ -1661,12 +2075,12 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 6, + "w": 4, "x": 0, - "y": 13 + "y": 8 }, "hiddenSeries": false, - "id": 34, + "id": 90, "legend": { "avg": false, "current": false, @@ -1683,7 +2097,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1695,7 +2109,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\".*api_traces_traceid\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1710,7 +2124,7 @@ ], "timeShift": null, - "title": "Queries/Sec (cortex-gw)", + "title": "Queries/Sec (Query Frontend)", "tooltip": { "shared": true, "sort": 0, @@ -1761,12 +2175,12 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 9, - "x": 6, - "y": 13 + "w": 6, + "x": 4, + "y": 8 }, "hiddenSeries": false, - "id": 78, + "id": 17, "legend": { "avg": false, "current": false, @@ -1783,7 +2197,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1795,19 +2209,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1821,7 +2235,7 @@ ], "timeShift": null, - "title": "Query Latency (Gateway)", + "title": "Query Latency (Query Frontend)", "tooltip": { "shared": true, "sort": 0, @@ -1872,12 +2286,12 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 6, - "x": 0, - "y": 18 + "w": 4, + "x": 12, + "y": 8 }, "hiddenSeries": false, - "id": 90, + "id": 98, "legend": { "avg": false, "current": false, @@ -1894,7 +2308,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1906,11 +2320,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", route=~\".*api_search.*\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", - "refId": "A" + "refId": "A", + "stepMode": "min" } ], "thresholds": [ @@ -1921,7 +2337,7 @@ ], "timeShift": null, - "title": "Queries/Sec (Query Frontend)", + "title": "Search Queries/Sec (Query Frontend)", "tooltip": { "shared": true, "sort": 0, @@ -1972,12 +2388,12 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 9, - "x": 6, - "y": 18 + "w": 6, + "x": 16, + "y": 8 }, "hiddenSeries": false, - "id": 17, + "id": 94, "legend": { "avg": false, "current": false, @@ -1994,7 +2410,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2006,22 +2422,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", - "refId": "A" + "refId": "A", + "stepMode": "min" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", - "refId": "B" + "refId": "B", + "stepMode": "min" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_traces_traceid\"}[$__rate_interval])) by (le))", + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/query-frontend\", route=~\".*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", - "refId": "C" + "refId": "C", + "stepMode": "min" } ], "thresholds": [ @@ -2032,7 +2454,7 @@ ], "timeShift": null, - "title": "Query Latency (Query Frontend)", + "title": "Search Query Latency (Query Frontend)", "tooltip": { "shared": true, "sort": 0, @@ -2083,9 +2505,9 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 6, + "w": 4, "x": 0, - "y": 23 + "y": 13 }, "hiddenSeries": false, "id": 91, @@ -2105,7 +2527,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2183,9 +2605,9 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 9, - "x": 6, - "y": 23 + "w": 6, + "x": 4, + "y": 13 }, "hiddenSeries": false, "id": 89, @@ -2205,7 +2627,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2294,12 +2716,12 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 6, - "x": 0, - "y": 28 + "w": 4, + "x": 12, + "y": 13 }, "hiddenSeries": false, - "id": 92, + "id": 99, "legend": { "avg": false, "current": false, @@ -2316,7 +2738,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2328,11 +2750,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"querier_.*api_search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", - "refId": "A" + "refId": "A", + "stepMode": "min" } ], "thresholds": [ @@ -2343,7 +2767,7 @@ ], "timeShift": null, - "title": "Queries/Sec (Ingester)", + "title": "Search Queries/Sec (Querier)", "tooltip": { "shared": true, "sort": 0, @@ -2394,12 +2818,12 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 9, - "x": 6, - "y": 28 + "w": 6, + "x": 16, + "y": 13 }, "hiddenSeries": false, - "id": 3, + "id": 95, "legend": { "avg": false, "current": false, @@ -2416,7 +2840,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2428,22 +2852,28 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", - "refId": "A" + "refId": "A", + "stepMode": "min" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", - "refId": "B" + "refId": "B", + "stepMode": "min" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/querier\", route=~\"querier_.*api_search.*\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", - "refId": "C" + "refId": "C", + "stepMode": "min" } ], "thresholds": [ @@ -2454,7 +2884,7 @@ ], "timeShift": null, - "title": "Query Latency (Ingester)", + "title": "Search Query Latency (Querier)", "tooltip": { "shared": true, "sort": 0, @@ -2493,30 +2923,6 @@ "alignLevel": null } }, - { - "collapsed": false, - "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 33 - }, - "id": 19, - "panels": [ - - ], - "title": "Write", - "type": "row" - }, { "aliasColors": { @@ -2525,22 +2931,22 @@ "dashLength": 10, "dashes": false, "datasource": "$ds", - "fill": 0, + "fill": 1, "fillGradient": 0, "gridPos": { "h": 5, - "w": 3, + "w": 4, "x": 0, - "y": 34 + "y": 18 }, "hiddenSeries": false, - "id": 10, + "id": 92, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -2551,7 +2957,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2559,13 +2965,14 @@ ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "rate(tempo_distributor_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "hide": false, "interval": "", - "legendFormat": "{{pod}}", + "legendFormat": "{{status_code}}", "refId": "A" } ], @@ -2577,7 +2984,7 @@ ], "timeShift": null, - "title": "Spans Received", + "title": "Queries/Sec (Ingester)", "tooltip": { "shared": true, "sort": 0, @@ -2628,18 +3035,18 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 3, - "x": 3, - "y": 34 + "w": 6, + "x": 4, + "y": 18 }, "hiddenSeries": false, - "id": 9, + "id": 3, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -2650,7 +3057,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2658,14 +3065,26 @@ ], "spaceLength": 10, - "stack": true, + "stack": false, "steppedLine": false, "targets": [ { - "expr": "rate(tempo_ingester_traces_created_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", "interval": "", - "legendFormat": "", + "legendFormat": ".99", "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=\"/tempopb.Querier/FindTraceByID\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" } ], "thresholds": [ @@ -2676,7 +3095,7 @@ ], "timeShift": null, - "title": "Traces Created", + "title": "Query Latency (Ingester)", "tooltip": { "shared": true, "sort": 0, @@ -2694,7 +3113,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -2727,18 +3146,18 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 3, - "x": 6, - "y": 34 + "w": 4, + "x": 12, + "y": 18 }, "hiddenSeries": false, - "id": 8, + "id": 100, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -2749,7 +3168,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2761,10 +3180,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tempo_ingester_blocks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[1h]))", + "exemplar": true, + "expr": "sum(rate(tempo_request_duration_seconds_count{route=~\"/tempopb.Querier/Search.*\", cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "hide": false, "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" + "legendFormat": "{{status_code}}", + "refId": "A", + "stepMode": "min" } ], "thresholds": [ @@ -2775,7 +3197,7 @@ ], "timeShift": null, - "title": "Blocks Flushed (1h)", + "title": "Search Queries/Sec (Ingester)", "tooltip": { "shared": true, "sort": 0, @@ -2826,18 +3248,18 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 3, - "x": 9, - "y": 34 + "w": 6, + "x": 16, + "y": 18 }, "hiddenSeries": false, - "id": 12, + "id": 96, "legend": { "avg": false, "current": false, "max": false, "min": false, - "show": false, + "show": true, "total": false, "values": false }, @@ -2848,7 +3270,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "8.1.0-27525pre", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2860,10 +3282,28 @@ "steppedLine": false, "targets": [ { - "expr": "increase(tempo_ingester_failed_flushes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0", + "exemplar": true, + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B", + "stepMode": "min" + }, + { + "exemplar": true, + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Querier/Search.*\"}[$__rate_interval])) by (le))", "interval": "", - "legendFormat": "{{pod}}", - "refId": "A" + "legendFormat": ".5", + "refId": "C", + "stepMode": "min" } ], "thresholds": [ @@ -2874,7 +3314,7 @@ ], "timeShift": null, - "title": "Failed Flushes", + "title": "Search Query Latency (Ingester)", "tooltip": { "shared": true, "sort": 0, @@ -2892,7 +3332,7 @@ }, "yaxes": [ { - "format": "short", + "format": "s", "label": null, "logBase": 1, "max": null, @@ -2914,911 +3354,1305 @@ } }, { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, + "collapsed": true, + "datasource": null, "gridPos": { - "h": 5, - "w": 3, - "x": 12, - "y": 34 - }, - "hiddenSeries": false, - "id": 26, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true + "h": 1, + "w": 24, + "x": 0, + "y": 23 }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + "id": 19, + "panels": [ + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 0, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 0, + "y": 24 + }, + "hiddenSeries": false, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(tempo_distributor_spans_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Spans Received", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 3, + "y": 24 + }, + "hiddenSeries": false, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(tempo_ingester_traces_created_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Traces Created", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 6, + "y": 24 + }, + "hiddenSeries": false, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(increase(tempo_ingester_blocks_cleared_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h]))", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(tempo_ingester_blocks_flushed_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\"}[1h]))", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "Blocks Cleared (1h)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "Blocks Flushed (1h)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 15, - "y": 34 - }, - "hiddenSeries": false, - "id": 36, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 9, + "y": 24 + }, + "hiddenSeries": false, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(tempo_ingester_failed_flushes_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]) > 0", + "interval": "", + "legendFormat": "{{pod}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Failed Flushes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "thresholds": [ + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 12, + "y": 24 + }, + "hiddenSeries": false, + "id": 26, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(tempo_ingester_blocks_cleared_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h]))", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeShift": null, - "title": "Flush Duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeFrom": null, + "timeRegions": [ - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ], + "timeShift": null, + "title": "Blocks Cleared (1h)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 7, - "x": 0, - "y": 39 - }, - "hiddenSeries": false, - "id": 71, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 3, + "x": 15, + "y": 24 + }, + "hiddenSeries": false, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (status_code)", - "interval": "", - "legendFormat": "{{status_code}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_ingester_flush_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "Pushes/sec (gateway)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "Flush Duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 10, - "w": 6, - "x": 7, - "y": 39 - }, - "hiddenSeries": false, - "id": 72, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 7, + "x": 0, + "y": 29 + }, + "hiddenSeries": false, + "id": 71, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempo_receiver_accepted_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "accepted", - "refId": "A" - }, - { - "expr": "sum(rate(tempo_receiver_refused_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "refused", - "refId": "B" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (status_code)", + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "OTEL Distributor Spans/second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "Pushes/sec (gateway)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "none", - "label": "", - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 6, - "x": 13, - "y": 39 - }, - "hiddenSeries": false, - "id": 79, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 6, + "x": 7, + "y": 29 + }, + "hiddenSeries": false, + "id": 72, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_receiver_accepted_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "accepted", + "refId": "A" + }, + { + "expr": "sum(rate(tempo_receiver_refused_spans{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "refused", + "refId": "B" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "Push Latency (Gateway)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "OTEL Distributor Spans/second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 6, - "x": 13, - "y": 44 - }, - "hiddenSeries": false, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 13, + "y": 29 + }, + "hiddenSeries": false, + "id": 79, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".99", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".9", - "refId": "B" + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/cortex-gw\", route=\"opentelemetry_proto_collector_trace_v1_traceservice_export\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Push Latency (Gateway)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", - "interval": "", - "legendFormat": ".5", - "refId": "C" - } - ], - "thresholds": [ + "aliasColors": { - ], - "timeFrom": null, - "timeRegions": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 13, + "y": 34 + }, + "hiddenSeries": false, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "timeShift": null, - "title": "Push Latency (Ingester)", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".99", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".9", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/ingester\", route=~\"/tempopb.Pusher/Push.*\"}[$__rate_interval])) by (le))", + "interval": "", + "legendFormat": ".5", + "refId": "C" + } + ], + "thresholds": [ - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Push Latency (Ingester)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Write", + "type": "row" }, { - "collapsed": false, + "collapsed": true, "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 24 }, "id": 74, "panels": [ + { + "aliasColors": { - ], - "title": "Memcached", - "type": "row" - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fieldConfig": { - "defaults": { - "links": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": { + "links": [ - ] - }, - "overrides": [ + ] + }, + "overrides": [ - ] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 50 - }, - "hiddenSeries": false, - "id": 75, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 25 + }, + "hiddenSeries": false, + "id": 75, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(cortex_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, method)", - "interval": "", - "legendFormat": "{{status_code}}-{{method}}", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(cortex_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (status_code, method)", + "interval": "", + "legendFormat": "{{status_code}}-{{method}}", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "Requests/Second", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "Requests/Second", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fieldConfig": { - "defaults": { - "links": [ + "aliasColors": { - ] - }, - "overrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": { + "links": [ - ] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 50 - }, - "hiddenSeries": false, - "id": 76, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + ] + }, + "overrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(cortex_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", - "interval": "", - "legendFormat": ".99-{{method}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(cortex_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", - "interval": "", - "legendFormat": ".9-{{method}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(cortex_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", - "interval": "", - "legendFormat": ".5-{{method}}", - "refId": "C" - } - ], - "thresholds": [ + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 25 + }, + "hiddenSeries": false, + "id": 76, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(.99, sum(rate(cortex_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "interval": "", + "legendFormat": ".99-{{method}}", + "refId": "A" + }, + { + "expr": "histogram_quantile(.9, sum(rate(cortex_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "interval": "", + "legendFormat": ".9-{{method}}", + "refId": "B" + }, + { + "expr": "histogram_quantile(.5, sum(rate(cortex_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"}[$__rate_interval])) by (method, le))", + "interval": "", + "legendFormat": ".5-{{method}}", + "refId": "C" + } + ], + "thresholds": [ - ], - "timeShift": null, - "title": "Latency By Operation", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeFrom": null, + "timeRegions": [ - ] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ], + "timeShift": null, + "title": "Latency By Operation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Memcached", + "type": "row" }, { "collapsed": true, "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 58 + "y": 25 }, "id": 28, "panels": [ @@ -4062,19 +4896,11 @@ { "collapsed": true, "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 59 + "y": 26 }, "id": 56, "panels": [ @@ -4395,31 +5221,328 @@ "show": true, "values": [ - ] + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": { + "custom": { + + }, + "links": [ + + ] + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 42 + }, + "hiddenSeries": false, + "id": 60, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.2.1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "min(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "interval": "", + "legendFormat": "min", + "refId": "A" + }, + { + "expr": "max(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "interval": "", + "legendFormat": "max", + "refId": "B" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "KV Store Count", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "title": "Ring/Memberlist", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 69, + "panels": [ + { + "datasource": "$ds", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [ + + ], + "thresholds": { + "mode": "absolute", + "steps": [ + + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 77, + "options": { + "graph": { + + }, + "legend": { + "calcs": [ + + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "7.5.0-14141pre", + "targets": [ + { + "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{error}}", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Vulture Query Errors", + "type": "timeseries" + }, + { + "datasource": "$ds", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [ + + ], + "max": 1, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "percentunit" + }, + "overrides": [ + + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 67, + "options": { + "displayMode": "gradient", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": { + + } }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, + "pluginVersion": "8.2.0-32962pre", + "targets": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", + "interval": "", + "legendFormat": "{{secondsago}}", + "refId": "A" } ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, + "timeFrom": null, + "timeShift": null, + "title": "Average Vulture Query Errors", + "type": "bargauge" + } + ], + "title": "Vulture", + "type": "row" + }, + { + "collapsed": true, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 28 + }, + "id": 81, + "panels": [ { "aliasColors": { @@ -4428,29 +5551,16 @@ "dashLength": 10, "dashes": false, "datasource": "$ds", - "fieldConfig": { - "defaults": { - "custom": { - - }, - "links": [ - - ] - }, - "overrides": [ - - ] - }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, - "w": 6, - "x": 18, - "y": 42 + "h": 6, + "w": 4, + "x": 0, + "y": 29 }, "hiddenSeries": false, - "id": 60, + "id": 83, "legend": { "avg": false, "current": false, @@ -4467,7 +5577,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.2.1", + "pluginVersion": "8.2.0-32962pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -4479,16 +5589,10 @@ "steppedLine": false, "targets": [ { - "expr": "min(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", + "expr": "sum(rate(tempodb_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (level)", "interval": "", - "legendFormat": "min", + "legendFormat": "", "refId": "A" - }, - { - "expr": "max(tempo_memberlist_client_kv_store_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/$component\"})", - "interval": "", - "legendFormat": "max", - "refId": "B" } ], "thresholds": [ @@ -4499,10 +5603,10 @@ ], "timeShift": null, - "title": "KV Store Count", + "title": "Objects Combined / s", "tooltip": { "shared": true, - "sort": 2, + "sort": 0, "value_type": "individual" }, "type": "graph", @@ -4517,6 +5621,7 @@ }, "yaxes": [ { + "$$hashKey": "object:150", "format": "short", "label": null, "logBase": 1, @@ -4525,6 +5630,7 @@ "show": true }, { + "$$hashKey": "object:151", "format": "short", "label": null, "logBase": 1, @@ -4537,610 +5643,312 @@ "align": false, "alignLevel": null } - } - ], - "title": "Ring/Memberlist", - "type": "row" - }, - { - "collapsed": false, - "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 60 - }, - "id": 69, - "panels": [ - - ], - "title": "Vulture", - "type": "row" - }, - { - "datasource": "$ds", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [ - - ], - "thresholds": { - "mode": "absolute", - "steps": [ - - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 61 - }, - "id": 77, - "options": { - "graph": { - - }, - "legend": { - "calcs": [ - - ], - "displayMode": "list", - "placement": "bottom" - }, - "tooltip": { - "mode": "single" - } - }, - "pluginVersion": "7.5.0-14141pre", - "targets": [ - { - "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{error}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Vulture Query Errors", - "type": "timeseries" - }, - { - "datasource": "$ds", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [ - - ], - "max": 1, - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "percentunit" - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 61 - }, - "id": 67, - "options": { - "displayMode": "gradient", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": { - - } - }, - "pluginVersion": "8.1.0-27525pre", - "targets": [ - { - "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval])) by (error) / ignoring (error) group_left sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[$__rate_interval]))", - "interval": "", - "legendFormat": "{{secondsago}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Average Vulture Query Errors", - "type": "bargauge" - }, - { - "collapsed": false, - "datasource": null, - "fieldConfig": { - "defaults": { - - }, - "overrides": [ - - ] - }, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 70 - }, - "id": 81, - "panels": [ - - ], - "title": "Compactor", - "type": "row" - }, - { - "aliasColors": { - - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 4, - "x": 0, - "y": 71 - }, - "hiddenSeries": false, - "id": 83, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempodb_compaction_objects_combined_total{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"$namespace/compactor\"}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [ - - ], - "timeFrom": null, - "timeRegions": [ - - ], - "timeShift": null, - "title": "Objects Combined / s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ - - ] - }, - "yaxes": [ - { - "$$hashKey": "object:150", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true }, { - "$$hashKey": "object:151", - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "description": "", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 4, - "x": 4, - "y": 71 - }, - "hiddenSeries": false, - "id": 85, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "description": "", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 4, + "x": 4, + "y": 29 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempodb_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempodb_compaction_objects_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "Objects Written / s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "Objects Written / s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 4, - "x": 8, - "y": 71 - }, - "hiddenSeries": false, - "id": 88, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 29 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "sum(rate(tempodb_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempodb_compaction_bytes_written_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "Bytes Written / s", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "Bytes Written / s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } }, { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": { + "aliasColors": { - }, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 6, - "w": 4, - "x": 12, - "y": 71 - }, - "hiddenSeries": false, - "id": 86, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "nullPointMode": "null", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.1.0-27525pre", - "pointradius": 2, - "points": false, - "renderer": "flot", - "seriesOverrides": [ + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 4, + "x": 12, + "y": 29 + }, + "hiddenSeries": false, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "8.2.0-32962pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ - ], - "spaceLength": 10, - "stack": true, - "steppedLine": false, - "targets": [ - { - "expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[5m])) by (level)", - "interval": "", - "legendFormat": "", - "refId": "A" - } - ], - "thresholds": [ + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/compactor\"}[5m])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [ - ], - "timeFrom": null, - "timeRegions": [ + ], + "timeFrom": null, + "timeRegions": [ - ], - "timeShift": null, - "title": "Blocks Compacted", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [ + ], + "timeShift": null, + "title": "Blocks Compacted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ - ] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], - "yaxis": { - "align": false, - "alignLevel": null - } + "title": "Compactor", + "type": "row" } ], "refresh": "", - "schemaVersion": 30, + "schemaVersion": 31, "style": "dark", "tags": [ "tempo" @@ -5172,8 +5980,8 @@ { "current": { "selected": false, - "text": "Loki-ops", - "value": "Loki-ops" + "text": "loki-ops", + "value": "loki-ops" }, "description": null, "error": null,