diff --git a/CHANGELOG.md b/CHANGELOG.md index b6301515ec0..27d27fbe99d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ ## master / unreleased * [CHANGE] Update to Go 1.16, latest OpenTelemetry proto definition and collector [#546](https://github.com/grafana/tempo/pull/546) +* [CHANGE] Tempo Query Frontend now accepts queries at `/tempo/api/traces/{traceID}` as opposed to `/api/traces/{traceID}`. + This is a **breaking change**, make sure to change the Grafana Datasource endpoint accordingly. [#574](https://github.com/grafana/tempo/pull/574) * [FEATURE] Add page based access to the index file. [#557](https://github.com/grafana/tempo/pull/557) * [ENHANCEMENT] Add a Shutdown handler to flush data to backend, at "/shutdown". [#526](https://github.com/grafana/tempo/pull/526) * [ENHANCEMENT] Queriers now query all (healthy) ingesters for a trace to mitigate 404s on ingester rollouts/scaleups. diff --git a/docs/tempo/website/configuration/_index.md b/docs/tempo/website/configuration/_index.md index 6ca419277e3..df4b299f2a1 100644 --- a/docs/tempo/website/configuration/_index.md +++ b/docs/tempo/website/configuration/_index.md @@ -78,7 +78,7 @@ querier: frontend_address: query-frontend-discovery.default.svc.cluster.local:9095 # the address of the query frontend to connect to, and process queries ``` -The Querier also queries compacted blocks that fall within (2 * BlocklistPoll) where the value of Blocklist poll duration +It also queries compacted blocks that fall within the (2 * BlocklistPoll) range where the value of Blocklist poll duration is defined in the storage section below. ## Compactor diff --git a/operations/tempo-mixin/out/tempo-operational.json b/operations/tempo-mixin/out/tempo-operational.json index 3aa3bed0a99..e52465dfee5 100644 --- a/operations/tempo-mixin/out/tempo-operational.json +++ b/operations/tempo-mixin/out/tempo-operational.json @@ -27,8 +27,7 @@ "editable": true, "gnetId": null, "graphTooltip": 1, - "id": 115, - "iteration": 1612469301683, + "iteration": 1615543281712, "links": [ ], @@ -59,9 +58,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -93,7 +90,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -168,9 +165,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -202,7 +197,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -277,9 +272,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -311,7 +304,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -385,9 +378,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -419,7 +410,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -495,9 +486,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -529,7 +518,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -604,9 +593,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -638,7 +625,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -721,9 +708,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -755,7 +740,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -829,9 +814,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -863,7 +846,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -954,9 +937,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -988,7 +969,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1063,9 +1044,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -1097,7 +1076,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1181,9 +1160,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -1215,7 +1192,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": true, "renderer": "flot", @@ -1301,9 +1278,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -1335,7 +1310,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1411,9 +1386,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -1445,7 +1418,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": true, "renderer": "flot", @@ -1532,9 +1505,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -1566,7 +1537,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1647,9 +1618,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -1681,7 +1650,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1773,9 +1742,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -1784,7 +1751,7 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 10, + "h": 5, "w": 6, "x": 0, "y": 13 @@ -1807,7 +1774,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1819,7 +1786,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"tempo_api_traces_traceid\", job=\"$namespace/cortex-gw\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1834,7 +1801,7 @@ ], "timeShift": null, - "title": "Queries/Sec (Query Frontend)", + "title": "Queries/Sec (cortex-gw)", "tooltip": { "shared": true, "sort": 0, @@ -1883,9 +1850,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -1894,8 +1859,8 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 10, - "w": 6, + "h": 5, + "w": 9, "x": 6, "y": 13 }, @@ -1917,7 +1882,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2004,9 +1969,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2016,9 +1979,117 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 4, - "x": 12, - "y": 13 + "w": 6, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 90, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.0-14141pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"tempo_api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Queries/Sec (Query Frontend)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": { + + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 9, + "x": 6, + "y": 18 }, "hiddenSeries": false, "id": 17, @@ -2038,7 +2109,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2050,19 +2121,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -2125,9 +2196,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2137,9 +2206,117 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 4, - "x": 16, - "y": 13 + "w": 6, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.0-14141pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"querier_tempo_api_traces_traceid\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Queries/Sec (Querier)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": { + + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 9, + "x": 6, + "y": 23 }, "hiddenSeries": false, "id": 89, @@ -2159,7 +2336,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2171,19 +2348,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -2246,9 +2423,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2258,9 +2433,117 @@ "fillGradient": 0, "gridPos": { "h": 5, - "w": 4, - "x": 12, - "y": 18 + "w": 6, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 92, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.0-14141pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [ + + ], + "timeFrom": null, + "timeRegions": [ + + ], + "timeShift": null, + "title": "Queries/Sec (Ingester)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + + ] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": { + + }, + "overrides": [ + + ] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 9, + "x": 6, + "y": 28 }, "hiddenSeries": false, "id": 3, @@ -2280,7 +2563,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2364,7 +2647,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 33 }, "id": 19, "panels": [ @@ -2383,9 +2666,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2397,7 +2678,7 @@ "h": 5, "w": 3, "x": 0, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 10, @@ -2417,7 +2698,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2492,9 +2773,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2506,7 +2785,7 @@ "h": 5, "w": 3, "x": 3, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 9, @@ -2526,7 +2805,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2601,9 +2880,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2615,7 +2892,7 @@ "h": 5, "w": 3, "x": 6, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 8, @@ -2635,7 +2912,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2710,9 +2987,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2724,7 +2999,7 @@ "h": 5, "w": 3, "x": 9, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 12, @@ -2744,7 +3019,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2819,9 +3094,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2833,7 +3106,7 @@ "h": 5, "w": 3, "x": 12, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 26, @@ -2853,7 +3126,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2928,9 +3201,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -2942,7 +3213,7 @@ "h": 5, "w": 3, "x": 15, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 36, @@ -2962,7 +3233,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": true, "renderer": "flot", @@ -3049,9 +3320,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -3063,7 +3332,7 @@ "h": 10, "w": 7, "x": 0, - "y": 29 + "y": 39 }, "hiddenSeries": false, "id": 71, @@ -3083,7 +3352,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3158,9 +3427,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -3172,7 +3439,7 @@ "h": 10, "w": 6, "x": 7, - "y": 29 + "y": 39 }, "hiddenSeries": false, "id": 72, @@ -3192,7 +3459,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3273,9 +3540,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -3287,7 +3552,7 @@ "h": 5, "w": 6, "x": 13, - "y": 29 + "y": 39 }, "hiddenSeries": false, "id": 79, @@ -3307,7 +3572,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3394,9 +3659,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -3408,7 +3671,7 @@ "h": 5, "w": 6, "x": 13, - "y": 34 + "y": 44 }, "hiddenSeries": false, "id": 2, @@ -3428,7 +3691,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3512,7 +3775,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 49 }, "id": 74, "panels": [ @@ -3531,9 +3794,6 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - - }, "links": [ ] @@ -3548,7 +3808,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 50 }, "hiddenSeries": false, "id": 75, @@ -3568,7 +3828,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3643,9 +3903,6 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - - }, "links": [ ] @@ -3660,7 +3917,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 50 }, "hiddenSeries": false, "id": 76, @@ -3680,7 +3937,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3764,7 +4021,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 58 }, "id": 28, "panels": [ @@ -4012,7 +4269,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 59 }, "id": 56, "panels": [ @@ -4487,7 +4744,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 60 }, "id": 69, "panels": [ @@ -4502,9 +4759,6 @@ "defaults": { "color": { "mode": "thresholds" - }, - "custom": { - }, "mappings": [ @@ -4603,7 +4857,7 @@ "h": 9, "w": 12, "x": 0, - "y": 51 + "y": 61 }, "id": 77, "options": { @@ -4621,7 +4875,7 @@ } }, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "targets": [ { "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\", error=\"notfound\"}[1h])) / sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h]))", @@ -4641,9 +4895,6 @@ "defaults": { "color": { "mode": "thresholds" - }, - "custom": { - }, "mappings": [ @@ -4742,7 +4993,7 @@ "h": 9, "w": 12, "x": 12, - "y": 51 + "y": 61 }, "id": 67, "options": { @@ -4760,10 +5011,10 @@ } }, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "targets": [ { - "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\", error=\"missingspans\"}[1h])) / sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h]))", + "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\", error=\"missingspans\"}[1h])) by (secondsago) / sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h])) by (secondsago)", "interval": "", "legendFormat": "{{secondsago}}", "refId": "A" @@ -4781,7 +5032,7 @@ "h": 1, "w": 24, "x": 0, - "y": 60 + "y": 70 }, "id": 81, "panels": [ @@ -4800,9 +5051,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -4814,7 +5063,7 @@ "h": 6, "w": 4, "x": 0, - "y": 61 + "y": 71 }, "hiddenSeries": false, "id": 83, @@ -4834,7 +5083,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -4912,9 +5161,7 @@ "description": "", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -4926,7 +5173,7 @@ "h": 6, "w": 4, "x": 4, - "y": 61 + "y": 71 }, "hiddenSeries": false, "id": 85, @@ -4946,7 +5193,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -5021,9 +5268,7 @@ "datasource": null, "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -5035,7 +5280,7 @@ "h": 6, "w": 4, "x": 8, - "y": 61 + "y": 71 }, "hiddenSeries": false, "id": 88, @@ -5055,7 +5300,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -5130,9 +5375,7 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": { - } }, "overrides": [ @@ -5144,7 +5387,7 @@ "h": 6, "w": 4, "x": 12, - "y": 61 + "y": 71 }, "hiddenSeries": false, "id": 86, @@ -5164,7 +5407,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -5425,6 +5668,6 @@ }, "timezone": "", "title": "Tempo Operational", - "uid": "AOFXrRUZz", + "uid": "a6175b9cc7ec20591890117c39580030", "version": 1 } diff --git a/operations/tempo-mixin/tempo-operational.json b/operations/tempo-mixin/tempo-operational.json index 232551924b7..bc53cd644df 100644 --- a/operations/tempo-mixin/tempo-operational.json +++ b/operations/tempo-mixin/tempo-operational.json @@ -25,8 +25,7 @@ "editable": true, "gnetId": null, "graphTooltip": 1, - "id": 115, - "iteration": 1612469301683, + "iteration": 1615543281712, "links": [], "panels": [ { @@ -50,9 +49,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -81,7 +78,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -145,9 +142,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -176,7 +171,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -240,9 +235,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -271,7 +264,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -334,9 +327,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -365,7 +356,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -430,9 +421,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -461,7 +450,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -525,9 +514,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -556,7 +543,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -628,9 +615,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -659,7 +644,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -722,9 +707,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -753,7 +736,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -831,9 +814,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -862,7 +843,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -926,9 +907,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -957,7 +936,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1030,9 +1009,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1061,7 +1038,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": true, "renderer": "flot", @@ -1136,9 +1113,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1167,7 +1142,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1232,9 +1207,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1263,7 +1236,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": true, "renderer": "flot", @@ -1339,9 +1312,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1370,7 +1341,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1440,9 +1411,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -1471,7 +1440,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1550,15 +1519,13 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 10, + "h": 5, "w": 6, "x": 0, "y": 13 @@ -1581,7 +1548,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1591,7 +1558,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"tempo_api_traces_traceid\", job=\"$namespace/cortex-gw\"}[$__rate_interval])) by (status_code)", "hide": false, "interval": "", "legendFormat": "{{status_code}}", @@ -1602,7 +1569,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Queries/Sec (Query Frontend)", + "title": "Queries/Sec (cortex-gw)", "tooltip": { "shared": true, "sort": 0, @@ -1646,16 +1613,14 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { - "h": 10, - "w": 6, + "h": 5, + "w": 9, "x": 6, "y": 13 }, @@ -1677,7 +1642,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1753,18 +1718,110 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 18 + }, + "hiddenSeries": false, + "id": 90, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.0-14141pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"tempo_api_traces_traceid\", job=\"$namespace/query-frontend\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queries/Sec (Query Frontend)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 5, - "w": 4, - "x": 12, - "y": 13 + "w": 9, + "x": 6, + "y": 18 }, "hiddenSeries": false, "id": 17, @@ -1784,7 +1841,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1794,19 +1851,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/query-frontend\", route=\"tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1860,18 +1917,110 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 23 + }, + "hiddenSeries": false, + "id": 91, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.0-14141pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"querier_tempo_api_traces_traceid\", job=\"$namespace/querier\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queries/Sec (Querier)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 5, - "w": 4, - "x": 16, - "y": 13 + "w": 9, + "x": 6, + "y": 23 }, "hiddenSeries": false, "id": 89, @@ -1891,7 +2040,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -1901,19 +2050,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.99, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.9, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_api_traces_traceid\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(.5, sum(rate(tempo_request_duration_seconds_bucket{job=\"$namespace/querier\", route=\"querier_tempo_api_traces_traceid\"}[$__rate_interval])) by (le))", "interval": "", "legendFormat": ".5", "refId": "C" @@ -1967,18 +2116,110 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 28 + }, + "hiddenSeries": false, + "id": 92, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.5.0-14141pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempo_request_duration_seconds_count{route=\"/tempopb.Querier/FindTraceByID\", job=\"$namespace/ingester\"}[$__rate_interval])) by (status_code)", + "hide": false, + "interval": "", + "legendFormat": "{{status_code}}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Queries/Sec (Ingester)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": {}, "overrides": [] }, "fill": 1, "fillGradient": 0, "gridPos": { "h": 5, - "w": 4, - "x": 12, - "y": 18 + "w": 9, + "x": 6, + "y": 28 }, "hiddenSeries": false, "id": 3, @@ -1998,7 +2239,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2074,7 +2315,7 @@ "h": 1, "w": 24, "x": 0, - "y": 23 + "y": 33 }, "id": 19, "panels": [], @@ -2088,9 +2329,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 0, @@ -2099,7 +2338,7 @@ "h": 5, "w": 3, "x": 0, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 10, @@ -2119,7 +2358,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2183,9 +2422,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2194,7 +2431,7 @@ "h": 5, "w": 3, "x": 3, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 9, @@ -2214,7 +2451,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2278,9 +2515,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2289,7 +2524,7 @@ "h": 5, "w": 3, "x": 6, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 8, @@ -2309,7 +2544,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2373,9 +2608,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2384,7 +2617,7 @@ "h": 5, "w": 3, "x": 9, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 12, @@ -2404,7 +2637,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2468,9 +2701,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2479,7 +2710,7 @@ "h": 5, "w": 3, "x": 12, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 26, @@ -2499,7 +2730,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2563,9 +2794,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2574,7 +2803,7 @@ "h": 5, "w": 3, "x": 15, - "y": 24 + "y": 34 }, "hiddenSeries": false, "id": 36, @@ -2594,7 +2823,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": true, "renderer": "flot", @@ -2670,9 +2899,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2681,7 +2908,7 @@ "h": 10, "w": 7, "x": 0, - "y": 29 + "y": 39 }, "hiddenSeries": false, "id": 71, @@ -2701,7 +2928,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2765,9 +2992,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2776,7 +3001,7 @@ "h": 10, "w": 6, "x": 7, - "y": 29 + "y": 39 }, "hiddenSeries": false, "id": 72, @@ -2796,7 +3021,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2866,9 +3091,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2877,7 +3100,7 @@ "h": 5, "w": 6, "x": 13, - "y": 29 + "y": 39 }, "hiddenSeries": false, "id": 79, @@ -2897,7 +3120,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -2973,9 +3196,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -2984,7 +3205,7 @@ "h": 5, "w": 6, "x": 13, - "y": 34 + "y": 44 }, "hiddenSeries": false, "id": 2, @@ -3004,7 +3225,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3080,7 +3301,7 @@ "h": 1, "w": 24, "x": 0, - "y": 39 + "y": 49 }, "id": 74, "panels": [], @@ -3095,7 +3316,6 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3106,7 +3326,7 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 50 }, "hiddenSeries": false, "id": 75, @@ -3126,7 +3346,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3191,7 +3411,6 @@ "datasource": "$ds", "fieldConfig": { "defaults": { - "custom": {}, "links": [] }, "overrides": [] @@ -3202,7 +3421,7 @@ "h": 8, "w": 12, "x": 12, - "y": 40 + "y": 50 }, "hiddenSeries": false, "id": 76, @@ -3222,7 +3441,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -3298,7 +3517,7 @@ "h": 1, "w": 24, "x": 0, - "y": 48 + "y": 58 }, "id": 28, "panels": [ @@ -3514,7 +3733,7 @@ "h": 1, "w": 24, "x": 0, - "y": 49 + "y": 59 }, "id": 56, "panels": [ @@ -3925,7 +4144,7 @@ "h": 1, "w": 24, "x": 0, - "y": 50 + "y": 60 }, "id": 69, "panels": [], @@ -3939,7 +4158,6 @@ "color": { "mode": "thresholds" }, - "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", @@ -4035,7 +4253,7 @@ "h": 9, "w": 12, "x": 0, - "y": 51 + "y": 61 }, "id": 77, "options": { @@ -4051,7 +4269,7 @@ "showUnfilled": true, "text": {} }, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "targets": [ { "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\", error=\"notfound\"}[1h])) / sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h]))", @@ -4072,7 +4290,6 @@ "color": { "mode": "thresholds" }, - "custom": {}, "mappings": [], "thresholds": { "mode": "absolute", @@ -4168,7 +4385,7 @@ "h": 9, "w": 12, "x": 12, - "y": 51 + "y": 61 }, "id": 67, "options": { @@ -4184,7 +4401,7 @@ "showUnfilled": true, "text": {} }, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "targets": [ { "expr": "sum(rate(tempo_vulture_trace_error_total{cluster=\"$cluster\", namespace=\"$namespace\", error=\"missingspans\"}[1h])) by (secondsago) / sum(rate(tempo_vulture_trace_total{cluster=\"$cluster\", namespace=\"$namespace\"}[1h])) by (secondsago)", @@ -4205,7 +4422,7 @@ "h": 1, "w": 24, "x": 0, - "y": 60 + "y": 70 }, "id": 81, "panels": [], @@ -4219,9 +4436,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -4230,7 +4445,7 @@ "h": 6, "w": 4, "x": 0, - "y": 61 + "y": 71 }, "hiddenSeries": false, "id": 83, @@ -4250,7 +4465,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -4317,9 +4532,7 @@ "datasource": "$ds", "description": "", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -4328,7 +4541,7 @@ "h": 6, "w": 4, "x": 4, - "y": 61 + "y": 71 }, "hiddenSeries": false, "id": 85, @@ -4348,7 +4561,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -4412,9 +4625,7 @@ "dashes": false, "datasource": null, "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -4423,7 +4634,7 @@ "h": 6, "w": 4, "x": 8, - "y": 61 + "y": 71 }, "hiddenSeries": false, "id": 88, @@ -4443,7 +4654,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -4507,9 +4718,7 @@ "dashes": false, "datasource": "$ds", "fieldConfig": { - "defaults": { - "custom": {} - }, + "defaults": {}, "overrides": [] }, "fill": 1, @@ -4518,7 +4727,7 @@ "h": 6, "w": 4, "x": 12, - "y": 61 + "y": 71 }, "hiddenSeries": false, "id": 86, @@ -4538,7 +4747,7 @@ "alertThreshold": true }, "percentage": false, - "pluginVersion": "7.5.0-11433pre", + "pluginVersion": "7.5.0-14141pre", "pointradius": 2, "points": false, "renderer": "flot", @@ -4777,6 +4986,6 @@ }, "timezone": "", "title": "Tempo Operational", - "uid": "AOFXrRUZz", + "uid": "a6175b9cc7ec20591890117c39580030", "version": 1 } \ No newline at end of file diff --git a/tempodb/tempodb.go b/tempodb/tempodb.go index 6adf2b4a966..0e0e437bb42 100644 --- a/tempodb/tempodb.go +++ b/tempodb/tempodb.go @@ -242,6 +242,13 @@ func (rw *readerWriter) Find(ctx context.Context, tenantID string, id common.ID, copiedBlocklist = append(copiedBlocklist, b) } } + + compactedBlocklist := rw.compactedBlockLists[tenantID] + for _, c := range compactedBlocklist { + if includeCompactedBlock(c, id, blockStartBytes, blockEndBytes, rw.cfg.BlocklistPoll) { + copiedBlocklist = append(copiedBlocklist, &c.BlockMeta) + } + } rw.blockListsMtx.Unlock() // deliberately placed outside the blocklist mtx unlock @@ -460,23 +467,21 @@ func (rw *readerWriter) cleanMissingTenants(tenants []string) { tenantSet[tenantID] = struct{}{} } + rw.blockListsMtx.Lock() for tenantID := range rw.blockLists { if _, present := tenantSet[tenantID]; !present { - rw.blockListsMtx.Lock() delete(rw.blockLists, tenantID) - rw.blockListsMtx.Unlock() level.Info(rw.logger).Log("msg", "deleted in-memory blocklists", "tenantID", tenantID) } } for tenantID := range rw.compactedBlockLists { if _, present := tenantSet[tenantID]; !present { - rw.blockListsMtx.Lock() delete(rw.compactedBlockLists, tenantID) - rw.blockListsMtx.Unlock() level.Info(rw.logger).Log("msg", "deleted in-memory compacted blocklists", "tenantID", tenantID) } } + rw.blockListsMtx.Unlock() } // updateBlocklist Add and remove regular or compacted blocks from the in-memory blocklist. @@ -529,3 +534,12 @@ func includeBlock(b *backend.BlockMeta, id common.ID, blockStart []byte, blockEn return true } + +// if block is compacted within lookback period, and is within shard ranges, include it in search +func includeCompactedBlock(c *backend.CompactedBlockMeta, id common.ID, blockStart []byte, blockEnd []byte, poll time.Duration) bool { + lookback := time.Now().Add(-(2 * poll)) + if c.CompactedTime.Before(lookback) { + return false + } + return includeBlock(&c.BlockMeta, id, blockStart, blockEnd) +} diff --git a/tempodb/tempodb_test.go b/tempodb/tempodb_test.go index 8b5ad62d454..065a33b946f 100644 --- a/tempodb/tempodb_test.go +++ b/tempodb/tempodb_test.go @@ -796,3 +796,182 @@ func TestIncludeBlock(t *testing.T) { }) } } + +func TestIncludeCompactedBlock(t *testing.T) { + blocklistPoll := 5 * time.Minute + tests := []struct { + name string + searchID common.ID + blockStart uuid.UUID + blockEnd uuid.UUID + meta *backend.CompactedBlockMeta + expected bool + }{ + { + name: "include recent", + searchID: []byte{0x05}, + blockStart: uuid.MustParse(BlockIDMin), + blockEnd: uuid.MustParse(BlockIDMax), + meta: &backend.CompactedBlockMeta{ + BlockMeta: backend.BlockMeta{ + BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"), + MinID: []byte{0x00}, + MaxID: []byte{0x10}, + }, + CompactedTime: time.Now().Add(-(1 * blocklistPoll)), + }, + expected: true, + }, + { + name: "skip old", + searchID: []byte{0x05}, + blockStart: uuid.MustParse(BlockIDMin), + blockEnd: uuid.MustParse(BlockIDMax), + meta: &backend.CompactedBlockMeta{ + BlockMeta: backend.BlockMeta{ + BlockID: uuid.MustParse("50000000-0000-0000-0000-000000000000"), + MinID: []byte{0x00}, + MaxID: []byte{0x10}, + }, + CompactedTime: time.Now().Add(-(3 * blocklistPoll)), + }, + expected: false, + }, + { + name: "skip recent but out of range", + searchID: []byte{0x05}, + blockStart: uuid.MustParse("40000000-0000-0000-0000-000000000000"), + blockEnd: uuid.MustParse("50000000-0000-0000-0000-000000000000"), + meta: &backend.CompactedBlockMeta{ + BlockMeta: backend.BlockMeta{ + BlockID: uuid.MustParse("51000000-0000-0000-0000-000000000000"), + MinID: []byte{0x00}, + MaxID: []byte{0x10}, + }, + CompactedTime: time.Now().Add(-(1 * blocklistPoll)), + }, + expected: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, err := tc.blockStart.MarshalBinary() + require.NoError(t, err) + e, err := tc.blockEnd.MarshalBinary() + require.NoError(t, err) + + assert.Equal(t, tc.expected, includeCompactedBlock(tc.meta, tc.searchID, s, e, blocklistPoll)) + }) + } + +} + +func TestSearchCompactedBlocks(t *testing.T) { + tempDir, err := ioutil.TempDir("/tmp", "") + defer os.RemoveAll(tempDir) + assert.NoError(t, err, "unexpected error creating temp dir") + + r, w, c, err := New(&Config{ + Backend: "local", + Local: &local.Config{ + Path: path.Join(tempDir, "traces"), + }, + Block: &encoding.BlockConfig{ + IndexDownsampleBytes: 17, + BloomFP: .01, + Encoding: backend.EncLZ4_256k, + IndexPageSizeBytes: 1000, + }, + WAL: &wal.Config{ + Filepath: path.Join(tempDir, "wal"), + }, + BlocklistPoll: time.Minute, + }, log.NewNopLogger()) + assert.NoError(t, err) + + c.EnableCompaction(&CompactorConfig{ + ChunkSizeBytes: 10, + MaxCompactionRange: time.Hour, + BlockRetention: 0, + CompactedBlockRetention: 0, + }, &mockSharder{}, &mockOverrides{}) + + wal := w.WAL() + assert.NoError(t, err) + + head, err := wal.NewBlock(uuid.New(), testTenantID) + assert.NoError(t, err) + + // write + numMsgs := 10 + reqs := make([]*tempopb.PushRequest, 0, numMsgs) + ids := make([][]byte, 0, numMsgs) + for i := 0; i < numMsgs; i++ { + id := make([]byte, 16) + rand.Read(id) + req := test.MakeRequest(rand.Int()%1000, id) + reqs = append(reqs, req) + ids = append(ids, id) + + bReq, err := proto.Marshal(req) + assert.NoError(t, err) + err = head.Write(id, bReq) + assert.NoError(t, err, "unexpected error writing req") + } + + complete, err := w.CompleteBlock(head, &mockSharder{}) + assert.NoError(t, err) + + blockID := complete.BlockMeta().BlockID.String() + + err = w.WriteBlock(context.Background(), complete) + assert.NoError(t, err) + + rw := r.(*readerWriter) + + // poll + rw.pollBlocklist() + + // read + for i, id := range ids { + bFound, err := r.Find(context.Background(), testTenantID, id, blockID, blockID) + assert.NoError(t, err) + + out := &tempopb.PushRequest{} + err = proto.Unmarshal(bFound[0], out) + assert.NoError(t, err) + + assert.True(t, proto.Equal(out, reqs[i])) + } + + // compact + var blockMetas []*backend.BlockMeta + blockMetas = append(blockMetas, complete.BlockMeta()) + assert.NoError(t, rw.compact(blockMetas, testTenantID)) + + // poll + rw.pollBlocklist() + + // make sure the block is compacted + compactedBlocks, ok := rw.compactedBlockLists[testTenantID] + require.True(t, ok) + require.Len(t, compactedBlocks, 1) + assert.Equal(t, compactedBlocks[0].BlockID.String(), blockID) + blocks, ok := rw.blockLists[testTenantID] + require.True(t, ok) + require.Len(t, blocks, 1) + assert.NotEqual(t, blocks[0].BlockID.String(), blockID) + + // find should succeed with old block range + for i, id := range ids { + bFound, err := r.Find(context.Background(), testTenantID, id, blockID, blockID) + assert.NoError(t, err) + + out := &tempopb.PushRequest{} + err = proto.Unmarshal(bFound[0], out) + assert.NoError(t, err) + + assert.True(t, proto.Equal(out, reqs[i])) + } +}