From 2147324722a0ff116fafe231029390f686d9b213 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Thu, 19 Nov 2020 15:40:07 -0500 Subject: [PATCH 1/9] Add metric tempodb_compaction_objects_processed_total --- tempodb/compactor.go | 6 ++++++ tempodb/compactor_test.go | 8 ++++++++ 2 files changed, 14 insertions(+) diff --git a/tempodb/compactor.go b/tempodb/compactor.go index 369fe20798f..79d205124c8 100644 --- a/tempodb/compactor.go +++ b/tempodb/compactor.go @@ -27,6 +27,11 @@ var ( Help: "Records the amount of time to compact a set of blocks.", Buckets: prometheus.ExponentialBuckets(30, 2, 10), }, []string{"level"}) + metricCompactionObjectsProcessed = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "tempodb", + Name: "compaction_objects_processed_total", + Help: "Total number of objects processed during compaction.", + }) metricCompactionErrors = promauto.NewCounter(prometheus.CounterOpts{ Namespace: "tempodb", Name: "compaction_errors_total", @@ -188,6 +193,7 @@ func (rw *readerWriter) compact(blockMetas []*encoding.BlockMeta, tenantID strin return err } lowestBookmark.clear() + metricCompactionObjectsProcessed.Inc() // write partial block if currentBlock.CurrentBufferLength() >= int(rw.compactorCfg.FlushSizeBytes) { diff --git a/tempodb/compactor_test.go b/tempodb/compactor_test.go index f0f0a4bc8b9..35cb02b186e 100644 --- a/tempodb/compactor_test.go +++ b/tempodb/compactor_test.go @@ -124,6 +124,9 @@ func TestCompaction(t *testing.T) { rw.pollBlocklist() + processedStart, err := GetCounterValue(metricCompactionObjectsProcessed) + assert.NoError(t, err) + blocklist := rw.blocklist(testTenantID) blockSelector := newTimeWindowBlockSelector(blocklist, rw.compactorCfg.MaxCompactionRange, 10000, defaultMinInputBlocks, 2) @@ -154,6 +157,11 @@ func TestCompaction(t *testing.T) { } assert.Equal(t, blockCount*recordCount, records) + // Check metric + processedEnd, err := GetCounterValue(metricCompactionObjectsProcessed) + assert.NoError(t, err) + assert.Equal(t, float64(blockCount*recordCount), processedEnd-processedStart) + // now see if we can find our ids for i, id := range allIds { b, _, err := rw.Find(context.Background(), testTenantID, id) From 0b28b0c27d51741ef77fb05beda54e569d426ee9 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Fri, 20 Nov 2020 09:12:23 -0500 Subject: [PATCH 2/9] Add level label to tempodb_compaction_objects_processed_total metric --- tempodb/compactor.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tempodb/compactor.go b/tempodb/compactor.go index 79d205124c8..ddfbe235c47 100644 --- a/tempodb/compactor.go +++ b/tempodb/compactor.go @@ -27,11 +27,11 @@ var ( Help: "Records the amount of time to compact a set of blocks.", Buckets: prometheus.ExponentialBuckets(30, 2, 10), }, []string{"level"}) - metricCompactionObjectsProcessed = promauto.NewCounter(prometheus.CounterOpts{ + metricCompactionObjectsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "tempodb", Name: "compaction_objects_processed_total", Help: "Total number of objects processed during compaction.", - }) + }, []string{"level"}) metricCompactionErrors = promauto.NewCounter(prometheus.CounterOpts{ Namespace: "tempodb", Name: "compaction_errors_total", @@ -113,12 +113,13 @@ func (rw *readerWriter) compact(blockMetas []*encoding.BlockMeta, tenantID strin } compactionLevel := compactionLevelForBlocks(blockMetas) + compactionLevelLabel := strconv.Itoa(int(compactionLevel)) nextCompactionLevel := compactionLevel + 1 start := time.Now() defer func() { level.Info(rw.logger).Log("msg", "compaction complete") - metricCompactionDuration.WithLabelValues(strconv.Itoa(int(compactionLevel))).Observe(time.Since(start).Seconds()) + metricCompactionDuration.WithLabelValues(compactionLevelLabel).Observe(time.Since(start).Seconds()) }() var err error @@ -193,7 +194,7 @@ func (rw *readerWriter) compact(blockMetas []*encoding.BlockMeta, tenantID strin return err } lowestBookmark.clear() - metricCompactionObjectsProcessed.Inc() + metricCompactionObjectsProcessed.WithLabelValues(compactionLevelLabel).Inc() // write partial block if currentBlock.CurrentBufferLength() >= int(rw.compactorCfg.FlushSizeBytes) { From 2cad4c4f514699b668c040335153c1ad561bf2f9 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Fri, 20 Nov 2020 10:27:49 -0500 Subject: [PATCH 3/9] Add metric tempodb_compaction_blocks_total --- tempodb/compactor.go | 7 +++++++ tempodb/compactor_test.go | 19 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/tempodb/compactor.go b/tempodb/compactor.go index ddfbe235c47..78f6f98d29d 100644 --- a/tempodb/compactor.go +++ b/tempodb/compactor.go @@ -21,6 +21,11 @@ import ( ) var ( + metricCompactionBlocks = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "tempodb", + Name: "compaction_blocks_total", + Help: "Total number of blocks compacted.", + }, []string{"level"}) metricCompactionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ Namespace: "tempodb", Name: "compaction_duration_seconds", @@ -226,6 +231,8 @@ func (rw *readerWriter) compact(blockMetas []*encoding.BlockMeta, tenantID strin // mark old blocks compacted so they don't show up in polling markCompacted(rw, tenantID, blockMetas, newCompactedBlocks) + metricCompactionBlocks.WithLabelValues(compactionLevelLabel).Add(float64(len(blockMetas))) + return nil } diff --git a/tempodb/compactor_test.go b/tempodb/compactor_test.go index 35cb02b186e..9f854f58d11 100644 --- a/tempodb/compactor_test.go +++ b/tempodb/compactor_test.go @@ -124,7 +124,10 @@ func TestCompaction(t *testing.T) { rw.pollBlocklist() - processedStart, err := GetCounterValue(metricCompactionObjectsProcessed) + processedStart, err := GetCounterVecValue(metricCompactionObjectsProcessed, "0") + assert.NoError(t, err) + + blocksStart, err := GetCounterVecValue(metricCompactionBlocks, "0") assert.NoError(t, err) blocklist := rw.blocklist(testTenantID) @@ -158,10 +161,14 @@ func TestCompaction(t *testing.T) { assert.Equal(t, blockCount*recordCount, records) // Check metric - processedEnd, err := GetCounterValue(metricCompactionObjectsProcessed) + processedEnd, err := GetCounterVecValue(metricCompactionObjectsProcessed, "0") assert.NoError(t, err) assert.Equal(t, float64(blockCount*recordCount), processedEnd-processedStart) + blocksEnd, err := GetCounterVecValue(metricCompactionBlocks, "0") + assert.NoError(t, err) + assert.Equal(t, float64(blockCount), blocksEnd-blocksStart) + // now see if we can find our ids for i, id := range allIds { b, _, err := rw.Find(context.Background(), testTenantID, id) @@ -268,6 +275,14 @@ func GetCounterValue(metric prometheus.Counter) (float64, error) { return m.Counter.GetValue(), nil } +func GetCounterVecValue(metric *prometheus.CounterVec, label string) (float64, error) { + var m = &dto.Metric{} + if err := metric.WithLabelValues(label).Write(m); err != nil { + return 0, err + } + return m.Counter.GetValue(), nil +} + func TestCompactionUpdatesBlocklist(t *testing.T) { tempDir, err := ioutil.TempDir("/tmp", "") defer os.RemoveAll(tempDir) From c7b24c20da3914af9f55bd737d62272eaeac677f Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Fri, 20 Nov 2020 11:36:34 -0500 Subject: [PATCH 4/9] Add tempodb_compaction_blocks_total and _objects_processed_total to operational dashboard --- operations/tempo-mixin/tempo-operational.json | 190 ++++++++++++++++++ 1 file changed, 190 insertions(+) diff --git a/operations/tempo-mixin/tempo-operational.json b/operations/tempo-mixin/tempo-operational.json index 7378fc95611..127729951c7 100644 --- a/operations/tempo-mixin/tempo-operational.json +++ b/operations/tempo-mixin/tempo-operational.json @@ -4603,6 +4603,196 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 7, + "x": 7, + "y": 77 + }, + "hiddenSeries": false, + "id": 85, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempodb_compaction_objects_processed_total[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Objects Processed / s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$ds", + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 7, + "x": 14, + "y": 77 + }, + "hiddenSeries": false, + "id": 86, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.3.0-beta1", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(tempodb_compaction_blocks_total[5m])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Blocks Compacted", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } } ], "refresh": "30s", From 9a82020f2a18fb0d139939a4716b1aa3e7e844d0 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Fri, 20 Nov 2020 11:54:06 -0500 Subject: [PATCH 5/9] Update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 705af7f1bcd..1d8a9740f3e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,8 @@ * [CHANGE] Upgrade Cortex from v1.3.0 to v.1.4.0 [#341](https://github.com/grafana/tempo/pull/341) * [CHANGE] Compact more than 2 blocks at a time [#348](https://github.com/grafana/tempo/pull/348) * [ENHANCEMENT] Add tempodb_compaction_objects_combined metric. [#339](https://github.com/grafana/tempo/pull/339) +* [ENHANCEMENT] Add tempodb_compaction_objects_processed_total metric. [#360](https://github.com/grafana/tempo/pull/360) +* [ENHANCEMENT] Add tempodb_compaction_blocks_total metric. [#360](https://github.com/grafana/tempo/pull/360) * [BUGFIX] Frequent errors logged by compactor regarding meta not found [#327](https://github.com/grafana/tempo/pull/327) * [BUGFIX] Fix distributors panicking on rollout [#343](https://github.com/grafana/tempo/pull/343) From 47d7f9771a273c0818fbef627a9976a00fb97cca Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Fri, 20 Nov 2020 18:08:31 -0500 Subject: [PATCH 6/9] Add tempodb_compaction_bytes_written metric, update compaction_bytes/objects_written metrics when flushing for efficiency --- tempodb/compactor.go | 17 ++++-- tempodb/compactor_test.go | 88 +++++++++++++++++++++++------ tempodb/wal/compactor_block.go | 11 +++- tempodb/wal/compactor_block_test.go | 4 +- 4 files changed, 97 insertions(+), 23 deletions(-) diff --git a/tempodb/compactor.go b/tempodb/compactor.go index 78f6f98d29d..342ebcaa4f0 100644 --- a/tempodb/compactor.go +++ b/tempodb/compactor.go @@ -32,10 +32,15 @@ var ( Help: "Records the amount of time to compact a set of blocks.", Buckets: prometheus.ExponentialBuckets(30, 2, 10), }, []string{"level"}) - metricCompactionObjectsProcessed = promauto.NewCounterVec(prometheus.CounterOpts{ + metricCompactionObjectsWritten = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "tempodb", - Name: "compaction_objects_processed_total", - Help: "Total number of objects processed during compaction.", + Name: "compaction_objects_written", + Help: "Total number of objects written to backend during compaction.", + }, []string{"level"}) + metricCompactionBytesWritten = promauto.NewCounterVec(prometheus.CounterOpts{ + Namespace: "tempodb", + Name: "compaction_bytes_written", + Help: "Total number of bytes written to backend during compaction.", }, []string{"level"}) metricCompactionErrors = promauto.NewCounter(prometheus.CounterOpts{ Namespace: "tempodb", @@ -199,7 +204,6 @@ func (rw *readerWriter) compact(blockMetas []*encoding.BlockMeta, tenantID strin return err } lowestBookmark.clear() - metricCompactionObjectsProcessed.WithLabelValues(compactionLevelLabel).Inc() // write partial block if currentBlock.CurrentBufferLength() >= int(rw.compactorCfg.FlushSizeBytes) { @@ -241,6 +245,11 @@ func appendBlock(rw *readerWriter, tracker backend.AppendTracker, block *wal.Com if err != nil { return nil, err } + + compactionLevelLabel := strconv.Itoa(int(block.BlockMeta().CompactionLevel - 1)) + metricCompactionObjectsWritten.WithLabelValues(compactionLevelLabel).Add(float64(block.CurrentBufferedObjects())) + metricCompactionBytesWritten.WithLabelValues(compactionLevelLabel).Add(float64(block.CurrentBufferLength())) + block.ResetBuffer() return tracker, nil diff --git a/tempodb/compactor_test.go b/tempodb/compactor_test.go index 9f854f58d11..1c45265aabc 100644 --- a/tempodb/compactor_test.go +++ b/tempodb/compactor_test.go @@ -94,7 +94,7 @@ func TestCompaction(t *testing.T) { _, err = rand.Read(id) assert.NoError(t, err, "unexpected creating random id") - req := test.MakeRequest(i*10, id) + req := test.MakeRequest(10, id) reqs = append(reqs, req) ids = append(ids, id) @@ -124,12 +124,6 @@ func TestCompaction(t *testing.T) { rw.pollBlocklist() - processedStart, err := GetCounterVecValue(metricCompactionObjectsProcessed, "0") - assert.NoError(t, err) - - blocksStart, err := GetCounterVecValue(metricCompactionBlocks, "0") - assert.NoError(t, err) - blocklist := rw.blocklist(testTenantID) blockSelector := newTimeWindowBlockSelector(blocklist, rw.compactorCfg.MaxCompactionRange, 10000, defaultMinInputBlocks, 2) @@ -160,15 +154,6 @@ func TestCompaction(t *testing.T) { } assert.Equal(t, blockCount*recordCount, records) - // Check metric - processedEnd, err := GetCounterVecValue(metricCompactionObjectsProcessed, "0") - assert.NoError(t, err) - assert.Equal(t, float64(blockCount*recordCount), processedEnd-processedStart) - - blocksEnd, err := GetCounterVecValue(metricCompactionBlocks, "0") - assert.NoError(t, err) - assert.Equal(t, float64(blockCount), blocksEnd-blocksStart) - // now see if we can find our ids for i, id := range allIds { b, _, err := rw.Find(context.Background(), testTenantID, id) @@ -345,6 +330,77 @@ func TestCompactionUpdatesBlocklist(t *testing.T) { } } +func TestCompactionMetrics(t *testing.T) { + tempDir, err := ioutil.TempDir("/tmp", "") + defer os.RemoveAll(tempDir) + assert.NoError(t, err, "unexpected error creating temp dir") + + r, w, c, err := New(&Config{ + Backend: "local", + Pool: &pool.Config{ + MaxWorkers: 10, + QueueDepth: 100, + }, + Local: &local.Config{ + Path: path.Join(tempDir, "traces"), + }, + WAL: &wal.Config{ + Filepath: path.Join(tempDir, "wal"), + IndexDownsample: rand.Int()%20 + 1, + BloomFP: .01, + }, + BlocklistPoll: 0, + }, log.NewNopLogger()) + assert.NoError(t, err) + + c.EnableCompaction(&CompactorConfig{ + ChunkSizeBytes: 10, + MaxCompactionRange: 24 * time.Hour, + BlockRetention: 0, + CompactedBlockRetention: 0, + }, &mockSharder{}) + + // Cut x blocks with y records each + blockCount := 5 + recordCount := 1 + cutTestBlocks(t, w, blockCount, recordCount) + + rw := r.(*readerWriter) + rw.pollBlocklist() + + // Get starting metrics + processedStart, err := GetCounterVecValue(metricCompactionObjectsWritten, "0") + assert.NoError(t, err) + + blocksStart, err := GetCounterVecValue(metricCompactionBlocks, "0") + assert.NoError(t, err) + + bytesStart, err := GetCounterVecValue(metricCompactionBytesWritten, "0") + assert.NoError(t, err) + + // compact everything + err = rw.compact(rw.blocklist(testTenantID), testTenantID) + assert.NoError(t, err) + + // Check metric + processedEnd, err := GetCounterVecValue(metricCompactionObjectsWritten, "0") + assert.NoError(t, err) + assert.Equal(t, float64(blockCount*recordCount), processedEnd-processedStart) + + blocksEnd, err := GetCounterVecValue(metricCompactionBlocks, "0") + assert.NoError(t, err) + assert.Equal(t, float64(blockCount), blocksEnd-blocksStart) + + bytesEnd, err := GetCounterVecValue(metricCompactionBytesWritten, "0") + assert.NoError(t, err) + bytesPerRecord := + 4 /* total length */ + + 4 /* id length */ + + 16 /* id */ + + 3 /* test record length */ + assert.Equal(t, float64(blockCount*recordCount*bytesPerRecord), bytesEnd-bytesStart) +} + func cutTestBlocks(t *testing.T, w Writer, blockCount int, recordCount int) { wal := w.WAL() for i := 0; i < blockCount; i++ { diff --git a/tempodb/wal/compactor_block.go b/tempodb/wal/compactor_block.go index bf0d9a02963..ef8a211c713 100644 --- a/tempodb/wal/compactor_block.go +++ b/tempodb/wal/compactor_block.go @@ -17,8 +17,9 @@ type CompactorBlock struct { bloom *bloom.ShardedBloomFilter - appendBuffer *bytes.Buffer - appender encoding.Appender + bufferedObjects int + appendBuffer *bytes.Buffer + appender encoding.Appender } func newCompactorBlock(id uuid.UUID, tenantID string, bloomFP float64, indexDownsample int, metas []*encoding.BlockMeta, filepath string, estimatedObjects int) (*CompactorBlock, error) { @@ -56,6 +57,7 @@ func (c *CompactorBlock) Write(id encoding.ID, object []byte) error { if err != nil { return err } + c.bufferedObjects++ c.meta.ObjectAdded(id) c.bloom.Add(id) return nil @@ -69,8 +71,13 @@ func (c *CompactorBlock) CurrentBufferLength() int { return c.appendBuffer.Len() } +func (c *CompactorBlock) CurrentBufferedObjects() int { + return c.bufferedObjects +} + func (c *CompactorBlock) ResetBuffer() { c.appendBuffer.Reset() + c.bufferedObjects = 0 } func (c *CompactorBlock) Length() int { diff --git a/tempodb/wal/compactor_block_test.go b/tempodb/wal/compactor_block_test.go index c34aa4f1da0..b3b37cc35a0 100644 --- a/tempodb/wal/compactor_block_test.go +++ b/tempodb/wal/compactor_block_test.go @@ -95,5 +95,7 @@ func TestCompactorBlockWrite(t *testing.T) { } records := cb.Records() - assert.Equal(t, math.Ceil(float64(numObjects)/3), float64(len(records))) + assert.Equal(t, math.Ceil(float64(numObjects)/float64(walCfg.IndexDownsample)), float64(len(records))) + + assert.Equal(t, numObjects, cb.CurrentBufferedObjects()) } From a08af5a6988d10424ca4abd3c9ab42c26f5c798b Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 23 Nov 2020 09:32:04 -0500 Subject: [PATCH 7/9] Update operational dashboard for bytes/objects written metrics, resize graphs --- operations/tempo-mixin/tempo-operational.json | 116 ++++++++++++++++-- 1 file changed, 106 insertions(+), 10 deletions(-) diff --git a/operations/tempo-mixin/tempo-operational.json b/operations/tempo-mixin/tempo-operational.json index 127729951c7..8eb8061a92a 100644 --- a/operations/tempo-mixin/tempo-operational.json +++ b/operations/tempo-mixin/tempo-operational.json @@ -4522,8 +4522,8 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, - "w": 7, + "h": 6, + "w": 4, "x": 0, "y": 77 }, @@ -4610,6 +4610,7 @@ "dashLength": 10, "dashes": false, "datasource": "$ds", + "description": "", "fieldConfig": { "defaults": { "custom": {} @@ -4619,9 +4620,9 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, - "w": 7, - "x": 7, + "h": 6, + "w": 4, + "x": 4, "y": 77 }, "hiddenSeries": false, @@ -4652,7 +4653,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempodb_compaction_objects_processed_total[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_objects_written{job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "refId": "A" @@ -4662,7 +4663,7 @@ "timeFrom": null, "timeRegions": [], "timeShift": null, - "title": "Objects Processed / s", + "title": "Objects Written / s", "tooltip": { "shared": true, "sort": 0, @@ -4704,6 +4705,101 @@ "bars": false, "dashLength": 10, "dashes": false, + "datasource": null, + "fieldConfig": { + "defaults": { + "custom": {} + }, + "overrides": [] + }, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 6, + "w": 4, + "x": 8, + "y": 77 + }, + "hiddenSeries": false, + "id": 88, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "nullPointMode": "null", + "options": { + "alertThreshold": true + }, + "percentage": false, + "pluginVersion": "7.4.0-7095pre", + "pointradius": 2, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(tempodb_compaction_bytes_written{job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "interval": "", + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Bytes Written / s", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, "datasource": "$ds", "fieldConfig": { "defaults": { @@ -4714,9 +4810,9 @@ "fill": 1, "fillGradient": 0, "gridPos": { - "h": 8, - "w": 7, - "x": 14, + "h": 6, + "w": 4, + "x": 12, "y": 77 }, "hiddenSeries": false, From a64e4648a8ea91a66d62dd0856bec9d9ea572696 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 23 Nov 2020 09:43:19 -0500 Subject: [PATCH 8/9] Respect cluster and namespace variables in compactor panels --- operations/tempo-mixin/tempo-operational.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/operations/tempo-mixin/tempo-operational.json b/operations/tempo-mixin/tempo-operational.json index 8eb8061a92a..2558e4026d5 100644 --- a/operations/tempo-mixin/tempo-operational.json +++ b/operations/tempo-mixin/tempo-operational.json @@ -4555,7 +4555,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(tempodb_compaction_objects_combined_total{job=~\"$namespace/compactor\"}[$__rate_interval])", + "expr": "increase(tempodb_compaction_objects_combined_total{cluster=\"$cluster\",job=~\"$namespace/compactor\"}[$__rate_interval])", "interval": "", "legendFormat": "", "refId": "A" @@ -4653,7 +4653,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempodb_compaction_objects_written{job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_objects_written{cluster=\"$cluster\",job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "refId": "A" @@ -4748,7 +4748,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(tempodb_compaction_bytes_written{job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", + "expr": "sum(rate(tempodb_compaction_bytes_written{cluster=\"$cluster\",job=\"$namespace/compactor\"}[$__rate_interval])) by (level)", "interval": "", "legendFormat": "", "refId": "A" @@ -4843,7 +4843,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(tempodb_compaction_blocks_total[5m])) by (level)", + "expr": "sum(increase(tempodb_compaction_blocks_total{cluster=\"$cluster\",job=\"$namespace/compactor\"}[5m])) by (level)", "interval": "", "legendFormat": "", "refId": "A" From 096d205fb6af729658b8344f497aa2e18e535b61 Mon Sep 17 00:00:00 2001 From: Martin Disibio Date: Mon, 23 Nov 2020 10:00:08 -0500 Subject: [PATCH 9/9] Remove tempodb_compaction_duration_seconds metric --- operations/tempo-mixin/tempo-operational.json | 111 +----------------- tempodb/compactor.go | 8 -- 2 files changed, 1 insertion(+), 118 deletions(-) diff --git a/operations/tempo-mixin/tempo-operational.json b/operations/tempo-mixin/tempo-operational.json index 2558e4026d5..64c771d1e31 100644 --- a/operations/tempo-mixin/tempo-operational.json +++ b/operations/tempo-mixin/tempo-operational.json @@ -1353,115 +1353,6 @@ "y": 7 }, "hiddenSeries": false, - "id": 52, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": false, - "linewidth": 1, - "nullPointMode": "connected", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "7.2.1", - "pointradius": 2, - "points": true, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [ - { - "expr": "histogram_quantile(.99, sum(rate(tempodb_compaction_duration_seconds_bucket{job=~\"$namespace/compactor\"}[$__rate_interval])) by (le,level))", - "interval": "", - "legendFormat": ".99-{{level}}", - "refId": "A" - }, - { - "expr": "histogram_quantile(.9, sum(rate(tempodb_compaction_duration_seconds_bucket{job=~\"$namespace/compactor\"}[$__rate_interval])) by (le,level))", - "hide": true, - "interval": "", - "legendFormat": ".9-{{level}}", - "refId": "B" - }, - { - "expr": "histogram_quantile(.5, sum(rate(tempodb_compaction_duration_seconds_bucket{job=~\"$namespace/compactor\"}[$__rate_interval])) by (le,level))", - "hide": true, - "interval": "", - "legendFormat": ".5-{{level}}", - "refId": "C" - } - ], - "thresholds": [], - "timeFrom": null, - "timeRegions": [], - "timeShift": null, - "title": "compaction duration", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - } - ], - "yaxis": { - "align": false, - "alignLevel": null - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$ds", - "fieldConfig": { - "defaults": { - "custom": {} - }, - "overrides": [] - }, - "fill": 1, - "fillGradient": 0, - "gridPos": { - "h": 5, - "w": 3, - "x": 18, - "y": 7 - }, - "hiddenSeries": false, "id": 53, "legend": { "avg": false, @@ -1559,7 +1450,7 @@ "gridPos": { "h": 5, "w": 3, - "x": 21, + "x": 18, "y": 7 }, "hiddenSeries": false, diff --git a/tempodb/compactor.go b/tempodb/compactor.go index 342ebcaa4f0..814f4244257 100644 --- a/tempodb/compactor.go +++ b/tempodb/compactor.go @@ -26,12 +26,6 @@ var ( Name: "compaction_blocks_total", Help: "Total number of blocks compacted.", }, []string{"level"}) - metricCompactionDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ - Namespace: "tempodb", - Name: "compaction_duration_seconds", - Help: "Records the amount of time to compact a set of blocks.", - Buckets: prometheus.ExponentialBuckets(30, 2, 10), - }, []string{"level"}) metricCompactionObjectsWritten = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "tempodb", Name: "compaction_objects_written", @@ -126,10 +120,8 @@ func (rw *readerWriter) compact(blockMetas []*encoding.BlockMeta, tenantID strin compactionLevelLabel := strconv.Itoa(int(compactionLevel)) nextCompactionLevel := compactionLevel + 1 - start := time.Now() defer func() { level.Info(rw.logger).Log("msg", "compaction complete") - metricCompactionDuration.WithLabelValues(compactionLevelLabel).Observe(time.Since(start).Seconds()) }() var err error