From 3a136ff844f24da5817b40520e17a08b579ea688 Mon Sep 17 00:00:00 2001 From: Jennie Pham Date: Tue, 28 Jun 2022 10:06:02 -0500 Subject: [PATCH 1/3] Adding Blocklist backend metrics --- CHANGELOG.md | 1 + tempodb/blocklist/poller.go | 40 ++++++++++ tempodb/blocklist/poller_test.go | 128 +++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 039ceab2582..a8a85ed1585 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,7 @@ Additionally, default label `span_status` is renamed to `status_code`. * [BUGFIX] Update tempo microservices Helm values example which missed the 'enabled' key for thriftHttp. [#1472](https://github.com/grafana/tempo/pull/1472) (@hajowieland) * [BUGFIX] Fix race condition in forwarder overrides loop. [1468](https://github.com/grafana/tempo/pull/1468) (@mapno) * [ENHANCEMENT] Add a config to query single ingester instance based on trace id hash for Trace By ID API. (1484)[https://github.com/grafana/tempo/pull/1484] (@sagarwala, @bikashmishra100, @ashwinidulams) +* [ENHANCEMENT] Add blocklist metrics for total backend objects and total backend bytes [#1519](https://github.com/grafana/tempo/pull/1519) (@ie-pham) ## v1.4.1 / 2022-05-05 diff --git a/tempodb/blocklist/poller.go b/tempodb/blocklist/poller.go index 31da9eadee6..4b2bb4dfdee 100644 --- a/tempodb/blocklist/poller.go +++ b/tempodb/blocklist/poller.go @@ -17,7 +17,22 @@ import ( "go.uber.org/atomic" ) +const ( + blockStatusLiveLabel = "live" + blockStatusCompactedLabel = "compacted" +) + var ( + metricBackendObjects = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "tempodb", + Name: "backend_objects_total", + Help: "Total number of objects (traces) in the backend", + }, []string{"tenant", "status"}) + metricBackendBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: "tempodb", + Name: "backend_bytes_total", + Help: "Total number of bytes in the backend.", + }, []string{"tenant", "status"}) metricBlocklistErrors = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "tempodb", Name: "blocklist_poll_errors_total", @@ -126,6 +141,12 @@ func (p *Poller) Do() (PerTenant, PerTenantCompacted, error) { blocklist[tenantID] = newBlockList compactedBlocklist[tenantID] = newCompactedBlockList + + totalObjectsBlockMeta, totalObjectsCompactedBlockMeta, totalBytesBlockMeta, totalBytesCompactedBlockMeta := p.sumTotalBackendMetas(newBlockList, newCompactedBlockList) + metricBackendObjects.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(totalObjectsBlockMeta)) + metricBackendObjects.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(totalObjectsCompactedBlockMeta)) + metricBackendBytes.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(totalBytesBlockMeta)) + metricBackendBytes.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(totalBytesCompactedBlockMeta)) } return blocklist, compactedBlocklist, nil @@ -275,3 +296,22 @@ func (p *Poller) tenantIndexPollError(idx *backend.TenantIndex, err error) error return nil } + +func (p *Poller) sumTotalBackendMetas(blockMeta []*backend.BlockMeta, compactedBlockMeta []*backend.CompactedBlockMeta) (int, int, uint64, uint64) { + var sumTotalObjectsBM int + var sumTotalObjectsCBM int + var sumTotalBytesBM uint64 + var sumTotalBytesCBM uint64 + + for _, bm := range blockMeta { + sumTotalObjectsBM += bm.TotalObjects + sumTotalBytesBM += bm.Size + } + + for _, cbm := range compactedBlockMeta { + sumTotalObjectsCBM += cbm.TotalObjects + sumTotalBytesCBM += cbm.Size + } + + return sumTotalObjectsBM, sumTotalObjectsCBM, sumTotalBytesBM, sumTotalBytesCBM +} diff --git a/tempodb/blocklist/poller_test.go b/tempodb/blocklist/poller_test.go index 2f8929aab44..e6a26f594e8 100644 --- a/tempodb/blocklist/poller_test.go +++ b/tempodb/blocklist/poller_test.go @@ -381,6 +381,134 @@ func TestTenantIndexPollError(t *testing.T) { }, nil)) } +func TestBlockListBackendMetrics(t *testing.T) { + tests := []struct { + name string + list PerTenant + compactedList PerTenantCompacted + testType string + expectedBackendObjectsTotal int + expectedBackendBytesTotal uint64 + expectedCompactedBackendObjectsTotal int + expectedCompacteddBackendByteTotal uint64 + }{ + { + name: "total backend objects calculation is correct", + list: PerTenant{ + "test": []*backend.BlockMeta{ + { + TotalObjects: 10, + }, + { + TotalObjects: 7, + }, + { + TotalObjects: 8, + }, + }, + }, + compactedList: PerTenantCompacted{ + "test": []*backend.CompactedBlockMeta{ + { + BlockMeta: backend.BlockMeta{ + TotalObjects: 7, + }, + }, + { + BlockMeta: backend.BlockMeta{ + TotalObjects: 8, + }, + }, + { + BlockMeta: backend.BlockMeta{ + TotalObjects: 5, + }, + }, + { + BlockMeta: backend.BlockMeta{ + TotalObjects: 15, + }, + }, + }, + }, + expectedBackendObjectsTotal: 25, + expectedBackendBytesTotal: 0, + expectedCompactedBackendObjectsTotal: 35, + expectedCompacteddBackendByteTotal: 0, + testType: "backend objects", + }, + { + name: "total backend bytes calculation is correct", + list: PerTenant{ + "test": []*backend.BlockMeta{ + { + Size: 250, + }, + { + Size: 500, + }, + { + Size: 250, + }, + }, + }, + compactedList: PerTenantCompacted{ + "test": []*backend.CompactedBlockMeta{ + { + BlockMeta: backend.BlockMeta{ + Size: 300, + }, + }, + { + BlockMeta: backend.BlockMeta{ + Size: 200, + }, + }, + { + BlockMeta: backend.BlockMeta{ + Size: 250, + }, + }, + { + BlockMeta: backend.BlockMeta{ + Size: 500, + }, + }, + }, + }, + expectedBackendObjectsTotal: 0, + expectedBackendBytesTotal: 1000, + expectedCompactedBackendObjectsTotal: 0, + expectedCompacteddBackendByteTotal: 1250, + testType: "backend bytes", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + c := newMockCompactor(tc.compactedList, false) + r := newMockReader(tc.list, tc.compactedList, false) + w := &backend.MockWriter{} + + poller := NewPoller(&PollerConfig{ + PollConcurrency: testPollConcurrency, + PollFallback: testPollFallback, + TenantIndexBuilders: testBuilders, + }, &mockJobSharder{ + owns: true, + }, r, c, w, log.NewNopLogger()) + + newBlockList := tc.list["test"] + newCompactedBlockList := tc.compactedList["test"] + totalObjectsBlockMeta, totalObjectsCompactedBlockMeta, totalBytesBlockMeta, totalBytesCompactedBlockMeta := poller.sumTotalBackendMetas(newBlockList, newCompactedBlockList) + assert.Equal(t, tc.expectedBackendObjectsTotal, totalObjectsBlockMeta) + assert.Equal(t, tc.expectedCompactedBackendObjectsTotal, totalObjectsCompactedBlockMeta) + assert.Equal(t, tc.expectedBackendBytesTotal, totalBytesBlockMeta) + assert.Equal(t, tc.expectedCompacteddBackendByteTotal, totalBytesCompactedBlockMeta) + }) + } + +} + func newMockCompactor(list PerTenantCompacted, expectsError bool) backend.Compactor { return &backend.MockCompactor{ BlockMetaFn: func(blockID uuid.UUID, tenantID string) (*backend.CompactedBlockMeta, error) { From 246d49571f0094b0da90cb80a2f2ba8f2d3df585 Mon Sep 17 00:00:00 2001 From: Jennie Pham Date: Thu, 30 Jun 2022 13:31:15 -0500 Subject: [PATCH 2/3] refactoring function to return struct --- tempodb/blocklist/poller.go | 26 +++++++++++++++++++------- tempodb/blocklist/poller_test.go | 26 +++++++------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/tempodb/blocklist/poller.go b/tempodb/blocklist/poller.go index 4b2bb4dfdee..9322901b55c 100644 --- a/tempodb/blocklist/poller.go +++ b/tempodb/blocklist/poller.go @@ -142,11 +142,11 @@ func (p *Poller) Do() (PerTenant, PerTenantCompacted, error) { blocklist[tenantID] = newBlockList compactedBlocklist[tenantID] = newCompactedBlockList - totalObjectsBlockMeta, totalObjectsCompactedBlockMeta, totalBytesBlockMeta, totalBytesCompactedBlockMeta := p.sumTotalBackendMetas(newBlockList, newCompactedBlockList) - metricBackendObjects.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(totalObjectsBlockMeta)) - metricBackendObjects.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(totalObjectsCompactedBlockMeta)) - metricBackendBytes.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(totalBytesBlockMeta)) - metricBackendBytes.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(totalBytesCompactedBlockMeta)) + backendMetaMetrics := sumTotalBackendMetaMetrics(newBlockList, newCompactedBlockList) + metricBackendObjects.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalObjects)) + metricBackendObjects.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalObjects)) + metricBackendBytes.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalBytes)) + metricBackendBytes.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalBytes)) } return blocklist, compactedBlocklist, nil @@ -297,7 +297,14 @@ func (p *Poller) tenantIndexPollError(idx *backend.TenantIndex, err error) error return nil } -func (p *Poller) sumTotalBackendMetas(blockMeta []*backend.BlockMeta, compactedBlockMeta []*backend.CompactedBlockMeta) (int, int, uint64, uint64) { +type backendMetaMetrics struct { + blockMetaTotalObjects int + compactedBlockMetaTotalObjects int + blockMetaTotalBytes int + compactedBlockMetaTotalBytes int +} + +func sumTotalBackendMetaMetrics(blockMeta []*backend.BlockMeta, compactedBlockMeta []*backend.CompactedBlockMeta) backendMetaMetrics { var sumTotalObjectsBM int var sumTotalObjectsCBM int var sumTotalBytesBM uint64 @@ -313,5 +320,10 @@ func (p *Poller) sumTotalBackendMetas(blockMeta []*backend.BlockMeta, compactedB sumTotalBytesCBM += cbm.Size } - return sumTotalObjectsBM, sumTotalObjectsCBM, sumTotalBytesBM, sumTotalBytesCBM + return backendMetaMetrics{ + blockMetaTotalObjects: sumTotalObjectsBM, + compactedBlockMetaTotalObjects: sumTotalObjectsCBM, + blockMetaTotalBytes: int(sumTotalBytesBM), + compactedBlockMetaTotalBytes: int(sumTotalBytesCBM), + } } diff --git a/tempodb/blocklist/poller_test.go b/tempodb/blocklist/poller_test.go index e6a26f594e8..f4a9faa4eca 100644 --- a/tempodb/blocklist/poller_test.go +++ b/tempodb/blocklist/poller_test.go @@ -388,9 +388,9 @@ func TestBlockListBackendMetrics(t *testing.T) { compactedList PerTenantCompacted testType string expectedBackendObjectsTotal int - expectedBackendBytesTotal uint64 + expectedBackendBytesTotal int expectedCompactedBackendObjectsTotal int - expectedCompacteddBackendByteTotal uint64 + expectedCompacteddBackendByteTotal int }{ { name: "total backend objects calculation is correct", @@ -485,25 +485,13 @@ func TestBlockListBackendMetrics(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - c := newMockCompactor(tc.compactedList, false) - r := newMockReader(tc.list, tc.compactedList, false) - w := &backend.MockWriter{} - - poller := NewPoller(&PollerConfig{ - PollConcurrency: testPollConcurrency, - PollFallback: testPollFallback, - TenantIndexBuilders: testBuilders, - }, &mockJobSharder{ - owns: true, - }, r, c, w, log.NewNopLogger()) - newBlockList := tc.list["test"] newCompactedBlockList := tc.compactedList["test"] - totalObjectsBlockMeta, totalObjectsCompactedBlockMeta, totalBytesBlockMeta, totalBytesCompactedBlockMeta := poller.sumTotalBackendMetas(newBlockList, newCompactedBlockList) - assert.Equal(t, tc.expectedBackendObjectsTotal, totalObjectsBlockMeta) - assert.Equal(t, tc.expectedCompactedBackendObjectsTotal, totalObjectsCompactedBlockMeta) - assert.Equal(t, tc.expectedBackendBytesTotal, totalBytesBlockMeta) - assert.Equal(t, tc.expectedCompacteddBackendByteTotal, totalBytesCompactedBlockMeta) + backendMetaMetrics := sumTotalBackendMetaMetrics(newBlockList, newCompactedBlockList) + assert.Equal(t, tc.expectedBackendObjectsTotal, backendMetaMetrics.blockMetaTotalObjects) + assert.Equal(t, tc.expectedCompactedBackendObjectsTotal, backendMetaMetrics.compactedBlockMetaTotalObjects) + assert.Equal(t, tc.expectedBackendBytesTotal, backendMetaMetrics.blockMetaTotalBytes) + assert.Equal(t, tc.expectedCompacteddBackendByteTotal, backendMetaMetrics.compactedBlockMetaTotalBytes) }) } From c1e024f84c56d6cde581d53413124459af7b9255 Mon Sep 17 00:00:00 2001 From: Jennie Pham Date: Thu, 30 Jun 2022 14:36:23 -0500 Subject: [PATCH 3/3] changing total bytes variables back to uint64 --- tempodb/blocklist/poller.go | 8 ++++---- tempodb/blocklist/poller_test.go | 10 +++++----- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tempodb/blocklist/poller.go b/tempodb/blocklist/poller.go index 9322901b55c..30232605093 100644 --- a/tempodb/blocklist/poller.go +++ b/tempodb/blocklist/poller.go @@ -300,8 +300,8 @@ func (p *Poller) tenantIndexPollError(idx *backend.TenantIndex, err error) error type backendMetaMetrics struct { blockMetaTotalObjects int compactedBlockMetaTotalObjects int - blockMetaTotalBytes int - compactedBlockMetaTotalBytes int + blockMetaTotalBytes uint64 + compactedBlockMetaTotalBytes uint64 } func sumTotalBackendMetaMetrics(blockMeta []*backend.BlockMeta, compactedBlockMeta []*backend.CompactedBlockMeta) backendMetaMetrics { @@ -323,7 +323,7 @@ func sumTotalBackendMetaMetrics(blockMeta []*backend.BlockMeta, compactedBlockMe return backendMetaMetrics{ blockMetaTotalObjects: sumTotalObjectsBM, compactedBlockMetaTotalObjects: sumTotalObjectsCBM, - blockMetaTotalBytes: int(sumTotalBytesBM), - compactedBlockMetaTotalBytes: int(sumTotalBytesCBM), + blockMetaTotalBytes: sumTotalBytesBM, + compactedBlockMetaTotalBytes: sumTotalBytesCBM, } } diff --git a/tempodb/blocklist/poller_test.go b/tempodb/blocklist/poller_test.go index f4a9faa4eca..4a4eee887f0 100644 --- a/tempodb/blocklist/poller_test.go +++ b/tempodb/blocklist/poller_test.go @@ -388,9 +388,9 @@ func TestBlockListBackendMetrics(t *testing.T) { compactedList PerTenantCompacted testType string expectedBackendObjectsTotal int - expectedBackendBytesTotal int + expectedBackendBytesTotal uint64 expectedCompactedBackendObjectsTotal int - expectedCompacteddBackendByteTotal int + expectedCompacteddBackendBytesTotal uint64 }{ { name: "total backend objects calculation is correct", @@ -434,7 +434,7 @@ func TestBlockListBackendMetrics(t *testing.T) { expectedBackendObjectsTotal: 25, expectedBackendBytesTotal: 0, expectedCompactedBackendObjectsTotal: 35, - expectedCompacteddBackendByteTotal: 0, + expectedCompacteddBackendBytesTotal: 0, testType: "backend objects", }, { @@ -479,7 +479,7 @@ func TestBlockListBackendMetrics(t *testing.T) { expectedBackendObjectsTotal: 0, expectedBackendBytesTotal: 1000, expectedCompactedBackendObjectsTotal: 0, - expectedCompacteddBackendByteTotal: 1250, + expectedCompacteddBackendBytesTotal: 1250, testType: "backend bytes", }, } @@ -491,7 +491,7 @@ func TestBlockListBackendMetrics(t *testing.T) { assert.Equal(t, tc.expectedBackendObjectsTotal, backendMetaMetrics.blockMetaTotalObjects) assert.Equal(t, tc.expectedCompactedBackendObjectsTotal, backendMetaMetrics.compactedBlockMetaTotalObjects) assert.Equal(t, tc.expectedBackendBytesTotal, backendMetaMetrics.blockMetaTotalBytes) - assert.Equal(t, tc.expectedCompacteddBackendByteTotal, backendMetaMetrics.compactedBlockMetaTotalBytes) + assert.Equal(t, tc.expectedCompacteddBackendBytesTotal, backendMetaMetrics.compactedBlockMetaTotalBytes) }) }