Adding Blocklist backend metrics

CHANGELOG.md

@@ -30,6 +30,7 @@ Additionally, default label `span_status` is renamed to `status_code`.
 * [BUGFIX] Update tempo microservices Helm values example which missed the 'enabled' key for thriftHttp. [#1472](https://github.com/grafana/tempo/pull/1472) (@hajowieland)
 * [BUGFIX] Fix race condition in forwarder overrides loop. [1468](https://github.com/grafana/tempo/pull/1468) (@mapno)
 * [ENHANCEMENT] Add a config to query single ingester instance based on trace id hash for Trace By ID API. (1484)[https://github.com/grafana/tempo/pull/1484] (@sagarwala, @bikashmishra100, @ashwinidulams)
+* [ENHANCEMENT] Add blocklist metrics for total backend objects and total backend bytes [#1519](https://github.com/grafana/tempo/pull/1519) (@ie-pham)
 
 ## v1.4.1 / 2022-05-05
 

tempodb/blocklist/poller.go

@@ -17,7 +17,22 @@ import (
 	"go.uber.org/atomic"
 )
 
+const (
+	blockStatusLiveLabel      = "live"
+	blockStatusCompactedLabel = "compacted"
+)
+
 var (
+	metricBackendObjects = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "tempodb",
+		Name:      "backend_objects_total",
+		Help:      "Total number of objects (traces) in the backend",
+	}, []string{"tenant", "status"})
+	metricBackendBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
+		Namespace: "tempodb",
+		Name:      "backend_bytes_total",
+		Help:      "Total number of bytes in the backend.",
+	}, []string{"tenant", "status"})
 	metricBlocklistErrors = promauto.NewCounterVec(prometheus.CounterOpts{
 		Namespace: "tempodb",
 		Name:      "blocklist_poll_errors_total",
@@ -126,6 +141,12 @@ func (p *Poller) Do() (PerTenant, PerTenantCompacted, error) {
 
 		blocklist[tenantID] = newBlockList
 		compactedBlocklist[tenantID] = newCompactedBlockList
+
+		backendMetaMetrics := sumTotalBackendMetaMetrics(newBlockList, newCompactedBlockList)
+		metricBackendObjects.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalObjects))
+		metricBackendObjects.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalObjects))
+		metricBackendBytes.WithLabelValues(tenantID, blockStatusLiveLabel).Set(float64(backendMetaMetrics.blockMetaTotalBytes))
+		metricBackendBytes.WithLabelValues(tenantID, blockStatusCompactedLabel).Set(float64(backendMetaMetrics.compactedBlockMetaTotalBytes))
 	}
 
 	return blocklist, compactedBlocklist, nil
@@ -275,3 +296,34 @@ func (p *Poller) tenantIndexPollError(idx *backend.TenantIndex, err error) error
 
 	return nil
 }
+
+type backendMetaMetrics struct {
+	blockMetaTotalObjects          int
+	compactedBlockMetaTotalObjects int
+	blockMetaTotalBytes            uint64
+	compactedBlockMetaTotalBytes   uint64
+}
+
+func sumTotalBackendMetaMetrics(blockMeta []*backend.BlockMeta, compactedBlockMeta []*backend.CompactedBlockMeta) backendMetaMetrics {
+	var sumTotalObjectsBM int
+	var sumTotalObjectsCBM int
+	var sumTotalBytesBM uint64
+	var sumTotalBytesCBM uint64
+
+	for _, bm := range blockMeta {
+		sumTotalObjectsBM += bm.TotalObjects
+		sumTotalBytesBM += bm.Size
+	}
+
+	for _, cbm := range compactedBlockMeta {
+		sumTotalObjectsCBM += cbm.TotalObjects
+		sumTotalBytesCBM += cbm.Size
+	}
+
+	return backendMetaMetrics{
+		blockMetaTotalObjects:          sumTotalObjectsBM,
+		compactedBlockMetaTotalObjects: sumTotalObjectsCBM,
+		blockMetaTotalBytes:            sumTotalBytesBM,
+		compactedBlockMetaTotalBytes:   sumTotalBytesCBM,
+	}
+}

tempodb/blocklist/poller_test.go

@@ -381,6 +381,122 @@ func TestTenantIndexPollError(t *testing.T) {
 	}, nil))
 }
 
+func TestBlockListBackendMetrics(t *testing.T) {
+	tests := []struct {
+		name                                 string
+		list                                 PerTenant
+		compactedList                        PerTenantCompacted
+		testType                             string
+		expectedBackendObjectsTotal          int
+		expectedBackendBytesTotal            uint64
+		expectedCompactedBackendObjectsTotal int
+		expectedCompacteddBackendBytesTotal  uint64
+	}{
+		{
+			name: "total backend objects calculation is correct",
+			list: PerTenant{
+				"test": []*backend.BlockMeta{
+					{
+						TotalObjects: 10,
+					},
+					{
+						TotalObjects: 7,
+					},
+					{
+						TotalObjects: 8,
+					},
+				},
+			},
+			compactedList: PerTenantCompacted{
+				"test": []*backend.CompactedBlockMeta{
+					{
+						BlockMeta: backend.BlockMeta{
+							TotalObjects: 7,
+						},
+					},
+					{
+						BlockMeta: backend.BlockMeta{
+							TotalObjects: 8,
+						},
+					},
+					{
+						BlockMeta: backend.BlockMeta{
+							TotalObjects: 5,
+						},
+					},
+					{
+						BlockMeta: backend.BlockMeta{
+							TotalObjects: 15,
+						},
+					},
+				},
+			},
+			expectedBackendObjectsTotal:          25,
+			expectedBackendBytesTotal:            0,
+			expectedCompactedBackendObjectsTotal: 35,
+			expectedCompacteddBackendBytesTotal:  0,
+			testType:                             "backend objects",
+		},
+		{
+			name: "total backend bytes calculation is correct",
+			list: PerTenant{
+				"test": []*backend.BlockMeta{
+					{
+						Size: 250,
+					},
+					{
+						Size: 500,
+					},
+					{
+						Size: 250,
+					},
+				},
+			},
+			compactedList: PerTenantCompacted{
+				"test": []*backend.CompactedBlockMeta{
+					{
+						BlockMeta: backend.BlockMeta{
+							Size: 300,
+						},
+					},
+					{
+						BlockMeta: backend.BlockMeta{
+							Size: 200,
+						},
+					},
+					{
+						BlockMeta: backend.BlockMeta{
+							Size: 250,
+						},
+					},
+					{
+						BlockMeta: backend.BlockMeta{
+							Size: 500,
+						},
+					},
+				},
+			},
+			expectedBackendObjectsTotal:          0,
+			expectedBackendBytesTotal:            1000,
+			expectedCompactedBackendObjectsTotal: 0,
+			expectedCompacteddBackendBytesTotal:  1250,
+			testType:                             "backend bytes",
+		},
+	}
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			newBlockList := tc.list["test"]
+			newCompactedBlockList := tc.compactedList["test"]
+			backendMetaMetrics := sumTotalBackendMetaMetrics(newBlockList, newCompactedBlockList)
+			assert.Equal(t, tc.expectedBackendObjectsTotal, backendMetaMetrics.blockMetaTotalObjects)
+			assert.Equal(t, tc.expectedCompactedBackendObjectsTotal, backendMetaMetrics.compactedBlockMetaTotalObjects)
+			assert.Equal(t, tc.expectedBackendBytesTotal, backendMetaMetrics.blockMetaTotalBytes)
+			assert.Equal(t, tc.expectedCompacteddBackendBytesTotal, backendMetaMetrics.compactedBlockMetaTotalBytes)
+		})
+	}
+
+}
+
 func newMockCompactor(list PerTenantCompacted, expectsError bool) backend.Compactor {
 	return &backend.MockCompactor{
 		BlockMetaFn: func(blockID uuid.UUID, tenantID string) (*backend.CompactedBlockMeta, error) {