feat: Improved data storage metrics (#10020)

This PR increases the level of telemetry around all of the node's data
stores. For every LMDB instance it reports the size of the specified
mapping, the actual DB size and the number of items. Additionally, for
the world state we report the number of leaves for every tree along with
the pending and proven chain heights.

---------

Co-authored-by: ludamad <adam.domurad@gmail.com>

barretenberg/cpp/src/barretenberg/crypto/merkle_tree/lmdb_store/lmdb_tree_store.test.cpp

@@ -254,4 +254,4 @@ TEST_F(LMDBTreeStoreTest, can_write_and_read_leaves_by_hash)
         success = store.read_leaf_by_hash(VALUES[9], readBack, *transaction);
         EXPECT_FALSE(success);
     }
-}
+}

yarn-project/archiver/src/archiver/archiver.ts

@@ -272,6 +272,9 @@ export class Archiver implements ArchiveSource {
       // the chain locally before we start unwinding stuff. This can be optimized by figuring out
       // up to which point we're pruning, and then requesting L2 blocks up to that point only.
       await this.handleEpochPrune(provenBlockNumber, currentL1BlockNumber);
+
+      const storeSizes = this.store.estimateSize();
+      this.instrumentation.recordDBMetrics(storeSizes);
     }
   }
 
@@ -1040,6 +1043,9 @@ class ArchiverStoreHelper
   getTotalL1ToL2MessageCount(): Promise<bigint> {
     return this.store.getTotalL1ToL2MessageCount();
   }
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+    return this.store.estimateSize();
+  }
 }
 
 type L1RollupConstants = {

yarn-project/archiver/src/archiver/archiver_store.ts

@@ -268,4 +268,9 @@ export interface ArchiverDataStore {
 
   addContractArtifact(address: AztecAddress, contract: ContractArtifact): Promise<void>;
   getContractArtifact(address: AztecAddress): Promise<ContractArtifact | undefined>;
+
+  /**
+   * Estimates the size of the store in bytes.
+   */
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number };
 }

yarn-project/archiver/src/archiver/instrumentation.ts

@@ -4,6 +4,7 @@ import {
   Attributes,
   type Gauge,
   type Histogram,
+  LmdbMetrics,
   Metrics,
   type TelemetryClient,
   type UpDownCounter,
@@ -18,6 +19,7 @@ export class ArchiverInstrumentation {
   private syncDuration: Histogram;
   private proofsSubmittedDelay: Histogram;
   private proofsSubmittedCount: UpDownCounter;
+  private dbMetrics: LmdbMetrics;
 
   private log = createDebugLogger('aztec:archiver:instrumentation');
 
@@ -55,6 +57,26 @@ export class ArchiverInstrumentation {
         explicitBucketBoundaries: millisecondBuckets(1, 80), // 10ms -> ~3hs
       },
     });
+
+    this.dbMetrics = new LmdbMetrics(
+      meter,
+      {
+        name: Metrics.ARCHIVER_DB_MAP_SIZE,
+        description: 'Database map size for the archiver',
+      },
+      {
+        name: Metrics.ARCHIVER_DB_USED_SIZE,
+        description: 'Database used size for the archiver',
+      },
+      {
+        name: Metrics.ARCHIVER_DB_NUM_ITEMS,
+        description: 'Num items in the archiver database',
+      },
+    );
+  }
+
+  public recordDBMetrics(metrics: { mappingSize: number; numItems: number; actualSize: number }) {
+    this.dbMetrics.recordDBMetrics(metrics);
   }
 
   public isEnabled(): boolean {

yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts

@@ -49,7 +49,7 @@ export class KVArchiverDataStore implements ArchiverDataStore {
 
   #log = createDebugLogger('aztec:archiver:data-store');
 
-  constructor(db: AztecKVStore, logsMaxPageSize: number = 1000) {
+  constructor(private db: AztecKVStore, logsMaxPageSize: number = 1000) {
     this.#blockStore = new BlockStore(db);
     this.#logStore = new LogStore(db, this.#blockStore, logsMaxPageSize);
     this.#messageStore = new MessageStore(db);
@@ -344,4 +344,8 @@ export class KVArchiverDataStore implements ArchiverDataStore {
       messagesSynchedTo: this.#messageStore.getSynchedL1BlockNumber(),
     });
   }
+
+  public estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+    return this.db.estimateSize();
+  }
 }

yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts

@@ -736,4 +736,8 @@ export class MemoryArchiverStore implements ArchiverDataStore {
   public getContractArtifact(address: AztecAddress): Promise<ContractArtifact | undefined> {
     return Promise.resolve(this.contractArtifacts.get(address.toString()));
   }
+
+  public estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
+    return { mappingSize: 0, actualSize: 0, numItems: 0 };
+  }
 }

yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts

@@ -146,7 +146,12 @@ describe('L1Publisher integration', () => {
       worldStateProvenBlocksOnly: false,
       worldStateDbMapSizeKb: 10 * 1024 * 1024,
     };
-    worldStateSynchronizer = new ServerWorldStateSynchronizer(builderDb, blockSource, worldStateConfig);
+    worldStateSynchronizer = new ServerWorldStateSynchronizer(
+      builderDb,
+      blockSource,
+      worldStateConfig,
+      new NoopTelemetryClient(),
+    );
     await worldStateSynchronizer.start();
     fork = await worldStateSynchronizer.fork();
     builder = new LightweightBlockBuilder(fork, new NoopTelemetryClient());

yarn-project/kv-store/src/interfaces/store.ts

@@ -72,5 +72,5 @@ export interface AztecKVStore {
   /**
    * Estimates the size of the store in bytes.
    */
-  estimateSize(): { bytes: number };
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number };
 }

yarn-project/kv-store/src/lmdb/store.ts

@@ -182,15 +182,52 @@ export class AztecLmdbStore implements AztecKVStore {
     }
   }
 
-  estimateSize(): { bytes: number } {
+  estimateSize(): { mappingSize: number; actualSize: number; numItems: number } {
     const stats = this.#rootDb.getStats();
-    // `mapSize` represents to total amount of memory currently being used by the database.
-    // since the database is mmap'd, this is a good estimate of the size of the database for now.
+    // The 'mapSize' is the total amount of virtual address space allocated to the DB (effectively the maximum possible size)
     // http://www.lmdb.tech/doc/group__mdb.html#a4bde3c8b676457342cba2fe27aed5fbd
+    let mapSize = 0;
     if ('mapSize' in stats && typeof stats.mapSize === 'number') {
-      return { bytes: stats.mapSize };
-    } else {
-      return { bytes: 0 };
+      mapSize = stats.mapSize;
     }
+    const dataResult = this.estimateSubDBSize(this.#data);
+    const multiResult = this.estimateSubDBSize(this.#multiMapData);
+    return {
+      mappingSize: mapSize,
+      actualSize: dataResult.actualSize + multiResult.actualSize,
+      numItems: dataResult.numItems + multiResult.numItems,
+    };
+  }
+
+  private estimateSubDBSize(db: Database<unknown, Key>): { actualSize: number; numItems: number } {
+    const stats = db.getStats();
+    let branchPages = 0;
+    let leafPages = 0;
+    let overflowPages = 0;
+    let pageSize = 0;
+    let totalSize = 0;
+    let numItems = 0;
+    // This is the total number of key/value pairs present in the DB
+    if ('entryCount' in stats && typeof stats.entryCount === 'number') {
+      numItems = stats.entryCount;
+    }
+    // The closest value we can get to the actual size of the database is the number of consumed pages * the page size
+    if (
+      'treeBranchPageCount' in stats &&
+      typeof stats.treeBranchPageCount === 'number' &&
+      'treeLeafPageCount' in stats &&
+      typeof stats.treeLeafPageCount === 'number' &&
+      'overflowPages' in stats &&
+      typeof stats.overflowPages === 'number' &&
+      'pageSize' in stats &&
+      typeof stats.pageSize === 'number'
+    ) {
+      branchPages = stats.treeBranchPageCount;
+      leafPages = stats.treeLeafPageCount;
+      overflowPages = stats.overflowPages;
+      pageSize = stats.pageSize;
+      totalSize = (branchPages + leafPages + overflowPages) * pageSize;
+    }
+    return { actualSize: totalSize, numItems };
   }
 }

yarn-project/p2p/src/mem_pools/instrumentation.ts

@@ -1,5 +1,12 @@
 import { type Gossipable } from '@aztec/circuit-types';
-import { Attributes, type Histogram, Metrics, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client';
+import {
+  Attributes,
+  type Histogram,
+  LmdbMetrics,
+  Metrics,
+  type TelemetryClient,
+  type UpDownCounter,
+} from '@aztec/telemetry-client';
 
 /**
  * Instrumentation class for the Pools (TxPool, AttestationPool, etc).
@@ -10,6 +17,8 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
   /** Tracks tx size */
   private objectSize: Histogram;
 
+  private dbMetrics: LmdbMetrics;
+
   private defaultAttributes;
 
   constructor(telemetry: TelemetryClient, name: string) {
@@ -35,6 +44,26 @@ export class PoolInstrumentation<PoolObject extends Gossipable> {
         ],
       },
     });
+
+    this.dbMetrics = new LmdbMetrics(
+      meter,
+      {
+        name: Metrics.MEMPOOL_DB_MAP_SIZE,
+        description: 'Database map size for the Tx mempool',
+      },
+      {
+        name: Metrics.MEMPOOL_DB_USED_SIZE,
+        description: 'Database used size for the Tx mempool',
+      },
+      {
+        name: Metrics.MEMPOOL_DB_NUM_ITEMS,
+        description: 'Num items in database for the Tx mempool',
+      },
+    );
+  }
+
+  public recordDBMetrics(metrics: { mappingSize: number; numItems: number; actualSize: number }) {
+    this.dbMetrics.recordDBMetrics(metrics);
   }
 
   public recordSize(poolObject: PoolObject) {

yarn-project/p2p/src/mem_pools/tx_pool/aztec_kv_tx_pool.ts

@@ -53,6 +53,8 @@ export class AztecKVTxPool implements TxPool {
       }
       this.#metrics.recordRemovedObjects(deleted, 'pending');
       this.#metrics.recordAddedObjects(txHashes.length, 'mined');
+      const storeSizes = this.#store.estimateSize();
+      this.#metrics.recordDBMetrics(storeSizes);
     });
   }
 

yarn-project/telemetry-client/src/index.ts

@@ -2,3 +2,4 @@ export * from './telemetry.js';
 export * from './histogram_utils.js';
 export * from './with_tracer.js';
 export * from './prom_otel_adapter.js';
+export * from './lmdb_metrics.js';

yarn-project/telemetry-client/src/lmdb_metrics.ts

@@ -0,0 +1,38 @@
+import { type Gauge, type Meter, type Metrics, ValueType } from './telemetry.js';
+
+export type LmdbMetricDescriptor = {
+  name: Metrics;
+  description: string;
+};
+
+export class LmdbMetrics {
+  private dbMapSize: Gauge;
+  private dbUsedSize: Gauge;
+  private dbNumItems: Gauge;
+
+  constructor(
+    meter: Meter,
+    dbMapSizeDescriptor: LmdbMetricDescriptor,
+    dbUsedSizeDescriptor: LmdbMetricDescriptor,
+    dbNumItemsDescriptor: LmdbMetricDescriptor,
+  ) {
+    this.dbMapSize = meter.createGauge(dbMapSizeDescriptor.name, {
+      description: dbMapSizeDescriptor.description,
+      valueType: ValueType.INT,
+    });
+    this.dbUsedSize = meter.createGauge(dbUsedSizeDescriptor.name, {
+      description: dbUsedSizeDescriptor.description,
+      valueType: ValueType.INT,
+    });
+    this.dbNumItems = meter.createGauge(dbNumItemsDescriptor.name, {
+      description: dbNumItemsDescriptor.description,
+      valueType: ValueType.INT,
+    });
+  }
+
+  public recordDBMetrics(metrics: { mappingSize: number; numItems: number; actualSize: number }) {
+    this.dbMapSize.record(metrics.mappingSize);
+    this.dbNumItems.record(metrics.actualSize);
+    this.dbUsedSize.record(metrics.actualSize);
+  }
+}