chore: Clean up data configuration (#9973)

This PR changes the following related to data storage configuration

- `PXE_DATA_DIRECTORY` has been removed. The PXE uses `DATA_DIRECTORY`
along with everything else.
- `DATA_DIRECTORY` is deemed to be the root of all state directories.
This was already partly implemented. Each component stores it's state in
one or more sub-directories of this root.
- `DATA_STORE_MAP_SIZE_KB` is a new environment variable and represented
the `mmap` size provided to all environments of LMDB.
- `WS_DATA_DIRECTORY` is a new optional environment variable. If
provided, represents an alternative root directory for the world state
only.
- `WS_DB_MAP_SIZE_KB` is a new optional environment variable. If
provided, represent the `mmap` size provided to all LMDB environment
used in the world state.

---------

Co-authored-by: ludamad <adam.domurad@gmail.com>

cspell.json

@@ -120,6 +120,7 @@
     "ierc",
     "indexeddb",
     "initialise",
+    "initialising",
     "interruptible",
     "isequal",
     "ivpk",
@@ -154,6 +155,7 @@
     "monomorphize",
     "mplex",
     "msgpack",
+    "msgpackr",
     "muldiv",
     "multiaddr",
     "multiaddrs",
@@ -163,6 +165,7 @@
     "muxers",
     "nada",
     "namespacing",
+    "napi",
     "Nargo",
     "nixpkgs",
     "nodebuffer",

yarn-project/archiver/src/archiver/config.ts

@@ -38,11 +38,6 @@ export type ArchiverConfig = {
    */
   l1Contracts: L1ContractAddresses;
 
-  /**
-   * Optional dir to store data. If omitted will store in memory.
-   */
-  dataDirectory: string | undefined;
-
   /** The max number of logs that can be obtained in 1 "getUnencryptedLogs" call. */
   maxLogs?: number;
 } & L1ReaderConfig &
@@ -57,11 +52,7 @@ export const archiverConfigMappings: ConfigMappingsType<ArchiverConfig> = {
   archiverPollingIntervalMS: {
     env: 'ARCHIVER_POLLING_INTERVAL_MS',
     description: 'The polling interval in ms for retrieving new L2 blocks and encrypted logs.',
-    ...numberConfigHelper(1000),
-  },
-  dataDirectory: {
-    env: 'DATA_DIRECTORY',
-    description: 'Optional dir to store data. If omitted will store in memory.',
+    ...numberConfigHelper(1_000),
   },
   maxLogs: {
     env: 'ARCHIVER_MAX_LOGS',

yarn-project/archiver/src/factory.ts

@@ -2,6 +2,7 @@ import { type ArchiverApi, type Service } from '@aztec/circuit-types';
 import { type ContractClassPublic } from '@aztec/circuits.js';
 import { createDebugLogger } from '@aztec/foundation/log';
 import { type Maybe } from '@aztec/foundation/types';
+import { type DataStoreConfig } from '@aztec/kv-store/config';
 import { createStore } from '@aztec/kv-store/utils';
 import { getCanonicalProtocolContract, protocolContractNames } from '@aztec/protocol-contracts';
 import { type TelemetryClient } from '@aztec/telemetry-client';
@@ -13,7 +14,7 @@ import { KVArchiverDataStore } from './archiver/index.js';
 import { createArchiverClient } from './rpc/index.js';
 
 export async function createArchiver(
-  config: ArchiverConfig,
+  config: ArchiverConfig & DataStoreConfig,
   telemetry: TelemetryClient = new NoopTelemetryClient(),
   opts: { blockUntilSync: boolean } = { blockUntilSync: true },
 ): Promise<ArchiverApi & Maybe<Service>> {

yarn-project/aztec-node/src/aztec-node/config.ts

@@ -1,5 +1,6 @@
 import { type ArchiverConfig, archiverConfigMappings } from '@aztec/archiver';
 import { type ConfigMappingsType, booleanConfigHelper, getConfigFromMappings } from '@aztec/foundation/config';
+import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config';
 import { type P2PConfig, p2pConfigMappings } from '@aztec/p2p';
 import { type ProverClientConfig, proverClientConfigMappings } from '@aztec/prover-client';
 import { type SequencerClientConfig, sequencerClientConfigMappings } from '@aztec/sequencer-client';
@@ -24,7 +25,7 @@ export type AztecNodeConfig = ArchiverConfig &
   P2PConfig & {
     /** Whether the validator is disabled for this node */
     disableValidator: boolean;
-  };
+  } & DataStoreConfig;
 
 export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
   ...archiverConfigMappings,
@@ -33,6 +34,7 @@ export const aztecNodeConfigMappings: ConfigMappingsType<AztecNodeConfig> = {
   ...proverClientConfigMappings,
   ...worldStateConfigMappings,
   ...p2pConfigMappings,
+  ...dataConfigMappings,
   disableValidator: {
     env: 'VALIDATOR_DISABLED',
     description: 'Whether the validator is disabled for this node.',

yarn-project/aztec/src/cli/aztec_start_options.ts

@@ -218,12 +218,6 @@ export const aztecStartOptions: { [key: string]: AztecStartOption[] } = {
       defaultValue: undefined,
       envVar: undefined,
     },
-    {
-      flag: '--pxe.dataDirectory <value>',
-      description: 'Where to store PXE data. If not set, will store in memory',
-      defaultValue: undefined,
-      envVar: 'PXE_DATA_DIRECTORY',
-    },
     ...getOptions('pxe', allPxeConfigMappings),
   ],
   ARCHIVER: [

yarn-project/aztec/src/cli/cmds/start_archiver.ts

@@ -2,6 +2,7 @@ import { Archiver, type ArchiverConfig, KVArchiverDataStore, archiverConfigMappi
 import { createDebugLogger } from '@aztec/aztec.js';
 import { ArchiverApiSchema } from '@aztec/circuit-types';
 import { type NamespacedApiHandlers } from '@aztec/foundation/json-rpc/server';
+import { type DataStoreConfig, dataConfigMappings } from '@aztec/kv-store/config';
 import { createStore } from '@aztec/kv-store/utils';
 import {
   createAndStartTelemetryClient,
@@ -16,7 +17,14 @@ export async function startArchiver(
   signalHandlers: (() => Promise<void>)[],
   services: NamespacedApiHandlers,
 ) {
-  const archiverConfig = extractRelevantOptions<ArchiverConfig>(options, archiverConfigMappings, 'archiver');
+  const archiverConfig = extractRelevantOptions<ArchiverConfig & DataStoreConfig>(
+    options,
+    {
+      ...archiverConfigMappings,
+      ...dataConfigMappings,
+    },
+    'archiver',
+  );
 
   const storeLog = createDebugLogger('aztec:archiver:lmdb');
   const store = await createStore('archiver', archiverConfig, storeLog);

yarn-project/end-to-end/scripts/e2e_test_config.yml

@@ -111,5 +111,6 @@ tests:
     use_compose: true
   pxe:
     use_compose: true
-  uniswap_trade_on_l1_from_l2:
-    use_compose: true
+  # https://github.com/AztecProtocol/aztec-packages/issues/10030
+  # uniswap_trade_on_l1_from_l2:
+  #   use_compose: true

yarn-project/end-to-end/src/benchmarks/bench_prover.test.ts

@@ -5,7 +5,7 @@ import { BBCircuitVerifier } from '@aztec/bb-prover';
 import { CompleteAddress, Fq, Fr, GasSettings } from '@aztec/circuits.js';
 import { FPCContract, FeeJuiceContract, TestContract, TokenContract } from '@aztec/noir-contracts.js';
 import { ProtocolContractAddress } from '@aztec/protocol-contracts';
-import { type PXEService, createPXEService } from '@aztec/pxe';
+import { type PXEService, type PXEServiceConfig, createPXEService } from '@aztec/pxe';
 
 import { jest } from '@jest/globals';
 
@@ -142,17 +142,18 @@ describe('benchmarks/proving', () => {
     ctx.logger.info('Starting PXEs configured with real proofs');
     provingPxes = [];
     for (let i = 0; i < 4; i++) {
-      const pxe = await createPXEService(
-        ctx.aztecNode,
-        {
-          proverEnabled: true,
-          bbBinaryPath: bbConfig.bbBinaryPath,
-          bbWorkingDirectory: bbConfig.bbWorkingDirectory,
-          l2BlockPollingIntervalMS: 1000,
-          l2StartingBlock: 1,
-        },
-        `proving-pxe-${i}`,
-      );
+      const l1Contracts = await ctx.aztecNode.getL1ContractAddresses();
+      const pxeConfig = {
+        proverEnabled: true,
+        bbBinaryPath: bbConfig.bbBinaryPath,
+        bbWorkingDirectory: bbConfig.bbWorkingDirectory,
+        l2BlockPollingIntervalMS: 1000,
+        l2StartingBlock: 1,
+        dataDirectory: undefined,
+        dataStoreMapSizeKB: 1024 * 1024,
+        l1Contracts,
+      } as PXEServiceConfig;
+      const pxe = await createPXEService(ctx.aztecNode, pxeConfig, `proving-pxe-${i}`);
 
       await getSchnorrAccount(pxe, schnorrWalletEncKey, schnorrWalletSigningKey, schnorrWalletSalt).register();
       await pxe.registerContract(initialTokenContract);

yarn-project/end-to-end/src/benchmarks/utils.ts

@@ -3,7 +3,7 @@ import { type AztecNode, BatchCall, INITIAL_L2_BLOCK_NUM, type SentTx, retryUnti
 import { times } from '@aztec/foundation/collection';
 import { randomInt } from '@aztec/foundation/crypto';
 import { BenchmarkingContract } from '@aztec/noir-contracts.js/Benchmarking';
-import { type PXEService, createPXEService } from '@aztec/pxe';
+import { type PXEService, type PXEServiceConfig, createPXEService } from '@aztec/pxe';
 
 import { mkdirpSync } from 'fs-extra';
 import { globSync } from 'glob';
@@ -102,10 +102,15 @@ export async function waitNewPXESynced(
   contract: BenchmarkingContract,
   startingBlock: number = INITIAL_L2_BLOCK_NUM,
 ): Promise<PXEService> {
-  const pxe = await createPXEService(node, {
-    l2BlockPollingIntervalMS: 100,
+  const l1Contracts = await node.getL1ContractAddresses();
+  const pxeConfig = {
     l2StartingBlock: startingBlock,
-  });
+    l2BlockPollingIntervalMS: 100,
+    dataDirectory: undefined,
+    dataStoreMapSizeKB: 1024 * 1024,
+    l1Contracts,
+  } as PXEServiceConfig;
+  const pxe = await createPXEService(node, pxeConfig);
   await pxe.registerContract(contract);
   await retryUntil(() => pxe.isGlobalStateSynchronized(), 'pxe-global-sync');
   return pxe;

yarn-project/end-to-end/src/prover-coordination/e2e_prover_coordination.test.ts

@@ -359,6 +359,12 @@ describe('e2e_prover_coordination', () => {
   });
 
   it('Can claim proving rights after a prune', async () => {
+    await logState();
+
+    const tips = await ctx.cheatCodes.rollup.getTips();
+
+    let currentPending = tips.pending;
+    let currentProven = tips.proven;
     // Here we are creating a proof quote for epoch 0
     const quoteForEpoch0 = await makeEpochProofQuote({
       epochToProve: 0n,
@@ -373,9 +379,11 @@ describe('e2e_prover_coordination', () => {
     // Build a block in epoch 1, we should see the quote for epoch 0 submitted earlier published to L1
     await contract.methods.create_note(recipient, recipient, 10).send().wait();
 
+    currentPending++;
+
     // Verify that we can claim the current epoch
     await expectProofClaimOnL1({ ...quoteForEpoch0.payload, proposer: publisherAddress });
-    await expectTips({ pending: 3n, proven: 0n });
+    await expectTips({ pending: currentPending, proven: currentProven });
 
     // Now go to epoch 1
     await advanceToNextEpoch();
@@ -384,19 +392,23 @@ describe('e2e_prover_coordination', () => {
     const epoch0BlockNumber = await getPendingBlockNumber();
     await rollupContract.write.setAssumeProvenThroughBlockNumber([BigInt(epoch0BlockNumber)]);
 
+    currentProven = epoch0BlockNumber;
+
     // Go to epoch 2
     await advanceToNextEpoch();
 
     // Progress epochs with a block in each until we hit a reorg
     // Note tips are block numbers, not slots
-    await expectTips({ pending: 3n, proven: 3n });
+    await expectTips({ pending: currentPending, proven: currentProven });
     const tx2BeforeReorg = await contract.methods.create_note(recipient, recipient, 10).send().wait();
-    await expectTips({ pending: 4n, proven: 3n });
+    currentPending++;
+    await expectTips({ pending: currentPending, proven: currentProven });
 
     // Go to epoch 3
     await advanceToNextEpoch();
     const tx3BeforeReorg = await contract.methods.create_note(recipient, recipient, 10).send().wait();
-    await expectTips({ pending: 5n, proven: 3n });
+    currentPending++;
+    await expectTips({ pending: currentPending, proven: currentProven });
 
     // Go to epoch 4 !!! REORG !!! ay caramba !!!
     await advanceToNextEpoch();
@@ -430,12 +442,15 @@ describe('e2e_prover_coordination', () => {
     const newWallet = await createAccount(newPxe);
     const newWalletAddress = newWallet.getAddress();
 
-    // The chain will prune back to block 3
+    // after the re-org the pending chain has moved on by 2 blocks
+    currentPending = currentProven + 2n;
+
+    // The chain will prune back to the proven block number
     // then include the txs from the pruned epochs that are still valid
-    // bringing us back to block 4 (same number, different hash)
+    // bringing us back to block proven + 1 (same number, different hash)
     // creating a new account will produce another block
-    // so we expect 5 blocks in the pending chain here!
-    await expectTips({ pending: 5n, proven: 3n });
+    // so we expect proven + 2 blocks in the pending chain here!
+    await expectTips({ pending: currentPending, proven: currentProven });
 
     // Submit proof claim for the new epoch
     const quoteForEpoch4 = await makeEpochProofQuote({
@@ -453,7 +468,8 @@ describe('e2e_prover_coordination', () => {
 
     logger.info('Sending new tx on reorged chain');
     await contractFromNewPxe.methods.create_note(newWalletAddress, newWalletAddress, 10).send().wait();
-    await expectTips({ pending: 6n, proven: 3n });
+    currentPending++;
+    await expectTips({ pending: currentPending, proven: currentProven });
 
     // Expect the proof claim to be accepted for the chain after the reorg
     await expectProofClaimOnL1({ ...quoteForEpoch4.payload, proposer: publisherAddress });

yarn-project/foundation/src/config/env_var.ts

@@ -37,6 +37,7 @@ export type EnvVar =
   | 'BOT_STOP_WHEN_UNHEALTHY'
   | 'COINBASE'
   | 'DATA_DIRECTORY'
+  | 'DATA_STORE_MAP_SIZE_KB'
   | 'DEBUG'
   | 'DEPLOY_AZTEC_CONTRACTS_SALT'
   | 'DEPLOY_AZTEC_CONTRACTS'
@@ -112,7 +113,6 @@ export type EnvVar =
   | 'PROVER_REQUIRED_CONFIRMATIONS'
   | 'PROVER_TEST_DELAY_MS'
   | 'PXE_BLOCK_POLLING_INTERVAL_MS'
-  | 'PXE_DATA_DIRECTORY'
   | 'PXE_L2_STARTING_BLOCK'
   | 'PXE_PROVER_ENABLED'
   | 'QUOTE_PROVIDER_BASIS_POINT_FEE'
@@ -152,6 +152,7 @@ export type EnvVar =
   | 'PROVER_VIEM_POLLING_INTERVAL_MS'
   | 'SEQ_VIEM_POLLING_INTERVAL_MS'
   | 'WS_DB_MAP_SIZE_KB'
+  | 'WS_DATA_DIRECTORY'
   | 'ETHEREUM_SLOT_DURATION'
   | 'AZTEC_SLOT_DURATION'
   | 'AZTEC_EPOCH_DURATION'

yarn-project/kv-store/package.json

@@ -5,7 +5,8 @@
   "exports": {
     ".": "./dest/interfaces/index.js",
     "./lmdb": "./dest/lmdb/index.js",
-    "./utils": "./dest/utils.js"
+    "./utils": "./dest/utils.js",
+    "./config": "./dest/config.js"
   },
   "scripts": {
     "build": "yarn clean && tsc -b",
@@ -55,6 +56,7 @@
     ]
   },
   "dependencies": {
+    "@aztec/ethereum": "workspace:^",
     "@aztec/foundation": "workspace:^",
     "lmdb": "^3.0.6"
   },

yarn-project/kv-store/src/config.ts

@@ -0,0 +1,34 @@
+import { l1ContractAddressesMapping } from '@aztec/ethereum';
+import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
+import { type EthAddress } from '@aztec/foundation/eth-address';
+
+export type DataStoreConfig = {
+  dataDirectory: string | undefined;
+  dataStoreMapSizeKB: number;
+  l1Contracts: { rollupAddress: EthAddress };
+};
+
+export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
+  dataDirectory: {
+    env: 'DATA_DIRECTORY',
+    description: 'Optional dir to store data. If omitted will store in memory.',
+  },
+  dataStoreMapSizeKB: {
+    env: 'DATA_STORE_MAP_SIZE_KB',
+    description: 'DB mapping size to be applied to all key/value stores',
+    ...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
+  },
+  l1Contracts: {
+    description: 'The deployed L1 contract addresses',
+    defaultValue: l1ContractAddressesMapping,
+  },
+};
+
+/**
+ * Returns the archiver configuration from the environment variables.
+ * Note: If an environment variable is not set, the default value is used.
+ * @returns The archiver configuration.
+ */
+export function getDataConfigFromEnv(): DataStoreConfig {
+  return getConfigFromMappings<DataStoreConfig>(dataConfigMappings);
+}

yarn-project/kv-store/src/lmdb/store.test.ts

@@ -4,6 +4,8 @@ import { join } from 'path';
 
 import { AztecLmdbStore } from './store.js';
 
+const defaultMapSize = 1024 * 1024 * 1024 * 10;
+
 describe('AztecLmdbStore', () => {
   const itForks = async (store: AztecLmdbStore) => {
     const singleton = store.openSingleton('singleton');
@@ -21,17 +23,17 @@ describe('AztecLmdbStore', () => {
 
   it('forks a persistent store', async () => {
     const path = await mkdtemp(join(tmpdir(), 'aztec-store-test-'));
-    const store = AztecLmdbStore.open(path, false);
+    const store = AztecLmdbStore.open(path, defaultMapSize, false);
     await itForks(store);
   });
 
   it('forks a persistent store with no path', async () => {
-    const store = AztecLmdbStore.open(undefined, false);
+    const store = AztecLmdbStore.open(undefined, defaultMapSize, false);
     await itForks(store);
   });
 
   it('forks an ephemeral store', async () => {
-    const store = AztecLmdbStore.open(undefined, true);
+    const store = AztecLmdbStore.open(undefined, defaultMapSize, true);
     await itForks(store);
   });
 });