From 2876b00c5fa38f69c07d9d99caa506a8c6fbc138 Mon Sep 17 00:00:00 2001 From: NC Date: Tue, 20 Feb 2024 02:51:21 +0800 Subject: [PATCH] feat: implement EIP-6110 (#6042) * Add immutable in the dependencies * Initial change to pubkeyCache * Added todos * Moved unfinalized cache to epochCache * Move populating finalized cache to afterProcessEpoch * Specify unfinalized cache during state cloning * Move from unfinalized to finalized cache in afterProcessEpoch * Confused myself * Clean up * Change logic * Fix cloning issue * Clean up redundant code * Add CarryoverData in epochCtx.createFromState * Fix typo * Update usage of pubkeyCache * Update pubkeyCache usage * Fix lint * Fix lint * Add 6110 to ChainConfig * Add 6110 to BeaconPreset * Define 6110 fork and container * Add V6110 api to execution engine * Update test * Add depositReceiptsRoot to process_execution_payload * State transitioning to EIP6110 * State transitioning to EIP6110 * Light client change in EIP-6110 * Update tests * produceBlock * Refactor processDeposit to match the spec * Implement processDepositReceipt * Implement 6110 fork guard for pubkeyCache * Handle changes in eth1 deposit * Update eth1 deposit test * Fix typo * Lint * Remove embarassing comments * Address comments * Modify applyDeposit signature * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update packages/state-transition/src/cache/pubkeyCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Remove old code * Rename fields in epochCache and immutableData * Remove CarryoverData * Move isAfter6110 from var to method * Fix cyclic import * Fix operations spec runner * Fix for spec test * Fix spec test * state.depositReceiptsStartIndex to BigInt * getDeposit requires cached state * default depositReceiptsStartIndex value in genesis * Fix pubkeyCache bug * newUnfinalizedPubkeyIndexMap in createCachedBeaconState * Lint * Pass epochCache instead of pubkey2IndexFn in apis * Address comments * Add unit test on pubkey cache cloning * Add unfinalizedPubkeyCacheSize to metrics * Add unfinalizedPubkeyCacheSize to metrics * Clean up code * Add besu to el-interop * Add 6110 genesis file * Template for sim test * Add unit test for getEth1DepositCount * Update sim test * Update besudocker * Finish beacon api calls in sim test * Update epochCache.createFromState() * Fix bug unfinalized validators are not finalized * Add sim test to run a few blocks * Lint * Merge branch 'unstable' into 611 * Add more check to sim test * Update besu docker image instruction * Update sim test with correct tx * Address comment + cleanup * Clean up code * Properly handle promise rejection * Lint * Update packages/beacon-node/src/execution/engine/types.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Update comments * Accept type undefined in ExecutionPayloadBodyRpc * Update comment and semantic * Remove if statement when adding finalized validator * Comment on repeated insert on finalized cache * rename createFromState * Add comment on getPubkey() * Stash change to reduce diffs * Stash change to reduce diffs * Lint * addFinalizedPubkey on finalized checkpoint * Update comment * Use OrderedMap for unfinalized cache * Pull out logic of deleting pubkeys for batch op * Add updateUnfinalizedPubkeys in regen * Update updateUnfinalizedPubkeys logic * Add comment * Add metrics for state context caches * Address comment * Address comment * Deprecate eth1Data polling when condition is reached * Fix conflicts * Fix sim test * Lint * Fix type * Fix test * Fix test * Lint * Update packages/light-client/src/spec/utils.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Fix spec test * Address comments * Improve cache logic on checkpoint finalized * Update sim test according to new cache logic * Update comment * Lint * Finalized pubkey cache only update once per checkpoint * Add perf test for updateUnfinalizedPubkeys * Add perf test for updateUnfinalizedPubkeys * Tweak params for perf test * Freeze besu docker image version for 6110 * Add benchmark result * Use Map instead of OrderedMap. Update benchmark * Minor optimization * Minor optimization * Add memory test for immutable.js * Update test * Reduce code duplication * Lint * Remove try/catch in updateUnfinalizedPubkeys * Introduce EpochCache metric * Add historicalValidatorLengths * Polish code * Migrate state-transition unit tests to vitest * Fix calculation of pivot index * `historicalValidatorLengths` only activate post 6110 * Update sim test * Lint * Update packages/state-transition/src/cache/epochCache.ts Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> * Improve readability on historicalValidatorLengths * Update types * Fix calculation * Add eth1data poll todo * Add epochCache.getValidatorCountAtEpoch * Add todo * Add getStateIterator for state cache * Partial commit * Update perf test * updateUnfinalizedPubkeys directly modify states from regen * Update sim test. Lint * Add todo * some improvements and a fix for effectiveBalanceIncrements fork safeness * rename eip6110 to elctra * fix electra-interop.test.ts --------- Co-authored-by: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Co-authored-by: gajinder lint and tsc small cleanup fix rebase issue --- packages/beacon-node/src/chain/chain.ts | 37 ++ .../beacon-node/src/chain/regen/queued.ts | 50 +- .../chain/stateCache/blockStateCacheImpl.ts | 6 +- .../chain/stateCache/fifoBlockStateCache.ts | 4 + .../stateCache/inMemoryCheckpointsCache.ts | 4 + .../stateCache/persistentCheckpointsCache.ts | 4 + .../beacon-node/src/chain/stateCache/types.ts | 2 + .../src/eth1/eth1DepositDataTracker.ts | 32 +- packages/beacon-node/src/eth1/index.ts | 8 + packages/beacon-node/src/eth1/interface.ts | 5 + .../beacon-node/src/eth1/utils/deposits.ts | 16 +- .../beacon-node/src/execution/engine/http.ts | 10 +- .../src/execution/engine/interface.ts | 4 +- .../beacon-node/src/execution/engine/mock.ts | 4 + .../src/execution/engine/payloadIdCache.ts | 8 + .../beacon-node/src/execution/engine/types.ts | 68 ++- .../beacon-node/src/metrics/metrics/beacon.ts | 7 + .../src/metrics/metrics/lodestar.ts | 31 ++ .../test/memory/unfinalizedPubkey2Index.ts | 54 +++ .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../test/perf/chain/opPools/opPool.test.ts | 2 +- .../produceBlock/produceBlockBody.test.ts | 2 +- .../updateUnfinalizedPubkeys.test.ts | 110 +++++ .../scripts/el-interop/besu/common-setup.sh | 19 + .../test/scripts/el-interop/besu/electra.tmpl | 77 +++ .../scripts/el-interop/besu/post-merge.sh | 8 + .../el-interop/besudocker/common-setup.sh | 22 + .../el-interop/besudocker/electra.tmpl | 77 +++ .../el-interop/besudocker/post-merge.sh | 8 + .../test/sim/electra-interop.test.ts | 457 ++++++++++++++++++ .../test/spec/presets/fork.test.ts | 3 +- .../test/spec/presets/genesis.test.ts | 2 +- .../test/spec/presets/operations.test.ts | 9 +- .../test/spec/presets/ssz_static.test.ts | 1 + .../test/spec/utils/specTestIterator.ts | 1 - .../upgradeLightClientHeader.test.ts | 34 +- .../opPools/aggregatedAttestationPool.test.ts | 4 +- .../test/unit/chain/shufflingCache.test.ts | 8 +- .../stateCache/fifoBlockStateCache.test.ts | 3 +- .../test/unit/eth1/utils/deposits.test.ts | 52 +- .../test/unit/executionEngine/http.test.ts | 4 + .../beaconBlocksMaybeBlobsByRange.test.ts | 1 + packages/beacon-node/test/utils/state.ts | 23 +- .../test/utils/validationData/attestation.ts | 7 +- .../config/src/chainConfig/configs/mainnet.ts | 2 +- .../config/src/chainConfig/configs/minimal.ts | 3 +- packages/light-client/src/spec/utils.ts | 12 +- packages/params/src/index.ts | 5 + packages/params/src/presets/mainnet.ts | 3 + packages/params/src/presets/minimal.ts | 3 + packages/params/src/types.ts | 6 + packages/state-transition/package.json | 7 +- .../src/block/processDeposit.ts | 110 +++-- .../src/block/processDepositReceipt.ts | 17 + .../src/block/processOperations.ts | 17 +- .../state-transition/src/cache/epochCache.ts | 182 ++++++- .../state-transition/src/cache/pubkeyCache.ts | 22 +- .../state-transition/src/cache/stateCache.ts | 5 +- packages/state-transition/src/cache/types.ts | 1 + packages/state-transition/src/index.ts | 9 +- packages/state-transition/src/metrics.ts | 5 + .../src/signatureSets/attesterSlashings.ts | 3 +- packages/state-transition/src/slot/index.ts | 1 + .../src/slot/upgradeStateToElectra.ts | 33 ++ .../state-transition/src/stateTransition.ts | 5 + packages/state-transition/src/types.ts | 2 + packages/state-transition/src/util/deposit.ts | 24 + .../state-transition/src/util/execution.ts | 6 + packages/state-transition/src/util/genesis.ts | 12 + packages/state-transition/src/util/index.ts | 1 + .../test/unit/cachedBeaconState.test.ts | 62 ++- .../test/unit/upgradeState.test.ts | 16 + .../test/unit/util/deposit.test.ts | 99 ++++ packages/types/package.json | 3 + packages/types/src/electra/sszTypes.ts | 147 +++++- packages/types/src/electra/types.ts | 8 +- packages/types/src/primitive/sszTypes.ts | 1 + packages/validator/src/util/params.ts | 4 +- yarn.lock | 5 + 79 files changed, 1960 insertions(+), 171 deletions(-) create mode 100644 packages/beacon-node/test/memory/unfinalizedPubkey2Index.ts create mode 100644 packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts create mode 100755 packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh create mode 100644 packages/beacon-node/test/scripts/el-interop/besu/electra.tmpl create mode 100755 packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh create mode 100644 packages/beacon-node/test/scripts/el-interop/besudocker/common-setup.sh create mode 100644 packages/beacon-node/test/scripts/el-interop/besudocker/electra.tmpl create mode 100755 packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh create mode 100644 packages/beacon-node/test/sim/electra-interop.test.ts create mode 100644 packages/state-transition/src/block/processDepositReceipt.ts create mode 100644 packages/state-transition/src/slot/upgradeStateToElectra.ts create mode 100644 packages/state-transition/src/util/deposit.ts create mode 100644 packages/state-transition/test/unit/util/deposit.test.ts diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 75608ab33b2e..9f67857fe530 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -1027,6 +1027,9 @@ export class BeaconChain implements IBeaconChain { metrics.forkChoice.balancesLength.set(forkChoiceMetrics.balancesLength); metrics.forkChoice.nodes.set(forkChoiceMetrics.nodes); metrics.forkChoice.indices.set(forkChoiceMetrics.indices); + + const headState = this.getHeadState(); + metrics.headState.unfinalizedPubkeyCacheSize.set(headState.epochCtx.unfinalizedPubkey2index.size); } private onClockSlot(slot: Slot): void { @@ -1115,6 +1118,40 @@ export class BeaconChain implements IBeaconChain { if (headState) { this.opPool.pruneAll(headBlock, headState); } + + const cpEpoch = cp.epoch; + const electraEpoch = headState?.config.ELECTRA_FORK_EPOCH ?? Infinity; + + if (headState === null) { + this.logger.verbose("Head state is null"); + } else if (cpEpoch >= electraEpoch) { + // Get the validator.length from the state at cpEpoch + // We are confident the last element in the list is from headEpoch + // Thus we query from the end of the list. (cpEpoch - headEpoch - 1) is negative number + const pivotValidatorIndex = headState.epochCtx.getValidatorCountAtEpoch(cpEpoch); + + if (pivotValidatorIndex !== undefined) { + // Note EIP-6914 will break this logic + const newFinalizedValidators = headState.epochCtx.unfinalizedPubkey2index.filter( + (index, _pubkey) => index < pivotValidatorIndex + ); + + // Populate finalized pubkey cache and remove unfinalized pubkey cache + if (!newFinalizedValidators.isEmpty()) { + this.regen.updateUnfinalizedPubkeys(newFinalizedValidators); + } + } + } + + // TODO-Electra: Deprecating eth1Data poll requires a check on a finalized checkpoint state. + // Will resolve this later + // if (cpEpoch >= (this.config.ELECTRA_FORK_EPOCH ?? Infinity)) { + // // finalizedState can be safely casted to Electra state since cp is already post-Electra + // if (finalizedState.eth1DepositIndex >= (finalizedState as CachedBeaconStateElectra).depositReceiptsStartIndex) { + // // Signal eth1 to stop polling eth1Data + // this.eth1.stopPollingEth1Data(); + // } + // } } async updateBeaconProposerData(epoch: Epoch, proposers: ProposerPreparationData[]): Promise { diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 082ffbe271e4..84907ea163af 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -1,6 +1,6 @@ import {phase0, Slot, RootHex, Epoch, BeaconBlock} from "@lodestar/types"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; +import {CachedBeaconStateAllForks, UnfinalizedPubkeyIndexMap, computeEpochAtSlot} from "@lodestar/state-transition"; import {Logger, toRootHex} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {CheckpointHex, toCheckpointHex} from "../stateCache/index.js"; @@ -206,6 +206,54 @@ export class QueuedStateRegenerator implements IStateRegenerator { return this.checkpointStateCache.updatePreComputedCheckpoint(rootHex, epoch); } + /** + * Remove `validators` from all unfinalized cache's epochCtx.UnfinalizedPubkey2Index, + * and add them to epochCtx.pubkey2index and epochCtx.index2pubkey + */ + updateUnfinalizedPubkeys(validators: UnfinalizedPubkeyIndexMap): void { + let numStatesUpdated = 0; + const states = this.stateCache.getStates(); + const cpStates = this.checkpointStateCache.getStates(); + + // Add finalized pubkeys to all states. + const addTimer = this.metrics?.regenFnAddPubkeyTime.startTimer(); + + // We only need to add pubkeys to any one of the states since the finalized caches is shared globally across all states + const firstState = (states.next().value ?? cpStates.next().value) as CachedBeaconStateAllForks | undefined; + + if (firstState !== undefined) { + firstState.epochCtx.addFinalizedPubkeys(validators, this.metrics?.epochCache ?? undefined); + } else { + this.logger.warn("Attempt to delete finalized pubkey from unfinalized pubkey cache. But no state is available"); + } + + addTimer?.(); + + // Delete finalized pubkeys from unfinalized pubkey cache for all states + const deleteTimer = this.metrics?.regenFnDeletePubkeyTime.startTimer(); + const pubkeysToDelete = Array.from(validators.keys()); + + for (const s of states) { + s.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + numStatesUpdated++; + } + + for (const s of cpStates) { + s.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + numStatesUpdated++; + } + + // Since first state is consumed from the iterator. Will need to perform delete explicitly + if (firstState !== undefined) { + firstState?.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + numStatesUpdated++; + } + + deleteTimer?.(); + + this.metrics?.regenFnNumStatesUpdated.observe(numStatesUpdated); + } + /** * Get the state to run with `block`. * - State after `block.parentRoot` dialed forward to block.slot diff --git a/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts b/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts index 05073d0e515f..1cb67cd6cf09 100644 --- a/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts +++ b/packages/beacon-node/src/chain/stateCache/blockStateCacheImpl.ts @@ -34,7 +34,7 @@ export class BlockStateCacheImpl implements BlockStateCache { this.maxStates = maxStates; this.cache = new MapTracker(metrics?.stateCache); if (metrics) { - this.metrics = metrics.stateCache; + this.metrics = {...metrics.stateCache, ...metrics.epochCache}; metrics.stateCache.size.addCollect(() => metrics.stateCache.size.set(this.cache.size)); } } @@ -137,6 +137,10 @@ export class BlockStateCacheImpl implements BlockStateCache { })); } + getStates(): IterableIterator { + return this.cache.values(); + } + private deleteAllEpochItems(epoch: Epoch): void { for (const rootHex of this.epochIndex.get(epoch) || []) { this.cache.delete(rootHex); diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 602b0abaee8d..f115806874f8 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -190,6 +190,10 @@ export class FIFOBlockStateCache implements BlockStateCache { })); } + getStates(): IterableIterator { + throw new Error("Method not implemented."); + } + /** * For unit test only. */ diff --git a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts index 03cdc84de166..38aeabb97955 100644 --- a/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/inMemoryCheckpointsCache.ts @@ -176,6 +176,10 @@ export class InMemoryCheckpointStateCache implements CheckpointStateCache { })); } + getStates(): IterableIterator { + return this.cache.values(); + } + /** ONLY FOR DEBUGGING PURPOSES. For spec tests on error */ dumpCheckpointKeys(): string[] { return Array.from(this.cache.keys()); diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 5c5901583ad8..c59c15de616d 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -594,6 +594,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { }); } + getStates(): IterableIterator { + throw new Error("Method not implemented."); + } + /** ONLY FOR DEBUGGING PURPOSES. For spec tests on error */ dumpCheckpointKeys(): string[] { return Array.from(this.cache.keys()); diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index 41e9b91aaa43..ad93def481e7 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -33,6 +33,7 @@ export interface BlockStateCache { prune(headStateRootHex: RootHex): void; deleteAllBeforeEpoch(finalizedEpoch: Epoch): void; dumpSummary(): routes.lodestar.StateCacheItem[]; + getStates(): IterableIterator; // Expose beacon states stored in cache. Use with caution } /** @@ -74,6 +75,7 @@ export interface CheckpointStateCache { processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise; clear(): void; dumpSummary(): routes.lodestar.StateCacheItem[]; + getStates(): IterableIterator; // Expose beacon states stored in cache. Use with caution } export enum CacheItemType { diff --git a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts index f36f70abbbc4..c0b3ab35a73a 100644 --- a/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts +++ b/packages/beacon-node/src/eth1/eth1DepositDataTracker.ts @@ -1,6 +1,11 @@ import {phase0, ssz} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; -import {BeaconStateAllForks, becomesNewEth1Data} from "@lodestar/state-transition"; +import { + BeaconStateAllForks, + CachedBeaconStateAllForks, + CachedBeaconStateElectra, + becomesNewEth1Data, +} from "@lodestar/state-transition"; import {ErrorAborted, TimeoutError, fromHex, Logger, isErrorAborted, sleep} from "@lodestar/utils"; import {IBeaconDb} from "../db/index.js"; @@ -67,6 +72,8 @@ export class Eth1DepositDataTracker { /** Dynamically adjusted batch size to fetch deposit logs */ private eth1GetLogsBatchSizeDynamic = MAX_BLOCKS_PER_LOG_QUERY; private readonly forcedEth1DataVote: phase0.Eth1Data | null; + /** To stop `runAutoUpdate()` in addition to AbortSignal */ + private stopPolling: boolean; constructor( opts: Eth1Options, @@ -81,6 +88,8 @@ export class Eth1DepositDataTracker { this.depositsCache = new Eth1DepositsCache(opts, config, db); this.eth1DataCache = new Eth1DataCache(config, db); this.eth1FollowDistance = config.ETH1_FOLLOW_DISTANCE; + // TODO Electra: fix scenario where node starts post-Electra and `stopPolling` will always be false + this.stopPolling = false; this.forcedEth1DataVote = opts.forcedEth1DataVote ? ssz.phase0.Eth1Data.deserialize(fromHex(opts.forcedEth1DataVote)) @@ -109,10 +118,22 @@ export class Eth1DepositDataTracker { } } + // TODO Electra: Figure out how an elegant way to stop eth1data polling + stopPollingEth1Data(): void { + this.stopPolling = true; + } + /** * Return eth1Data and deposits ready for block production for a given state */ - async getEth1DataAndDeposits(state: BeaconStateAllForks): Promise { + async getEth1DataAndDeposits(state: CachedBeaconStateAllForks): Promise { + if ( + state.epochCtx.isAfterElectra() && + state.eth1DepositIndex >= (state as CachedBeaconStateElectra).depositReceiptsStartIndex + ) { + // No need to poll eth1Data since Electra deprecates the mechanism after depositReceiptsStartIndex is reached + return {eth1Data: state.eth1Data, deposits: []}; + } const eth1Data = this.forcedEth1DataVote ?? (await this.getEth1Data(state)); const deposits = await this.getDeposits(state, eth1Data); return {eth1Data, deposits}; @@ -141,7 +162,10 @@ export class Eth1DepositDataTracker { * Returns deposits to be included for a given state and eth1Data vote. * Requires internal caches to be updated regularly to return good results */ - private async getDeposits(state: BeaconStateAllForks, eth1DataVote: phase0.Eth1Data): Promise { + private async getDeposits( + state: CachedBeaconStateAllForks, + eth1DataVote: phase0.Eth1Data + ): Promise { // No new deposits have to be included, continue if (eth1DataVote.depositCount === state.eth1DepositIndex) { return []; @@ -162,7 +186,7 @@ export class Eth1DepositDataTracker { private async runAutoUpdate(): Promise { let lastRunMs = 0; - while (!this.signal.aborted) { + while (!this.signal.aborted && !this.stopPolling) { lastRunMs = Date.now(); try { diff --git a/packages/beacon-node/src/eth1/index.ts b/packages/beacon-node/src/eth1/index.ts index 9fdba90258a2..a8ba55c54141 100644 --- a/packages/beacon-node/src/eth1/index.ts +++ b/packages/beacon-node/src/eth1/index.ts @@ -106,6 +106,10 @@ export class Eth1ForBlockProduction implements IEth1ForBlockProduction { startPollingMergeBlock(): void { return this.eth1MergeBlockTracker.startPollingMergeBlock(); } + + stopPollingEth1Data(): void { + return this.eth1DepositDataTracker?.stopPollingEth1Data(); + } } /** @@ -140,4 +144,8 @@ export class Eth1ForBlockProductionDisabled implements IEth1ForBlockProduction { startPollingMergeBlock(): void { // Ignore } + + stopPollingEth1Data(): void { + // Ignore + } } diff --git a/packages/beacon-node/src/eth1/interface.ts b/packages/beacon-node/src/eth1/interface.ts index fc9626eb5b8a..898041ac8947 100644 --- a/packages/beacon-node/src/eth1/interface.ts +++ b/packages/beacon-node/src/eth1/interface.ts @@ -62,6 +62,11 @@ export interface IEth1ForBlockProduction { * - head state not isMergeTransitionComplete */ startPollingMergeBlock(): void; + + /** + * Should stop polling eth1Data after a Electra block is finalized AND deposit_receipts_start_index is reached + */ + stopPollingEth1Data(): void; } /** Different Eth1Block from phase0.Eth1Block with blockHash */ diff --git a/packages/beacon-node/src/eth1/utils/deposits.ts b/packages/beacon-node/src/eth1/utils/deposits.ts index 24916264e6d8..8d0331fc01d6 100644 --- a/packages/beacon-node/src/eth1/utils/deposits.ts +++ b/packages/beacon-node/src/eth1/utils/deposits.ts @@ -1,9 +1,9 @@ import {toGindex, Tree} from "@chainsafe/persistent-merkle-tree"; -import {MAX_DEPOSITS} from "@lodestar/params"; -import {BeaconStateAllForks} from "@lodestar/state-transition"; +import {toRootHex} from "@lodestar/utils"; +import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {phase0, ssz} from "@lodestar/types"; import {FilterOptions} from "@lodestar/db"; -import {toRootHex} from "@lodestar/utils"; +import {getEth1DepositCount} from "@lodestar/state-transition"; import {Eth1Error, Eth1ErrorCode} from "../errors.js"; import {DepositTree} from "../../db/repositories/depositDataRoot.js"; @@ -11,7 +11,7 @@ export type DepositGetter = (indexRange: FilterOptions, eth1Data: pha export async function getDeposits( // eth1_deposit_index represents the next deposit index to be added - state: BeaconStateAllForks, + state: CachedBeaconStateAllForks, eth1Data: phase0.Eth1Data, depositsGetter: DepositGetter ): Promise { @@ -22,9 +22,11 @@ export async function getDeposits( throw new Eth1Error({code: Eth1ErrorCode.DEPOSIT_INDEX_TOO_HIGH, depositIndex, depositCount}); } - // Spec v0.12.2 - // assert len(body.deposits) == min(MAX_DEPOSITS, state.eth1_data.deposit_count - state.eth1_deposit_index) - const depositsLen = Math.min(MAX_DEPOSITS, depositCount - depositIndex); + const depositsLen = getEth1DepositCount(state, eth1Data); + + if (depositsLen === 0) { + return []; // If depositsLen === 0, we can return early since no deposit with be returned from depositsGetter + } const indexRange = {gte: depositIndex, lt: depositIndex + depositsLen}; const deposits = await depositsGetter(indexRange, eth1Data); diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index c64a9715589f..d29387794327 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -198,7 +198,9 @@ export class ExecutionEngineHttp implements IExecutionEngine { parentBlockRoot?: Root ): Promise { const method = - ForkSeq[fork] >= ForkSeq.deneb + ForkSeq[fork] >= ForkSeq.electra + ? "engine_newPayloadV6110" + : ForkSeq[fork] >= ForkSeq.deneb ? "engine_newPayloadV3" : ForkSeq[fork] >= ForkSeq.capella ? "engine_newPayloadV2" @@ -218,7 +220,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { const serializedVersionedHashes = serializeVersionedHashes(versionedHashes); const parentBeaconBlockRoot = serializeBeaconBlockRoot(parentBlockRoot); - const method = "engine_newPayloadV3"; + const method = ForkSeq[fork] >= ForkSeq.electra ? "engine_newPayloadV6110" : "engine_newPayloadV3"; engineRequest = { method, params: [serializedExecutionPayload, serializedVersionedHashes, parentBeaconBlockRoot], @@ -392,7 +394,9 @@ export class ExecutionEngineHttp implements IExecutionEngine { shouldOverrideBuilder?: boolean; }> { const method = - ForkSeq[fork] >= ForkSeq.deneb + ForkSeq[fork] >= ForkSeq.electra + ? "engine_getPayloadV6110" + : ForkSeq[fork] >= ForkSeq.deneb ? "engine_getPayloadV3" : ForkSeq[fork] >= ForkSeq.capella ? "engine_getPayloadV2" diff --git a/packages/beacon-node/src/execution/engine/interface.ts b/packages/beacon-node/src/execution/engine/interface.ts index fa1da210cd12..91a8ce1cffe3 100644 --- a/packages/beacon-node/src/execution/engine/interface.ts +++ b/packages/beacon-node/src/execution/engine/interface.ts @@ -3,10 +3,10 @@ import {KZGCommitment, Blob, KZGProof} from "@lodestar/types/deneb"; import {Root, RootHex, capella, Wei, ExecutionPayload} from "@lodestar/types"; import {DATA} from "../../eth1/provider/utils.js"; -import {PayloadIdCache, PayloadId, WithdrawalV1} from "./payloadIdCache.js"; +import {PayloadIdCache, PayloadId, WithdrawalV1, DepositReceiptV1} from "./payloadIdCache.js"; import {ExecutionPayloadBody} from "./types.js"; -export {PayloadIdCache, type PayloadId, type WithdrawalV1}; +export {PayloadIdCache, type PayloadId, type WithdrawalV1, type DepositReceiptV1}; export enum ExecutionPayloadStatus { /** given payload is valid */ diff --git a/packages/beacon-node/src/execution/engine/mock.ts b/packages/beacon-node/src/execution/engine/mock.ts index a99a76508df8..5a50f1697de9 100644 --- a/packages/beacon-node/src/execution/engine/mock.ts +++ b/packages/beacon-node/src/execution/engine/mock.ts @@ -35,6 +35,7 @@ export type ExecutionEngineMockOpts = { onlyPredefinedResponses?: boolean; capellaForkTimestamp?: number; denebForkTimestamp?: number; + electraForkTimestamp?: number; }; type ExecutionBlock = { @@ -88,12 +89,14 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { engine_newPayloadV1: this.notifyNewPayload.bind(this), engine_newPayloadV2: this.notifyNewPayload.bind(this), engine_newPayloadV3: this.notifyNewPayload.bind(this), + engine_newPayloadV6110: this.notifyNewPayload.bind(this), engine_forkchoiceUpdatedV1: this.notifyForkchoiceUpdate.bind(this), engine_forkchoiceUpdatedV2: this.notifyForkchoiceUpdate.bind(this), engine_forkchoiceUpdatedV3: this.notifyForkchoiceUpdate.bind(this), engine_getPayloadV1: this.getPayload.bind(this), engine_getPayloadV2: this.getPayload.bind(this), engine_getPayloadV3: this.getPayload.bind(this), + engine_getPayloadV6110: this.getPayload.bind(this), engine_getPayloadBodiesByHashV1: this.getPayloadBodiesByHash.bind(this), engine_getPayloadBodiesByRangeV1: this.getPayloadBodiesByRange.bind(this), engine_getClientVersionV1: this.getClientVersionV1.bind(this), @@ -394,6 +397,7 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend { } private timestampToFork(timestamp: number): ForkExecution { + if (timestamp > (this.opts.electraForkTimestamp ?? Infinity)) return ForkName.electra; if (timestamp > (this.opts.denebForkTimestamp ?? Infinity)) return ForkName.deneb; if (timestamp > (this.opts.capellaForkTimestamp ?? Infinity)) return ForkName.capella; return ForkName.bellatrix; diff --git a/packages/beacon-node/src/execution/engine/payloadIdCache.ts b/packages/beacon-node/src/execution/engine/payloadIdCache.ts index ea37e0922e9c..e5baa9fba92d 100644 --- a/packages/beacon-node/src/execution/engine/payloadIdCache.ts +++ b/packages/beacon-node/src/execution/engine/payloadIdCache.ts @@ -18,6 +18,14 @@ export type WithdrawalV1 = { amount: QUANTITY; }; +export type DepositReceiptV1 = { + pubkey: DATA; + withdrawalCredentials: DATA; + amount: QUANTITY; + signature: DATA; + index: QUANTITY; +}; + type FcuAttributes = {headBlockHash: DATA; finalizedBlockHash: DATA} & Omit; export class PayloadIdCache { diff --git a/packages/beacon-node/src/execution/engine/types.ts b/packages/beacon-node/src/execution/engine/types.ts index 85f514c953b0..999c77f86b9c 100644 --- a/packages/beacon-node/src/execution/engine/types.ts +++ b/packages/beacon-node/src/execution/engine/types.ts @@ -1,4 +1,4 @@ -import {capella, deneb, Wei, bellatrix, Root, ExecutionPayload} from "@lodestar/types"; +import {capella, deneb, electra, Wei, bellatrix, Root, ExecutionPayload} from "@lodestar/types"; import { BYTES_PER_LOGS_BLOOM, FIELD_ELEMENTS_PER_BLOB, @@ -17,7 +17,7 @@ import { quantityToBigint, } from "../../eth1/provider/utils.js"; import {ExecutionPayloadStatus, BlobsBundle, PayloadAttributes, VersionedHashes} from "./interface.js"; -import {WithdrawalV1} from "./payloadIdCache.js"; +import {WithdrawalV1, DepositReceiptV1} from "./payloadIdCache.js"; /* eslint-disable @typescript-eslint/naming-convention */ @@ -28,6 +28,7 @@ export type EngineApiRpcParamTypes = { engine_newPayloadV1: [ExecutionPayloadRpc]; engine_newPayloadV2: [ExecutionPayloadRpc]; engine_newPayloadV3: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; + engine_newPayloadV6110: [ExecutionPayloadRpc, VersionedHashesRpc, DATA]; /** * 1. Object - Payload validity status with respect to the consensus rules: * - blockHash: DATA, 32 Bytes - block hash value of the payload @@ -51,6 +52,7 @@ export type EngineApiRpcParamTypes = { engine_getPayloadV1: [QUANTITY]; engine_getPayloadV2: [QUANTITY]; engine_getPayloadV3: [QUANTITY]; + engine_getPayloadV6110: [QUANTITY]; /** * 1. Array of DATA - Array of block_hash field values of the ExecutionPayload structure @@ -83,6 +85,7 @@ export type EngineApiRpcReturnTypes = { engine_newPayloadV1: PayloadStatus; engine_newPayloadV2: PayloadStatus; engine_newPayloadV3: PayloadStatus; + engine_newPayloadV6110: PayloadStatus; engine_forkchoiceUpdatedV1: { payloadStatus: PayloadStatus; payloadId: QUANTITY | null; @@ -101,6 +104,7 @@ export type EngineApiRpcReturnTypes = { engine_getPayloadV1: ExecutionPayloadRpc; engine_getPayloadV2: ExecutionPayloadResponse; engine_getPayloadV3: ExecutionPayloadResponse; + engine_getPayloadV6110: ExecutionPayloadResponse; engine_getPayloadBodiesByHashV1: (ExecutionPayloadBodyRpc | null)[]; @@ -118,9 +122,17 @@ type ExecutionPayloadRpcWithValue = { }; type ExecutionPayloadResponse = ExecutionPayloadRpc | ExecutionPayloadRpcWithValue; -export type ExecutionPayloadBodyRpc = {transactions: DATA[]; withdrawals: WithdrawalV1[] | null}; +export type ExecutionPayloadBodyRpc = { + transactions: DATA[]; + withdrawals: WithdrawalV1[] | null | undefined; + depositReceipts: DepositReceiptV1[] | null | undefined; +}; -export type ExecutionPayloadBody = {transactions: bellatrix.Transaction[]; withdrawals: capella.Withdrawals | null}; +export type ExecutionPayloadBody = { + transactions: bellatrix.Transaction[]; + withdrawals: capella.Withdrawals | null; + depositReceipts: electra.DepositReceipts | null; +}; export type ExecutionPayloadRpc = { parentHash: DATA; // 32 bytes @@ -141,6 +153,7 @@ export type ExecutionPayloadRpc = { blobGasUsed?: QUANTITY; // DENEB excessBlobGas?: QUANTITY; // DENEB parentBeaconBlockRoot?: QUANTITY; // DENEB + depositReceipts?: DepositReceiptRpc[]; // ELECTRA }; export type WithdrawalRpc = { @@ -150,6 +163,14 @@ export type WithdrawalRpc = { amount: QUANTITY; }; +export type DepositReceiptRpc = { + pubkey: DATA; + withdrawalCredentials: DATA; + amount: QUANTITY; + signature: DATA; + index: QUANTITY; +}; + export type VersionedHashesRpc = DATA[]; export type PayloadAttributesRpc = { @@ -212,6 +233,12 @@ export function serializeExecutionPayload(fork: ForkName, data: ExecutionPayload payload.excessBlobGas = numToQuantity(excessBlobGas); } + // ELECTRA adds depositReceipts to the ExecutionPayload + if (ForkSeq[fork] >= ForkSeq.electra) { + const {depositReceipts} = data as electra.ExecutionPayload; + payload.depositReceipts = depositReceipts.map(serializeDepositReceipt); + } + return payload; } @@ -297,6 +324,17 @@ export function parseExecutionPayload( (executionPayload as deneb.ExecutionPayload).excessBlobGas = quantityToBigint(excessBlobGas); } + if (ForkSeq[fork] >= ForkSeq.electra) { + const {depositReceipts} = data; + // Geth can also reply with null + if (depositReceipts == null) { + throw Error( + `depositReceipts missing for ${fork} >= electra executionPayload number=${executionPayload.blockNumber} hash=${data.blockHash}` + ); + } + (executionPayload as electra.ExecutionPayload).depositReceipts = depositReceipts.map(deserializeDepositReceipts); + } + return {executionPayload, executionPayloadValue, blobsBundle, shouldOverrideBuilder}; } @@ -363,11 +401,32 @@ export function deserializeWithdrawal(serialized: WithdrawalRpc): capella.Withdr } as capella.Withdrawal; } +export function serializeDepositReceipt(depositReceipt: electra.DepositReceipt): DepositReceiptRpc { + return { + pubkey: bytesToData(depositReceipt.pubkey), + withdrawalCredentials: bytesToData(depositReceipt.withdrawalCredentials), + amount: numToQuantity(depositReceipt.amount), + signature: bytesToData(depositReceipt.signature), + index: numToQuantity(depositReceipt.index), + }; +} + +export function deserializeDepositReceipts(serialized: DepositReceiptRpc): electra.DepositReceipt { + return { + pubkey: dataToBytes(serialized.pubkey, 48), + withdrawalCredentials: dataToBytes(serialized.withdrawalCredentials, 32), + amount: quantityToNum(serialized.amount), + signature: dataToBytes(serialized.signature, 96), + index: quantityToNum(serialized.index), + } as electra.DepositReceipt; +} + export function deserializeExecutionPayloadBody(data: ExecutionPayloadBodyRpc | null): ExecutionPayloadBody | null { return data ? { transactions: data.transactions.map((tran) => dataToBytes(tran, null)), withdrawals: data.withdrawals ? data.withdrawals.map(deserializeWithdrawal) : null, + depositReceipts: data.depositReceipts ? data.depositReceipts.map(deserializeDepositReceipts) : null, } : null; } @@ -377,6 +436,7 @@ export function serializeExecutionPayloadBody(data: ExecutionPayloadBody | null) ? { transactions: data.transactions.map((tran) => bytesToData(tran)), withdrawals: data.withdrawals ? data.withdrawals.map(serializeWithdrawal) : null, + depositReceipts: data.depositReceipts ? data.depositReceipts.map(serializeDepositReceipt) : null, } : null; } diff --git a/packages/beacon-node/src/metrics/metrics/beacon.ts b/packages/beacon-node/src/metrics/metrics/beacon.ts index 141121de9079..96bb6bea4174 100644 --- a/packages/beacon-node/src/metrics/metrics/beacon.ts +++ b/packages/beacon-node/src/metrics/metrics/beacon.ts @@ -120,6 +120,13 @@ export function createBeaconMetrics(register: RegistryMetricCreator) { }), }, + headState: { + unfinalizedPubkeyCacheSize: register.gauge({ + name: "head_state_unfinalized_pubkey_cache_size", + help: "Current size of the unfinalizedPubkey2Index cache in the head state", + }), + }, + parentBlockDistance: register.histogram({ name: "beacon_imported_block_parent_distance", help: "Histogram of distance to parent block of valid imported blocks", diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index f43a3f1cdbe6..595e4a7a45fc 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -374,6 +374,17 @@ export function createLodestarMetrics( help: "Total count state.validators nodesPopulated is false on stfn for post state", }), + epochCache: { + finalizedPubkeyDuplicateInsert: register.gauge({ + name: "lodestar_epoch_cache_finalized_pubkey_duplicate_insert", + help: "Total count of duplicate insert of finalized pubkeys", + }), + newUnFinalizedPubkey: register.gauge({ + name: "lodestar_epoch_cache_new_unfinalized_pubkey", + help: "Total count of unfinalized pubkeys added", + }), + }, + // BLS verifier thread pool and queue bls: { @@ -1205,6 +1216,11 @@ export function createLodestarMetrics( help: "Histogram of time to serialize state to db", buckets: [0.1, 0.5, 1, 2, 3, 4], }), + numStatesUpdated: register.histogram({ + name: "lodestar_cp_state_cache_state_updated_count", + help: "Histogram of number of state cache items updated every time removing and adding pubkeys to pubkey cache", + buckets: [1, 2, 5, 10, 50, 250], + }), statePruneFromMemoryCount: register.gauge({ name: "lodestar_cp_state_cache_state_prune_from_memory_count", help: "Total number of states pruned from memory", @@ -1373,6 +1389,21 @@ export function createLodestarMetrics( help: "regen function total errors", labelNames: ["entrypoint", "caller"], }), + regenFnAddPubkeyTime: register.histogram({ + name: "lodestar_regen_fn_add_pubkey_time_seconds", + help: "Historgram of time spent on adding pubkeys to all state cache items in seconds", + buckets: [0.01, 0.1, 0.5, 1, 2, 5], + }), + regenFnDeletePubkeyTime: register.histogram({ + name: "lodestar_regen_fn_delete_pubkey_time_seconds", + help: "Histrogram of time spent on deleting pubkeys from all state cache items in seconds", + buckets: [0.01, 0.1, 0.5, 1, 2, 5], + }), + regenFnNumStatesUpdated: register.histogram({ + name: "lodestar_regen_state_cache_state_updated_count", + help: "Histogram of number of state cache items updated every time removing pubkeys from unfinalized cache", + buckets: [1, 2, 5, 10, 50, 250], + }), unhandledPromiseRejections: register.gauge({ name: "lodestar_unhandled_promise_rejections_total", help: "UnhandledPromiseRejection total count", diff --git a/packages/beacon-node/test/memory/unfinalizedPubkey2Index.ts b/packages/beacon-node/test/memory/unfinalizedPubkey2Index.ts new file mode 100644 index 000000000000..b37967d16ca4 --- /dev/null +++ b/packages/beacon-node/test/memory/unfinalizedPubkey2Index.ts @@ -0,0 +1,54 @@ +import crypto from "node:crypto"; +import {Map} from "immutable"; +import {ValidatorIndex} from "@lodestar/types"; +import {toMemoryEfficientHexStr} from "@lodestar/state-transition/src/cache/pubkeyCache.js"; +import {testRunnerMemory} from "./testRunnerMemory.js"; + +// Results in MacOS Nov 2023 +// +// UnfinalizedPubkey2Index 1000 keys - 274956.5 bytes / instance +// UnfinalizedPubkey2Index 10000 keys - 2591129.3 bytes / instance +// UnfinalizedPubkey2Index 100000 keys - 27261443.4 bytes / instance + +testRunnerMemoryBpi([ + { + id: "UnfinalizedPubkey2Index 1000 keys", + getInstance: () => getRandomMap(1000, () => toMemoryEfficientHexStr(crypto.randomBytes(48))), + }, + { + id: "UnfinalizedPubkey2Index 10000 keys", + getInstance: () => getRandomMap(10000, () => toMemoryEfficientHexStr(crypto.randomBytes(48))), + }, + { + id: "UnfinalizedPubkey2Index 100000 keys", + getInstance: () => getRandomMap(100000, () => toMemoryEfficientHexStr(crypto.randomBytes(48))), + }, +]); + +function getRandomMap(n: number, getKey: (i: number) => string): Map { + const map = Map(); + + return map.withMutations((m) => { + for (let i = 0; i < n; i++) { + m.set(getKey(i), i); + } + }); +} + +/** + * Test bytes per instance in different representations of raw binary data + */ +function testRunnerMemoryBpi(testCases: {getInstance: (bytes: number) => unknown; id: string}[]): void { + const longestId = Math.max(...testCases.map(({id}) => id.length)); + + for (const {id, getInstance} of testCases) { + const bpi = testRunnerMemory({ + getInstance, + convergeFactor: 1 / 100, + sampleEvery: 5, + }); + + // eslint-disable-next-line no-console + console.log(`${id.padEnd(longestId)} - ${bpi.toFixed(1)} bytes / instance`); + } +} diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 60ff6ce48302..7a882b03af0e 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -32,7 +32,7 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { before(function () { this.timeout(5 * 60 * 1000); // Generating the states for the first time is very slow - originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true, vc}); + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true, vc}) as unknown as CachedBeaconStateAltair; const {blockHeader, checkpoint} = computeAnchorCheckpoint(originalState.config, originalState); // TODO figure out why getBlockRootAtSlot(originalState, justifiedSlot) is not the same to justifiedCheckpoint.root diff --git a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts index 6e420f0e1011..7998a204f09d 100644 --- a/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/opPool.test.ts @@ -24,7 +24,7 @@ describe("opPool", () => { before(function () { this.timeout(2 * 60 * 1000); // Generating the states for the first time is very slow - originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}); + originalState = generatePerfTestCachedStateAltair({goBackOneSlot: true}) as unknown as CachedBeaconStateAltair; }); itBench({ diff --git a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts index 7bf8c2f7252f..2386f3538205 100644 --- a/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts +++ b/packages/beacon-node/test/perf/chain/produceBlock/produceBlockBody.test.ts @@ -25,7 +25,7 @@ describe("produceBlockBody", () => { before(async () => { db = new BeaconDb(config, await LevelDbController.create({name: ".tmpdb"}, {logger})); - state = stateOg.clone(); + state = stateOg.clone() as unknown as CachedBeaconStateAltair; chain = new BeaconChain( { proposerBoost: true, diff --git a/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts new file mode 100644 index 000000000000..900f6a6fb873 --- /dev/null +++ b/packages/beacon-node/test/perf/chain/stateCache/updateUnfinalizedPubkeys.test.ts @@ -0,0 +1,110 @@ +import {itBench, setBenchOpts} from "@dapplion/benchmark"; +import {Map} from "immutable"; +import {toBufferBE} from "bigint-buffer"; +import {digest} from "@chainsafe/as-sha256"; +import type {SecretKey} from "@chainsafe/bls/types"; +import bls from "@chainsafe/bls"; +import {ssz} from "@lodestar/types"; +import {type CachedBeaconStateAllForks, PubkeyIndexMap} from "@lodestar/state-transition"; +import {bytesToBigInt, intToBytes} from "@lodestar/utils"; +import {CheckpointStateCache, StateContextCache} from "../../../../src/chain/stateCache/index.js"; +import {generateCachedElectraState} from "../../../utils/state.js"; + +// Benchmark date from Mon Nov 21 2023 - Intel Core i7-9750H @ 2.60Ghz +// ✔ updateUnfinalizedPubkeys - updating 10 pubkeys 1444.173 ops/s 692.4380 us/op - 1057 runs 6.03 s +// ✔ updateUnfinalizedPubkeys - updating 100 pubkeys 189.5965 ops/s 5.274358 ms/op - 57 runs 1.15 s +// ✔ updateUnfinalizedPubkeys - updating 1000 pubkeys 12.90495 ops/s 77.48967 ms/op - 13 runs 1.62 s +describe("updateUnfinalizedPubkeys perf tests", function () { + setBenchOpts({noThreshold: true}); + + const numPubkeysToBeFinalizedCases = [10, 100, 1000]; + const numCheckpointStateCache = 8; + const numStateCache = 3 * 32; + + let checkpointStateCache: CheckpointStateCache; + let stateCache: StateContextCache; + + const unfinalizedPubkey2Index = generatePubkey2Index(0, Math.max.apply(null, numPubkeysToBeFinalizedCases)); + const baseState = generateCachedElectraState(); + + for (const numPubkeysToBeFinalized of numPubkeysToBeFinalizedCases) { + itBench({ + id: `updateUnfinalizedPubkeys - updating ${numPubkeysToBeFinalized} pubkeys`, + beforeEach: async () => { + baseState.epochCtx.unfinalizedPubkey2index = Map(unfinalizedPubkey2Index.map); + baseState.epochCtx.pubkey2index = new PubkeyIndexMap(); + baseState.epochCtx.index2pubkey = []; + + checkpointStateCache = new CheckpointStateCache({}); + stateCache = new StateContextCache({}); + + for (let i = 0; i < numCheckpointStateCache; i++) { + const clonedState = baseState.clone(); + const checkpoint = ssz.phase0.Checkpoint.defaultValue(); + + clonedState.slot = i; + checkpoint.epoch = i; // Assigning arbitrary non-duplicate values to ensure checkpointStateCache correctly saves all the states + + checkpointStateCache.add(checkpoint, clonedState); + } + + for (let i = 0; i < numStateCache; i++) { + const clonedState = baseState.clone(); + clonedState.slot = i; + stateCache.add(clonedState); + } + }, + fn: async () => { + const newFinalizedValidators = baseState.epochCtx.unfinalizedPubkey2index.filter( + (index, _pubkey) => index < numPubkeysToBeFinalized + ); + + const states = stateCache.getStates(); + const cpStates = checkpointStateCache.getStates(); + + const firstState = states.next().value as CachedBeaconStateAllForks; + firstState.epochCtx.addFinalizedPubkeys(newFinalizedValidators); + + const pubkeysToDelete = Array.from(newFinalizedValidators.keys()); + + firstState.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + + for (const s of states) { + s.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + } + + for (const s of cpStates) { + s.epochCtx.deleteUnfinalizedPubkeys(pubkeysToDelete); + } + }, + }); + } + + function generatePubkey2Index(startIndex: number, endIndex: number): PubkeyIndexMap { + const pubkey2Index = new PubkeyIndexMap(); + const pubkeys = generatePubkeys(endIndex - startIndex); + + for (let i = startIndex; i < endIndex; i++) { + pubkey2Index.set(pubkeys[i], i); + } + + return pubkey2Index; + } + + function generatePubkeys(validatorCount: number): Uint8Array[] { + const keys = []; + + for (let i = 0; i < validatorCount; i++) { + const sk = generatePrivateKey(i); + const pk = sk.toPublicKey().toBytes(); + keys.push(pk); + } + + return keys; + } + + function generatePrivateKey(index: number): SecretKey { + const secretKeyBytes = toBufferBE(bytesToBigInt(digest(intToBytes(index, 32))) % BigInt("38581184513"), 32); + return bls.SecretKey.fromBytes(secretKeyBytes); + } +}); diff --git a/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh b/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh new file mode 100755 index 000000000000..1444f6d3a479 --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besu/common-setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash -x + +echo $TTD +echo $DATA_DIR +echo $EL_BINARY_DIR +echo $JWT_SECRET_HEX +echo $TEMPLATE_FILE + +echo $scriptDir +echo $currentDir + + +env TTD=$TTD envsubst < $scriptDir/$TEMPLATE_FILE > $DATA_DIR/genesis.json +echo "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" > $DATA_DIR/sk.json +echo "12345678" > $DATA_DIR/password.txt +pubKey="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + +# echo a hex encoded 256 bit secret into a file +echo $JWT_SECRET_HEX> $DATA_DIR/jwtsecret \ No newline at end of file diff --git a/packages/beacon-node/test/scripts/el-interop/besu/electra.tmpl b/packages/beacon-node/test/scripts/el-interop/besu/electra.tmpl new file mode 100644 index 000000000000..7a63bfbe36d6 --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besu/electra.tmpl @@ -0,0 +1,77 @@ +{ + "config": { + "chainId":6110, + "homesteadBlock":0, + "eip150Block":0, + "eip155Block":0, + "eip158Block":0, + "byzantiumBlock":0, + "constantinopleBlock":0, + "petersburgBlock":0, + "istanbulBlock":0, + "muirGlacierBlock":0, + "berlinBlock":0, + "londonBlock":0, + "terminalTotalDifficulty":0, + "cancunTime":0, + "experimentalEipsTime":10, + "clique": { + "period": 5, + "epoch": 30000 + }, + "depositContractAddress": "0x4242424242424242424242424242424242424242" + }, + "nonce":"0x42", + "timestamp":"0x0", + "extraData":"0x0000000000000000000000000000000000000000000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit":"0x1C9C380", + "difficulty":"0x400000000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase":"0x0000000000000000000000000000000000000000", + "alloc":{ + "0xa4664C40AACeBD82A2Db79f0ea36C06Bc6A19Adb": { + "balance": "1000000000000000000000000000" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + } + }, + "number":"0x0", + "gasUsed":"0x0", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas":"0x7" +} diff --git a/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh b/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh new file mode 100755 index 000000000000..47bec71cf8bb --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besu/post-merge.sh @@ -0,0 +1,8 @@ +#!/bin/bash -x + +scriptDir=$(dirname $0) +currentDir=$(pwd) + +. $scriptDir/common-setup.sh + +$EL_BINARY_DIR/besu --engine-rpc-enabled --rpc-http-enabled --rpc-http-api ADMIN,ETH,MINER,NET --rpc-http-port $ETH_PORT --engine-rpc-port $ENGINE_PORT --engine-jwt-secret $currentDir/$DATA_DIR/jwtsecret --data-path $DATA_DIR --data-storage-format BONSAI --genesis-file $DATA_DIR/genesis.json \ No newline at end of file diff --git a/packages/beacon-node/test/scripts/el-interop/besudocker/common-setup.sh b/packages/beacon-node/test/scripts/el-interop/besudocker/common-setup.sh new file mode 100644 index 000000000000..b3d93190ef2d --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besudocker/common-setup.sh @@ -0,0 +1,22 @@ +#!/bin/bash -x + +echo $TTD +echo $DATA_DIR +echo $EL_BINARY_DIR +echo $JWT_SECRET_HEX +echo $TEMPLATE_FILE + +echo $scriptDir +echo $currentDir + + +env TTD=$TTD envsubst < $scriptDir/$TEMPLATE_FILE > $DATA_DIR/genesis.json +echo "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" > $DATA_DIR/sk.json +echo "12345678" > $DATA_DIR/password.txt +pubKey="0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b" + +# echo a hex encoded 256 bit secret into a file +echo $JWT_SECRET_HEX> $DATA_DIR/jwtsecret +# clear any previous docker dangling docker run +docker rm -f custom-execution +rm -rf $DATA_DIR/besu diff --git a/packages/beacon-node/test/scripts/el-interop/besudocker/electra.tmpl b/packages/beacon-node/test/scripts/el-interop/besudocker/electra.tmpl new file mode 100644 index 000000000000..7a63bfbe36d6 --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besudocker/electra.tmpl @@ -0,0 +1,77 @@ +{ + "config": { + "chainId":6110, + "homesteadBlock":0, + "eip150Block":0, + "eip155Block":0, + "eip158Block":0, + "byzantiumBlock":0, + "constantinopleBlock":0, + "petersburgBlock":0, + "istanbulBlock":0, + "muirGlacierBlock":0, + "berlinBlock":0, + "londonBlock":0, + "terminalTotalDifficulty":0, + "cancunTime":0, + "experimentalEipsTime":10, + "clique": { + "period": 5, + "epoch": 30000 + }, + "depositContractAddress": "0x4242424242424242424242424242424242424242" + }, + "nonce":"0x42", + "timestamp":"0x0", + "extraData":"0x0000000000000000000000000000000000000000000000000000000000000000a94f5374fce5edbc8e2a8697c15331677e6ebf0b0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "gasLimit":"0x1C9C380", + "difficulty":"0x400000000", + "mixHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "coinbase":"0x0000000000000000000000000000000000000000", + "alloc":{ + "0xa4664C40AACeBD82A2Db79f0ea36C06Bc6A19Adb": { + "balance": "1000000000000000000000000000" + }, + "0x4242424242424242424242424242424242424242": { + "balance": "0", + "code": "0x60806040526004361061003f5760003560e01c806301ffc9a71461004457806322895118146100a4578063621fd130146101ba578063c5f2892f14610244575b600080fd5b34801561005057600080fd5b506100906004803603602081101561006757600080fd5b50357fffffffff000000000000000000000000000000000000000000000000000000001661026b565b604080519115158252519081900360200190f35b6101b8600480360360808110156100ba57600080fd5b8101906020810181356401000000008111156100d557600080fd5b8201836020820111156100e757600080fd5b8035906020019184600183028401116401000000008311171561010957600080fd5b91939092909160208101903564010000000081111561012757600080fd5b82018360208201111561013957600080fd5b8035906020019184600183028401116401000000008311171561015b57600080fd5b91939092909160208101903564010000000081111561017957600080fd5b82018360208201111561018b57600080fd5b803590602001918460018302840111640100000000831117156101ad57600080fd5b919350915035610304565b005b3480156101c657600080fd5b506101cf6110b5565b6040805160208082528351818301528351919283929083019185019080838360005b838110156102095781810151838201526020016101f1565b50505050905090810190601f1680156102365780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b34801561025057600080fd5b506102596110c7565b60408051918252519081900360200190f35b60007fffffffff0000000000000000000000000000000000000000000000000000000082167f01ffc9a70000000000000000000000000000000000000000000000000000000014806102fe57507fffffffff0000000000000000000000000000000000000000000000000000000082167f8564090700000000000000000000000000000000000000000000000000000000145b92915050565b6030861461035d576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118056026913960400191505060405180910390fd5b602084146103b6576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252603681526020018061179c6036913960400191505060405180910390fd5b6060821461040f576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260298152602001806118786029913960400191505060405180910390fd5b670de0b6b3a7640000341015610470576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260268152602001806118526026913960400191505060405180910390fd5b633b9aca003406156104cd576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260338152602001806117d26033913960400191505060405180910390fd5b633b9aca00340467ffffffffffffffff811115610535576040517f08c379a000000000000000000000000000000000000000000000000000000000815260040180806020018281038252602781526020018061182b6027913960400191505060405180910390fd5b6060610540826114ba565b90507f649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c589898989858a8a6105756020546114ba565b6040805160a0808252810189905290819060208201908201606083016080840160c085018e8e80828437600083820152601f017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690910187810386528c815260200190508c8c808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe01690920188810386528c5181528c51602091820193918e019250908190849084905b83811015610648578181015183820152602001610630565b50505050905090810190601f1680156106755780820380516001836020036101000a031916815260200191505b5086810383528881526020018989808284376000838201819052601f9091017fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0169092018881038452895181528951602091820193918b019250908190849084905b838110156106ef5781810151838201526020016106d7565b50505050905090810190601f16801561071c5780820380516001836020036101000a031916815260200191505b509d505050505050505050505050505060405180910390a1600060028a8a600060801b604051602001808484808284377fffffffffffffffffffffffffffffffff0000000000000000000000000000000090941691909301908152604080517ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0818403018152601090920190819052815191955093508392506020850191508083835b602083106107fc57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016107bf565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610859573d6000803e3d6000fd5b5050506040513d602081101561086e57600080fd5b5051905060006002806108846040848a8c6116fe565b6040516020018083838082843780830192505050925050506040516020818303038152906040526040518082805190602001908083835b602083106108f857805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016108bb565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610955573d6000803e3d6000fd5b5050506040513d602081101561096a57600080fd5b5051600261097b896040818d6116fe565b60405160009060200180848480828437919091019283525050604080518083038152602092830191829052805190945090925082918401908083835b602083106109f457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe090920191602091820191016109b7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610a51573d6000803e3d6000fd5b5050506040513d6020811015610a6657600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610ada57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610a9d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610b37573d6000803e3d6000fd5b5050506040513d6020811015610b4c57600080fd5b50516040805160208101858152929350600092600292839287928f928f92018383808284378083019250505093505050506040516020818303038152906040526040518082805190602001908083835b60208310610bd957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610b9c565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610c36573d6000803e3d6000fd5b5050506040513d6020811015610c4b57600080fd5b50516040518651600291889160009188916020918201918291908601908083835b60208310610ca957805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610c6c565b6001836020036101000a0380198251168184511680821785525050505050509050018367ffffffffffffffff191667ffffffffffffffff1916815260180182815260200193505050506040516020818303038152906040526040518082805190602001908083835b60208310610d4e57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610d11565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610dab573d6000803e3d6000fd5b5050506040513d6020811015610dc057600080fd5b5051604080516020818101949094528082019290925280518083038201815260609092019081905281519192909182918401908083835b60208310610e3457805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610df7565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015610e91573d6000803e3d6000fd5b5050506040513d6020811015610ea657600080fd5b50519050858114610f02576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260548152602001806117486054913960600191505060405180910390fd5b60205463ffffffff11610f60576040517f08c379a00000000000000000000000000000000000000000000000000000000081526004018080602001828103825260218152602001806117276021913960400191505060405180910390fd5b602080546001019081905560005b60208110156110a9578160011660011415610fa0578260008260208110610f9157fe5b0155506110ac95505050505050565b600260008260208110610faf57fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061102557805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101610fe8565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa158015611082573d6000803e3d6000fd5b5050506040513d602081101561109757600080fd5b50519250600282049150600101610f6e565b50fe5b50505050505050565b60606110c26020546114ba565b905090565b6020546000908190815b60208110156112f05781600116600114156111e6576002600082602081106110f557fe5b01548460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061116b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161112e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156111c8573d6000803e3d6000fd5b5050506040513d60208110156111dd57600080fd5b505192506112e2565b600283602183602081106111f657fe5b015460405160200180838152602001828152602001925050506040516020818303038152906040526040518082805190602001908083835b6020831061126b57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161122e565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa1580156112c8573d6000803e3d6000fd5b5050506040513d60208110156112dd57600080fd5b505192505b6002820491506001016110d1565b506002826112ff6020546114ba565b600060401b6040516020018084815260200183805190602001908083835b6020831061135a57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe0909201916020918201910161131d565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790527fffffffffffffffffffffffffffffffffffffffffffffffff000000000000000095909516920191825250604080518083037ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff8018152601890920190819052815191955093508392850191508083835b6020831061143f57805182527fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe09092019160209182019101611402565b51815160209384036101000a7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff01801990921691161790526040519190930194509192505080830381855afa15801561149c573d6000803e3d6000fd5b5050506040513d60208110156114b157600080fd5b50519250505090565b60408051600880825281830190925260609160208201818036833701905050905060c082901b8060071a60f81b826000815181106114f457fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060061a60f81b8260018151811061153757fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060051a60f81b8260028151811061157a57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060041a60f81b826003815181106115bd57fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060031a60f81b8260048151811061160057fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060021a60f81b8260058151811061164357fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060011a60f81b8260068151811061168657fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a9053508060001a60f81b826007815181106116c957fe5b60200101907effffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff1916908160001a90535050919050565b6000808585111561170d578182fd5b83861115611719578182fd5b505082019391909203915056fe4465706f736974436f6e74726163743a206d65726b6c6520747265652066756c6c4465706f736974436f6e74726163743a207265636f6e7374727563746564204465706f7369744461746120646f6573206e6f74206d6174636820737570706c696564206465706f7369745f646174615f726f6f744465706f736974436f6e74726163743a20696e76616c6964207769746864726177616c5f63726564656e7469616c73206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c7565206e6f74206d756c7469706c65206f6620677765694465706f736974436f6e74726163743a20696e76616c6964207075626b6579206c656e6774684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f20686967684465706f736974436f6e74726163743a206465706f7369742076616c756520746f6f206c6f774465706f736974436f6e74726163743a20696e76616c6964207369676e6174757265206c656e677468a26469706673582212201dd26f37a621703009abf16e77e69c93dc50c79db7f6cc37543e3e0e3decdc9764736f6c634300060b0033", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000022": "0xf5a5fd42d16a20302798ef6ed309979b43003d2320d9f0e8ea9831a92759fb4b", + "0x0000000000000000000000000000000000000000000000000000000000000023": "0xdb56114e00fdd4c1f85c892bf35ac9a89289aaecb1ebd0a96cde606a748b5d71", + "0x0000000000000000000000000000000000000000000000000000000000000024": "0xc78009fdf07fc56a11f122370658a353aaa542ed63e44c4bc15ff4cd105ab33c", + "0x0000000000000000000000000000000000000000000000000000000000000025": "0x536d98837f2dd165a55d5eeae91485954472d56f246df256bf3cae19352a123c", + "0x0000000000000000000000000000000000000000000000000000000000000026": "0x9efde052aa15429fae05bad4d0b1d7c64da64d03d7a1854a588c2cb8430c0d30", + "0x0000000000000000000000000000000000000000000000000000000000000027": "0xd88ddfeed400a8755596b21942c1497e114c302e6118290f91e6772976041fa1", + "0x0000000000000000000000000000000000000000000000000000000000000028": "0x87eb0ddba57e35f6d286673802a4af5975e22506c7cf4c64bb6be5ee11527f2c", + "0x0000000000000000000000000000000000000000000000000000000000000029": "0x26846476fd5fc54a5d43385167c95144f2643f533cc85bb9d16b782f8d7db193", + "0x000000000000000000000000000000000000000000000000000000000000002a": "0x506d86582d252405b840018792cad2bf1259f1ef5aa5f887e13cb2f0094f51e1", + "0x000000000000000000000000000000000000000000000000000000000000002b": "0xffff0ad7e659772f9534c195c815efc4014ef1e1daed4404c06385d11192e92b", + "0x000000000000000000000000000000000000000000000000000000000000002c": "0x6cf04127db05441cd833107a52be852868890e4317e6a02ab47683aa75964220", + "0x000000000000000000000000000000000000000000000000000000000000002d": "0xb7d05f875f140027ef5118a2247bbb84ce8f2f0f1123623085daf7960c329f5f", + "0x000000000000000000000000000000000000000000000000000000000000002e": "0xdf6af5f5bbdb6be9ef8aa618e4bf8073960867171e29676f8b284dea6a08a85e", + "0x000000000000000000000000000000000000000000000000000000000000002f": "0xb58d900f5e182e3c50ef74969ea16c7726c549757cc23523c369587da7293784", + "0x0000000000000000000000000000000000000000000000000000000000000030": "0xd49a7502ffcfb0340b1d7885688500ca308161a7f96b62df9d083b71fcc8f2bb", + "0x0000000000000000000000000000000000000000000000000000000000000031": "0x8fe6b1689256c0d385f42f5bbe2027a22c1996e110ba97c171d3e5948de92beb", + "0x0000000000000000000000000000000000000000000000000000000000000032": "0x8d0d63c39ebade8509e0ae3c9c3876fb5fa112be18f905ecacfecb92057603ab", + "0x0000000000000000000000000000000000000000000000000000000000000033": "0x95eec8b2e541cad4e91de38385f2e046619f54496c2382cb6cacd5b98c26f5a4", + "0x0000000000000000000000000000000000000000000000000000000000000034": "0xf893e908917775b62bff23294dbbe3a1cd8e6cc1c35b4801887b646a6f81f17f", + "0x0000000000000000000000000000000000000000000000000000000000000035": "0xcddba7b592e3133393c16194fac7431abf2f5485ed711db282183c819e08ebaa", + "0x0000000000000000000000000000000000000000000000000000000000000036": "0x8a8d7fe3af8caa085a7639a832001457dfb9128a8061142ad0335629ff23ff9c", + "0x0000000000000000000000000000000000000000000000000000000000000037": "0xfeb3c337d7a51a6fbf00b9e34c52e1c9195c969bd4e7a0bfd51d5c5bed9c1167", + "0x0000000000000000000000000000000000000000000000000000000000000038": "0xe71f0aa83cc32edfbefa9f4d3e0174ca85182eec9f3a09f6a6c0df6377a510d7", + "0x0000000000000000000000000000000000000000000000000000000000000039": "0x31206fa80a50bb6abe29085058f16212212a60eec8f049fecb92d8c8e0a84bc0", + "0x000000000000000000000000000000000000000000000000000000000000003a": "0x21352bfecbeddde993839f614c3dac0a3ee37543f9b412b16199dc158e23b544", + "0x000000000000000000000000000000000000000000000000000000000000003b": "0x619e312724bb6d7c3153ed9de791d764a366b389af13c58bf8a8d90481a46765", + "0x000000000000000000000000000000000000000000000000000000000000003c": "0x7cdd2986268250628d0c10e385c58c6191e6fbe05191bcc04f133f2cea72c1c4", + "0x000000000000000000000000000000000000000000000000000000000000003d": "0x848930bd7ba8cac54661072113fb278869e07bb8587f91392933374d017bcbe1", + "0x000000000000000000000000000000000000000000000000000000000000003e": "0x8869ff2c22b28cc10510d9853292803328be4fb0e80495e8bb8d271f5b889636", + "0x000000000000000000000000000000000000000000000000000000000000003f": "0xb5fe28e79f1b850f8658246ce9b6a1e7b49fc06db7143e8fe0b4f2b0c5523a5c", + "0x0000000000000000000000000000000000000000000000000000000000000040": "0x985e929f70af28d0bdd1a90a808f977f597c7c778c489e98d3bd8910d31ac0f7" + } + } + }, + "number":"0x0", + "gasUsed":"0x0", + "parentHash":"0x0000000000000000000000000000000000000000000000000000000000000000", + "baseFeePerGas":"0x7" +} diff --git a/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh b/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh new file mode 100755 index 000000000000..fb091a7d838b --- /dev/null +++ b/packages/beacon-node/test/scripts/el-interop/besudocker/post-merge.sh @@ -0,0 +1,8 @@ +#!/bin/bash -x + +scriptDir=$(dirname $0) +currentDir=$(pwd) + +. $scriptDir/common-setup.sh + +docker run --rm -u $(id -u ${USER}):$(id -g ${USER}) --name custom-execution -p $ETH_PORT:$ETH_PORT -p $ENGINE_PORT:$ENGINE_PORT -v $currentDir/$DATA_DIR:/data $EL_BINARY_DIR --engine-rpc-enabled --rpc-http-enabled --rpc-http-api ADMIN,ETH,MINER,NET --rpc-http-port $ETH_PORT --engine-rpc-port $ENGINE_PORT --engine-jwt-secret /data/jwtsecret --data-path /data/besu --data-storage-format BONSAI --genesis-file /data/genesis.json \ No newline at end of file diff --git a/packages/beacon-node/test/sim/electra-interop.test.ts b/packages/beacon-node/test/sim/electra-interop.test.ts new file mode 100644 index 000000000000..c1663211cbd9 --- /dev/null +++ b/packages/beacon-node/test/sim/electra-interop.test.ts @@ -0,0 +1,457 @@ +import fs from "node:fs"; +import {describe, it, vi, afterAll, afterEach} from "vitest"; +/* eslint-disable @typescript-eslint/naming-convention */ +import _ from "lodash"; +import {LogLevel, sleep} from "@lodestar/utils"; +import {ForkName, SLOTS_PER_EPOCH, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; +import {electra, Epoch, Slot} from "@lodestar/types"; +import {ValidatorProposerConfig} from "@lodestar/validator"; + +import {ChainConfig} from "@lodestar/config"; +import {TimestampFormatCode} from "@lodestar/logger"; +import {CachedBeaconStateElectra} from "@lodestar/state-transition"; +import {initializeExecutionEngine} from "../../src/execution/index.js"; +import {ExecutionPayloadStatus, PayloadAttributes} from "../../src/execution/engine/interface.js"; + +import {testLogger, TestLoggerOpts} from "../utils/logger.js"; +import {runEL, ELStartMode, ELClient, sendRawTransactionBig} from "../utils/runEl.js"; +import {defaultExecutionEngineHttpOpts} from "../../src/execution/engine/http.js"; +import {getDevBeaconNode} from "../utils/node/beacon.js"; +import {BeaconRestApiServerOpts} from "../../src/api/index.js"; +import {simTestInfoTracker} from "../utils/node/simTest.js"; +import {getAndInitDevValidators} from "../utils/node/validator.js"; +import {ClockEvent} from "../../src/util/clock.js"; +import {dataToBytes} from "../../src/eth1/provider/utils.js"; +import {bytesToData} from "../../lib/eth1/provider/utils.js"; +import {BeaconNode} from "../../src/index.js"; +import {logFilesDir} from "./params.js"; +import {shell} from "./shell.js"; + +// NOTE: How to run +// DEV_RUN=true EL_BINARY_DIR=naviechan/besu:v6110 EL_SCRIPT_DIR=besudocker yarn vitest --run test/sim/electra-interop.test.ts +// or +// DEV_RUN=true EL_BINARY_DIR=/Volumes/fast_boi/navie/Documents/workspace/besu/build/install/besu/bin EL_SCRIPT_DIR=besu yarn vitest --run test/sim/electra-interop.test.ts +// ``` + +/* eslint-disable no-console, @typescript-eslint/naming-convention */ + +const jwtSecretHex = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; +const retries = defaultExecutionEngineHttpOpts.retries; +const retryDelay = defaultExecutionEngineHttpOpts.retryDelay; +describe("executionEngine / ExecutionEngineHttp", function () { + if (!process.env.EL_BINARY_DIR || !process.env.EL_SCRIPT_DIR) { + throw Error( + `EL ENV must be provided, EL_BINARY_DIR: ${process.env.EL_BINARY_DIR}, EL_SCRIPT_DIR: ${process.env.EL_SCRIPT_DIR}` + ); + } + vi.setConfig({testTimeout: 1000 * 60 * 10, hookTimeout: 1000 * 60 * 10}); + + const dataPath = fs.mkdtempSync("lodestar-test-electra"); + const elSetupConfig = { + elScriptDir: process.env.EL_SCRIPT_DIR, + elBinaryDir: process.env.EL_BINARY_DIR, + }; + const elRunOptions = { + dataPath, + jwtSecretHex, + enginePort: parseInt(process.env.ENGINE_PORT ?? "8551"), + ethPort: parseInt(process.env.ETH_PORT ?? "8545"), + }; + + const controller = new AbortController(); + afterAll(async () => { + controller?.abort(); + await shell(`sudo rm -rf ${dataPath}`); + }); + + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + + it("Send and get payloads with depositReceipts to/from EL", async () => { + const {elClient, tearDownCallBack} = await runEL( + {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, + {...elRunOptions, ttd: BigInt(0)}, + controller.signal + ); + afterEachCallbacks.push(() => tearDownCallBack()); + const {genesisBlockHash, engineRpcUrl, ethRpcUrl} = elClient; + console.log({genesisBlockHash}); + + const loggerExecutionEngine = testLogger("executionEngine"); + + const executionEngine = initializeExecutionEngine( + {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retries, retryDelay}, + {signal: controller.signal, logger: loggerExecutionEngine} + ); + + // 1. Prepare payload + const preparePayloadParams: PayloadAttributes = { + // Note: this is created with a pre-defined genesis.json + timestamp: 10, + prevRandao: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + suggestedFeeRecipient: "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + withdrawals: [], + parentBeaconBlockRoot: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + }; + const payloadId = await executionEngine.notifyForkchoiceUpdate( + ForkName.electra, + genesisBlockHash, + //use finalizedBlockHash as safeBlockHash + genesisBlockHash, + genesisBlockHash, + preparePayloadParams + ); + if (!payloadId) throw Error("InvalidPayloadId"); + + // 2. Send raw deposit transaction A and B. tx A is to be imported via newPayload, tx B is to be included in payload via getPayload + const depositTransactionA = + "0x02f9021c8217de808459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120749715de5d1226545c6b3790f515d551a5cc5bf1d49c87a696860554d2fc4f14000000000000000000000000000000000000000000000000000000000000003096a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9c080a09f597089338d7f44f5c59f8230bb38f243849228a8d4e9d2e2956e6050f5b2c7a076486996c7e62802b8f95eee114783e4b403fd11093ba96286ff42c595f24452"; + const depositReceiptA = { + amount: 32000000000, + index: 0, + pubkey: dataToBytes( + "0x96a96086cff07df17668f35f7418ef8798079167e3f4f9b72ecde17b28226137cf454ab1dd20ef5d924786ab3483c2f9", + 48 + ), + signature: dataToBytes( + "0xb1acdb2c4d3df3f1b8d3bfd33421660df358d84d78d16c4603551935f4b67643373e7eb63dcb16ec359be0ec41fee33b03a16e80745f2374ff1d3c352508ac5d857c6476d3c3bcf7e6ca37427c9209f17be3af5264c0e2132b3dd1156c28b4e9", + 96 + ), + withdrawalCredentials: dataToBytes("0x003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef2", 32), + }; + + const depositTransactionB = + "0x02f9021c8217de018459682f008459682f0e830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120a18b4c7cab0afa273ea9504904521ea8421a4e32740b7611bd3d5095ca99f0cb0000000000000000000000000000000000000000000000000000000000000030a5c85a60ba2905c215f6a12872e62b1ee037051364244043a5f639aa81b04a204c55e7cc851f29c7c183be253ea1510b000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020001db70c485b6264692f26b8aeaab5b0c384180df8e2184a21a808a3ec8e86ca00000000000000000000000000000000000000000000000000000000000000609561731785b48cf1886412234531e4940064584463e96ac63a1a154320227e333fb51addc4a89b7e0d3f862d7c1fd4ea03bd8eb3d8806f1e7daf591cbbbb92b0beb74d13c01617f22c5026b4f9f9f294a8a7c32db895de3b01bee0132c9209e1c001a0644e0a763a34b4bfb9f56a677857b57fcf15e3db57e2f57060e92084f75f3d82a018ba8eaacbd8e6f6917675b1d0362b12ca82850ca8ef9c010430760c2b2e0cb5"; + const depositReceiptB = { + amount: 32000000000, + index: 1, + pubkey: dataToBytes( + "0xa5c85a60ba2905c215f6a12872e62b1ee037051364244043a5f639aa81b04a204c55e7cc851f29c7c183be253ea1510b", + 48 + ), + signature: dataToBytes( + "0x9561731785b48cf1886412234531e4940064584463e96ac63a1a154320227e333fb51addc4a89b7e0d3f862d7c1fd4ea03bd8eb3d8806f1e7daf591cbbbb92b0beb74d13c01617f22c5026b4f9f9f294a8a7c32db895de3b01bee0132c9209e1", + 96 + ), + withdrawalCredentials: dataToBytes("0x001db70c485b6264692f26b8aeaab5b0c384180df8e2184a21a808a3ec8e86ca", 32), + }; + + sendRawTransactionBig(ethRpcUrl, depositTransactionA, `${dataPath}/deposit.json`).catch((e: Error) => { + loggerExecutionEngine.error("Fail to send raw deposit transaction A", undefined, e); + }); + + sendRawTransactionBig(ethRpcUrl, depositTransactionB, `${dataPath}/deposit.json`).catch((e: Error) => { + loggerExecutionEngine.error("Fail to send raw deposit transaction B", undefined, e); + }); + + // 3. Import new payload with tx A and deposit receipt A + const newPayloadBlockHash = "0xfd1189e6ea0814b7d40d4e50b31ae5feabbb2acff39399457bbdda7cb5ccd490"; + const newPayload = { + parentHash: dataToBytes("0x26118cf71453320edcebbc4ebb34af5b578087a32385b80108bf691fa23efc42", 32), + feeRecipient: dataToBytes("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", 20), + stateRoot: dataToBytes("0x14208ac0e218167936e220b72d5d5887a963cb858ea2f2d268518f014a3da3fa", 32), + logsBloom: dataToBytes( + "0x10000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000400000000000000000000000020000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000008000000000000000000000000", + 256 + ), + prevRandao: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + gasLimit: 30000000, + gasUsed: 84846, + timestamp: 16, + extraData: dataToBytes("0x", 0), + baseFeePerGas: 7n, + excessBlobGas: 0n, + transactions: [dataToBytes(depositTransactionA, null)], + withdrawals: [], + depositReceipts: [depositReceiptA], + blockNumber: 1, + blockHash: dataToBytes(newPayloadBlockHash, 32), + receiptsRoot: dataToBytes("0x79ee3424eb720a3ad4b1c5a372bb8160580cbe4d893778660f34213c685627a9", 32), + blobGasUsed: 0n, + }; + const parentBeaconBlockRoot = dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32); + const payloadResult = await executionEngine.notifyNewPayload( + ForkName.electra, + newPayload, + [], + parentBeaconBlockRoot + ); + if (payloadResult.status !== ExecutionPayloadStatus.VALID) { + throw Error("getPayload returned payload that notifyNewPayload deems invalid"); + } + + // 4. Update fork choice + const preparePayloadParams2: PayloadAttributes = { + timestamp: 48, + prevRandao: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + suggestedFeeRecipient: "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", + withdrawals: [], + parentBeaconBlockRoot: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), + }; + + const payloadId2 = await executionEngine.notifyForkchoiceUpdate( + ForkName.electra, + newPayloadBlockHash, + //use finalizedBlockHash as safeBlockHash + newPayloadBlockHash, + newPayloadBlockHash, + preparePayloadParams2 + ); + if (!payloadId2) throw Error("InvalidPayloadId"); + + // 5. Get the payload. Check depositReceipts field contains deposit + // Wait a bit first for besu to pick up tx from the tx pool. + await sleep(1000); + const payloadAndBlockValue = await executionEngine.getPayload(ForkName.electra, payloadId2); + const payload = payloadAndBlockValue.executionPayload as electra.ExecutionPayload; + + if (payload.transactions.length !== 1) { + throw Error(`Number of transactions mismatched. Expected: 1, actual: ${payload.transactions.length}`); + } else { + const actualTransaction = bytesToData(payload.transactions[0]); + + if (actualTransaction !== depositTransactionB) { + throw Error(`Transaction mismatched. Expected: ${depositTransactionB}, actual: ${actualTransaction}`); + } + } + + if (payload.depositReceipts.length !== 1) { + throw Error(`Number of depositReceipts mismatched. Expected: 1, actual: ${payload.depositReceipts.length}`); + } + + const actualDepositReceipt = payload.depositReceipts[0]; + if (!_.isEqual(actualDepositReceipt, depositReceiptB)) { + throw Error( + `Deposit receipts mismatched. Expected: ${JSON.stringify(depositReceiptB)}, actual: ${JSON.stringify( + actualDepositReceipt + )}` + ); + } + }); + + it("Post-merge, run for a few blocks", async function () { + console.log("\n\nPost-merge, run for a few blocks\n\n"); + const {elClient, tearDownCallBack} = await runEL( + {...elSetupConfig, mode: ELStartMode.PostMerge, genesisTemplate: "electra.tmpl"}, + {...elRunOptions, ttd: BigInt(0)}, + controller.signal + ); + afterEachCallbacks.push(() => tearDownCallBack()); + + await runNodeWithEL({ + elClient, + electraEpoch: 0, + testName: "post-merge", + }); + }); + + /** + * Want to test two things: + * 1) Send two raw deposit transactions, and see if two new validators with corrent balances show up in the state.validators and unfinalized cache + * 2) Upon state-transition, see if the two new validators move from unfinalized cache to finalized cache + */ + async function runNodeWithEL({ + elClient, + electraEpoch, + testName, + }: { + elClient: ELClient; + electraEpoch: Epoch; + testName: string; + }): Promise { + const {genesisBlockHash, ttd, engineRpcUrl, ethRpcUrl} = elClient; + const validatorClientCount = 1; + const validatorsPerClient = 32; + + const testParams: Pick = { + SECONDS_PER_SLOT: 2, + }; + + // Just enough to have a checkpoint finalized + const expectedEpochsToFinish = 4; + // 1 epoch of margin of error + const epochsOfMargin = 1; + const timeoutSetupMargin = 30 * 1000; // Give extra 30 seconds of margin + + // delay a bit so regular sync sees it's up to date and sync is completed from the beginning + const genesisSlotsDelay = 8; + + const timeout = + ((epochsOfMargin + expectedEpochsToFinish) * SLOTS_PER_EPOCH + genesisSlotsDelay) * + testParams.SECONDS_PER_SLOT * + 1000; + + vi.setConfig({testTimeout: timeout + 2 * timeoutSetupMargin}); + + const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; + + const testLoggerOpts: TestLoggerOpts = { + level: LogLevel.info, + file: { + filepath: `${logFilesDir}/mergemock-${testName}.log`, + level: LogLevel.debug, + }, + timestampFormat: { + format: TimestampFormatCode.EpochSlot, + genesisTime, + slotsPerEpoch: SLOTS_PER_EPOCH, + secondsPerSlot: testParams.SECONDS_PER_SLOT, + }, + }; + const loggerNodeA = testLogger("Node-A", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: { + ...testParams, + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: electraEpoch, + TERMINAL_TOTAL_DIFFICULTY: ttd, + }, + options: { + api: {rest: {enabled: true} as BeaconRestApiServerOpts}, + sync: {isSingleNode: true}, + network: {allowPublishToZeroPeers: true, discv5: null}, + // Now eth deposit/merge tracker methods directly available on engine endpoints + eth1: {enabled: false, providerUrls: [engineRpcUrl], jwtSecretHex}, + executionEngine: {urls: [engineRpcUrl], jwtSecretHex}, + chain: {suggestedFeeRecipient: "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, + }, + validatorCount: validatorClientCount * validatorsPerClient, + logger: loggerNodeA, + genesisTime, + eth1BlockHash: dataToBytes(genesisBlockHash, 32), + withEth1Credentials: true, + }); + + afterEachCallbacks.push(async function () { + await bn.close(); + await sleep(1000); + }); + + const stopInfoTracker = simTestInfoTracker(bn, loggerNodeA); + const valProposerConfig = { + defaultConfig: { + feeRecipient: "0xcccccccccccccccccccccccccccccccccccccccc", + }, + } as ValidatorProposerConfig; + + const {validators} = await getAndInitDevValidators({ + node: bn, + logPrefix: "Node-A", + validatorsPerClient, + validatorClientCount, + startIndex: 0, + // At least one sim test must use the REST API for beacon <-> validator comms + useRestApi: true, + testLoggerOpts, + valProposerConfig, + }); + + afterEachCallbacks.push(async function () { + await Promise.all(validators.map((v) => v.close())); + }); + + await waitForSlot(bn, 1); + + // send raw tx at slot 1 + const depositTransaction = + "0x02f9021e8217de8085012a05f20085019254d380830271009442424242424242424242424242424242424242428901bc16d674ec800000b901a422895118000000000000000000000000000000000000000000000000000000000000008000000000000000000000000000000000000000000000000000000000000000e00000000000000000000000000000000000000000000000000000000000000120ef950826b191ebea0bbafa92a2c6bffa8239c6f456d92891ce2852b8360f0d30000000000000000000000000000000000000000000000000000000000000003095e4f91aea91a9e00387fad9d60997cff6cbf68d42d1b6629a7b248cdef255f94a2a2381e5d4125273fe42da5f7aa0e1000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020003f5102dabe0a27b1746098d1dc17a5d3fbd478759fea9287e4e419b3c3cef20000000000000000000000000000000000000000000000000000000000000060b6c06e65228046268aa918baf78e072c25e65aa0bcf258cefcac3371c47df81bc4d43ca942f5fc28f9a563e925fd9c5010bc8c300add3faf3af0d61fabaaf03694020feaafb03e47c1bc4fcf082684c7ed3f7d5839d1722214b24f95ad2b226cc080a0be1161617492e4ca2fcb89edcadf5e71e8cac0d6447d18cfde9b55e5a8412417a07ec8c47dd484036c745049bb2e2980d44e38d4dacac50dc4a14a2f23c52f2e5f"; + sendRawTransactionBig(ethRpcUrl, depositTransaction, `${dataPath}/deposit.json`).catch((e: Error) => { + loggerNodeA.error("Fail to send raw deposit transaction", undefined, e); + }); + + await waitForSlot(bn, 5); + // Expect new validator to be in unfinalized cache, in state.validators and not in finalized cache + let headState = bn.chain.getHeadState(); + let epochCtx = headState.epochCtx; + if (headState.validators.length !== 33 || headState.balances.length !== 33) { + throw Error("New validator is not reflected in the beacon state at slot 5"); + } + if (epochCtx.index2pubkey.length !== 32 || epochCtx.pubkey2index.size !== 32) { + throw Error("Finalized cache is modified."); + } + if (epochCtx.unfinalizedPubkey2index.size !== 1) { + throw Error( + `Unfinalized cache is missing the expected validator. Size: ${epochCtx.unfinalizedPubkey2index.size}` + ); + } + // validator count at epoch 1 should be empty at this point since no epoch transition has happened. + if (epochCtx.getValidatorCountAtEpoch(1) !== undefined) { + throw Error("Historical validator lengths is modified"); + } + + await new Promise((resolve, _reject) => { + bn.chain.clock.on(ClockEvent.epoch, (epoch) => { + // Resolve only if the finalized checkpoint includes execution payload + if (epoch >= expectedEpochsToFinish) { + console.log("\nGot event epoch, stopping validators and nodes\n"); + resolve(); + } + }); + }); + + // Stop chain and un-subscribe events so the execution engine won't update it's head + // Allow some time to broadcast finalized events and complete the importBlock routine + await Promise.all(validators.map((v) => v.close())); + await bn.close(); + await sleep(500); + + // Check if new validator is in finalized cache + headState = bn.chain.getHeadState() as CachedBeaconStateElectra; + epochCtx = headState.epochCtx; + + if (headState.validators.length !== 33 || headState.balances.length !== 33) { + throw Error("New validator is not reflected in the beacon state."); + } + if (epochCtx.index2pubkey.length !== 33 || epochCtx.pubkey2index.size !== 33) { + throw Error("New validator is not in finalized cache"); + } + if (!epochCtx.unfinalizedPubkey2index.isEmpty()) { + throw Error("Unfinalized cache still contains new validator"); + } + // After 4 epochs, headState's finalized cp epoch should be 2 + // epochCtx should only have validator count for epoch 3 and 4. + if (epochCtx.getValidatorCountAtEpoch(4) === undefined || epochCtx.getValidatorCountAtEpoch(3) === undefined) { + throw Error("Missing historical validator length for epoch 3 or 4"); + } + + if (epochCtx.getValidatorCountAtEpoch(4) !== 33 || epochCtx.getValidatorCountAtEpoch(3) !== 33) { + throw Error("Incorrect historical validator length for epoch 3 or 4"); + } + + if (epochCtx.getValidatorCountAtEpoch(2) !== undefined || epochCtx.getValidatorCountAtEpoch(1) !== undefined) { + throw Error("Historical validator length for epoch 1 or 2 is not dropped properly"); + } + + if (headState.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + throw Error("state.depositReceiptsStartIndex is not set upon processing new deposit receipt"); + } + + // wait for 1 slot to print current epoch stats + await sleep(1 * bn.config.SECONDS_PER_SLOT * 1000); + stopInfoTracker(); + console.log("\n\nDone\n\n"); + } +}); + +async function waitForSlot(bn: BeaconNode, targetSlot: Slot): Promise { + await new Promise((resolve, reject) => { + bn.chain.clock.on(ClockEvent.slot, (currentSlot) => { + if (currentSlot === targetSlot) { + resolve(); + return; + } + if (currentSlot > targetSlot) { + reject(Error(`Beacon node has passed target slot ${targetSlot}. Current slot ${currentSlot}`)); + } + }); + }); +} diff --git a/packages/beacon-node/test/spec/presets/fork.test.ts b/packages/beacon-node/test/spec/presets/fork.test.ts index c880d24bbbe3..c121e651fcea 100644 --- a/packages/beacon-node/test/spec/presets/fork.test.ts +++ b/packages/beacon-node/test/spec/presets/fork.test.ts @@ -5,6 +5,7 @@ import { CachedBeaconStateAltair, CachedBeaconStatePhase0, CachedBeaconStateCapella, + CachedBeaconStateDeneb, } from "@lodestar/state-transition"; import * as slotFns from "@lodestar/state-transition/slot"; import {phase0, ssz} from "@lodestar/types"; @@ -36,7 +37,7 @@ const fork: TestRunnerFn = (forkNext) => { case ForkName.deneb: return slotFns.upgradeStateToDeneb(preState as CachedBeaconStateCapella); case ForkName.electra: - throw Error("not Implemented"); + return slotFns.upgradeStateToElectra(preState as CachedBeaconStateDeneb); } }, options: { diff --git a/packages/beacon-node/test/spec/presets/genesis.test.ts b/packages/beacon-node/test/spec/presets/genesis.test.ts index f03f2595a566..d8f85dd9b5b8 100644 --- a/packages/beacon-node/test/spec/presets/genesis.test.ts +++ b/packages/beacon-node/test/spec/presets/genesis.test.ts @@ -61,7 +61,7 @@ const genesisInitialization: TestRunnerFn + testcase["execution_payload_header"] as ExecutionPayloadHeader ) ); }, diff --git a/packages/beacon-node/test/spec/presets/operations.test.ts b/packages/beacon-node/test/spec/presets/operations.test.ts index 4c1c10e0cb66..cc032bec6fa2 100644 --- a/packages/beacon-node/test/spec/presets/operations.test.ts +++ b/packages/beacon-node/test/spec/presets/operations.test.ts @@ -4,11 +4,12 @@ import { CachedBeaconStateAllForks, CachedBeaconStateBellatrix, CachedBeaconStateCapella, + CachedBeaconStateElectra, ExecutionPayloadStatus, getBlockRootAtSlot, } from "@lodestar/state-transition"; import * as blockFns from "@lodestar/state-transition/block"; -import {ssz, phase0, altair, bellatrix, capella, sszTypesFor} from "@lodestar/types"; +import {ssz, phase0, altair, bellatrix, capella, electra, sszTypesFor} from "@lodestar/types"; import {InputType} from "@lodestar/spec-test-util"; import {ACTIVE_PRESET, ForkName} from "@lodestar/params"; @@ -56,6 +57,11 @@ const operationFns: Record> = blockFns.processDeposit(fork, state, testCase.deposit); }, + deposit_receipt: (state, testCase: {deposit_receipt: electra.DepositReceipt}) => { + const fork = state.config.getForkSeq(state.slot); + blockFns.processDepositReceipt(fork, state as CachedBeaconStateElectra, testCase.deposit_receipt); + }, + proposer_slashing: (state, testCase: {proposer_slashing: phase0.ProposerSlashing}) => { const fork = state.config.getForkSeq(state.slot); blockFns.processProposerSlashing(fork, state, testCase.proposer_slashing); @@ -121,6 +127,7 @@ const operations: TestRunnerFn = (fork, block: ssz[fork].BeaconBlock, body: ssz[fork].BeaconBlockBody, deposit: ssz.phase0.Deposit, + deposit_receipt: ssz.electra.DepositReceipt, proposer_slashing: ssz.phase0.ProposerSlashing, voluntary_exit: ssz.phase0.SignedVoluntaryExit, // Altair diff --git a/packages/beacon-node/test/spec/presets/ssz_static.test.ts b/packages/beacon-node/test/spec/presets/ssz_static.test.ts index d81b9dee0098..d81b47ecf587 100644 --- a/packages/beacon-node/test/spec/presets/ssz_static.test.ts +++ b/packages/beacon-node/test/spec/presets/ssz_static.test.ts @@ -45,6 +45,7 @@ const sszStatic = /* eslint-disable @typescript-eslint/strict-boolean-expressions */ const sszType = (sszTypesFor(fork) as Types)[typeName] || + (ssz.deneb as Types)[typeName] || (ssz.capella as Types)[typeName] || (ssz.bellatrix as Types)[typeName] || (ssz.altair as Types)[typeName] || diff --git a/packages/beacon-node/test/spec/utils/specTestIterator.ts b/packages/beacon-node/test/spec/utils/specTestIterator.ts index 6aa683cb6530..3201fb15dbf3 100644 --- a/packages/beacon-node/test/spec/utils/specTestIterator.ts +++ b/packages/beacon-node/test/spec/utils/specTestIterator.ts @@ -57,7 +57,6 @@ const coveredTestRunners = [ // ], // ``` export const defaultSkipOpts: SkipOpts = { - skippedForks: ["eip6110"], // TODO: capella // BeaconBlockBody proof in lightclient is the new addition in v1.3.0-rc.2-hotfix // Skip them for now to enable subsequently diff --git a/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts b/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts index ba8014c47c9e..abb520bddafb 100644 --- a/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts +++ b/packages/beacon-node/test/unit/chain/lightclient/upgradeLightClientHeader.test.ts @@ -15,7 +15,7 @@ describe("UpgradeLightClientHeader", function () { BELLATRIX_FORK_EPOCH: 2, CAPELLA_FORK_EPOCH: 3, DENEB_FORK_EPOCH: 4, - ELECTRA_FORK_EPOCH: Infinity, + ELECTRA_FORK_EPOCH: 5, }); const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); @@ -37,18 +37,12 @@ describe("UpgradeLightClientHeader", function () { bellatrix: 17, capella: 25, deneb: 33, - electra: 0, + electra: 41, }; }); - // Since electra is not implemented for loop is till deneb (Object.values(ForkName).length-1) - // Once electra is implemnted run for loop till Object.values(ForkName).length - - // for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { - // for (let j = i + 1; j < Object.values(ForkName).length; j++) { - - for (let i = ForkSeq.altair; i < Object.values(ForkName).length - 1; i++) { - for (let j = i + 1; j < Object.values(ForkName).length - 1; j++) { + for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { + for (let j = i + 1; j < Object.values(ForkName).length; j++) { const fromFork = ForkName[ForkSeq[i] as ForkName]; const toFork = ForkName[ForkSeq[j] as ForkName]; @@ -62,27 +56,7 @@ describe("UpgradeLightClientHeader", function () { } } - // for electra not implemented for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { - const fromFork = ForkName[ForkSeq[i] as ForkName]; - const toFork = ForkName["electra"]; - - it(`Throw error ${fromFork}=>${toFork}`, function () { - lcHeaderByFork[fromFork].beacon.slot = testSlots[fromFork]; - lcHeaderByFork[toFork].beacon.slot = testSlots[fromFork]; - - expect(() => { - upgradeLightClientHeader(config, toFork, lcHeaderByFork[fromFork]); - }).toThrow("Not Implemented"); - }); - } - - // Since electra is not implemented for loop is till deneb (Object.values(ForkName).length-1) - // Once electra is implemnted run for loop till Object.values(ForkName).length - - // for (let i = ForkSeq.altair; i < Object.values(ForkName).length; i++) { - - for (let i = ForkSeq.altair; i < Object.values(ForkName).length - 1; i++) { for (let j = i; j > 0; j--) { const fromFork = ForkName[ForkSeq[i] as ForkName]; const toFork = ForkName[ForkSeq[j] as ForkName]; diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 800984fa84bc..41c205cdd0e9 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -61,9 +61,9 @@ describe("AggregatedAttestationPool", function () { epochParticipation[committee[i]] = 0b000; } } - (originalState as CachedBeaconStateAltair).previousEpochParticipation = + (originalState as unknown as CachedBeaconStateAltair).previousEpochParticipation = ssz.altair.EpochParticipation.toViewDU(epochParticipation); - (originalState as CachedBeaconStateAltair).currentEpochParticipation = + (originalState as unknown as CachedBeaconStateAltair).currentEpochParticipation = ssz.altair.EpochParticipation.toViewDU(epochParticipation); originalState.commit(); let altairState: CachedBeaconStateAllForks; diff --git a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts index 6295a993c072..e6a7e8706bbe 100644 --- a/packages/beacon-node/test/unit/chain/shufflingCache.test.ts +++ b/packages/beacon-node/test/unit/chain/shufflingCache.test.ts @@ -1,6 +1,6 @@ import {describe, it, expect, beforeEach} from "vitest"; -import {getShufflingDecisionBlock} from "@lodestar/state-transition"; +import {getShufflingDecisionBlock, CachedBeaconStateAllForks} from "@lodestar/state-transition"; // eslint-disable-next-line import/no-relative-packages import {generateTestCachedBeaconStateOnlyValidators} from "../../../../state-transition/test/perf/util.js"; import {ShufflingCache} from "../../../src/chain/shufflingCache.js"; @@ -14,7 +14,7 @@ describe("ShufflingCache", function () { beforeEach(() => { shufflingCache = new ShufflingCache(null, {maxShufflingCacheEpochs: 1}); - shufflingCache.processState(state, currentEpoch); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch); }); it("should get shuffling from cache", async function () { @@ -29,7 +29,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch, "0x00"); expect(await shufflingCache.get(currentEpoch, decisionRoot)).toEqual(state.epochCtx.currentShuffling); // insert shufflings at other epochs does prune the cache - shufflingCache.processState(state, currentEpoch + 1); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch + 1); // the current shuffling is not available anymore expect(await shufflingCache.get(currentEpoch, decisionRoot)).toBeNull(); }); @@ -39,7 +39,7 @@ describe("ShufflingCache", function () { shufflingCache.insertPromise(currentEpoch + 1, nextDecisionRoot); const shufflingRequest0 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); const shufflingRequest1 = shufflingCache.get(currentEpoch + 1, nextDecisionRoot); - shufflingCache.processState(state, currentEpoch + 1); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, currentEpoch + 1); expect(await shufflingRequest0).toEqual(state.epochCtx.nextShuffling); expect(await shufflingRequest1).toEqual(state.epochCtx.nextShuffling); }); diff --git a/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts index 7d3f34ddac36..b4aac92dd9bb 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts @@ -1,8 +1,7 @@ import {describe, it, expect, beforeEach} from "vitest"; import {toHexString} from "@chainsafe/ssz"; -import {EpochShuffling} from "@lodestar/state-transition"; +import {EpochShuffling, CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; -import {CachedBeaconStateAllForks} from "@lodestar/state-transition/src/types.js"; import {FIFOBlockStateCache} from "../../../../src/chain/stateCache/index.js"; import {generateCachedState} from "../../../utils/state.js"; diff --git a/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts b/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts index ce0d7fae1fad..6cc3ae8f1ce0 100644 --- a/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts +++ b/packages/beacon-node/test/unit/eth1/utils/deposits.test.ts @@ -1,13 +1,15 @@ import {describe, it, expect} from "vitest"; import {phase0, ssz} from "@lodestar/types"; -import {MAX_DEPOSITS} from "@lodestar/params"; +import {MAX_DEPOSITS, SLOTS_PER_EPOCH} from "@lodestar/params"; import {verifyMerkleBranch} from "@lodestar/utils"; +import {createChainForkConfig} from "@lodestar/config"; import {filterBy} from "../../../utils/db.js"; import {Eth1ErrorCode} from "../../../../src/eth1/errors.js"; import {generateState} from "../../../utils/state.js"; import {expectRejectedWithLodestarError} from "../../../utils/errors.js"; import {getDeposits, getDepositsWithProofs, DepositGetter} from "../../../../src/eth1/utils/deposits.js"; import {DepositTree} from "../../../../src/db/repositories/depositDataRoot.js"; +import {createCachedBeaconStateTest} from "../../../utils/cachedBeaconState.js"; describe("eth1 / util / deposits", function () { describe("getDeposits", () => { @@ -18,6 +20,7 @@ describe("eth1 / util / deposits", function () { depositIndexes: number[]; expectedReturnedIndexes?: number[]; error?: Eth1ErrorCode; + postElectra?: boolean; }; const testCases: TestCase[] = [ @@ -70,18 +73,59 @@ describe("eth1 / util / deposits", function () { depositIndexes: [], expectedReturnedIndexes: [], }, + { + id: "No deposits to be included post Electra after deposit_receipts_start_index", + depositCount: 2030, + eth1DepositIndex: 2025, + depositIndexes: Array.from({length: 2030}, (_, i) => i), + expectedReturnedIndexes: [], + postElectra: true, + }, + { + id: "Should return deposits post Electra before deposit_receipts_start_index", + depositCount: 2022, + eth1DepositIndex: 2018, + depositIndexes: Array.from({length: 2022}, (_, i) => i), + expectedReturnedIndexes: [2018, 2019, 2020, 2021], + postElectra: true, + }, + { + id: "Should return deposits less than MAX_DEPOSITS post Electra before deposit_receipts_start_index", + depositCount: 10 * MAX_DEPOSITS, + eth1DepositIndex: 0, + depositIndexes: Array.from({length: 10 * MAX_DEPOSITS}, (_, i) => i), + expectedReturnedIndexes: Array.from({length: MAX_DEPOSITS}, (_, i) => i), + postElectra: true, + }, ]; + /* eslint-disable @typescript-eslint/naming-convention */ + const postElectraConfig = createChainForkConfig({ + ALTAIR_FORK_EPOCH: 1, + BELLATRIX_FORK_EPOCH: 2, + CAPELLA_FORK_EPOCH: 3, + DENEB_FORK_EPOCH: 4, + ELECTRA_FORK_EPOCH: 5, + }); + const postElectraSlot = postElectraConfig.ELECTRA_FORK_EPOCH * SLOTS_PER_EPOCH + 1; + for (const testCase of testCases) { - const {id, depositIndexes, eth1DepositIndex, depositCount, expectedReturnedIndexes, error} = testCase; + const {id, depositIndexes, eth1DepositIndex, depositCount, expectedReturnedIndexes, error, postElectra} = + testCase; it(id, async function () { - const state = generateState({eth1DepositIndex}); + const state = postElectra + ? generateState({slot: postElectraSlot, eth1DepositIndex}, postElectraConfig) + : generateState({eth1DepositIndex}); + const cachedState = createCachedBeaconStateTest( + state, + postElectra ? postElectraConfig : createChainForkConfig({}) + ); const eth1Data = generateEth1Data(depositCount); const deposits = depositIndexes.map((index) => generateDepositEvent(index)); const depositsGetter: DepositGetter = async (indexRange) => filterBy(deposits, indexRange, (deposit) => deposit.index); - const resultPromise = getDeposits(state, eth1Data, depositsGetter); + const resultPromise = getDeposits(cachedState, eth1Data, depositsGetter); if (expectedReturnedIndexes) { const result = await resultPromise; diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index aa33c7dbbc40..b0e20ad1f2b5 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -193,6 +193,7 @@ describe("ExecutionEngine / http", () => { amount: "0x7b", }, ], + depositReceipts: null, // depositReceipts is null pre-electra }, null, // null returned for missing blocks { @@ -201,6 +202,7 @@ describe("ExecutionEngine / http", () => { "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", ], withdrawals: null, // withdrawals is null pre-capella + depositReceipts: null, // depositReceipts is null pre-electra }, ], }; @@ -248,6 +250,7 @@ describe("ExecutionEngine / http", () => { amount: "0x7b", }, ], + depositReceipts: null, // depositReceipts is null pre-electra }, null, // null returned for missing blocks { @@ -256,6 +259,7 @@ describe("ExecutionEngine / http", () => { "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", ], withdrawals: null, // withdrawals is null pre-capella + depositReceipts: null, // depositReceipts is null pre-electra }, ], }; diff --git a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts index 3fb9cb8e1c79..c7de35d468df 100644 --- a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts +++ b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts @@ -24,6 +24,7 @@ describe("beaconBlocksMaybeBlobsByRange", () => { BELLATRIX_FORK_EPOCH: 0, CAPELLA_FORK_EPOCH: 0, DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, }); const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); const config = createBeaconConfig(chainConfig, genesisValidatorsRoot); diff --git a/packages/beacon-node/test/utils/state.ts b/packages/beacon-node/test/utils/state.ts index 1e9f614e8093..ad4dd4b88dac 100644 --- a/packages/beacon-node/test/utils/state.ts +++ b/packages/beacon-node/test/utils/state.ts @@ -7,8 +7,10 @@ import { PubkeyIndexMap, CachedBeaconStateBellatrix, BeaconStateBellatrix, + CachedBeaconStateElectra, + BeaconStateElectra, } from "@lodestar/state-transition"; -import {BeaconState, altair, bellatrix, ssz} from "@lodestar/types"; +import {BeaconState, altair, bellatrix, electra, ssz} from "@lodestar/types"; import {createBeaconConfig, ChainForkConfig} from "@lodestar/config"; import {FAR_FUTURE_EPOCH, ForkName, ForkSeq, MAX_EFFECTIVE_BALANCE, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; @@ -64,6 +66,7 @@ export function generateState( : generateValidators(numValidators, validatorOpts)); state.genesisTime = Math.floor(Date.now() / 1000); + state.slot = stateSlot; state.fork.previousVersion = config.GENESIS_FORK_VERSION; state.fork.currentVersion = config.GENESIS_FORK_VERSION; state.latestBlockHeader.bodyRoot = ssz.phase0.BeaconBlockBody.hashTreeRoot(ssz.phase0.BeaconBlockBody.defaultValue()); @@ -92,6 +95,12 @@ export function generateState( }; } + if (forkSeq >= ForkSeq.electra) { + const stateElectra = state as electra.BeaconState; + stateElectra.depositReceiptsStartIndex = 2023n; + stateElectra.latestExecutionPayloadHeader = ssz.electra.ExecutionPayloadHeader.defaultValue(); + } + return config.getForkTypes(stateSlot).BeaconState.toViewDU(state); } @@ -137,6 +146,18 @@ export function generateCachedBellatrixState(opts?: TestBeaconState): CachedBeac }); } +/** + * This generates state with default pubkey + */ +export function generateCachedElectraState(opts?: TestBeaconState): CachedBeaconStateElectra { + const config = getConfig(ForkName.electra); + const state = generateState(opts, config); + return createCachedBeaconState(state as BeaconStateElectra, { + config: createBeaconConfig(config, state.genesisValidatorsRoot), + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }); +} export const zeroProtoBlock: ProtoBlock = { slot: 0, blockRoot: ZERO_HASH_HEX, diff --git a/packages/beacon-node/test/utils/validationData/attestation.ts b/packages/beacon-node/test/utils/validationData/attestation.ts index c33d942dabc5..335ec81870c6 100644 --- a/packages/beacon-node/test/utils/validationData/attestation.ts +++ b/packages/beacon-node/test/utils/validationData/attestation.ts @@ -4,6 +4,7 @@ import { computeSigningRoot, computeStartSlotAtEpoch, getShufflingDecisionBlock, + CachedBeaconStateAllForks, } from "@lodestar/state-transition"; import {ProtoBlock, IForkChoice, ExecutionStatus, DataAvailabilityStatus} from "@lodestar/fork-choice"; import {DOMAIN_BEACON_ATTESTER} from "@lodestar/params"; @@ -82,8 +83,8 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { }; const shufflingCache = new ShufflingCache(); - shufflingCache.processState(state, state.epochCtx.currentShuffling.epoch); - shufflingCache.processState(state, state.epochCtx.nextShuffling.epoch); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, state.epochCtx.currentShuffling.epoch); + shufflingCache.processState(state as unknown as CachedBeaconStateAllForks, state.epochCtx.nextShuffling.epoch); const dependentRoot = getShufflingDecisionBlock(state, state.epochCtx.currentShuffling.epoch); const forkChoice = { @@ -133,7 +134,7 @@ export function getAttestationValidData(opts: AttestationValidDataOpts): { getState: async () => state, // TODO: remove this once we have a better way to get state getStateSync: () => state, - } as Partial as IStateRegenerator; + } as unknown as Partial as IStateRegenerator; const chain = { clock, diff --git a/packages/config/src/chainConfig/configs/mainnet.ts b/packages/config/src/chainConfig/configs/mainnet.ts index 0de1bee666ec..6d1fd9b75952 100644 --- a/packages/config/src/chainConfig/configs/mainnet.ts +++ b/packages/config/src/chainConfig/configs/mainnet.ts @@ -49,7 +49,7 @@ export const chainConfig: ChainConfig = { DENEB_FORK_VERSION: b("0x04000000"), DENEB_FORK_EPOCH: 269568, // March 13, 2024, 01:55:35pm UTC - // Electra + // ELECTRA ELECTRA_FORK_VERSION: b("0x05000000"), ELECTRA_FORK_EPOCH: Infinity, diff --git a/packages/config/src/chainConfig/configs/minimal.ts b/packages/config/src/chainConfig/configs/minimal.ts index c99a76d1ee40..44a28ca36ec7 100644 --- a/packages/config/src/chainConfig/configs/minimal.ts +++ b/packages/config/src/chainConfig/configs/minimal.ts @@ -45,8 +45,7 @@ export const chainConfig: ChainConfig = { // Deneb DENEB_FORK_VERSION: b("0x04000001"), DENEB_FORK_EPOCH: Infinity, - - // Electra + // ELECTRA ELECTRA_FORK_VERSION: b("0x05000001"), ELECTRA_FORK_EPOCH: Infinity, diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index aafd81c9250a..152c55ac6d15 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -115,7 +115,11 @@ export function upgradeLightClientHeader( // eslint-disable-next-line no-fallthrough case ForkName.electra: - throw Error("Not Implemented"); + (upgradedHeader as LightClientHeader).execution.depositReceiptsRoot = + ssz.electra.LightClientHeader.fields.execution.fields.depositReceiptsRoot.defaultValue(); + + // Break if no further upgrades is required else fall through + if (ForkSeq[targetFork] <= ForkSeq.electra) break; } return upgradedHeader; } @@ -149,6 +153,12 @@ export function isValidLightClientHeader(config: ChainForkConfig, header: LightC } } + if (epoch < config.ELECTRA_FORK_EPOCH) { + if ((header as LightClientHeader).execution.depositReceiptsRoot !== undefined) { + return false; + } + } + return isValidMerkleBranch( config .getExecutionForkTypes(header.beacon.slot) diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 6a95e3ca632e..482e591123b6 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -93,6 +93,8 @@ export const { MAX_BLOB_COMMITMENTS_PER_BLOCK, MAX_BLOBS_PER_BLOCK, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH, + + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, } = activePreset; //////////// @@ -244,3 +246,6 @@ export const KZG_COMMITMENT_SUBTREE_INDEX0 = KZG_COMMITMENT_GINDEX0 - 2 ** KZG_C // ssz.deneb.BlobSidecars.elementType.fixedSize export const BLOBSIDECAR_FIXED_SIZE = ACTIVE_PRESET === PresetName.minimal ? 131672 : 131928; + +// Electra Misc +export const UNSET_DEPOSIT_RECEIPTS_START_INDEX = 2n ** 64n - 1n; diff --git a/packages/params/src/presets/mainnet.ts b/packages/params/src/presets/mainnet.ts index 42a705a07f03..86f5c39c539e 100644 --- a/packages/params/src/presets/mainnet.ts +++ b/packages/params/src/presets/mainnet.ts @@ -118,4 +118,7 @@ export const mainnetPreset: BeaconPreset = { MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096, MAX_BLOBS_PER_BLOCK: 6, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17, + + // ELECTRA + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192, }; diff --git a/packages/params/src/presets/minimal.ts b/packages/params/src/presets/minimal.ts index b940841a0429..6239a8c1a3eb 100644 --- a/packages/params/src/presets/minimal.ts +++ b/packages/params/src/presets/minimal.ts @@ -119,4 +119,7 @@ export const minimalPreset: BeaconPreset = { MAX_BLOB_COMMITMENTS_PER_BLOCK: 16, MAX_BLOBS_PER_BLOCK: 6, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9, + + // ELECTRA + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4, }; diff --git a/packages/params/src/types.ts b/packages/params/src/types.ts index 3c5ba6381131..57ee8230a77d 100644 --- a/packages/params/src/types.ts +++ b/packages/params/src/types.ts @@ -82,6 +82,9 @@ export type BeaconPreset = { MAX_BLOB_COMMITMENTS_PER_BLOCK: number; MAX_BLOBS_PER_BLOCK: number; KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: number; + + // ELECTRA + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: number; }; /** @@ -167,6 +170,9 @@ export const beaconPresetTypes: BeaconPresetTypes = { MAX_BLOB_COMMITMENTS_PER_BLOCK: "number", MAX_BLOBS_PER_BLOCK: "number", KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: "number", + + // ELECTRA + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: "number", }; type BeaconPresetTypes = { diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index b797b48654dc..7bad75278eb4 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -67,7 +67,12 @@ "@lodestar/params": "^1.21.0", "@lodestar/types": "^1.21.0", "@lodestar/utils": "^1.21.0", - "bigint-buffer": "^1.1.5" + "bigint-buffer": "^1.1.5", + "buffer-xor": "^2.0.2", + "immutable": "^4.3.2" + }, + "devDependencies": { + "@types/buffer-xor": "^2.0.0" }, "keywords": [ "ethereum", diff --git a/packages/state-transition/src/block/processDeposit.ts b/packages/state-transition/src/block/processDeposit.ts index 7ade79fd7739..5a2a892a5c9b 100644 --- a/packages/state-transition/src/block/processDeposit.ts +++ b/packages/state-transition/src/block/processDeposit.ts @@ -1,5 +1,5 @@ import {PublicKey, Signature, verify} from "@chainsafe/blst"; -import {phase0, ssz} from "@lodestar/types"; +import {BLSPubkey, Bytes32, UintNum64, phase0, ssz} from "@lodestar/types"; import {verifyMerkleBranch} from "@lodestar/utils"; import { @@ -11,6 +11,8 @@ import { MAX_EFFECTIVE_BALANCE, } from "@lodestar/params"; +import {DepositData} from "@lodestar/types/lib/phase0/types.js"; +import {DepositReceipt} from "@lodestar/types/lib/electra/types.js"; import {ZERO_HASH} from "../constants/index.js"; import {computeDomain, computeSigningRoot, increaseBalance} from "../util/index.js"; import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; @@ -22,8 +24,6 @@ import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "../types.js"; * PERF: Work depends on number of Deposit per block. On regular networks the average is 0 / block. */ export function processDeposit(fork: ForkSeq, state: CachedBeaconStateAllForks, deposit: phase0.Deposit): void { - const {config, validators, epochCtx} = state; - // verify the merkle branch if ( !verifyMerkleBranch( @@ -40,15 +40,29 @@ export function processDeposit(fork: ForkSeq, state: CachedBeaconStateAllForks, // deposits must be processed in order state.eth1DepositIndex += 1; - const pubkey = deposit.data.pubkey; // Drop tree - const amount = deposit.data.amount; - const cachedIndex = epochCtx.pubkey2index.get(pubkey); + applyDeposit(fork, state, deposit.data); +} + +/** + * Adds a new validator into the registry. Or increase balance if already exist. + * Follows applyDeposit() in consensus spec. Will be used by processDeposit() and processDepositReceipt() + * + */ +export function applyDeposit( + fork: ForkSeq, + state: CachedBeaconStateAllForks, + deposit: DepositData | DepositReceipt +): void { + const {config, validators, epochCtx} = state; + const {pubkey, withdrawalCredentials, amount} = deposit; + + const cachedIndex = epochCtx.getValidatorIndex(pubkey); if (cachedIndex === undefined || !Number.isSafeInteger(cachedIndex) || cachedIndex >= validators.length) { // verify the deposit signature (proof of posession) which is not checked by the deposit contract const depositMessage = { - pubkey: deposit.data.pubkey, // Retain tree for hashing - withdrawalCredentials: deposit.data.withdrawalCredentials, // Retain tree for hashing - amount: deposit.data.amount, + pubkey, + withdrawalCredentials, + amount, }; // fork-agnostic domain since deposits are valid across forks const domain = computeDomain(DOMAIN_DEPOSIT, config.GENESIS_FORK_VERSION, ZERO_HASH); @@ -63,45 +77,55 @@ export function processDeposit(fork: ForkSeq, state: CachedBeaconStateAllForks, } catch (e) { return; // Catch all BLS errors: failed key validation, failed signature validation, invalid signature } + addValidatorToRegistry(fork, state, pubkey, withdrawalCredentials, amount); + } else { + // increase balance by deposit amount + increaseBalance(state, cachedIndex, amount); + } +} - // add validator and balance entries - const effectiveBalance = Math.min(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); - validators.push( - ssz.phase0.Validator.toViewDU({ - pubkey, - withdrawalCredentials: deposit.data.withdrawalCredentials, - activationEligibilityEpoch: FAR_FUTURE_EPOCH, - activationEpoch: FAR_FUTURE_EPOCH, - exitEpoch: FAR_FUTURE_EPOCH, - withdrawableEpoch: FAR_FUTURE_EPOCH, - effectiveBalance, - slashed: false, - }) - ); - state.balances.push(amount); +function addValidatorToRegistry( + fork: ForkSeq, + state: CachedBeaconStateAllForks, + pubkey: BLSPubkey, + withdrawalCredentials: Bytes32, + amount: UintNum64 +): void { + const {validators, epochCtx} = state; + // add validator and balance entries + const effectiveBalance = Math.min(amount - (amount % EFFECTIVE_BALANCE_INCREMENT), MAX_EFFECTIVE_BALANCE); + validators.push( + ssz.phase0.Validator.toViewDU({ + pubkey, + withdrawalCredentials, + activationEligibilityEpoch: FAR_FUTURE_EPOCH, + activationEpoch: FAR_FUTURE_EPOCH, + exitEpoch: FAR_FUTURE_EPOCH, + withdrawableEpoch: FAR_FUTURE_EPOCH, + effectiveBalance, + slashed: false, + }) + ); + state.balances.push(amount); - const validatorIndex = validators.length - 1; - // Updating here is better than updating at once on epoch transition - // - Simplify genesis fn applyDeposits(): effectiveBalanceIncrements is populated immediately - // - Keep related code together to reduce risk of breaking this cache - // - Should have equal performance since it sets a value in a flat array - epochCtx.effectiveBalanceIncrementsSet(validatorIndex, effectiveBalance); + const validatorIndex = validators.length - 1; + // Updating here is better than updating at once on epoch transition + // - Simplify genesis fn applyDeposits(): effectiveBalanceIncrements is populated immediately + // - Keep related code together to reduce risk of breaking this cache + // - Should have equal performance since it sets a value in a flat array + epochCtx.effectiveBalanceIncrementsSet(validatorIndex, effectiveBalance); - // now that there is a new validator, update the epoch context with the new pubkey - epochCtx.addPubkey(validatorIndex, pubkey); + // now that there is a new validator, update the epoch context with the new pubkey + epochCtx.addPubkey(validatorIndex, pubkey); - // Only after altair: - if (fork >= ForkSeq.altair) { - const stateAltair = state as CachedBeaconStateAltair; + // Only after altair: + if (fork >= ForkSeq.altair) { + const stateAltair = state as CachedBeaconStateAltair; - stateAltair.inactivityScores.push(0); + stateAltair.inactivityScores.push(0); - // add participation caches - stateAltair.previousEpochParticipation.push(0); - stateAltair.currentEpochParticipation.push(0); - } - } else { - // increase balance by deposit amount - increaseBalance(state, cachedIndex, amount); + // add participation caches + stateAltair.previousEpochParticipation.push(0); + stateAltair.currentEpochParticipation.push(0); } } diff --git a/packages/state-transition/src/block/processDepositReceipt.ts b/packages/state-transition/src/block/processDepositReceipt.ts new file mode 100644 index 000000000000..140e38d634fd --- /dev/null +++ b/packages/state-transition/src/block/processDepositReceipt.ts @@ -0,0 +1,17 @@ +import {electra} from "@lodestar/types"; +import {ForkSeq, UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; + +import {CachedBeaconStateElectra} from "../types.js"; +import {applyDeposit} from "./processDeposit.js"; + +export function processDepositReceipt( + fork: ForkSeq, + state: CachedBeaconStateElectra, + depositReceipt: electra.DepositReceipt +): void { + if (state.depositReceiptsStartIndex === UNSET_DEPOSIT_RECEIPTS_START_INDEX) { + state.depositReceiptsStartIndex = BigInt(depositReceipt.index); + } + + applyDeposit(fork, state, depositReceipt); +} diff --git a/packages/state-transition/src/block/processOperations.ts b/packages/state-transition/src/block/processOperations.ts index 38716bb42a40..7b8b39efe743 100644 --- a/packages/state-transition/src/block/processOperations.ts +++ b/packages/state-transition/src/block/processOperations.ts @@ -1,13 +1,15 @@ -import {BeaconBlockBody, capella} from "@lodestar/types"; -import {ForkSeq, MAX_DEPOSITS} from "@lodestar/params"; +import {BeaconBlockBody, capella, electra} from "@lodestar/types"; +import {ForkSeq} from "@lodestar/params"; -import {CachedBeaconStateAllForks, CachedBeaconStateCapella} from "../types.js"; +import {CachedBeaconStateAllForks, CachedBeaconStateCapella, CachedBeaconStateElectra} from "../types.js"; +import {getEth1DepositCount} from "../util/deposit.js"; import {processAttestations} from "./processAttestations.js"; import {processProposerSlashing} from "./processProposerSlashing.js"; import {processAttesterSlashing} from "./processAttesterSlashing.js"; import {processDeposit} from "./processDeposit.js"; import {processVoluntaryExit} from "./processVoluntaryExit.js"; import {processBlsToExecutionChange} from "./processBlsToExecutionChange.js"; +import {processDepositReceipt} from "./processDepositReceipt.js"; import {ProcessBlockOpts} from "./types.js"; export { @@ -17,6 +19,7 @@ export { processDeposit, processVoluntaryExit, processBlsToExecutionChange, + processDepositReceipt, }; export function processOperations( @@ -26,7 +29,7 @@ export function processOperations( opts: ProcessBlockOpts = {verifySignatures: true} ): void { // verify that outstanding deposits are processed up to the maximum number of deposits - const maxDeposits = Math.min(MAX_DEPOSITS, state.eth1Data.depositCount - state.eth1DepositIndex); + const maxDeposits = getEth1DepositCount(state); if (body.deposits.length !== maxDeposits) { throw new Error( `Block contains incorrect number of deposits: depositCount=${body.deposits.length} expected=${maxDeposits}` @@ -54,4 +57,10 @@ export function processOperations( processBlsToExecutionChange(state as CachedBeaconStateCapella, blsToExecutionChange); } } + + if (fork >= ForkSeq.electra) { + for (const depositReceipt of (body as electra.BeaconBlockBody).executionPayload.depositReceipts) { + processDepositReceipt(fork, state as CachedBeaconStateElectra, depositReceipt); + } + } } diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 191aa7f3985c..41f8031d903e 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -1,4 +1,6 @@ import {PublicKey} from "@chainsafe/blst"; +import * as immutable from "immutable"; +import {fromHexString} from "@chainsafe/ssz"; import {BLSSignature, CommitteeIndex, Epoch, Slot, ValidatorIndex, phase0, SyncPeriod} from "@lodestar/types"; import {createBeaconConfig, BeaconConfig, ChainConfig} from "@lodestar/config"; import { @@ -29,8 +31,17 @@ import {computeEpochShuffling, EpochShuffling, getShufflingDecisionBlock} from " import {computeBaseRewardPerIncrement, computeSyncParticipantReward} from "../util/syncCommittee.js"; import {sumTargetUnslashedBalanceIncrements} from "../util/targetUnslashedBalance.js"; import {getTotalSlashingsByIncrement} from "../epoch/processSlashings.js"; +import {EpochCacheMetrics} from "../metrics.js"; import {EffectiveBalanceIncrements, getEffectiveBalanceIncrementsWithLen} from "./effectiveBalanceIncrements.js"; -import {Index2PubkeyCache, PubkeyIndexMap, syncPubkeys} from "./pubkeyCache.js"; +import { + Index2PubkeyCache, + PubkeyIndexMap, + UnfinalizedPubkeyIndexMap, + syncPubkeys, + toMemoryEfficientHexStr, + PubkeyHex, + newUnfinalizedPubkeyIndexMap, +} from "./pubkeyCache.js"; import {BeaconStateAllForks, BeaconStateAltair, ShufflingGetter} from "./types.js"; import { computeSyncCommitteeCache, @@ -82,23 +93,31 @@ type ProposersDeferred = {computed: false; seed: Uint8Array} | {computed: true; export class EpochCache { config: BeaconConfig; /** - * Unique globally shared pubkey registry. There should only exist one for the entire application. + * Unique globally shared finalized pubkey registry. There should only exist one for the entire application. * * TODO: this is a hack, we need a safety mechanism in case a bad eth1 majority vote is in, * or handle non finalized data differently, or use an immutable.js structure for cheap copies - * Warning: may contain pubkeys that do not yet exist in the current state, but do in a later processed state. + * + * New: This would include only validators whose activation_eligibility_epoch != FAR_FUTURE_EPOCH and hence it is + * insert only. Validators could be 1) Active 2) In the activation queue 3) Initialized but pending queued * * $VALIDATOR_COUNT x 192 char String -> Number Map */ pubkey2index: PubkeyIndexMap; /** - * Unique globally shared pubkey registry. There should only exist one for the entire application. + * Unique globally shared finalized pubkey registry. There should only exist one for the entire application. * - * Warning: may contain indices that do not yet exist in the current state, but do in a later processed state. + * New: This would include only validators whose activation_eligibility_epoch != FAR_FUTURE_EPOCH and hence it is + * insert only. Validators could be 1) Active 2) In the activation queue 3) Initialized but pending queued * * $VALIDATOR_COUNT x BLST deserialized pubkey (Jacobian coordinates) */ index2pubkey: Index2PubkeyCache; + /** + * Unique pubkey registry shared in the same fork. There should only exist one for the fork. + */ + unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; + /** * Indexes of the block proposers for the current epoch. * @@ -198,11 +217,21 @@ export class EpochCache { // TODO: Helper stats epoch: Epoch; syncPeriod: SyncPeriod; + /** + * state.validators.length of every state at epoch boundary + * They are saved in increasing order of epoch. + * The first validator length in the list corresponds to the state AFTER the latest finalized checkpoint state. ie. state.finalizedCheckpoint.epoch - 1 + * The last validator length corresponds to the latest epoch state ie. this.epoch + * eg. latest epoch = 105, latest finalized cp state epoch = 102 + * then the list will be (in terms of epoch) [103, 104, 105] + */ + private historicalValidatorLengths: immutable.List; constructor(data: { config: BeaconConfig; pubkey2index: PubkeyIndexMap; index2pubkey: Index2PubkeyCache; + unfinalizedPubkey2index: UnfinalizedPubkeyIndexMap; proposers: number[]; proposersPrevEpoch: number[] | null; proposersNextEpoch: ProposersDeferred; @@ -225,10 +254,12 @@ export class EpochCache { nextSyncCommitteeIndexed: SyncCommitteeCache; epoch: Epoch; syncPeriod: SyncPeriod; + historialValidatorLengths: immutable.List; }) { this.config = data.config; this.pubkey2index = data.pubkey2index; this.index2pubkey = data.index2pubkey; + this.unfinalizedPubkey2index = data.unfinalizedPubkey2index; this.proposers = data.proposers; this.proposersPrevEpoch = data.proposersPrevEpoch; this.proposersNextEpoch = data.proposersNextEpoch; @@ -251,11 +282,12 @@ export class EpochCache { this.nextSyncCommitteeIndexed = data.nextSyncCommitteeIndexed; this.epoch = data.epoch; this.syncPeriod = data.syncPeriod; + this.historicalValidatorLengths = data.historialValidatorLengths; } /** * Create an epoch cache - * @param validators cached validators that matches `state.validators` + * @param state a finalized beacon state. Passing in unfinalized state may cause unexpected behaviour eg. empty unfinalized cache * * SLOW CODE - 🐢 */ @@ -431,6 +463,8 @@ export class EpochCache { config, pubkey2index, index2pubkey, + // `createFromFinalizedState()` creates cache with empty unfinalizedPubkey2index. Be cautious to only pass in finalized state + unfinalizedPubkey2index: newUnfinalizedPubkeyIndexMap(), proposers, // On first epoch, set to null to prevent unnecessary work since this is only used for metrics proposersPrevEpoch: null, @@ -454,6 +488,7 @@ export class EpochCache { nextSyncCommitteeIndexed, epoch: currentEpoch, syncPeriod: computeSyncPeriodAtEpoch(currentEpoch), + historialValidatorLengths: immutable.List(), }); } @@ -469,6 +504,8 @@ export class EpochCache { // Common append-only structures shared with all states, no need to clone pubkey2index: this.pubkey2index, index2pubkey: this.index2pubkey, + // No need to clone this reference. On each mutation the `unfinalizedPubkey2index` reference is replaced, @see `addPubkey` + unfinalizedPubkey2index: this.unfinalizedPubkey2index, // Immutable data proposers: this.proposers, proposersPrevEpoch: this.proposersPrevEpoch, @@ -495,6 +532,7 @@ export class EpochCache { nextSyncCommitteeIndexed: this.nextSyncCommitteeIndexed, epoch: this.epoch, syncPeriod: this.syncPeriod, + historialValidatorLengths: this.historicalValidatorLengths, }); } @@ -505,6 +543,7 @@ export class EpochCache { afterProcessEpoch( state: BeaconStateAllForks, epochTransitionCache: { + indicesEligibleForActivationQueue: ValidatorIndex[]; nextEpochShufflingActiveValidatorIndices: ValidatorIndex[]; nextEpochShufflingActiveIndicesLength: number; nextEpochTotalActiveBalanceByIncrement: number; @@ -579,6 +618,25 @@ export class EpochCache { // ``` this.epoch = computeEpochAtSlot(state.slot); this.syncPeriod = computeSyncPeriodAtEpoch(this.epoch); + // ELECTRA Only: Add current cpState.validators.length + // Only keep validatorLength for epochs after finalized cpState.epoch + // eg. [100(epoch 1), 102(epoch 2)].push(104(epoch 3)), this.epoch = 3, finalized cp epoch = 1 + // We keep the last (3 - 1) items = [102, 104] + if (currEpoch >= this.config.ELECTRA_FORK_EPOCH) { + this.historicalValidatorLengths = this.historicalValidatorLengths.push(state.validators.length); + + // If number of validatorLengths we want to keep exceeds the current list size, it implies + // finalized checkpoint hasn't advanced, and no need to slice + const hasFinalizedCpAdvanced = + this.epoch - state.finalizedCheckpoint.epoch < this.historicalValidatorLengths.size; + + if (hasFinalizedCpAdvanced) { + // We use finalized cp epoch - this.epoch which is a negative number to keep the last n entries and discard the rest + this.historicalValidatorLengths = this.historicalValidatorLengths.slice( + state.finalizedCheckpoint.epoch - this.epoch + ); + } + } } beforeEpochTransition(): void { @@ -766,9 +824,75 @@ export class EpochCache { return isAggregatorFromCommitteeLength(committee.length, slotSignature); } + /** + * Return finalized pubkey given the validator index. + * Only finalized pubkey as we do not store unfinalized pubkey because no where in the spec has a + * need to make such enquiry + */ + getPubkey(index: ValidatorIndex): PublicKey | undefined { + return this.index2pubkey[index]; + } + + getValidatorIndex(pubkey: Uint8Array | PubkeyHex): ValidatorIndex | undefined { + if (this.isAfterElectra()) { + return this.pubkey2index.get(pubkey) ?? this.unfinalizedPubkey2index.get(toMemoryEfficientHexStr(pubkey)); + } else { + return this.pubkey2index.get(pubkey); + } + } + + /** + * + * Add unfinalized pubkeys + * + */ addPubkey(index: ValidatorIndex, pubkey: Uint8Array): void { + if (this.isAfterElectra()) { + this.addUnFinalizedPubkey(index, pubkey); + } else { + // deposit mechanism pre ELECTRA follows a safe distance with assumption + // that they are already canonical + this.addFinalizedPubkey(index, pubkey); + } + } + + addUnFinalizedPubkey(index: ValidatorIndex, pubkey: PubkeyHex | Uint8Array, metrics?: EpochCacheMetrics): void { + this.unfinalizedPubkey2index = this.unfinalizedPubkey2index.set(toMemoryEfficientHexStr(pubkey), index); + metrics?.newUnFinalizedPubkey.inc(); + } + + addFinalizedPubkeys(pubkeyMap: UnfinalizedPubkeyIndexMap, metrics?: EpochCacheMetrics): void { + pubkeyMap.forEach((index, pubkey) => this.addFinalizedPubkey(index, pubkey, metrics)); + } + + /** + * Add finalized validator index and pubkey into finalized cache. + * Since addFinalizedPubkey() primarily takes pubkeys from unfinalized cache, it can take pubkey hex string directly + */ + addFinalizedPubkey(index: ValidatorIndex, pubkey: PubkeyHex | Uint8Array, metrics?: EpochCacheMetrics): void { + const existingIndex = this.pubkey2index.get(pubkey); + + if (existingIndex !== undefined) { + if (existingIndex === index) { + // Repeated insert. + metrics?.finalizedPubkeyDuplicateInsert.inc(); + return; + } else { + // attempt to insert the same pubkey with different index, should never happen. + throw Error("inserted existing pubkey into finalizedPubkey2index cache with a different index"); + } + } + this.pubkey2index.set(pubkey, index); - this.index2pubkey[index] = PublicKey.fromBytes(pubkey); // Optimize for aggregation + const pubkeyBytes = pubkey instanceof Uint8Array ? pubkey : fromHexString(pubkey); + this.index2pubkey[index] = PublicKey.fromBytes(pubkeyBytes); // Optimize for aggregation + } + + /** + * Delete pubkeys from unfinalized cache + */ + deleteUnfinalizedPubkeys(pubkeys: Iterable): void { + this.unfinalizedPubkey2index = this.unfinalizedPubkey2index.deleteAll(pubkeys); } getShufflingAtSlot(slot: Slot): EpochShuffling { @@ -845,15 +969,53 @@ export class EpochCache { } effectiveBalanceIncrementsSet(index: number, effectiveBalance: number): void { - if (index >= this.effectiveBalanceIncrements.length) { - // Clone and extend effectiveBalanceIncrements + if (this.isAfterElectra()) { + // TODO: electra + // getting length and setting getEffectiveBalanceIncrementsByteLen is not fork safe + // so each time we add an index, we should new the Uint8Array to keep it forksafe + // one simple optimization could be to increment the length once per block rather + // on each add/set + // + // there could still be some unused length remaining from the prev ELECTRA padding + const newLength = + index >= this.effectiveBalanceIncrements.length ? index + 1 : this.effectiveBalanceIncrements.length; const effectiveBalanceIncrements = this.effectiveBalanceIncrements; - this.effectiveBalanceIncrements = new Uint8Array(getEffectiveBalanceIncrementsByteLen(index + 1)); + this.effectiveBalanceIncrements = new Uint8Array(newLength); this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); + } else { + if (index >= this.effectiveBalanceIncrements.length) { + // Clone and extend effectiveBalanceIncrements + const effectiveBalanceIncrements = this.effectiveBalanceIncrements; + this.effectiveBalanceIncrements = new Uint8Array(getEffectiveBalanceIncrementsByteLen(index + 1)); + this.effectiveBalanceIncrements.set(effectiveBalanceIncrements, 0); + } } this.effectiveBalanceIncrements[index] = Math.floor(effectiveBalance / EFFECTIVE_BALANCE_INCREMENT); } + + isAfterElectra(): boolean { + return this.epoch >= (this.config.ELECTRA_FORK_EPOCH ?? Infinity); + } + + getValidatorCountAtEpoch(targetEpoch: Epoch): number | undefined { + const currentEpoch = this.epoch; + + if (targetEpoch === currentEpoch) { + return this.historicalValidatorLengths.get(-1); + } + + // Attempt to get validator count from future epoch + if (targetEpoch > currentEpoch) { + return undefined; + } + + // targetEpoch is so far back that historicalValidatorLengths doesnt contain such info + if (targetEpoch < currentEpoch - this.historicalValidatorLengths.size + 1) { + return undefined; + } + return this.historicalValidatorLengths.get(targetEpoch - currentEpoch - 1); + } } function getEffectiveBalanceIncrementsByteLen(validatorCount: number): number { diff --git a/packages/state-transition/src/cache/pubkeyCache.ts b/packages/state-transition/src/cache/pubkeyCache.ts index b2b45ca09d25..a7eacb7961fd 100644 --- a/packages/state-transition/src/cache/pubkeyCache.ts +++ b/packages/state-transition/src/cache/pubkeyCache.ts @@ -1,9 +1,18 @@ import {PublicKey} from "@chainsafe/blst"; +import * as immutable from "immutable"; import {ValidatorIndex, phase0} from "@lodestar/types"; +import {BeaconStateAllForks} from "./types.js"; export type Index2PubkeyCache = PublicKey[]; +/** + * OrderedMap preserves the order of entries in which they are `set()`. + * We assume `values()` yields validator indices in strictly increasing order + * as new validator indices are assigned in increasing order. + * EIP-6914 will break this assumption. + */ +export type UnfinalizedPubkeyIndexMap = immutable.Map; -type PubkeyHex = string; +export type PubkeyHex = string; /** * toHexString() creates hex strings via string concatenation, which are very memory inefficient. @@ -13,7 +22,7 @@ type PubkeyHex = string; * * See https://github.com/ChainSafe/lodestar/issues/3446 */ -function toMemoryEfficientHexStr(hex: Uint8Array | string): string { +export function toMemoryEfficientHexStr(hex: Uint8Array | string): string { if (typeof hex === "string") { if (hex.startsWith("0x")) { hex = hex.slice(2); @@ -24,6 +33,13 @@ function toMemoryEfficientHexStr(hex: Uint8Array | string): string { return Buffer.from(hex.buffer, hex.byteOffset, hex.byteLength).toString("hex"); } +/** + * A wrapper for calling immutable.js. To abstract the initialization of UnfinalizedPubkeyIndexMap + */ +export function newUnfinalizedPubkeyIndexMap(): UnfinalizedPubkeyIndexMap { + return immutable.Map(); +} + export class PubkeyIndexMap { // We don't really need the full pubkey. We could just use the first 20 bytes like an Ethereum address readonly map = new Map(); @@ -39,7 +55,7 @@ export class PubkeyIndexMap { return this.map.get(toMemoryEfficientHexStr(key)); } - set(key: Uint8Array, value: ValidatorIndex): void { + set(key: Uint8Array | PubkeyHex, value: ValidatorIndex): void { this.map.set(toMemoryEfficientHexStr(key), value); } } diff --git a/packages/state-transition/src/cache/stateCache.ts b/packages/state-transition/src/cache/stateCache.ts index f4e637e5d665..0435e8829d21 100644 --- a/packages/state-transition/src/cache/stateCache.ts +++ b/packages/state-transition/src/cache/stateCache.ts @@ -10,6 +10,7 @@ import { BeaconStateBellatrix, BeaconStateCapella, BeaconStateDeneb, + BeaconStateElectra, } from "./types.js"; import {RewardCache, createEmptyRewardCache} from "./rewardCache.js"; @@ -130,11 +131,13 @@ export type CachedBeaconStateAltair = CachedBeaconState; export type CachedBeaconStateBellatrix = CachedBeaconState; export type CachedBeaconStateCapella = CachedBeaconState; export type CachedBeaconStateDeneb = CachedBeaconState; +export type CachedBeaconStateElectra = CachedBeaconState; export type CachedBeaconStateAllForks = CachedBeaconState; export type CachedBeaconStateExecutions = CachedBeaconState; /** * Create CachedBeaconState computing a new EpochCache instance + * TODO ELECTRA: rename this to createFinalizedCachedBeaconState() as it's intended for finalized state only */ export function createCachedBeaconState( state: T, @@ -158,7 +161,7 @@ export function createCachedBeaconState( * Create a CachedBeaconState given a cached seed state and state bytes * This guarantees that the returned state shares the same tree with the seed state * Check loadState() api for more details - * // TODO: rename to loadUnfinalizedCachedBeaconState() due to EIP-6110 + * // TODO: rename to loadUnfinalizedCachedBeaconState() due to ELECTRA */ export function loadCachedBeaconState( cachedSeedState: T, diff --git a/packages/state-transition/src/cache/types.ts b/packages/state-transition/src/cache/types.ts index d6d8a3c37904..b3fe6fc8ed5b 100644 --- a/packages/state-transition/src/cache/types.ts +++ b/packages/state-transition/src/cache/types.ts @@ -8,6 +8,7 @@ export type BeaconStateAltair = CompositeViewDU>; export type BeaconStateCapella = CompositeViewDU>; export type BeaconStateDeneb = CompositeViewDU>; +export type BeaconStateElectra = CompositeViewDU>; export type BeaconStateAllForks = CompositeViewDU>; export type BeaconStateExecutions = CompositeViewDU>; diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 0ef460e784af..4ed801e3c490 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -11,6 +11,7 @@ export type { CachedBeaconStateBellatrix, CachedBeaconStateCapella, CachedBeaconStateDeneb, + CachedBeaconStateElectra, CachedBeaconStateAllForks, CachedBeaconStateExecutions, // Non-cached states @@ -19,6 +20,7 @@ export type { BeaconStateBellatrix, BeaconStateCapella, BeaconStateDeneb, + BeaconStateElectra, BeaconStateAllForks, BeaconStateExecutions, } from "./types.js"; @@ -42,7 +44,12 @@ export { export {type EpochTransitionCache, beforeProcessEpoch} from "./cache/epochTransitionCache.js"; // Aux data-structures -export {PubkeyIndexMap, type Index2PubkeyCache} from "./cache/pubkeyCache.js"; +export { + PubkeyIndexMap, + type Index2PubkeyCache, + type UnfinalizedPubkeyIndexMap, + newUnfinalizedPubkeyIndexMap, +} from "./cache/pubkeyCache.js"; export { type EffectiveBalanceIncrements, diff --git a/packages/state-transition/src/metrics.ts b/packages/state-transition/src/metrics.ts index 12cec46d9a49..48e0bc921876 100644 --- a/packages/state-transition/src/metrics.ts +++ b/packages/state-transition/src/metrics.ts @@ -30,6 +30,11 @@ export type BeaconStateTransitionMetrics = { ) => void; }; +export type EpochCacheMetrics = { + finalizedPubkeyDuplicateInsert: Gauge; + newUnFinalizedPubkey: Gauge; +}; + export function onStateCloneMetrics( state: CachedBeaconStateAllForks, metrics: BeaconStateTransitionMetrics, diff --git a/packages/state-transition/src/signatureSets/attesterSlashings.ts b/packages/state-transition/src/signatureSets/attesterSlashings.ts index f0de50e5d0b2..256582afe368 100644 --- a/packages/state-transition/src/signatureSets/attesterSlashings.ts +++ b/packages/state-transition/src/signatureSets/attesterSlashings.ts @@ -27,13 +27,12 @@ export function getIndexedAttestationBigintSignatureSet( state: CachedBeaconStateAllForks, indexedAttestation: phase0.IndexedAttestationBigint ): ISignatureSet { - const {index2pubkey} = state.epochCtx; const slot = computeStartSlotAtEpoch(Number(indexedAttestation.data.target.epoch as bigint)); const domain = state.config.getDomain(state.slot, DOMAIN_BEACON_ATTESTER, slot); return { type: SignatureSetType.aggregate, - pubkeys: indexedAttestation.attestingIndices.map((i) => index2pubkey[i]), + pubkeys: indexedAttestation.attestingIndices.map((i) => state.epochCtx.index2pubkey[i]), signingRoot: computeSigningRoot(ssz.phase0.AttestationDataBigint, indexedAttestation.data, domain), signature: indexedAttestation.signature, }; diff --git a/packages/state-transition/src/slot/index.ts b/packages/state-transition/src/slot/index.ts index 6c4add1d1230..b05bd7ac93f2 100644 --- a/packages/state-transition/src/slot/index.ts +++ b/packages/state-transition/src/slot/index.ts @@ -7,6 +7,7 @@ export {upgradeStateToAltair} from "./upgradeStateToAltair.js"; export {upgradeStateToBellatrix} from "./upgradeStateToBellatrix.js"; export {upgradeStateToCapella} from "./upgradeStateToCapella.js"; export {upgradeStateToDeneb} from "./upgradeStateToDeneb.js"; +export {upgradeStateToElectra} from "./upgradeStateToElectra.js"; /** * Dial state to next slot. Common for all forks diff --git a/packages/state-transition/src/slot/upgradeStateToElectra.ts b/packages/state-transition/src/slot/upgradeStateToElectra.ts new file mode 100644 index 000000000000..19cac8811f77 --- /dev/null +++ b/packages/state-transition/src/slot/upgradeStateToElectra.ts @@ -0,0 +1,33 @@ +import {ssz} from "@lodestar/types"; +import {UNSET_DEPOSIT_RECEIPTS_START_INDEX} from "@lodestar/params"; +import {CachedBeaconStateDeneb} from "../types.js"; +import {CachedBeaconStateElectra, getCachedBeaconState} from "../cache/stateCache.js"; + +/** + * Upgrade a state from Capella to Deneb. + */ +export function upgradeStateToElectra(stateDeneb: CachedBeaconStateDeneb): CachedBeaconStateElectra { + const {config} = stateDeneb; + + const stateElectraNode = ssz.deneb.BeaconState.commitViewDU(stateDeneb); + const stateElectraView = ssz.electra.BeaconState.getViewDU(stateElectraNode); + + const stateElectra = getCachedBeaconState(stateElectraView, stateDeneb); + + stateElectra.fork = ssz.phase0.Fork.toViewDU({ + previousVersion: stateDeneb.fork.currentVersion, + currentVersion: config.ELECTRA_FORK_VERSION, + epoch: stateDeneb.epochCtx.epoch, + }); + + // latestExecutionPayloadHeader's depositReceiptsRoot set to zeros by default + // default value of depositReceiptsStartIndex is UNSET_DEPOSIT_RECEIPTS_START_INDEX + stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + + // Commit new added fields ViewDU to the root node + stateElectra.commit(); + // Clear cache to ensure the cache of deneb fields is not used by new ELECTRA fields + stateElectra["clearCache"](); + + return stateElectra; +} diff --git a/packages/state-transition/src/stateTransition.ts b/packages/state-transition/src/stateTransition.ts index a454bf4b34d0..5f433918415a 100644 --- a/packages/state-transition/src/stateTransition.ts +++ b/packages/state-transition/src/stateTransition.ts @@ -9,6 +9,7 @@ import { CachedBeaconStateAltair, CachedBeaconStateBellatrix, CachedBeaconStateCapella, + CachedBeaconStateDeneb, } from "./types.js"; import {computeEpochAtSlot} from "./util/index.js"; import {verifyProposerSignature} from "./signatureSets/index.js"; @@ -18,6 +19,7 @@ import { upgradeStateToBellatrix, upgradeStateToCapella, upgradeStateToDeneb, + upgradeStateToElectra, } from "./slot/index.js"; import {processBlock} from "./block/index.js"; import {EpochTransitionStep, processEpoch} from "./epoch/index.js"; @@ -239,6 +241,9 @@ function processSlotsWithTransientCache( if (stateSlot === config.DENEB_FORK_EPOCH) { postState = upgradeStateToDeneb(postState as CachedBeaconStateCapella) as CachedBeaconStateAllForks; } + if (stateSlot === config.ELECTRA_FORK_EPOCH) { + postState = upgradeStateToElectra(postState as CachedBeaconStateDeneb) as CachedBeaconStateAllForks; + } } else { postState.slot++; } diff --git a/packages/state-transition/src/types.ts b/packages/state-transition/src/types.ts index 6b6b1f6260b2..d3a1ed69a7a9 100644 --- a/packages/state-transition/src/types.ts +++ b/packages/state-transition/src/types.ts @@ -9,6 +9,7 @@ export type { CachedBeaconStateBellatrix, CachedBeaconStateCapella, CachedBeaconStateDeneb, + CachedBeaconStateElectra, } from "./cache/stateCache.js"; export type { @@ -19,4 +20,5 @@ export type { BeaconStateBellatrix, BeaconStateCapella, BeaconStateDeneb, + BeaconStateElectra, } from "./cache/types.js"; diff --git a/packages/state-transition/src/util/deposit.ts b/packages/state-transition/src/util/deposit.ts new file mode 100644 index 000000000000..493099fdd982 --- /dev/null +++ b/packages/state-transition/src/util/deposit.ts @@ -0,0 +1,24 @@ +import {ForkSeq, MAX_DEPOSITS} from "@lodestar/params"; +import {UintNum64, phase0} from "@lodestar/types"; +import {CachedBeaconStateAllForks, CachedBeaconStateElectra} from "../types.js"; + +export function getEth1DepositCount(state: CachedBeaconStateAllForks, eth1Data?: phase0.Eth1Data): UintNum64 { + const eth1DataToUse = eth1Data ?? state.eth1Data; + if (state.config.getForkSeq(state.slot) >= ForkSeq.electra) { + const electraState = state as CachedBeaconStateElectra; + // eth1DataIndexLimit = min(UintNum64, UintBn64) can be safely casted as UintNum64 + // since the result lies within upper and lower bound of UintNum64 + const eth1DataIndexLimit: UintNum64 = + eth1DataToUse.depositCount < electraState.depositReceiptsStartIndex + ? eth1DataToUse.depositCount + : Number(electraState.depositReceiptsStartIndex); + + if (state.eth1DepositIndex < eth1DataIndexLimit) { + return Math.min(MAX_DEPOSITS, eth1DataIndexLimit - state.eth1DepositIndex); + } else { + return 0; + } + } else { + return Math.min(MAX_DEPOSITS, eth1DataToUse.depositCount - state.eth1DepositIndex); + } +} diff --git a/packages/state-transition/src/util/execution.ts b/packages/state-transition/src/util/execution.ts index 1c5046354fcb..d9a88e5475b5 100644 --- a/packages/state-transition/src/util/execution.ts +++ b/packages/state-transition/src/util/execution.ts @@ -2,6 +2,7 @@ import { bellatrix, capella, deneb, + electra, isBlindedBeaconBlockBody, ssz, BeaconBlock, @@ -170,5 +171,10 @@ export function executionPayloadToPayloadHeader(fork: ForkSeq, payload: Executio ).excessBlobGas; } + if (fork >= ForkSeq.electra) { + (bellatrixPayloadFields as electra.ExecutionPayloadHeader).depositReceiptsRoot = + ssz.electra.DepositReceipts.hashTreeRoot((payload as electra.ExecutionPayload).depositReceipts); + } + return bellatrixPayloadFields; } diff --git a/packages/state-transition/src/util/genesis.ts b/packages/state-transition/src/util/genesis.ts index 1041c33d0eb3..6c700436d7f6 100644 --- a/packages/state-transition/src/util/genesis.ts +++ b/packages/state-transition/src/util/genesis.ts @@ -7,6 +7,7 @@ import { GENESIS_EPOCH, GENESIS_SLOT, MAX_EFFECTIVE_BALANCE, + UNSET_DEPOSIT_RECEIPTS_START_INDEX, } from "@lodestar/params"; import {Bytes32, phase0, Root, ssz, TimeSeconds} from "@lodestar/types"; @@ -214,6 +215,7 @@ export function initializeBeaconStateFromEth1( | typeof ssz.bellatrix.ExecutionPayloadHeader | typeof ssz.capella.ExecutionPayloadHeader | typeof ssz.deneb.ExecutionPayloadHeader + | typeof ssz.electra.ExecutionPayloadHeader > ): CachedBeaconStateAllForks { const stateView = getGenesisBeaconState( @@ -284,6 +286,16 @@ export function initializeBeaconStateFromEth1( ssz.deneb.ExecutionPayloadHeader.defaultViewDU(); } + if (GENESIS_SLOT >= config.ELECTRA_FORK_EPOCH) { + const stateElectra = state as CompositeViewDU; + stateElectra.fork.previousVersion = config.ELECTRA_FORK_VERSION; + stateElectra.fork.currentVersion = config.ELECTRA_FORK_VERSION; + stateElectra.latestExecutionPayloadHeader = + (executionPayloadHeader as CompositeViewDU) ?? + ssz.electra.ExecutionPayloadHeader.defaultViewDU(); + stateElectra.depositReceiptsStartIndex = UNSET_DEPOSIT_RECEIPTS_START_INDEX; + } + state.commit(); return state; diff --git a/packages/state-transition/src/util/index.ts b/packages/state-transition/src/util/index.ts index 3f2e91da9a77..ba998f65b254 100644 --- a/packages/state-transition/src/util/index.ts +++ b/packages/state-transition/src/util/index.ts @@ -23,3 +23,4 @@ export * from "./slot.js"; export * from "./syncCommittee.js"; export * from "./validator.js"; export * from "./weakSubjectivity.js"; +export * from "./deposit.js"; diff --git a/packages/state-transition/test/unit/cachedBeaconState.test.ts b/packages/state-transition/test/unit/cachedBeaconState.test.ts index 2891cd3e6216..9b05b8947f73 100644 --- a/packages/state-transition/test/unit/cachedBeaconState.test.ts +++ b/packages/state-transition/test/unit/cachedBeaconState.test.ts @@ -1,8 +1,9 @@ +import {fromHexString} from "@chainsafe/ssz"; import {describe, it, expect} from "vitest"; import {Epoch, ssz, RootHex} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {config as defaultConfig} from "@lodestar/config/default"; -import {createBeaconConfig} from "@lodestar/config"; +import {createBeaconConfig, createChainForkConfig} from "@lodestar/config"; import {createCachedBeaconStateTest} from "../utils/state.js"; import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; import {createCachedBeaconState, loadCachedBeaconState} from "../../src/cache/stateCache.js"; @@ -28,6 +29,65 @@ describe("CachedBeaconState", () => { expect(state2.epochCtx.epoch).toBe(0); }); + it("Clone and mutate cache pre-Electra", () => { + const stateView = ssz.altair.BeaconState.defaultViewDU(); + const state1 = createCachedBeaconStateTest(stateView); + + const pubkey1 = fromHexString( + "0x84105a985058fc8740a48bf1ede9d223ef09e8c6b1735ba0a55cf4a9ff2ff92376b778798365e488dab07a652eb04576" + ); + const index1 = 123; + const pubkey2 = fromHexString( + "0xa41726266b1d83ef609d759ba7796d54cfe549154e01e4730a3378309bc81a7638140d7e184b33593c072595f23f032d" + ); + const index2 = 456; + + state1.epochCtx.addPubkey(index1, pubkey1); + + const state2 = state1.clone(); + state2.epochCtx.addPubkey(index2, pubkey2); + + expect(state1.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); + expect(state2.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); + expect(state1.epochCtx.getValidatorIndex(pubkey2)).toBe(index2); + expect(state2.epochCtx.getValidatorIndex(pubkey2)).toBe(index2); + }); + + /* eslint-disable @typescript-eslint/naming-convention */ + it("Clone and mutate cache post-Electra", () => { + const stateView = ssz.electra.BeaconState.defaultViewDU(); + const state1 = createCachedBeaconStateTest( + stateView, + createChainForkConfig({ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + }), + {skipSyncCommitteeCache: true, skipSyncPubkeys: true} + ); + + const pubkey1 = fromHexString( + "0x84105a985058fc8740a48bf1ede9d223ef09e8c6b1735ba0a55cf4a9ff2ff92376b778798365e488dab07a652eb04576" + ); + const index1 = 123; + const pubkey2 = fromHexString( + "0xa41726266b1d83ef609d759ba7796d54cfe549154e01e4730a3378309bc81a7638140d7e184b33593c072595f23f032d" + ); + const index2 = 456; + + state1.epochCtx.addPubkey(index1, pubkey1); + + const state2 = state1.clone(); + state2.epochCtx.addPubkey(index2, pubkey2); + + expect(state1.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); + expect(state2.epochCtx.getValidatorIndex(pubkey1)).toBe(index1); + expect(state1.epochCtx.getValidatorIndex(pubkey2)).toBe(undefined); + expect(state2.epochCtx.getValidatorIndex(pubkey2)).toBe(index2); + }); + it("Auto-commit on hashTreeRoot", () => { // Use Checkpoint instead of BeaconState to speed up the test const cp1 = ssz.phase0.Checkpoint.defaultViewDU(); diff --git a/packages/state-transition/test/unit/upgradeState.test.ts b/packages/state-transition/test/unit/upgradeState.test.ts index 75ba415c1bea..df9b052542f9 100644 --- a/packages/state-transition/test/unit/upgradeState.test.ts +++ b/packages/state-transition/test/unit/upgradeState.test.ts @@ -5,6 +5,7 @@ import {createBeaconConfig, ChainForkConfig, createChainForkConfig} from "@lodes import {config as chainConfig} from "@lodestar/config/default"; import {upgradeStateToDeneb} from "../../src/slot/upgradeStateToDeneb.js"; +import {upgradeStateToElectra} from "../../src/slot/upgradeStateToElectra.js"; import {createCachedBeaconState} from "../../src/cache/stateCache.js"; import {PubkeyIndexMap} from "../../src/cache/pubkeyCache.js"; @@ -24,6 +25,21 @@ describe("upgradeState", () => { const newState = upgradeStateToDeneb(stateView); expect(() => newState.toValue()).not.toThrow(); }); + it("upgradeStateToElectra", () => { + const denebState = ssz.deneb.BeaconState.defaultViewDU(); + const config = getConfig(ForkName.deneb); + const stateView = createCachedBeaconState( + denebState, + { + config: createBeaconConfig(config, denebState.genesisValidatorsRoot), + pubkey2index: new PubkeyIndexMap(), + index2pubkey: [], + }, + {skipSyncCommitteeCache: true} + ); + const newState = upgradeStateToElectra(stateView); + expect(() => newState.toValue()).not.toThrow(); + }); }); const ZERO_HASH = Buffer.alloc(32, 0); diff --git a/packages/state-transition/test/unit/util/deposit.test.ts b/packages/state-transition/test/unit/util/deposit.test.ts new file mode 100644 index 000000000000..e8f7f7a86af8 --- /dev/null +++ b/packages/state-transition/test/unit/util/deposit.test.ts @@ -0,0 +1,99 @@ +import {describe, it, expect} from "vitest"; +import {ssz} from "@lodestar/types"; +import {createChainForkConfig} from "@lodestar/config"; +import {MAX_DEPOSITS} from "@lodestar/params"; +import {CachedBeaconStateElectra, getEth1DepositCount} from "../../../src/index.js"; +import {createCachedBeaconStateTest} from "../../utils/state.js"; + +describe("getEth1DepositCount", () => { + it("Pre Electra", () => { + const stateView = ssz.altair.BeaconState.defaultViewDU(); + const preElectraState = createCachedBeaconStateTest(stateView); + + if (preElectraState.epochCtx.isAfterElectra()) { + throw Error("Not a pre-Electra state"); + } + + preElectraState.eth1Data.depositCount = 123; + + // 1. Should get less than MAX_DEPOSIT + preElectraState.eth1DepositIndex = 120; + expect(getEth1DepositCount(preElectraState)).toBe(3); + + // 2. Should get MAX_DEPOSIT + preElectraState.eth1DepositIndex = 100; + expect(getEth1DepositCount(preElectraState)).toBe(MAX_DEPOSITS); + }); + it("Post Electra with eth1 deposit", () => { + const stateView = ssz.electra.BeaconState.defaultViewDU(); + const postElectraState = createCachedBeaconStateTest( + stateView, + createChainForkConfig({ + /* eslint-disable @typescript-eslint/naming-convention */ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + }), + {skipSyncCommitteeCache: true, skipSyncPubkeys: true} + ) as CachedBeaconStateElectra; + + if (!postElectraState.epochCtx.isAfterElectra()) { + throw Error("Not a post-Electra state"); + } + + postElectraState.depositReceiptsStartIndex = 1000n; + postElectraState.eth1Data.depositCount = 995; + + // 1. Should get less than MAX_DEPOSIT + postElectraState.eth1DepositIndex = 990; + expect(getEth1DepositCount(postElectraState)).toBe(5); + + // 2. Should get MAX_DEPOSIT + postElectraState.eth1DepositIndex = 100; + expect(getEth1DepositCount(postElectraState)).toBe(MAX_DEPOSITS); + + // 3. Should be 0 + postElectraState.eth1DepositIndex = 1000; + expect(getEth1DepositCount(postElectraState)).toBe(0); + }); + it("Post Electra without eth1 deposit", () => { + const stateView = ssz.electra.BeaconState.defaultViewDU(); + const postElectraState = createCachedBeaconStateTest( + stateView, + createChainForkConfig({ + /* eslint-disable @typescript-eslint/naming-convention */ + ALTAIR_FORK_EPOCH: 0, + BELLATRIX_FORK_EPOCH: 0, + CAPELLA_FORK_EPOCH: 0, + DENEB_FORK_EPOCH: 0, + ELECTRA_FORK_EPOCH: 0, + }), + {skipSyncCommitteeCache: true, skipSyncPubkeys: true} + ) as CachedBeaconStateElectra; + + if (!postElectraState.epochCtx.isAfterElectra()) { + throw Error("Not a post-Electra state"); + } + + postElectraState.depositReceiptsStartIndex = 1000n; + postElectraState.eth1Data.depositCount = 1005; + + // Before eth1DepositIndex reaching the start index + // 1. Should get less than MAX_DEPOSIT + postElectraState.eth1DepositIndex = 990; + expect(getEth1DepositCount(postElectraState)).toBe(10); + + // 2. Should get MAX_DEPOSIT + postElectraState.eth1DepositIndex = 983; + expect(getEth1DepositCount(postElectraState)).toBe(MAX_DEPOSITS); + + // After eth1DepositIndex reaching the start index + // 1. Should be 0 + postElectraState.eth1DepositIndex = 1000; + expect(getEth1DepositCount(postElectraState)).toBe(0); + postElectraState.eth1DepositIndex = 1003; + expect(getEth1DepositCount(postElectraState)).toBe(0); + }); +}); diff --git a/packages/types/package.json b/packages/types/package.json index eb2801c90156..9a83070358c7 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -29,6 +29,9 @@ "./deneb": { "import": "./lib/deneb/index.js" }, + "./electra": { + "import": "./lib/electra/index.js" + }, "./phase0": { "import": "./lib/phase0/index.js" } diff --git a/packages/types/src/electra/sszTypes.ts b/packages/types/src/electra/sszTypes.ts index 30690a499845..b347b20df9c7 100644 --- a/packages/types/src/electra/sszTypes.ts +++ b/packages/types/src/electra/sszTypes.ts @@ -1,12 +1,37 @@ -import {ContainerType} from "@chainsafe/ssz"; +import {ContainerType, ListCompositeType, VectorCompositeType} from "@chainsafe/ssz"; +import { + HISTORICAL_ROOTS_LIMIT, + BLOCK_BODY_EXECUTION_PAYLOAD_DEPTH as EXECUTION_PAYLOAD_DEPTH, + EPOCHS_PER_SYNC_COMMITTEE_PERIOD, + SLOTS_PER_EPOCH, + MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD, +} from "@lodestar/params"; import {ssz as primitiveSsz} from "../primitive/index.js"; +import {ssz as phase0Ssz} from "../phase0/index.js"; +import {ssz as altairSsz} from "../altair/index.js"; +import {ssz as bellatrixSsz} from "../bellatrix/index.js"; +import {ssz as capellaSsz} from "../capella/index.js"; import {ssz as denebSsz} from "../deneb/index.js"; -const {BLSSignature} = primitiveSsz; +const {UintNum64, Slot, Root, BLSSignature, UintBn256, Bytes32, BLSPubkey, DepositIndex, UintBn64} = primitiveSsz; + +export const DepositReceipt = new ContainerType( + { + pubkey: BLSPubkey, + withdrawalCredentials: Bytes32, + amount: UintNum64, + signature: BLSSignature, + index: DepositIndex, + }, + {typeName: "DepositReceipt", jsonCase: "eth2"} +); + +export const DepositReceipts = new ListCompositeType(DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD); export const ExecutionPayload = new ContainerType( { ...denebSsz.ExecutionPayload.fields, + depositReceipts: DepositReceipts, // New in ELECTRA }, {typeName: "ExecutionPayload", jsonCase: "eth2"} ); @@ -14,13 +39,18 @@ export const ExecutionPayload = new ContainerType( export const ExecutionPayloadHeader = new ContainerType( { ...denebSsz.ExecutionPayloadHeader.fields, + depositReceiptsRoot: Root, // New in ELECTRA }, {typeName: "ExecutionPayloadHeader", jsonCase: "eth2"} ); +// We have to preserve Fields ordering while changing the type of ExecutionPayload export const BeaconBlockBody = new ContainerType( { - ...denebSsz.BeaconBlockBody.fields, + ...altairSsz.BeaconBlockBody.fields, + executionPayload: ExecutionPayload, // Modified in ELECTRA + blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, + blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, }, {typeName: "BeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -28,28 +58,25 @@ export const BeaconBlockBody = new ContainerType( export const BeaconBlock = new ContainerType( { ...denebSsz.BeaconBlock.fields, + body: BeaconBlockBody, // Modified in ELECTRA }, {typeName: "BeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} ); export const SignedBeaconBlock = new ContainerType( { - message: BeaconBlock, + message: BeaconBlock, // Modified in ELECTRA signature: BLSSignature, }, {typeName: "SignedBeaconBlock", jsonCase: "eth2"} ); -export const BlobSidecar = new ContainerType( - { - ...denebSsz.BlobSidecar.fields, - }, - {typeName: "BlobSidecar", jsonCase: "eth2"} -); - export const BlindedBeaconBlockBody = new ContainerType( { - ...denebSsz.BlindedBeaconBlockBody.fields, + ...altairSsz.BeaconBlockBody.fields, + executionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA + blsToExecutionChanges: capellaSsz.BeaconBlockBody.fields.blsToExecutionChanges, + blobKzgCommitments: denebSsz.BeaconBlockBody.fields.blobKzgCommitments, }, {typeName: "BlindedBeaconBlockBody", jsonCase: "eth2", cachePermanentRootStruct: true} ); @@ -57,13 +84,14 @@ export const BlindedBeaconBlockBody = new ContainerType( export const BlindedBeaconBlock = new ContainerType( { ...denebSsz.BlindedBeaconBlock.fields, + body: BlindedBeaconBlockBody, // Modified in ELECTRA }, {typeName: "BlindedBeaconBlock", jsonCase: "eth2", cachePermanentRootStruct: true} ); export const SignedBlindedBeaconBlock = new ContainerType( { - message: BlindedBeaconBlock, + message: BlindedBeaconBlock, // Modified in ELECTRA signature: BLSSignature, }, {typeName: "SignedBlindedBeaconBlock", jsonCase: "eth2"} @@ -71,7 +99,10 @@ export const SignedBlindedBeaconBlock = new ContainerType( export const BuilderBid = new ContainerType( { - ...denebSsz.BuilderBid.fields, + header: ExecutionPayloadHeader, // Modified in ELECTRA + blindedBlobsBundle: denebSsz.BlobKzgCommitments, + value: UintBn256, + pubkey: BLSPubkey, }, {typeName: "BuilderBid", jsonCase: "eth2"} ); @@ -86,63 +117,133 @@ export const SignedBuilderBid = new ContainerType( export const ExecutionPayloadAndBlobsBundle = new ContainerType( { - ...denebSsz.ExecutionPayloadAndBlobsBundle.fields, + executionPayload: ExecutionPayload, // Modified in ELECTRA + blobsBundle: denebSsz.BlobsBundle, }, {typeName: "ExecutionPayloadAndBlobsBundle", jsonCase: "eth2"} ); +// We don't spread deneb.BeaconState fields since we need to replace +// latestExecutionPayloadHeader and we cannot keep order doing that export const BeaconState = new ContainerType( { - ...denebSsz.BeaconState.fields, + genesisTime: UintNum64, + genesisValidatorsRoot: Root, + slot: primitiveSsz.Slot, + fork: phase0Ssz.Fork, + // History + latestBlockHeader: phase0Ssz.BeaconBlockHeader, + blockRoots: phase0Ssz.HistoricalBlockRoots, + stateRoots: phase0Ssz.HistoricalStateRoots, + // historical_roots Frozen in Capella, replaced by historical_summaries + historicalRoots: new ListCompositeType(Root, HISTORICAL_ROOTS_LIMIT), + // Eth1 + eth1Data: phase0Ssz.Eth1Data, + eth1DataVotes: phase0Ssz.Eth1DataVotes, + eth1DepositIndex: UintNum64, + // Registry + validators: phase0Ssz.Validators, + balances: phase0Ssz.Balances, + randaoMixes: phase0Ssz.RandaoMixes, + // Slashings + slashings: phase0Ssz.Slashings, + // Participation + previousEpochParticipation: altairSsz.EpochParticipation, + currentEpochParticipation: altairSsz.EpochParticipation, + // Finality + justificationBits: phase0Ssz.JustificationBits, + previousJustifiedCheckpoint: phase0Ssz.Checkpoint, + currentJustifiedCheckpoint: phase0Ssz.Checkpoint, + finalizedCheckpoint: phase0Ssz.Checkpoint, + // Inactivity + inactivityScores: altairSsz.InactivityScores, + // Sync + currentSyncCommittee: altairSsz.SyncCommittee, + nextSyncCommittee: altairSsz.SyncCommittee, + // Execution + latestExecutionPayloadHeader: ExecutionPayloadHeader, // Modified in ELECTRA + // Withdrawals + nextWithdrawalIndex: capellaSsz.BeaconState.fields.nextWithdrawalIndex, + nextWithdrawalValidatorIndex: capellaSsz.BeaconState.fields.nextWithdrawalValidatorIndex, + // Deep history valid from Capella onwards + historicalSummaries: capellaSsz.BeaconState.fields.historicalSummaries, + depositReceiptsStartIndex: UintBn64, // New in ELECTRA }, {typeName: "BeaconState", jsonCase: "eth2"} ); export const LightClientHeader = new ContainerType( { - ...denebSsz.LightClientHeader.fields, + beacon: phase0Ssz.BeaconBlockHeader, + execution: ExecutionPayloadHeader, // Modified in ELECTRA + executionBranch: new VectorCompositeType(Bytes32, EXECUTION_PAYLOAD_DEPTH), }, {typeName: "LightClientHeader", jsonCase: "eth2"} ); export const LightClientBootstrap = new ContainerType( { - ...denebSsz.LightClientBootstrap.fields, + header: LightClientHeader, + currentSyncCommittee: altairSsz.SyncCommittee, + currentSyncCommitteeBranch: altairSsz.LightClientBootstrap.fields.currentSyncCommitteeBranch, }, {typeName: "LightClientBootstrap", jsonCase: "eth2"} ); export const LightClientUpdate = new ContainerType( { - ...denebSsz.LightClientUpdate.fields, + attestedHeader: LightClientHeader, + nextSyncCommittee: altairSsz.SyncCommittee, + nextSyncCommitteeBranch: altairSsz.LightClientUpdate.fields.nextSyncCommitteeBranch, + finalizedHeader: LightClientHeader, + finalityBranch: altairSsz.LightClientUpdate.fields.finalityBranch, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientUpdate", jsonCase: "eth2"} ); export const LightClientFinalityUpdate = new ContainerType( { - ...denebSsz.LightClientFinalityUpdate.fields, + attestedHeader: LightClientHeader, + finalizedHeader: LightClientHeader, + finalityBranch: altairSsz.LightClientFinalityUpdate.fields.finalityBranch, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientFinalityUpdate", jsonCase: "eth2"} ); export const LightClientOptimisticUpdate = new ContainerType( { - ...denebSsz.LightClientOptimisticUpdate.fields, + attestedHeader: LightClientHeader, + syncAggregate: altairSsz.SyncAggregate, + signatureSlot: Slot, }, {typeName: "LightClientOptimisticUpdate", jsonCase: "eth2"} ); export const LightClientStore = new ContainerType( { - ...denebSsz.LightClientStore.fields, + snapshot: LightClientBootstrap, + validUpdates: new ListCompositeType(LightClientUpdate, EPOCHS_PER_SYNC_COMMITTEE_PERIOD * SLOTS_PER_EPOCH), }, {typeName: "LightClientStore", jsonCase: "eth2"} ); +// PayloadAttributes primarily for SSE event +export const PayloadAttributes = new ContainerType( + { + ...capellaSsz.PayloadAttributes.fields, + parentBeaconBlockRoot: Root, + }, + {typeName: "PayloadAttributes", jsonCase: "eth2"} +); + export const SSEPayloadAttributes = new ContainerType( { - ...denebSsz.SSEPayloadAttributes.fields, + ...bellatrixSsz.SSEPayloadAttributesCommon.fields, + payloadAttributes: PayloadAttributes, }, {typeName: "SSEPayloadAttributes", jsonCase: "eth2"} ); diff --git a/packages/types/src/electra/types.ts b/packages/types/src/electra/types.ts index 198259eed1dd..3286c10a0334 100644 --- a/packages/types/src/electra/types.ts +++ b/packages/types/src/electra/types.ts @@ -1,12 +1,14 @@ import {ValueOf} from "@chainsafe/ssz"; import * as ssz from "./sszTypes.js"; -export type BlobSidecar = ValueOf; -export type ExecutionPayloadAndBlobsBundle = ValueOf; +export type DepositReceipt = ValueOf; +export type DepositReceipts = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; +export type ExecutionPayloadAndBlobsBundle = ValueOf; + export type BeaconBlockBody = ValueOf; export type BeaconBlock = ValueOf; export type SignedBeaconBlock = ValueOf; @@ -17,6 +19,8 @@ export type BlindedBeaconBlockBody = ValueOf; export type BlindedBeaconBlock = ValueOf; export type SignedBlindedBeaconBlock = ValueOf; +export type FullOrBlindedExecutionPayload = ExecutionPayload | ExecutionPayloadHeader; + export type BuilderBid = ValueOf; export type SignedBuilderBid = ValueOf; export type SSEPayloadAttributes = ValueOf; diff --git a/packages/types/src/primitive/sszTypes.ts b/packages/types/src/primitive/sszTypes.ts index 068a32e2cc17..376e17c3f1b6 100644 --- a/packages/types/src/primitive/sszTypes.ts +++ b/packages/types/src/primitive/sszTypes.ts @@ -50,6 +50,7 @@ export const SubcommitteeIndex = UintNum64; */ export const ValidatorIndex = UintNum64; export const WithdrawalIndex = UintNum64; +export const DepositIndex = UintNum64; export const Gwei = UintBn64; export const Wei = UintBn256; export const Root = new ByteVectorType(32); diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 0afede39b951..298707c07303 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -106,7 +106,6 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record