Merge branch 'unstable' into patch-4

dashboards/lodestar_vm_host.json

@@ -291,7 +291,7 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "editorMode": "code",
-          "expr": "sum by (space) ({__name__=~\"nodejs_heap_space_size_used_bytes|network_worker_nodejs_heap_space_size_used_bytes|discv5_worker_nodejs_heap_space_size_used_bytes\")",
+          "expr": "sum by (space) ({__name__=~\"nodejs_heap_space_size_used_bytes|network_worker_nodejs_heap_space_size_used_bytes|discv5_worker_nodejs_heap_space_size_used_bytes\"})",
           "hide": false,
           "interval": "",
           "legendFormat": "{{space}}",

docs/pages/getting-started/starting-a-node.md

@@ -115,6 +115,7 @@ In case you really trust `checkpointSyncUrl` then you may skip providing `wssChe
 :::warning
 Please use this option very carefully (and at your own risk), a malicious server URL can put you on the wrong chain with a danger of you losing your funds by social engineering.
 If possible, validate your `wssCheckpoint` from multiple places (e.g. different client distributions) or from other trusted sources. This will highly reduce the risk of starting off on a malicious chain.
+This list of [public endpoints](https://eth-clients.github.io/checkpoint-sync-endpoints/) maintained by the Ethereum community may be used for reference.
 :::
 
 **Taking too long to sync?**

packages/beacon-node/src/api/impl/validator/index.ts

@@ -1067,7 +1067,13 @@ export function getValidatorApi({
 
       await waitForSlot(slot); // Must never request for a future slot > currentSlot
 
-      const aggregate = chain.attestationPool.getAggregate(slot, attestationDataRoot);
+      const dataRootHex = toHexString(attestationDataRoot);
+      const aggregate = chain.attestationPool.getAggregate(slot, dataRootHex);
+
+      if (!aggregate) {
+        throw new ApiError(404, `No aggregated attestation for slot=${slot}, dataRoot=${dataRootHex}`);
+      }
+
       metrics?.production.producedAggregateParticipants.observe(aggregate.aggregationBits.getTrueBitIndexes().length);
 
       return {

packages/beacon-node/src/api/rest/base.ts

@@ -1,5 +1,5 @@
 import {parse as parseQueryString} from "qs";
-import {FastifyInstance, fastify} from "fastify";
+import {FastifyInstance, FastifyRequest, fastify} from "fastify";
 import {fastifyCors} from "@fastify/cors";
 import bearerAuthPlugin from "@fastify/bearer-auth";
 import {ErrorAborted, Gauge, Histogram, Logger} from "@lodestar/utils";
@@ -62,7 +62,7 @@ export class RestApiServer {
     this.activeSockets = new HttpActiveSocketsTracker(server.server, metrics);
 
     // To parse our ApiError -> statusCode
-    server.setErrorHandler((err, req, res) => {
+    server.setErrorHandler((err, _req, res) => {
       if (err.validation) {
         void res.status(400).send(err.validation);
       } else {
@@ -83,19 +83,19 @@ export class RestApiServer {
     // Log all incoming request to debug (before parsing). TODO: Should we hook latter in the lifecycle? https://www.fastify.io/docs/latest/Lifecycle/
     // Note: Must be an async method so fastify can continue the release lifecycle. Otherwise we must call done() or the request stalls
     server.addHook("onRequest", async (req, _res) => {
-      const operationId = req.routeSchema.operationId as string;
+      const operationId = getOperationId(req);
       this.logger.debug(`Req ${req.id} ${req.ip} ${operationId}`);
       metrics?.requests.inc({operationId});
     });
 
     server.addHook("preHandler", async (req, _res) => {
-      const operationId = req.routeSchema.operationId as string;
+      const operationId = getOperationId(req);
       this.logger.debug(`Exec ${req.id} ${req.ip} ${operationId}`);
     });
 
     // Log after response
     server.addHook("onResponse", async (req, res) => {
-      const operationId = req.routeSchema.operationId as string;
+      const operationId = getOperationId(req);
       this.logger.debug(`Res ${req.id} ${operationId} - ${res.raw.statusCode}`);
       metrics?.responseTime.observe({operationId}, res.elapsedTime / 1000);
     });
@@ -105,7 +105,7 @@ export class RestApiServer {
       // Don't log NodeISSyncing errors, they happen very frequently while syncing and the validator polls duties
       if (err instanceof ErrorAborted || err instanceof NodeIsSyncing) return;
 
-      const operationId = req.routeSchema.operationId as string;
+      const operationId = getOperationId(req);
 
       if (err instanceof ApiError) {
         this.logger.warn(`Req ${req.id} ${operationId} failed`, {reason: err.message});
@@ -157,3 +157,8 @@ export class RestApiServer {
     return false;
   }
 }
+
+function getOperationId(req: FastifyRequest): string {
+  // Note: `schema` will be `undefined` if route is not defined
+  return req.routeOptions.schema?.operationId ?? "unknown";
+}

packages/beacon-node/src/chain/opPools/attestationPool.ts

@@ -1,7 +1,7 @@
 import {PointFormat, Signature} from "@chainsafe/bls/types";
 import bls from "@chainsafe/bls";
-import {BitArray, toHexString} from "@chainsafe/ssz";
-import {phase0, Slot, Root, RootHex} from "@lodestar/types";
+import {BitArray} from "@chainsafe/ssz";
+import {phase0, Slot, RootHex} from "@lodestar/types";
 import {MapDef} from "@lodestar/utils";
 import {IClock} from "../../util/clock.js";
 import {InsertOutcome, OpPoolError, OpPoolErrorCode} from "./types.js";
@@ -128,12 +128,11 @@ export class AttestationPool {
   /**
    * For validator API to get an aggregate
    */
-  getAggregate(slot: Slot, dataRoot: Root): phase0.Attestation {
-    const dataRootHex = toHexString(dataRoot);
+  getAggregate(slot: Slot, dataRootHex: RootHex): phase0.Attestation | null {
     const aggregate = this.attestationByRootBySlot.get(slot)?.get(dataRootHex);
     if (!aggregate) {
       // TODO: Add metric for missing aggregates
-      throw Error(`No attestation for slot=${slot} dataRoot=${dataRootHex}`);
+      return null;
     }
 
     return fastToAttestation(aggregate);

packages/cli/package.json

@@ -32,11 +32,11 @@
     "lint:fix": "yarn run lint --fix",
     "test:unit": "vitest --run --dir test/unit/",
     "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e/",
-    "test:sim:multifork": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/multi_fork.test.ts",
-    "test:sim:mixedclient": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/mixed_client.test.ts",
+    "test:sim:multifork": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/multiFork.test.ts",
+    "test:sim:mixedclient": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/mixedClient.test.ts",
     "test:sim:endpoints": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/endpoints.test.ts",
     "test:sim:deneb": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/deneb.test.ts",
-    "test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/backup_eth_provider.test.ts",
+    "test:sim:backup_eth_provider": "LODESTAR_PRESET=minimal DOTENV_CONFIG_PATH=../../.env.test node -r dotenv/config --loader ts-node/esm test/sim/backupEthProvider.test.ts",
     "test": "yarn test:unit && yarn test:e2e",
     "check-readme": "typescript-docs-verifier"
   },

packages/cli/test/scripts/e2e_test_env.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
 import {BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
-import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {SimulationEnvironment} from "../utils/simulation/simulationEnvironment.js";
 import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";
 import {connectAllNodes} from "../utils/simulation/utils/network.js";
 

packages/cli/test/sim/backup_eth_provider.test.ts → packages/cli/test/sim/backupEthProvider.test.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
 import {activePreset} from "@lodestar/params";
-import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {SimulationEnvironment} from "../utils/simulation/simulationEnvironment.js";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
 import {AssertionMatch, BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
 import {defineSimTestConfig, logFilesDir, replaceIpFromUrl} from "../utils/simulation/utils/index.js";

packages/cli/test/sim/deneb.test.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
 import {activePreset} from "@lodestar/params";
-import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {SimulationEnvironment} from "../utils/simulation/simulationEnvironment.js";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
 import {AssertionMatch, BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
 import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";

packages/cli/test/sim/endpoints.test.ts

@@ -3,7 +3,7 @@ import path from "node:path";
 import assert from "node:assert";
 import {toHexString} from "@chainsafe/ssz";
 import {ApiError, routes} from "@lodestar/api";
-import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {SimulationEnvironment} from "../utils/simulation/simulationEnvironment.js";
 import {BeaconClient, ExecutionClient} from "../utils/simulation/interfaces.js";
 import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";
 import {waitForSlot} from "../utils/simulation/utils/network.js";

packages/cli/test/sim/mixed_client.test.ts → packages/cli/test/sim/mixedClient.test.ts

@@ -1,6 +1,6 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
-import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {SimulationEnvironment} from "../utils/simulation/simulationEnvironment.js";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";
 import {AssertionMatch, BeaconClient, ExecutionClient, ValidatorClient} from "../utils/simulation/interfaces.js";
 import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";

packages/cli/test/sim/multi_fork.test.ts → packages/cli/test/sim/multiFork.test.ts

@@ -1,7 +1,7 @@
 /* eslint-disable @typescript-eslint/naming-convention */
 import path from "node:path";
 import {AssertionMatch, BeaconClient, ExecutionClient, ValidatorClient} from "../utils/simulation/interfaces.js";
-import {SimulationEnvironment} from "../utils/simulation/SimulationEnvironment.js";
+import {SimulationEnvironment} from "../utils/simulation/simulationEnvironment.js";
 import {defineSimTestConfig, logFilesDir} from "../utils/simulation/utils/index.js";
 import {connectAllNodes, waitForSlot} from "../utils/simulation/utils/network.js";
 import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js";

packages/cli/test/utils/simulation/assertions/accountBalanceAssertion.ts

@@ -1,14 +1,6 @@
 import {EL_GENESIS_ACCOUNT} from "../constants.js";
 import {AssertionMatch, AssertionResult, NodePair, SimulationAssertion} from "../interfaces.js";
 
-function hexToBigInt(num: string): bigint {
-  return num.startsWith("0x") ? BigInt(num) : BigInt(`0x${num}`);
-}
-
-function bigIntToHex(num: bigint): string {
-  return `0x${num.toString(16)}`;
-}
-
 const transactionAmount = BigInt(2441406250);
 
 export function createAccountBalanceAssertion({
@@ -30,17 +22,12 @@ export function createAccountBalanceAssertion({
       return AssertionMatch.None;
     },
     async capture({node}) {
-      await node.execution.provider?.getRpc().fetch({
-        method: "eth_sendTransaction",
-        params: [
-          {
-            to: address,
-            from: EL_GENESIS_ACCOUNT,
-            gas: "0x76c0",
-            gasPrice: "0x9184e72a000",
-            value: bigIntToHex(transactionAmount),
-          },
-        ],
+      await node.execution.provider?.eth.sendTransaction({
+        to: address,
+        from: EL_GENESIS_ACCOUNT,
+        gas: "0x76c0",
+        gasPrice: "0x9184e72a000",
+        value: transactionAmount,
       });
 
       // Capture the value transferred to account
@@ -57,10 +44,7 @@ export function createAccountBalanceAssertion({
         expectedBalanceAtCurrentSlot += BigInt(store[captureSlot]);
       }
 
-      const balance = hexToBigInt(
-        (await node.execution.provider?.getRpc().fetch({method: "eth_getBalance", params: [address, "latest"]})) ??
-          "0x0"
-      );
+      const balance = await node.execution.provider?.eth.getBalance(address, "latest");
 
       if (balance !== expectedBalanceAtCurrentSlot) {
         errors.push(

packages/cli/test/utils/simulation/assertions/executionHeadAssertion.ts

@@ -18,9 +18,9 @@ export function createExecutionHeadAssertion({
       return AssertionMatch.None;
     },
     async capture({node}) {
-      const blockNumber = await node.execution.provider?.getBlockNumber();
+      const blockNumber = await node.execution.provider?.eth.getBlockNumber();
       if (blockNumber == null) throw new Error("Execution provider not available");
-      const executionHeadBlock = await node.execution.provider?.getBlockByNumber(blockNumber);
+      const executionHeadBlock = await node.execution.provider?.eth.getBlock(blockNumber);
 
       const consensusHead = await node.beacon.api.beacon.getBlockV2("head");
       ApiError.assert(consensusHead);

packages/cli/test/utils/simulation/beacon_clients/index.ts → packages/cli/test/utils/simulation/clients/beacon/index.ts

@@ -1,9 +1,9 @@
 import {writeFile} from "node:fs/promises";
 import {BeaconStateAllForks} from "@lodestar/state-transition";
-import {EL_ENGINE_BASE_PORT, SHARED_JWT_SECRET} from "../constants.js";
-import {AtLeast, BeaconClient, BeaconGeneratorOptions, BeaconNode} from "../interfaces.js";
-import {makeUniqueArray} from "../utils/index.js";
-import {ensureDirectories} from "../utils/paths.js";
+import {EL_ENGINE_BASE_PORT, SHARED_JWT_SECRET} from "../../constants.js";
+import {AtLeast, BeaconClient, BeaconGeneratorOptions, BeaconNode} from "../../interfaces.js";
+import {makeUniqueArray} from "../../utils/index.js";
+import {ensureDirectories} from "../../utils/paths.js";
 import {generateLighthouseBeaconNode} from "./lighthouse.js";
 import {generateLodestarBeaconNode} from "./lodestar.js";
 

packages/cli/test/utils/simulation/beacon_clients/lighthouse.ts → packages/cli/test/utils/simulation/clients/beacon/lighthouse.ts

@@ -5,9 +5,9 @@ import yaml from "js-yaml";
 import {HttpClient} from "@lodestar/api";
 import {getClient} from "@lodestar/api/beacon";
 import {chainConfigToJson} from "@lodestar/config";
-import {BeaconClient, BeaconNodeGenerator, LighthouseAPI, RunnerType} from "../interfaces.js";
-import {getNodeMountedPaths} from "../utils/paths.js";
-import {getNodePorts} from "../utils/ports.js";
+import {BeaconClient, BeaconNodeGenerator, LighthouseAPI, RunnerType} from "../../interfaces.js";
+import {getNodeMountedPaths} from "../../utils/paths.js";
+import {getNodePorts} from "../../utils/ports.js";
 
 export const generateLighthouseBeaconNode: BeaconNodeGenerator<BeaconClient.Lighthouse> = (opts, runner) => {
   if (!process.env.LIGHTHOUSE_BINARY_PATH && !process.env.LIGHTHOUSE_DOCKER_IMAGE) {

packages/cli/test/utils/simulation/beacon_clients/lodestar.ts → packages/cli/test/utils/simulation/clients/beacon/lodestar.ts

@@ -5,11 +5,11 @@ import got from "got";
 import {getClient} from "@lodestar/api/beacon";
 import {chainConfigToJson} from "@lodestar/config";
 import {LogLevel} from "@lodestar/utils";
-import {BeaconArgs} from "../../../../src/cmds/beacon/options.js";
-import {GlobalArgs} from "../../../../src/options/globalOptions.js";
-import {LODESTAR_BINARY_PATH} from "../constants.js";
-import {BeaconClient, BeaconNodeGenerator, RunnerType} from "../interfaces.js";
-import {getNodePorts} from "../utils/ports.js";
+import {BeaconArgs} from "../../../../../src/cmds/beacon/options.js";
+import {GlobalArgs} from "../../../../../src/options/globalOptions.js";
+import {LODESTAR_BINARY_PATH} from "../../constants.js";
+import {BeaconClient, BeaconNodeGenerator, RunnerType} from "../../interfaces.js";
+import {getNodePorts} from "../../utils/ports.js";
 
 export const generateLodestarBeaconNode: BeaconNodeGenerator<BeaconClient.Lodestar> = (opts, runner) => {
   const {

packages/cli/test/utils/simulation/execution_clients/geth.ts → packages/cli/test/utils/simulation/clients/execution/geth.ts

@@ -1,19 +1,18 @@
-/* eslint-disable @typescript-eslint/naming-convention */
 import {writeFile} from "node:fs/promises";
 import path from "node:path";
 import got from "got";
-import {ZERO_HASH} from "@lodestar/state-transition";
+import {Web3} from "web3";
 import {
   EL_GENESIS_ACCOUNT,
   EL_GENESIS_PASSWORD,
   EL_GENESIS_SECRET_KEY,
   SHARED_JWT_SECRET,
   SIM_ENV_NETWORK_ID,
-} from "../constants.js";
-import {Eth1ProviderWithAdmin} from "../Eth1ProviderWithAdmin.js";
-import {ExecutionClient, ExecutionNodeGenerator, ExecutionStartMode, JobOptions, RunnerType} from "../interfaces.js";
-import {getNodeMountedPaths} from "../utils/paths.js";
-import {getNodePorts} from "../utils/ports.js";
+} from "../../constants.js";
+import {registerWeb3JsPlugins} from "../../web3JsPlugins.js";
+import {ExecutionClient, ExecutionNodeGenerator, ExecutionStartMode, JobOptions, RunnerType} from "../../interfaces.js";
+import {getNodeMountedPaths} from "../../utils/paths.js";
+import {getNodePorts} from "../../utils/ports.js";
 
 export const generateGethNode: ExecutionNodeGenerator<ExecutionClient.Geth> = (opts, runner) => {
   if (!process.env.GETH_BINARY_DIR && !process.env.GETH_DOCKER_IMAGE) {
@@ -166,11 +165,8 @@ export const generateGethNode: ExecutionNodeGenerator<ExecutionClient.Geth> = (o
 
   const job = runner.create([{...initJobOptions, children: [{...importJobOptions, children: [startJobOptions]}]}]);
 
-  const provider = new Eth1ProviderWithAdmin(
-    {DEPOSIT_CONTRACT_ADDRESS: ZERO_HASH},
-    // To allow admin_* RPC methods had to add "ethRpcUrl"
-    {providerUrls: [ethRpcPublicUrl, engineRpcPublicUrl], jwtSecretHex: SHARED_JWT_SECRET}
-  );
+  const provider = new Web3(ethRpcPublicUrl);
+  registerWeb3JsPlugins(provider);
 
   return {
     client: ExecutionClient.Geth,