hive: pyspec simulator implementation

hive_integration/nodocker/build_sims.sh

@@ -13,11 +13,13 @@ ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/engine/engine_sim
 ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/consensus/consensus_sim
 ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/graphql/graphql_sim
 ${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/rpc/rpc_sim
+${ENV_SCRIPT} nim ${NIM_FLAGS} ${SIM_DIR}/pyspec/pyspec_sim
 
 ${SIM_DIR}/engine/engine_sim
 ${SIM_DIR}/consensus/consensus_sim
 ${SIM_DIR}/graphql/graphql_sim
 ${SIM_DIR}/rpc/rpc_sim
+${SIM_DIR}/pyspec/pyspec_sim
 
 echo "## ${1}" > simulators.md
-cat engine.md consensus.md graphql.md rpc.md >> simulators.md
+cat engine.md consensus.md graphql.md rpc.md pyspec.md >> simulators.md

hive_integration/nodocker/engine/engine_callsigs.nim

@@ -0,0 +1,3 @@
+import ethtypes, engine_api_types
+
+proc engine_newPayloadV2(payload: ExecutionPayloadV1OrV2): PayloadStatusV1

hive_integration/nodocker/engine/engine_client.nim

@@ -3,7 +3,7 @@ import
   stew/byteutils,
   eth/[common, common/eth_types, rlp], chronos,
   web3/engine_api_types,
-  json_rpc/[rpcclient, errors],
+  json_rpc/[rpcclient, errors, jsonmarshal],
   ../../../tests/rpcclient/eth_api,
   ../../../premix/parser,
   ../../../nimbus/rpc/hexstrings,
@@ -13,6 +13,12 @@ import web3/engine_api as web3_engine_api
 
 type Hash256 = eth_types.Hash256
 
+from os import DirSep, AltSep
+const
+  sourceDir = currentSourcePath.rsplit({DirSep, AltSep}, 1)[0]
+
+createRpcSigs(RpcClient, sourceDir & "/engine_callsigs.nim")
+
 template wrapTry(body: untyped) =
   try:
     body
@@ -33,6 +39,13 @@ proc forkchoiceUpdatedV1*(client: RpcClient,
   wrapTrySimpleRes:
     client.engine_forkchoiceUpdatedV1(update, payloadAttributes)
 
+proc forkchoiceUpdatedV2*(client: RpcClient,
+      update: ForkchoiceStateV1,
+      payloadAttributes = none(PayloadAttributesV2)):
+        Result[ForkchoiceUpdatedResponse, string] =
+  wrapTrySimpleRes:
+    client.engine_forkchoiceUpdatedV2(update, payloadAttributes)
+
 proc getPayloadV1*(client: RpcClient, payloadId: PayloadID): Result[ExecutionPayloadV1, string] =
   wrapTrySimpleRes:
     client.engine_getPayloadV1(payloadId)
@@ -49,6 +62,12 @@ proc newPayloadV2*(client: RpcClient,
   wrapTrySimpleRes:
     client.engine_newPayloadV2(payload)
 
+proc newPayloadV2*(client: RpcClient,
+      payload: ExecutionPayloadV1OrV2):
+        Result[PayloadStatusV1, string] =
+  wrapTrySimpleRes:
+    client.engine_newPayloadV2(payload)
+
 proc toBlockNumber(n: Option[HexQuantityStr]): common.BlockNumber =
   if n.isNone:
     return 0.toBlockNumber
@@ -181,6 +200,11 @@ proc balanceAt*(client: RpcClient, address: EthAddress): Result[UInt256, string]
     let res = waitFor client.eth_getBalance(ethAddressStr(address), "latest")
     return ok(UInt256.fromHex(res.string))
 
+proc nonceAt*(client: RpcClient, address: EthAddress): Result[AccountNonce, string] =
+  wrapTry:
+    let res = waitFor client.eth_getTransactionCount(ethAddressStr(address), "latest")
+    return ok(fromHex[AccountNonce](res.string))
+
 proc txReceipt*(client: RpcClient, txHash: Hash256): Result[eth_api.ReceiptObject, string] =
   wrapTry:
     let res = waitFor client.eth_getTransactionReceipt(txHash)

hive_integration/nodocker/pyspec/helpers.nim

@@ -0,0 +1,58 @@
+import
+  eth/[common],
+  json_rpc/[rpcclient],
+  web3/ethtypes,
+  ../../../nimbus/transaction
+
+import eth/common/eth_types as common_eth_types
+type Hash256 = common_eth_types.Hash256
+
+import web3/engine_api_types
+from web3/ethtypes as web3types import nil
+
+type
+  Web3BlockHash* = web3types.BlockHash
+  Web3Address* = web3types.Address
+  Web3Bloom* = web3types.FixedBytes[256]
+  Web3Quantity* = web3types.Quantity
+  Web3PrevRandao* = web3types.FixedBytes[32]
+  Web3ExtraData* = web3types.DynamicBytes[0, 32]
+
+func toWdV1(wd: Withdrawal): WithdrawalV1 =
+  result = WithdrawalV1(
+    index: Web3Quantity wd.index,
+    validatorIndex: Web3Quantity wd.validatorIndex,
+    address: Web3Address wd.address,
+    amount: Web3Quantity wd.amount
+  )
+
+func toPayloadV1OrV2*(blk: EthBlock): ExecutionPayloadV1OrV2 =
+  let header = blk.header
+
+  # Return the new payload
+  result = ExecutionPayloadV1OrV2(
+    parentHash:    Web3BlockHash header.parentHash.data,
+    feeRecipient:  Web3Address header.coinbase,
+    stateRoot:     Web3BlockHash header.stateRoot.data,
+    receiptsRoot:  Web3BlockHash header.receiptRoot.data,
+    logsBloom:     Web3Bloom header.bloom,
+    prevRandao:    Web3PrevRandao header.mixDigest.data,
+    blockNumber:   Web3Quantity header.blockNumber.truncate(uint64),
+    gasLimit:      Web3Quantity header.gasLimit,
+    gasUsed:       Web3Quantity header.gasUsed,
+    timestamp:     Web3Quantity toUnix(header.timestamp),
+    extraData:     Web3ExtraData header.extraData,
+    baseFeePerGas: header.baseFee,
+    blockHash:     Web3BlockHash header.blockHash.data
+  )
+
+  for tx in blk.txs:
+    let txData = rlp.encode(tx)
+    result.transactions.add TypedTransaction(txData)
+
+  if blk.withdrawals.isSome:
+    let withdrawals = blk.withdrawals.get
+    var wds = newSeqOfCap[WithdrawalV1](withdrawals.len)
+    for wd in withdrawals:
+      wds.add toWdV1(wd)
+    result.withdrawals = some(wds)

hive_integration/nodocker/pyspec/pyspec_sim.nim

@@ -0,0 +1,167 @@
+# Nimbus
+# Copyright (c) 2023 Status Research & Development GmbH
+# Licensed under either of
+#  * Apache License, version 2.0, ([LICENSE-APACHE](LICENSE-APACHE))
+#  * MIT license ([LICENSE-MIT](LICENSE-MIT))
+# at your option.
+# This file may not be copied, modified, or distributed except according to
+# those terms.
+
+import
+  std/[os, json, strutils, times, typetraits, options],
+  stew/[byteutils, results],
+  eth/common,
+  web3/engine_api_types,
+  ../sim_utils,
+  ../../../tools/common/helpers as chp,
+  ../../../tools/evmstate/helpers as ehp,
+  ../../../tests/test_helpers,
+  ../engine/engine_client,
+  ./test_env,
+  ./helpers
+
+const
+  baseFolder = "hive_integration/nodocker/pyspec"
+  caseFolder = baseFolder & "/testcases"
+  supportedNetwork = ["Merge", "Shanghai", "MergeToShanghaiAtTime15k"]
+
+type
+  Hash256 = common.Hash256
+
+proc getPayload(node: JsonNode): ExecutionPayloadV1OrV2 =
+  let rlpBytes = hexToSeqByte(node.getStr)
+  toPayloadV1OrV2(rlp.decode(rlpBytes, EthBlock))
+
+proc hash256(h: Web3BlockHash): Hash256 =
+  Hash256(data: distinctBase h)
+
+proc validatePostState(node: JsonNode, t: TestEnv): bool =
+  # check nonce, balance & storage of accounts in final block against fixture values
+  for account, genesisAccount in postState(node["postState"]):
+    # get nonce & balance from last block (end of test execution)
+    let nonceRes = t.rpcClient.nonceAt(account)
+    if nonceRes.isErr:
+      echo "unable to call nonce from account: " & account.toHex
+      echo nonceRes.error
+      return false
+
+    let balanceRes = t.rpcClient.balanceAt(account)
+    if balanceRes.isErr:
+      echo "unable to call balance from account: " & account.toHex
+      echo balanceRes.error
+      return false
+
+    # check final nonce & balance matches expected in fixture
+    if genesisAccount.nonce != nonceRes.value:
+      echo "nonce recieved from account 0x",
+        account.toHex,
+        " doesn't match expected ",
+        genesisAccount.nonce,
+        " got ",
+        nonceRes.value
+      return false
+
+    if genesisAccount.balance != balanceRes.value:
+      echo "balance recieved from account 0x",
+        account.toHex,
+        " doesn't match expected ",
+        genesisAccount.balance,
+        " got ",
+        balanceRes.value
+      return false
+
+    # check final storage
+    if genesisAccount.storage.len > 0:
+      for slot, val in genesisAccount.storage:
+        let sRes = t.rpcClient.storageAt(account, slot)
+        if sRes.isErr:
+          echo "unable to call storage from account: 0x",
+            account.toHex,
+            " at slot 0x",
+            slot.toHex
+          echo sRes.error
+          return false
+
+        if val != sRes.value:
+          echo "storage recieved from account 0x",
+            account.toHex,
+            " at slot 0x",
+            slot.toHex,
+            " doesn't match expected 0x",
+            val.toHex,
+            " got 0x",
+            sRes.value.toHex
+          return false
+
+  return true
+
+proc runTest(node: JsonNode, network: string): TestStatus =
+  let conf = getChainConfig(network)
+  var t = TestEnv(conf: makeTestConfig())
+  t.setupELClient(conf, node)
+
+  let blks = node["blocks"]
+  var latestValidHash = Hash256()
+  result = TestStatus.OK
+  for blkNode in blks:
+    let expectedStatus = if "expectException" in blkNode:
+                           PayloadExecutionStatus.invalid
+                         else:
+                           PayloadExecutionStatus.valid
+    let payload = getPayload(blkNode["rlp"])
+    let res = t.rpcClient.newPayloadV2(payload)
+    if res.isErr:
+      result = TestStatus.Failed
+      echo "unable to send block ", payload.blockNumber.uint64, ": ", res.error
+      break
+
+    let pStatus = res.value
+    if pStatus.status == PayloadExecutionStatus.valid:
+      latestValidHash = hash256(pStatus.latestValidHash.get)
+
+    if pStatus.status != expectedStatus:
+      result = TestStatus.Failed
+      echo "payload status mismatch for block ", payload.blockNumber.uint64, ", status: ", pStatus.status
+      if pStatus.validationError.isSome:
+        echo pStatus.validationError.get
+      break
+
+  block:
+    # only update head of beacon chain if valid response occurred
+    if latestValidHash != Hash256():
+      # update with latest valid response
+      let fcState = ForkchoiceStateV1(headBlockHash: BlockHash latestValidHash.data)
+      let res = t.rpcClient.forkchoiceUpdatedV2(fcState)
+      if res.isErr:
+        result = TestStatus.Failed
+        echo "unable to update head of beacon chain: ", res.error
+        break
+
+    if not validatePostState(node, t):
+      result = TestStatus.Failed
+      break
+
+  t.stopELClient()
+
+proc main() =
+  var stat: SimStat
+  let start = getTime()
+
+  for fileName in walkDirRec(caseFolder):
+    if not fileName.endsWith(".json"):
+      continue
+
+    let fixtureTests = json.parseFile(fileName)
+    for name, fixture in fixtureTests:
+      let network = fixture["network"].getStr
+      if network notin supportedNetwork:
+        # skip pre Merge tests
+        continue
+
+      let status = runTest(fixture, network)
+      stat.inc(name, status)
+
+  let elpd = getTime() - start
+  print(stat, elpd, "pyspec")
+
+main()

hive_integration/nodocker/pyspec/test_env.nim

@@ -0,0 +1,84 @@
+import
+  std/[json],
+  eth/p2p as eth_p2p,
+  eth/trie/trie_defs,
+  stew/[byteutils],
+  json_rpc/[rpcserver, rpcclient],
+  ../../../nimbus/[
+    config,
+    constants,
+    transaction,
+    db/accounts_cache,
+    core/sealer,
+    core/chain,
+    core/tx_pool,
+    rpc,
+    sync/protocol,
+    rpc/merge/merger,
+    common
+  ],
+  ../../../tests/test_helpers,
+  ../../../tools/evmstate/helpers
+
+type
+  TestEnv* = ref object
+    conf*: NimbusConf
+    ctx: EthContext
+    ethNode: EthereumNode
+    com: CommonRef
+    chainRef: ChainRef
+    rpcServer: RpcHttpServer
+    sealingEngine: SealingEngineRef
+    rpcClient*: RpcHttpClient
+
+const
+  engineSigner = hexToByteArray[20]("0x658bdf435d810c91414ec09147daa6db62406379")
+
+proc genesisHeader(node: JsonNode): BlockHeader =
+  let genesisRLP = hexToSeqByte(node["genesisRLP"].getStr)
+  rlp.decode(genesisRLP, EthBlock).header
+
+proc setupELClient*(t: TestEnv, conf: ChainConfig, node: JsonNode) =
+  let memDB = newMemoryDb()
+  t.ctx  = newEthContext()
+  t.ethNode = setupEthNode(t.conf, t.ctx, eth)
+  t.com = CommonRef.new(
+      memDB,
+      conf,
+      t.conf.pruneMode == PruneMode.Full
+    )
+  t.chainRef = newChain(t.com)
+  let
+    stateDB = AccountsCache.init(memDB, emptyRlpHash, t.conf.pruneMode == PruneMode.Full)
+    genesisHeader = node.genesisHeader
+
+  setupStateDB(node["pre"], stateDB)
+  stateDB.persist()
+
+  doAssert stateDB.rootHash == genesisHeader.stateRoot
+
+  discard t.com.db.persistHeaderToDb(genesisHeader,
+    t.com.consensus == ConsensusType.POS)
+  doAssert(t.com.db.getCanonicalHead().blockHash == genesisHeader.blockHash)
+
+  let txPool  = TxPoolRef.new(t.com, engineSigner)
+  t.rpcServer = newRpcHttpServer(["localhost:8545"])
+  t.sealingEngine = SealingEngineRef.new(
+    t.chainRef, t.ctx, engineSigner,
+    txPool, EngineStopped
+  )
+
+  let merger = MergerRef.new(t.com.db)
+  setupEthRpc(t.ethNode, t.ctx, t.com, txPool, t.rpcServer)
+  setupEngineAPI(t.sealingEngine, t.rpcServer, merger)
+  #setupDebugRpc(t.com, t.rpcServer)
+
+  t.rpcServer.start()
+
+  t.rpcClient = newRpcHttpClient()
+  waitFor t.rpcClient.connect("localhost", 8545.Port, false)
+
+proc stopELClient*(t: TestEnv) =
+  waitFor t.rpcClient.close()
+  waitFor t.sealingEngine.stop()
+  waitFor t.rpcServer.closeWait()