Updates to Fluffy to support hive tests. (#2333)

* Updates to Fluffy book for hive tests. * Add support to disable state root checks for state content from the command line. * Update portal_stateStore endpoint to support decoding offer and storing retrieval value.
status-im · Jun 11, 2024 · 9c26fa3 · 9c26fa3
1 parent eb041ab
commit 9c26fa3
Show file tree

Hide file tree

Showing 10 changed files with 236 additions and 96 deletions.
diff --git a/fluffy/conf.nim b/fluffy/conf.nim
@@ -299,6 +299,13 @@ type
       name: "disable-poke"
     .}: bool
 
+    disableStateRootValidation* {.
+      hidden,
+      desc: "Disables state root validation for content received by the state network.",
+      defaultValue: false,
+      name: "disable-state-root-validation"
+    .}: bool
+
     case cmd* {.command, defaultValue: noCommand.}: PortalCmd
     of noCommand:
       discard

diff --git a/fluffy/docs/the_fluffy_book/docs/fluffy-with-portal-hive.md b/fluffy/docs/the_fluffy_book/docs/fluffy-with-portal-hive.md
@@ -1,27 +1,27 @@
 # Fluffy with Portal-hive
 
-Fluffy is one of the Portal clients that is being tested with [portal-hive](https://github.com/ethereum/portal-hive).
+Fluffy is one of the Portal clients that is being tested with [hive](https://github.com/ethereum/hive).
 
 To see the status of the tests for the current version you can access [https://portal-hive.ethdevops.io/](https://portal-hive.ethdevops.io/).
 
 ## Run the hive tests locally
 
-Build portal-hive:
+Build hive:
 
 ```sh
-git clone https://github.com/ethereum/portal-hive.git
-cd ./portal-hive
+git clone https://github.com/ethereum/hive.git
+cd ./hive
 go build .
 ```
 
 Example commands for running test suites:
 
 ```sh
-# Run the rpc-compat tests with the 3 different clients
-./hive --sim rpc-compat --client fluffy,trin,ultralight
+# Run the history tests with the 3 different clients
+./hive --sim history --client fluffy,trin,ultralight
 
-# Run the portal-interop tests with only the fluffy client
-./hive --sim portal-interop --client fluffy
+# Run the state tests with only the fluffy client
+./hive --sim state --client fluffy
 
 # Access results through web-ui:
 ```sh
@@ -30,7 +30,7 @@ go build ./cmd/hiveview
 ```
 
 !!! note
-    You can see all the implemented simulators in [https://github.com/ethereum/portal-hive/tree/main/simulators](https://github.com/ethereum/portal-hive/tree/main/simulators)
+    You can see all the implemented simulators in [https://github.com/ethereum/hive/tree/master/simulators](https://github.com/ethereum/hive/tree/master/simulators)
 
 ## Build a local development Docker image for portal-hive
 
@@ -55,4 +55,3 @@ docker build --tag fluffy-dev --file ./fluffy/tools/docker/Dockerfile.portalhive
     The `./vendors` dir is dockerignored and cached. If you have to make local
     changes to one of the dependencies in that directory you will have to remove
     `vendors/` from `./fluffy/tools/docker/Dockerfile.portalhive.dockerignore`.
-
diff --git a/fluffy/fluffy.nim b/fluffy/fluffy.nim
@@ -234,6 +234,7 @@ proc run(config: PortalConf) {.raises: [CatchableError].} =
             bootstrapRecords = bootstrapRecords,
             portalConfig = portalConfig,
             historyNetwork = historyNetwork,
+            not config.disableStateRootValidation,
           )
         )
       else:

diff --git a/fluffy/network/state/state_network.nim b/fluffy/network/state/state_network.nim
@@ -32,6 +32,7 @@ type StateNetwork* = ref object
   contentQueue*: AsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])]
   processContentLoop: Future[void]
   historyNetwork: Opt[HistoryNetwork]
+  validateStateIsCanonical: bool
 
 func toContentIdHandler(contentKey: ByteList): results.Opt[ContentId] =
   ok(toContentId(contentKey))
@@ -44,6 +45,7 @@ proc new*(
     bootstrapRecords: openArray[Record] = [],
     portalConfig: PortalProtocolConfig = defaultPortalProtocolConfig,
     historyNetwork = Opt.none(HistoryNetwork),
+    validateStateIsCanonical = true,
 ): T =
   let cq = newAsyncQueue[(Opt[NodeId], ContentKeysList, seq[seq[byte]])](50)
 
@@ -67,6 +69,7 @@ proc new*(
     contentDB: contentDB,
     contentQueue: cq,
     historyNetwork: historyNetwork,
+    validateStateIsCanonical: validateStateIsCanonical,
   )
 
 proc getContent(
@@ -146,10 +149,15 @@ proc processOffer*(
   let contentValue = V.decode(contentValueBytes).valueOr:
     return err("Unable to decode offered content value")
 
-  let stateRoot = (await n.getStateRootByBlockHash(contentValue.blockHash)).valueOr:
-    return err("Failed to get state root by block hash")
+  let res =
+    if n.validateStateIsCanonical:
+      let stateRoot = (await n.getStateRootByBlockHash(contentValue.blockHash)).valueOr:
+        return err("Failed to get state root by block hash")
+      validateOffer(Opt.some(stateRoot), contentKey, contentValue)
+    else:
+      # Skip state root validation
+      validateOffer(Opt.none(KeccakHash), contentKey, contentValue)
 
-  let res = validateOffer(stateRoot, contentKey, contentValue)
   if res.isErr():
     return err("Offered content failed validation: " & res.error())
 

diff --git a/fluffy/network/state/state_validation.nim b/fluffy/network/state/state_validation.nim
@@ -30,16 +30,18 @@ proc isValidNextNode(thisNodeRlp: Rlp, rlpIdx: int, nextNode: TrieNode): bool =
   nextNode.hashEquals(nextHash)
 
 proc validateTrieProof*(
-    expectedRootHash: KeccakHash,
+    expectedRootHash: Opt[KeccakHash],
     path: Nibbles,
     proof: TrieProof,
     allowKeyEndInPathForLeafs = false,
 ): Result[void, string] =
   if proof.len() == 0:
     return err("proof is empty")
 
-  if not proof[0].hashEquals(expectedRootHash):
-    return err("hash of proof root node doesn't match the expected root hash")
+  # TODO: Remove this once the hive tests support passing in state roots from the history network
+  if expectedRootHash.isSome():
+    if not proof[0].hashEquals(expectedRootHash.get()):
+      return err("hash of proof root node doesn't match the expected root hash")
 
   let nibbles = path.unpackNibbles()
   if nibbles.len() == 0:
@@ -131,14 +133,18 @@ proc validateRetrieval*(
     err("hash of bytecode doesn't match the expected code hash")
 
 proc validateOffer*(
-    trustedStateRoot: KeccakHash, key: AccountTrieNodeKey, offer: AccountTrieNodeOffer
+    trustedStateRoot: Opt[KeccakHash],
+    key: AccountTrieNodeKey,
+    offer: AccountTrieNodeOffer,
 ): Result[void, string] =
   ?validateTrieProof(trustedStateRoot, key.path, offer.proof)
 
   validateRetrieval(key, offer.toRetrievalValue())
 
 proc validateOffer*(
-    trustedStateRoot: KeccakHash, key: ContractTrieNodeKey, offer: ContractTrieNodeOffer
+    trustedStateRoot: Opt[KeccakHash],
+    key: ContractTrieNodeKey,
+    offer: ContractTrieNodeOffer,
 ): Result[void, string] =
   ?validateTrieProof(
     trustedStateRoot,
@@ -149,12 +155,12 @@ proc validateOffer*(
 
   let account = ?offer.accountProof.toAccount()
 
-  ?validateTrieProof(account.storageRoot, key.path, offer.storageProof)
+  ?validateTrieProof(Opt.some(account.storageRoot), key.path, offer.storageProof)
 
   validateRetrieval(key, offer.toRetrievalValue())
 
 proc validateOffer*(
-    trustedStateRoot: KeccakHash, key: ContractCodeKey, offer: ContractCodeOffer
+    trustedStateRoot: Opt[KeccakHash], key: ContractCodeKey, offer: ContractCodeOffer
 ): Result[void, string] =
   ?validateTrieProof(
     trustedStateRoot,

diff --git a/fluffy/rpc/rpc_portal_api.nim b/fluffy/rpc/rpc_portal_api.nim
@@ -13,6 +13,7 @@ import
   json_serialization/std/tables,
   stew/byteutils,
   ../network/wire/portal_protocol,
+  ../network/state/state_content,
   ./rpc_types
 
 {.warning[UnusedImport]: off.}
@@ -42,6 +43,12 @@ TraceResponse.useDefaultSerializationIn JrpcConv
 proc installPortalApiHandlers*(
     rpcServer: RpcServer | RpcProxy, p: PortalProtocol, network: static string
 ) =
+  let
+    invalidKeyErr =
+      (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+    invalidValueErr =
+      (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content value")
+
   rpcServer.rpc("portal_" & network & "NodeInfo") do() -> NodeInfo:
     return p.routingTable.getNodeInfo()
 
@@ -212,14 +219,39 @@ proc installPortalApiHandlers*(
   rpcServer.rpc("portal_" & network & "Store") do(
     contentKey: string, contentValue: string
   ) -> bool:
-    let key = ByteList.init(hexToSeqByte(contentKey))
-    let contentId = p.toContentId(key)
+    let
+      key = ByteList.init(hexToSeqByte(contentKey))
+      contentValueBytes = hexToSeqByte(contentValue)
+
+    let valueToStore =
+      if network == "state":
+        let decodedKey = ContentKey.decode(key).valueOr:
+          raise invalidKeyErr
+
+        case decodedKey.contentType
+        of unused:
+          raise invalidKeyErr
+        of accountTrieNode:
+          let offerValue = AccountTrieNodeOffer.decode(contentValueBytes).valueOr:
+            raise invalidValueErr
+          offerValue.toRetrievalValue.encode()
+        of contractTrieNode:
+          let offerValue = ContractTrieNodeOffer.decode(contentValueBytes).valueOr:
+            raise invalidValueErr
+          offerValue.toRetrievalValue.encode()
+        of contractCode:
+          let offerValue = ContractCodeOffer.decode(contentValueBytes).valueOr:
+            raise invalidValueErr
+          offerValue.toRetrievalValue.encode()
+      else:
+        contentValueBytes
 
+    let contentId = p.toContentId(key)
     if contentId.isSome():
-      p.storeContent(key, contentId.get(), hexToSeqByte(contentValue))
+      p.storeContent(key, contentId.get(), valueToStore)
       return true
     else:
-      raise (ref errors.InvalidRequest)(code: -32602, msg: "Invalid content key")
+      raise invalidKeyErr
 
   rpcServer.rpc("portal_" & network & "LocalContent") do(contentKey: string) -> string:
     let

diff --git a/fluffy/tests/state_network_tests/test_state_gossip_getparent_genesis.nim b/fluffy/tests/state_network_tests/test_state_gossip_getparent_genesis.nim
@@ -42,12 +42,16 @@ suite "State Gossip getParent - Genesis JSON Files":
 
         # validate each parent offer until getting to the root node
         var parent = offer.withKey(key).getParent()
-        check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+        check validateOffer(Opt.some(accountState.rootHash()), parent.key, parent.offer)
+        .isOk()
         db.put(parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq())
 
         for i in proof.low ..< proof.high - 1:
           parent = parent.getParent()
-          check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+          check validateOffer(
+            Opt.some(accountState.rootHash()), parent.key, parent.offer
+          )
+          .isOk()
           db.put(parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq())
 
         # after putting all parent nodes into the trie, verify can lookup the leaf
@@ -89,14 +93,19 @@ suite "State Gossip getParent - Genesis JSON Files":
 
             # validate each parent offer until getting to the root node
             var parent = offer.withKey(key).getParent()
-            check validateOffer(accountState.rootHash(), parent.key, parent.offer).isOk()
+            check validateOffer(
+              Opt.some(accountState.rootHash()), parent.key, parent.offer
+            )
+            .isOk()
             db.put(
               parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq()
             )
 
             for i in storageProof.low ..< storageProof.high - 1:
               parent = parent.getParent()
-              check validateOffer(accountState.rootHash(), parent.key, parent.offer)
+              check validateOffer(
+                Opt.some(accountState.rootHash()), parent.key, parent.offer
+              )
               .isOk()
               db.put(
                 parent.key.nodeHash.data, parent.offer.toRetrievalValue().node.asSeq()

diff --git a/fluffy/tests/state_network_tests/test_state_validation_genesis.nim b/fluffy/tests/state_network_tests/test_state_validation_genesis.nim
@@ -34,15 +34,17 @@ template checkValidProofsForExistingLeafs(
         path: accountPath, nodeHash: keccakHash(accountProof[^1].asSeq())
       )
       accountTrieOffer = AccountTrieNodeOffer(proof: accountProof)
-      proofResult =
-        validateOffer(accountState.rootHash(), accountTrieNodeKey, accountTrieOffer)
+      proofResult = validateOffer(
+        Opt.some(accountState.rootHash()), accountTrieNodeKey, accountTrieOffer
+      )
     check proofResult.isOk()
 
     let
       contractCodeKey = ContractCodeKey(address: address, codeHash: acc.codeHash)
       contractCode =
         ContractCodeOffer(code: Bytecode.init(account.code), accountProof: accountProof)
-      codeResult = validateOffer(accountState.rootHash(), contractCodeKey, contractCode)
+      codeResult =
+        validateOffer(Opt.some(accountState.rootHash()), contractCodeKey, contractCode)
     check codeResult.isOk()
 
     if account.code.len() > 0:
@@ -64,7 +66,7 @@ template checkValidProofsForExistingLeafs(
             storageProof: storageProof, accountProof: accountProof
           )
           proofResult = validateOffer(
-            accountState.rootHash(), contractTrieNodeKey, contractTrieOffer
+            Opt.some(accountState.rootHash()), contractTrieNodeKey, contractTrieOffer
           )
         check proofResult.isOk()
 
@@ -88,8 +90,9 @@ template checkInvalidProofsWithBadValue(
     accountProof[^1][^1] += 1 # bad account leaf value
     let
       accountTrieOffer = AccountTrieNodeOffer(proof: accountProof)
-      proofResult =
-        validateOffer(accountState.rootHash(), accountTrieNodeKey, accountTrieOffer)
+      proofResult = validateOffer(
+        Opt.some(accountState.rootHash()), accountTrieNodeKey, accountTrieOffer
+      )
     check proofResult.isErr()
 
     let
@@ -98,7 +101,8 @@ template checkInvalidProofsWithBadValue(
         code: Bytecode.init(@[1u8, 2, 3]), # bad code value
         accountProof: accountProof,
       )
-      codeResult = validateOffer(accountState.rootHash(), contractCodeKey, contractCode)
+      codeResult =
+        validateOffer(Opt.some(accountState.rootHash()), contractCodeKey, contractCode)
     check codeResult.isErr()
 
     if account.code.len() > 0:
@@ -122,7 +126,7 @@ template checkInvalidProofsWithBadValue(
             storageProof: storageProof, accountProof: accountProof
           )
           proofResult = validateOffer(
-            accountState.rootHash(), contractTrieNodeKey, contractTrieOffer
+            Opt.some(accountState.rootHash()), contractTrieNodeKey, contractTrieOffer
           )
         check proofResult.isErr()