Feature/etcm 533 non membership proof

src/it/scala/io/iohk/ethereum/txExecTest/util/DumpChainApp.scala

@@ -2,7 +2,6 @@ package io.iohk.ethereum.txExecTest.util
 
 import java.time.Clock
 import java.util.concurrent.atomic.AtomicReference
-
 import akka.actor.ActorSystem
 import akka.util.ByteString
 import com.typesafe.config.ConfigFactory
@@ -15,6 +14,7 @@ import io.iohk.ethereum.db.storage.pruning.{ArchivePruning, PruningMode}
 import io.iohk.ethereum.db.storage.{AppStateStorage, StateStorage}
 import io.iohk.ethereum.domain.BlockHeader.HeaderExtraFields.HefEmpty
 import io.iohk.ethereum.domain.{Blockchain, UInt256, _}
+import io.iohk.ethereum.jsonrpc.ProofService.{EmptyStorageValueProof, StorageProof, StorageProofKey, StorageValueProof}
 import io.iohk.ethereum.ledger.{InMemoryWorldStateProxy, InMemoryWorldStateProxyStorage}
 import io.iohk.ethereum.mpt.MptNode
 import io.iohk.ethereum.network.EtcPeerManagerActor.PeerInfo
@@ -150,7 +150,7 @@ class BlockchainMock(genesisHash: ByteString) extends Blockchain {
       rootHash: NodeHash,
       position: BigInt,
       ethCompatibleStorage: Boolean
-  ): Option[(BigInt, Seq[MptNode])] = None
+  ): StorageProof = EmptyStorageValueProof(StorageProofKey(position))
 
   override protected def getHashByBlockNumber(number: BigInt): Option[ByteString] = Some(genesisHash)
 

src/main/scala/io/iohk/ethereum/domain/Blockchain.scala

@@ -1,7 +1,6 @@
 package io.iohk.ethereum.domain
 
 import java.util.concurrent.atomic.AtomicReference
-
 import akka.util.ByteString
 import cats.syntax.flatMap._
 import cats.instances.option._
@@ -13,6 +12,7 @@ import io.iohk.ethereum.db.storage._
 import io.iohk.ethereum.db.storage.pruning.PruningMode
 import io.iohk.ethereum.domain
 import io.iohk.ethereum.domain.BlockchainImpl.BestBlockLatestCheckpointNumbers
+import io.iohk.ethereum.jsonrpc.ProofService.StorageProof
 import io.iohk.ethereum.ledger.{InMemoryWorldStateProxy, InMemoryWorldStateProxyStorage}
 import io.iohk.ethereum.mpt.{MerklePatriciaTrie, MptNode}
 import io.iohk.ethereum.utils.{ByteStringUtils, Logger}
@@ -95,11 +95,17 @@ trait Blockchain {
     */
   def getAccountStorageAt(rootHash: ByteString, position: BigInt, ethCompatibleStorage: Boolean): ByteString
 
+  /**
+    * Get a storage-value and its proof being the path from the root node until the last matching node.
+    *
+    * @param rootHash storage root hash
+    * @param position storage position
+    */
   def getStorageProofAt(
       rootHash: ByteString,
       position: BigInt,
       ethCompatibleStorage: Boolean
-  ): Option[(BigInt, Seq[MptNode])]
+  ): StorageProof
 
   /**
     * Returns the receipts based on a block hash
@@ -307,16 +313,15 @@ class BlockchainImpl(
       rootHash: ByteString,
       position: BigInt,
       ethCompatibleStorage: Boolean
-  ): Option[(BigInt, Seq[MptNode])] = {
+  ): StorageProof = {
     val storage: MptStorage = stateStorage.getBackingStorage(0)
     val mpt: MerklePatriciaTrie[BigInt, BigInt] = {
       if (ethCompatibleStorage) domain.EthereumUInt256Mpt.storageMpt(rootHash, storage)
       else domain.ArbitraryIntegerMpt.storageMpt(rootHash, storage)
     }
-    for {
-      value <- mpt.get(position)
-      proof <- mpt.getProof(position)
-    } yield (value, proof)
+    val value: Option[BigInt] = mpt.get(position)
+    val proof: Option[Vector[MptNode]] = mpt.getProof(position)
+    StorageProof(position, value, proof)
   }
 
   private def persistBestBlocksData(): Unit = {

src/main/scala/io/iohk/ethereum/jsonrpc/EthProofJsonMethodsImplicits.scala

@@ -4,9 +4,7 @@ import io.iohk.ethereum.jsonrpc.JsonRpcError.InvalidParams
 import io.iohk.ethereum.jsonrpc.ProofService.{GetProofRequest, GetProofResponse, StorageProofKey}
 import io.iohk.ethereum.jsonrpc.serialization.JsonEncoder
 import io.iohk.ethereum.jsonrpc.serialization.JsonMethodDecoder
-import org.json4s.Extraction
 import org.json4s.JsonAST.{JArray, JString, JValue, _}
-import org.json4s.JsonDSL._
 
 object EthProofJsonMethodsImplicits extends JsonMethodsImplicits {
   def extractStorageKeys(input: JValue): Either[JsonRpcError, Seq[StorageProofKey]] = {

src/main/scala/io/iohk/ethereum/jsonrpc/EthProofService.scala

@@ -4,12 +4,14 @@ import akka.util.ByteString
 import cats.implicits._
 import io.iohk.ethereum.consensus.blocks.BlockGenerator
 import io.iohk.ethereum.domain.{Account, Address, Block, Blockchain, UInt256}
+import io.iohk.ethereum.jsonrpc.ProofService.StorageProof.asRlpSerializedNode
 import io.iohk.ethereum.jsonrpc.ProofService.{
   GetProofRequest,
   GetProofResponse,
   ProofAccount,
   StorageProof,
-  StorageProofKey
+  StorageProofKey,
+  StorageValueProof
 }
 import io.iohk.ethereum.mpt.{MptNode, MptTraversals}
 import monix.eval.Task
@@ -30,8 +32,26 @@ object ProofService {
 
   case class GetProofResponse(proofAccount: ProofAccount)
 
-  /** The key used to get the storage slot in its account tree */
-  case class StorageProofKey(v: BigInt) extends AnyVal
+  sealed trait StorageProof {
+    def key: StorageProofKey
+    def value: BigInt
+    def proof: Seq[ByteString]
+  }
+
+  object StorageProof {
+    def apply(position: BigInt, value: Option[BigInt], proof: Option[Vector[MptNode]]): StorageProof =
+      (value, proof) match {
+        case (Some(value), Some(proof)) =>
+          StorageValueProof(StorageProofKey(position), value, proof.map(asRlpSerializedNode))
+        case (None, Some(proof)) =>
+          EmptyStorageValue(StorageProofKey(position), proof.map(asRlpSerializedNode))
+        case (Some(value), None) => EmptyStorageProof(StorageProofKey(position), value)
+        case (None, None) => EmptyStorageValueProof(StorageProofKey(position))
+      }
+
+    def asRlpSerializedNode(node: MptNode): ByteString =
+      ByteString(MptTraversals.encodeNode(node))
+  }
 
   /**
     * Object proving a relationship of a storage value to an account's storageHash
@@ -40,11 +60,20 @@ object ProofService {
     * @param value the value of the storage slot in its account tree
     * @param proof the set of node values needed to traverse a patricia merkle tree (from root to leaf) to retrieve a value
     */
-  case class StorageProof(
-      key: StorageProofKey,
-      value: BigInt,
-      proof: Seq[ByteString]
-  )
+  case class EmptyStorageValueProof(key: StorageProofKey) extends StorageProof {
+    val value: BigInt = BigInt(0)
+    val proof: Seq[ByteString] = Seq.empty[MptNode].map(asRlpSerializedNode)
+  }
+  case class EmptyStorageValue(key: StorageProofKey, proof: Seq[ByteString]) extends StorageProof {
+    val value: BigInt = BigInt(0)
+  }
+  case class EmptyStorageProof(key: StorageProofKey, value: BigInt) extends StorageProof {
+    val proof: Seq[ByteString] = Seq.empty[MptNode].map(asRlpSerializedNode)
+  }
+  case class StorageValueProof(key: StorageProofKey, value: BigInt, proof: Seq[ByteString]) extends StorageProof
+
+  /** The key used to get the storage slot in its account tree */
+  case class StorageProofKey(v: BigInt) extends AnyVal
 
   /**
     * The merkle proofs of the specified account connecting them to the blockhash of the block specified.
@@ -143,14 +172,14 @@ class EthProofService(blockchain: Blockchain, blockGenerator: BlockGenerator, et
         blockchain.getAccountProof(address, blockNumber).map(_.map(asRlpSerializedNode)),
         noAccountProof(address, blockNumber)
       )
-      storageProof <- getStorageProof(account, storageKeys)
+      storageProof = getStorageProof(account, storageKeys)
     } yield ProofAccount(account, accountProof, storageProof, address)
   }
 
   def getStorageProof(
       account: Account,
       storageKeys: Seq[StorageProofKey]
-  ): Either[JsonRpcError, Seq[StorageProof]] = {
+  ): Seq[StorageProof] = {
     storageKeys.toList
       .map { storageKey =>
         blockchain
@@ -159,21 +188,14 @@ class EthProofService(blockchain: Blockchain, blockGenerator: BlockGenerator, et
             position = storageKey.v,
             ethCompatibleStorage = ethCompatibleStorage
           )
-          .map { case (value, proof) => StorageProof(storageKey, value, proof.map(asRlpSerializedNode)) }
-          .toRight(noStorageProof(account, storageKey))
       }
-      .sequence
-      .map(_.toSeq)
   }
 
-  private def noStorageProof(account: Account, storagekey: StorageProofKey): JsonRpcError =
-    JsonRpcError.LogicError(s"No storage proof for [${account.toString}] storage key [${storagekey.toString}]")
-
   private def noAccount(address: Address, blockNumber: BigInt): JsonRpcError =
-    JsonRpcError.LogicError(s"No storage proof for Address [${address.toString}] blockNumber [${blockNumber.toString}]")
+    JsonRpcError.LogicError(s"No account found for Address [${address.toString}] blockNumber [${blockNumber.toString}]")
 
   private def noAccountProof(address: Address, blockNumber: BigInt): JsonRpcError =
-    JsonRpcError.LogicError(s"No storage proof for Address [${address.toString}] blockNumber [${blockNumber.toString}]")
+    JsonRpcError.LogicError(s"No account proof for Address [${address.toString}] blockNumber [${blockNumber.toString}]")
 
   private def asRlpSerializedNode(node: MptNode): ByteString =
     ByteString(MptTraversals.encodeNode(node))

src/main/scala/io/iohk/ethereum/mpt/MerklePatriciaTrie.scala

@@ -9,7 +9,6 @@ import io.iohk.ethereum.rlp.RLPImplicits._
 import io.iohk.ethereum.rlp.{encode => encodeRLP}
 import org.bouncycastle.util.encoders.Hex
 import io.iohk.ethereum.utils.ByteUtils.matchingLength
-
 import scala.annotation.tailrec
 
 object MerklePatriciaTrie {
@@ -82,16 +81,14 @@ class MerklePatriciaTrie[K, V] private (private[mpt] val rootNode: Option[MptNod
     * @throws io.iohk.ethereum.mpt.MerklePatriciaTrie.MPTException if there is any inconsistency in how the trie is build.
     */
   def get(key: K): Option[V] = {
-    pathTraverse[Option[V]](None, mkKeyNibbles(key)) { case (_, node) =>
-      node match {
-        case LeafNode(_, value, _, _, _) =>
-          Some(vSerializer.fromBytes(value.toArray[Byte]))
+    pathTraverse[Option[V]](None, mkKeyNibbles(key)) {
+      case (_, Some(LeafNode(_, value, _, _, _))) =>
+        Some(vSerializer.fromBytes(value.toArray[Byte]))
 
-        case BranchNode(_, terminator, _, _, _) =>
-          terminator.map(term => vSerializer.fromBytes(term.toArray[Byte]))
+      case (_, Some(BranchNode(_, terminator, _, _, _))) =>
+        terminator.map(term => vSerializer.fromBytes(term.toArray[Byte]))
 
-        case _ => None
-      }
+      case _ => None
     }.flatten
   }
 
@@ -105,7 +102,8 @@ class MerklePatriciaTrie[K, V] private (private[mpt] val rootNode: Option[MptNod
   def getProof(key: K): Option[Vector[MptNode]] = {
     pathTraverse[Vector[MptNode]](Vector.empty, mkKeyNibbles(key)) { case (acc, node) =>
       node match {
-        case nextNodeOnExt @ (_: BranchNode | _: ExtensionNode | _: LeafNode) => acc :+ nextNodeOnExt
+        case Some(nextNodeOnExt @ (_: BranchNode | _: ExtensionNode | _: LeafNode | _: HashNode)) =>
+          acc :+ nextNodeOnExt
         case _ => acc
       }
     }
@@ -121,25 +119,25 @@ class MerklePatriciaTrie[K, V] private (private[mpt] val rootNode: Option[MptNod
     * @tparam T accumulator type
     * @return accumulated data or None if key doesn't exist
     */
-  private def pathTraverse[T](acc: T, searchKey: Array[Byte])(op: (T, MptNode) => T): Option[T] = {
+  private def pathTraverse[T](acc: T, searchKey: Array[Byte])(op: (T, Option[MptNode]) => T): Option[T] = {
 
     @tailrec
-    def pathTraverse(acc: T, node: MptNode, searchKey: Array[Byte], op: (T, MptNode) => T): Option[T] = {
+    def pathTraverse(acc: T, node: MptNode, searchKey: Array[Byte], op: (T, Option[MptNode]) => T): Option[T] = {
       node match {
         case LeafNode(key, _, _, _, _) =>
-          if (key.toArray[Byte] sameElements searchKey) Some(op(acc, node)) else None
+          if (key.toArray[Byte] sameElements searchKey) Some(op(acc, Some(node))) else Some(op(acc, None))
 
         case extNode @ ExtensionNode(sharedKey, _, _, _, _) =>
           val (commonKey, remainingKey) = searchKey.splitAt(sharedKey.length)
           if (searchKey.length >= sharedKey.length && (sharedKey.toArray[Byte] sameElements commonKey)) {
-            pathTraverse(op(acc, node), extNode.next, remainingKey, op)
-          } else None
+            pathTraverse(op(acc, Some(node)), extNode.next, remainingKey, op)
+          } else Some(op(acc, None))
 
         case branch: BranchNode =>
-          if (searchKey.isEmpty) Some(op(acc, node))
+          if (searchKey.isEmpty) Some(op(acc, Some(node)))
           else
             pathTraverse(
-              op(acc, node),
+              op(acc, Some(node)),
               branch.children(searchKey(0)),
               searchKey.slice(1, searchKey.length),
               op
@@ -149,13 +147,13 @@ class MerklePatriciaTrie[K, V] private (private[mpt] val rootNode: Option[MptNod
           pathTraverse(acc, getFromHash(bytes, nodeStorage), searchKey, op)
 
         case NullNode =>
-          None
+          Some(op(acc, None))
       }
     }
 
     rootNode match {
       case Some(root) =>
-        pathTraverse(acc, root, searchKey, op)
+        pathTraverse(op(acc, Some(root)), root, searchKey, op)
       case None =>
         None
     }

src/test/scala/io/iohk/ethereum/ObjectGenerators.scala

@@ -50,9 +50,10 @@ trait ObjectGenerators {
     } yield (aByteList.toArray, t)
   }
 
-  def keyValueListGen(): Gen[List[(Int, Int)]] = {
+  def keyValueListGen(minValue: Int = Int.MinValue, maxValue: Int = Int.MaxValue): Gen[List[(Int, Int)]] = {
     for {
-      aKeyList <- Gen.nonEmptyListOf(Arbitrary.arbitrary[Int]).map(_.distinct)
+      values <- Gen.chooseNum(minValue, maxValue)
+      aKeyList <- Gen.nonEmptyListOf(values).map(_.distinct)
     } yield aKeyList.zip(aKeyList)
   }
 

src/test/scala/io/iohk/ethereum/domain/BlockchainSpec.scala

@@ -6,7 +6,7 @@ import io.iohk.ethereum.consensus.blocks.CheckpointBlockGenerator
 import io.iohk.ethereum.db.dataSource.EphemDataSource
 import io.iohk.ethereum.db.storage.StateStorage
 import io.iohk.ethereum.domain.BlockHeader.HeaderExtraFields.HefPostEcip1097
-import io.iohk.ethereum.mpt.MerklePatriciaTrie
+import io.iohk.ethereum.mpt.{HashNode, MerklePatriciaTrie}
 import io.iohk.ethereum.{BlockHelpers, Fixtures, ObjectGenerators}
 import io.iohk.ethereum.ObjectGenerators._
 import io.iohk.ethereum.proof.MptProofVerifier
@@ -152,7 +152,10 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh
     //unhappy path
     val wrongAddress = Address(666)
     val retrievedAccountProofWrong = blockchain.getAccountProof(wrongAddress, headerWithAcc.number)
-    retrievedAccountProofWrong.isDefined shouldBe false
+    //the account doesn't exist, so we can't retrieve it, but we do receive a proof of non-existence with a full path of nodes that we iterated
+    retrievedAccountProofWrong.isDefined shouldBe true
+    retrievedAccountProofWrong.size shouldBe 1
+    mptWithAcc.get(wrongAddress) shouldBe None
 
     //happy path
     val retrievedAccountProof = blockchain.getAccountProof(address, headerWithAcc.number)
@@ -162,6 +165,26 @@ class BlockchainSpec extends AnyFlatSpec with Matchers with ScalaCheckPropertyCh
     }
   }
 
+  it should "return proof for non-existent account" in new EphemBlockchainTestSetup {
+    val emptyMpt = MerklePatriciaTrie[Address, Account](
+      storagesInstance.storages.stateStorage.getBackingStorage(0)
+    )
+    val mptWithAcc = emptyMpt.put(Address(42), Account.empty(UInt256(7)))
+
+    val headerWithAcc = Fixtures.Blocks.ValidBlock.header.copy(stateRoot = ByteString(mptWithAcc.getRootHash))
+
+    blockchain.storeBlockHeader(headerWithAcc).commit()
+
+    val wrongAddress = Address(666)
+    val retrievedAccountProofWrong = blockchain.getAccountProof(wrongAddress, headerWithAcc.number)
+    //the account doesn't exist, so we can't retrieve it, but we do receive a proof of non-existence with a full path of nodes(root node) that we iterated
+    (retrievedAccountProofWrong.getOrElse(Vector.empty).toList match {
+      case _ @HashNode(_) :: Nil => true
+      case _ => false
+    }) shouldBe true
+    mptWithAcc.get(wrongAddress) shouldBe None
+  }
+
   it should "return correct best block number after applying and rollbacking blocks" in new TestSetup {
     forAll(intGen(min = 1: Int, max = maxNumberBlocksToImport)) { numberBlocksToImport =>
       val testSetup = newSetup()