Merkle Patricia Trie

build.sbt

@@ -1,13 +1,23 @@
-name := "etc-client"
 
-version := "0.1"
-
-scalaVersion := "2.11.8"
+val commonSettings = Seq(
+  name := "etc-client",
+  version := "0.1",
+  scalaVersion := "2.11.8"
+)
 
-libraryDependencies ++= Seq(
+val dep = Seq(
   "com.typesafe.akka" %% "akka-actor" % "2.4.16",
   "org.consensusresearch" %% "scrypto" % "1.2.0-RC3",
   "com.madgag.spongycastle" % "core" % "1.54.0.0",
-  "org.scalatest" %% "scalatest" % "3.0.1" % "test",
-  "org.scalacheck" %% "scalacheck" % "1.13.4" % "test"
+  "org.scalatest" %% "scalatest" % "3.0.1" % "it,test",
+  "org.scalacheck" %% "scalacheck" % "1.13.4" % "it,test",
+  "org.scorexfoundation" %% "iodb" % "0.1.1"
 )
+
+val Integration = config("it") extend Test
+
+val root = project.in(file("."))
+    .configs(Integration)
+    .settings(commonSettings: _*)
+    .settings(libraryDependencies ++= dep)
+    .settings(inConfig(Integration)(Defaults.testSettings) : _*)

src/it/scala/io/iohk/ethereum/mpt/IodbDataSourceIntegrationSuite.scala

@@ -0,0 +1,42 @@
+package io.iohk.ethereum.mpt
+
+import org.scalatest.FunSuite
+import org.scalatest.prop.PropertyChecks
+import io.iohk.ethereum.ObjectGenerators
+import io.iohk.iodb.LSMStore
+import java.io.File
+
+class IodbDataSourceIntegrationSuite extends FunSuite
+  with PropertyChecks
+  with ObjectGenerators {
+
+  val KeySize: Int = 32
+  val KeyNumberLimit: Int = 40
+  val DefaultRootHash: Array[Byte] = Array.emptyByteArray
+
+  def putMultiple(dataSource: DataSource, toInsert: Seq[(Array[Byte], Array[Byte])]): DataSource = {
+    toInsert.foldLeft(dataSource){ case (recDB, keyValuePair) =>
+      recDB.update(DefaultRootHash, Seq(), Seq(keyValuePair))
+    }
+  }
+
+  test("IodbDataSource insert"){
+    forAll(seqByteArrayOfNItemsGen(KeySize)) { unFilteredKeyList: Seq[Array[Byte]] =>
+      //create temporary dir
+      val dir = File.createTempFile("iodbInsert", "iodbInsert")
+      dir.delete()
+      dir.mkdir()
+
+      val keyList = unFilteredKeyList.filter(_.length == KeySize).take(KeyNumberLimit)
+      val db = putMultiple(
+        dataSource = new IodbDataSource(new LSMStore(dir = dir, keySize = KeySize)),
+        toInsert = keyList.zip(keyList)
+      )
+      keyList.foreach { key =>
+        val obtained = db.get(key)
+        assert(obtained.isDefined)
+        assert(obtained.get sameElements key)
+      }
+    }
+  }
+}

src/it/scala/io/iohk/ethereum/mpt/MerklePatriciaTreeIntegrationSuite.scala

@@ -0,0 +1,208 @@
+package io.iohk.ethereum.mpt
+
+import java.io.File
+import java.nio.ByteBuffer
+import java.security.MessageDigest
+
+import akka.util.ByteString
+import io.iohk.ethereum.ObjectGenerators
+import io.iohk.ethereum.crypto.sha3
+import io.iohk.ethereum.mpt.MerklePatriciaTrie.defaultByteArraySerializable
+import io.iohk.ethereum.rlp.{decode => decodeRLP, encode => encodeRLP}
+import io.iohk.iodb.LSMStore
+import org.scalacheck.{Arbitrary, Gen}
+import org.scalatest.FunSuite
+import org.scalatest.prop.PropertyChecks
+import org.spongycastle.util.encoders.Hex
+
+import scala.util.Random
+
+class MerklePatriciaTreeIntegrationSuite extends FunSuite
+  with PropertyChecks
+  with ObjectGenerators {
+  val hashFn = (input: Array[Byte]) => sha3(input)
+
+  val EmptyTrie = MerklePatriciaTrie[Array[Byte], Array[Byte]](EphemDataSource(), hashFn)
+
+  implicit val intByteArraySerializable = new ByteArraySerializable[Int] {
+    override def toBytes(input: Int): Array[Byte] = {
+      val b: ByteBuffer = ByteBuffer.allocate(4)
+      b.putInt(input)
+      b.array
+    }
+
+    override def fromBytes(bytes: Array[Byte]): Int = ByteBuffer.wrap(bytes).getInt()
+  }
+
+  def md5(bytes: Array[Byte]): Array[Byte] = {
+    MessageDigest.getInstance("MD5").digest(bytes)
+  }
+
+
+  test("IODB test - Insert of the first 5000 numbers hashed and then remove half of them"){
+    //create temporary dir
+    val dir = File.createTempFile("iodb", "iodb")
+    dir.delete()
+    dir.mkdir()
+
+    val dataSource = new IodbDataSource(new LSMStore(dir = dir, keySize = 32))
+    val emptyTrie = MerklePatriciaTrie[Array[Byte], Array[Byte]](dataSource, hashFn)
+
+    val keys = (0 to 100).map(intByteArraySerializable.toBytes)
+    val trie = Random.shuffle(keys).foldLeft(emptyTrie) { case (recTrie, key) => recTrie.put(md5(key), key) }
+
+    // We delete have of the (key-value) pairs we had inserted
+    val trieAfterDelete = Random.shuffle(keys.take(100/2)).foldLeft(trie) { case (recTrie, key) => recTrie.remove(md5(key)) }
+
+    // We delete keys with no effect so as to test that is the case (and for more code coverage)
+    val trieAfterDeleteNoEffect = keys.take(100/2).foldLeft(trieAfterDelete) { case (recTrie, key) => recTrie.remove(md5(key)) }
+    assert(Hex.toHexString(trieAfterDeleteNoEffect.getRootHash) == "b0bfbf4d2d6f3c9863c27f41a087208131f775edd9de2cb66242d1e0981aa94c")
+  }
+
+  test("IODB Test - PatriciaTrie insert and get") {
+    forAll(keyValueListGen()) { keyValueList: Seq[(Int, Int)] =>
+      //create temporary dir
+      val dir = File.createTempFile("iodb", "iodb")
+      dir.delete()
+      dir.mkdir()
+
+      val dataSource = new IodbDataSource(new LSMStore(dir = dir, keySize = 32))
+      val trie = keyValueList.foldLeft(MerklePatriciaTrie[Int, Int](dataSource, hashFn)) {
+        case (recTrie, (key, value)) => recTrie.put(key, value)
+      }
+      keyValueList.foreach { case (key, value) =>
+        val obtained = trie.get(key)
+        assert(obtained.isDefined)
+        assert(obtained.get == value)
+      }
+    }
+  }
+
+  test("IODB Test - PatriciaTrie delete") {
+    forAll(Gen.nonEmptyListOf(Arbitrary.arbitrary[Int])) { keyList: List[Int] =>
+      //create temporary dir
+      val dirWithDelete = File.createTempFile("iodb", "iodb1")
+      dirWithDelete.delete()
+      dirWithDelete.mkdir()
+      val dataSourceWithDelete = new IodbDataSource(new LSMStore(dir = dirWithDelete, keySize = 32))
+
+      val keyValueList = keyList.distinct.zipWithIndex
+      val trieAfterInsert = keyValueList.foldLeft(MerklePatriciaTrie[Int, Int](dataSourceWithDelete, hashFn)) {
+        case (recTrie, (key, value)) => recTrie.put(key, value)
+      }
+      val (keyValueToDelete, keyValueLeft) = Random.shuffle(keyValueList).splitAt(Gen.choose(0, keyValueList.size).sample.get)
+      val trieAfterDelete = keyValueToDelete.foldLeft(trieAfterInsert) {
+        case (recTrie, (key, value)) => recTrie.remove(key)
+      }
+
+      keyValueLeft.foreach { case (key, value) =>
+        val obtained = trieAfterDelete.get(key)
+        assert(obtained.isDefined)
+        assert(obtained.get == value)
+      }
+      keyValueToDelete.foreach { case (key, value) =>
+        val obtained = trieAfterDelete.get(key)
+        assert(obtained.isEmpty)
+      }
+
+      val dirOnlyInsert = File.createTempFile("iodb", "iodb2")
+      dirOnlyInsert.delete()
+      dirOnlyInsert.mkdir()
+      val dataSourceOnlyInsert = new IodbDataSource(new LSMStore(dir = dirOnlyInsert, keySize = 32))
+
+      val trieWithKeyValueLeft = keyValueLeft.foldLeft(MerklePatriciaTrie[Int, Int](dataSourceOnlyInsert, hashFn)) {
+        case (recTrie, (key, value)) => recTrie.put(key, value)
+      }
+      assert(trieAfterDelete.getRootHash sameElements trieWithKeyValueLeft.getRootHash)
+    }
+  }
+
+  test("EthereumJ compatibility - Insert of the first 40000 numbers"){
+    val shuffledKeys = Random.shuffle(0 to 40000).map(intByteArraySerializable.toBytes)
+    val trie = shuffledKeys.foldLeft(EmptyTrie) { case (recTrie, key) => recTrie.put(key, key) }
+    assert(Hex.toHexString(trie.getRootHash) == "3f8b75707975e5c16588fa1ba3e69f8da39f4e7bf3ca28b029c7dcb589923463")
+  }
+
+  test("EthereumJ compatibility - Insert of the first 20000 numbers hashed"){
+    val shuffledKeys = Random.shuffle(0 to 20000).map(intByteArraySerializable.toBytes)
+    val trie = shuffledKeys.foldLeft(EmptyTrie) { case (recTrie, key) => recTrie.put(md5(key), key) }
+
+    // We insert keys that should have no effect so as to test that is the case (and for more code coverage)
+    val trieAfterInsertNoEffect = shuffledKeys.take(20000/2).foldLeft(trie) { case (recTrie, key) => recTrie.put(md5(key), key) }
+    assert(Hex.toHexString(trieAfterInsertNoEffect.getRootHash) == "a522b23a640c5fdb726e3f9644863e8913fe86339909fe881957efa0c23cebaa")
+  }
+
+  test("EthereumJ compatibility - Insert of the first 20000 numbers hashed and then remove half of them"){
+    val keys = (0 to 20000).map(intByteArraySerializable.toBytes)
+    val trie = Random.shuffle(keys).foldLeft(EmptyTrie) { case (recTrie, key) => recTrie.put(md5(key), key) }
+
+    // We delete have of the (key-value) pairs we had inserted
+    val trieAfterDelete = Random.shuffle(keys.take(20000/2)).foldLeft(trie) { case (recTrie, key) => recTrie.remove(md5(key)) }
+
+    // We delete keys with no effect so as to test that is the case (and for more code coverage)
+    val trieAfterDeleteNoEffect = keys.take(20000/2).foldLeft(trieAfterDelete) { case (recTrie, key) => recTrie.remove(md5(key)) }
+    assert(Hex.toHexString(trieAfterDeleteNoEffect.getRootHash) == "a693b82dcc5a9e581e9bf9aa7af3aed31fe3eb61f97fd733ce44c9f9df2d7f45")
+  }
+
+  test("EthereumJ compatibility - Insert of the first 20000 numbers hashed (with some sliced)"){
+    val keys = (0 to 20000).map(intByteArraySerializable.toBytes)
+
+    // We slice some of the keys so that me test more code coverage (if not we only test keys with the same length)
+    val slicedKeys = keys.zipWithIndex.map{case (key, index) =>
+      val hashedKey = md5(key)
+      if(index%2==0) hashedKey.take(hashedKey.length/2) else hashedKey
+    }
+    val keyValuePairs = slicedKeys.zip(keys)
+
+    val trie = Random.shuffle(keyValuePairs).foldLeft(EmptyTrie) { case (recTrie, (key, value)) => recTrie.put(key, value) }
+    assert(Hex.toHexString(trie.getRootHash) == "46cde8656f3be6ce93ba9dcb1017548f44c65d1ea659ac827fac8c9ac77cf6b3")
+  }
+
+  test("EthereumJ compatibility - Insert of the first 20000 numbers hashed (with some sliced) and then remove half of them") {
+    val keys = (0 to 20000).map(intByteArraySerializable.toBytes)
+
+    // We slice some of the keys so that me test more code coverage (if not we only test keys with the same length)
+    val slicedKeys = keys.zipWithIndex.map { case (key, index) =>
+      val hashedKey = md5(key)
+      if (index % 2 == 0) hashedKey.take(hashedKey.length / 2) else hashedKey }
+    val keyValuePairs = slicedKeys.zip(keys)
+
+    val trie = Random.shuffle(keyValuePairs).foldLeft(EmptyTrie) { case (recTrie, (key, value)) => recTrie.put(key, value) }
+
+    assert(Hex.toHexString(trie.getRootHash) == "46cde8656f3be6ce93ba9dcb1017548f44c65d1ea659ac827fac8c9ac77cf6b3")
+
+    // We delete have of the (key-value) pairs we had inserted
+    val trieAfterDelete = Random.shuffle(keyValuePairs.take(20000 / 2)).foldLeft(trie) { case (recTrie, (key, _)) => recTrie.remove(key) }
+
+    assert(Hex.toHexString(trieAfterDelete.getRootHash) == "ae7b65dddd3ac0428082160cf3ceff0276cf6e6deaa23b42c4c156b50a459822")
+  }
+
+  /* Performance test */
+  test("Performance test (From: https://github.com/ethereum/wiki/wiki/Benchmarks)"){
+    val debug = false
+    val Rounds = 1000
+    val Symmetric = true
+
+    val start: Long = System.currentTimeMillis
+    val emptyTrie = MerklePatriciaTrie[Array[Byte], Array[Byte]](EphemDataSource(), hashFn)
+    var seed: Array[Byte] = Array.fill(32)(0.toByte)
+
+    val trieResult = (0 until Rounds).foldLeft(emptyTrie){ case (recTrie, i) =>
+      seed = hashFn(seed)
+      if(!Symmetric) recTrie.put(seed, seed)
+      else{
+        val mykey = seed
+        seed = hashFn(seed)
+        val myval = if((seed(0) & 0xFF) % 2 == 1) Array[Byte](seed.last) else seed
+        recTrie.put(mykey, myval)
+      }
+    }
+    val rootHash = Hex.toHexString(trieResult.getRootHash)
+    if(debug){
+      println("Time taken(ms): " + (System.currentTimeMillis - start))
+      println("Root hash obtained: " + rootHash)
+    }
+    if(Symmetric) assert(rootHash.take(4) == "36f6" && rootHash.drop(rootHash.length-4) == "93a3")
+    else assert(rootHash.take(4) == "da8a" && rootHash.drop(rootHash.length-4) == "0ca4")
+  }
+}

src/main/scala/io/iohk/ethereum/mpt/EphemDataSource.scala

@@ -0,0 +1,18 @@
+package io.iohk.ethereum.mpt
+
+import akka.util.ByteString
+
+case class EphemDataSource(storage: Map[ByteString, Array[Byte]]) extends DataSource {
+
+  override def get(key: Array[Byte]): Option[Array[Byte]] = storage.get(ByteString(key))
+
+  override def update(rootHash: Array[Byte], toRemove: Seq[Key], toUpdate: Seq[(Key, Value)]): DataSource = {
+    val afterRemoval = toRemove.foldLeft(storage)((storage, key) => storage - ByteString(key))
+    val afterUpdate = toUpdate.foldLeft(afterRemoval)((storage, toUpdate) => storage + (ByteString(toUpdate._1) -> toUpdate._2))
+    EphemDataSource(afterUpdate)
+  }
+}
+
+object EphemDataSource {
+  def apply(): EphemDataSource = EphemDataSource(Map())
+}

src/main/scala/io/iohk/ethereum/mpt/HexPrefix.scala

@@ -0,0 +1,66 @@
+package io.iohk.ethereum.mpt
+
+object HexPrefix {
+  /**
+    * Pack nibbles to binary
+    *
+    * @param nibbles sequence
+    * @param isLeaf boolean used to encode whether or not the data being encoded corresponds to a LeafNode or an ExtensionNode
+    * @return hex-encoded byte array
+    *
+    */
+  def encode(nibbles: Array[Byte], isLeaf: Boolean): Array[Byte] = {
+    val hasOddLength = nibbles.length % 2 == 1
+    val firstByteFlag: Byte = (2*(if(isLeaf) 1 else 0) + (if(hasOddLength) 1 else 0)).toByte
+    val lengthFlag = if(hasOddLength) 1 else 2
+
+    val nibblesWithFlag = new Array[Byte](nibbles.length + lengthFlag)
+    Array.copy(nibbles, 0, nibblesWithFlag, lengthFlag, nibbles.length)
+    nibblesWithFlag(0) = firstByteFlag
+    if(!hasOddLength) nibblesWithFlag(1) = 0
+    nibblesToBytes(nibblesWithFlag)
+  }
+
+  /**
+    * Unpack a binary string to its nibbles equivalent
+    *
+    * @param src of binary data
+    * @return array of nibbles in byte-format and
+    *         boolean used to encode whether or not the data being decoded corresponds to a LeafNode or an ExtensionNode
+    *
+    */
+  def decode(src: Array[Byte]): (Array[Byte], Boolean) = {
+    val srcNibbles: Array[Byte] = bytesToNibbles(bytes = src)
+    val t = (srcNibbles(0) & 2) != 0
+    val hasOddLength = (srcNibbles(0) & 1) != 0
+    val flagLength = if(hasOddLength) 1 else 2
+
+    val res = new Array[Byte](srcNibbles.length - flagLength)
+    Array.copy(srcNibbles, flagLength, res, 0, srcNibbles.length-flagLength)
+    (res, t)
+  }
+
+  /**
+    * Transforms an array of 8bit values to the corresponding array of 4bit values (hexadecimal format)
+    *
+    * @param bytes byte[]
+    * @return array with each individual nibble
+    *
+    */
+  def bytesToNibbles(bytes: Array[Byte]): Array[Byte] = {
+    bytes.foldRight[List[Byte]](List()){(elem, rec) =>
+      ((elem >> 4) & 0xF).toByte :: (elem & 0xF).toByte :: rec}.toArray
+  }
+
+  /**
+    * Transforms an array of 4bit values (hexadecimal format) to the corresponding array of 8bit values
+    *
+    * @param nibbles byte[]
+    * @return array with bytes combining pairs of nibbles
+    *
+    */
+  def nibblesToBytes(nibbles: Array[Byte]): Array[Byte] = {
+    require(nibbles.length % 2 == 0)
+    nibbles.grouped(2).map{case Array(n1,n2) => (16*n1 + n2).toByte}.toArray
+  }
+}