Merge pull request #367 from ergoplatform/i363

Node optimizations

src/main/scala/org/ergoplatform/ErgoApp.scala

@@ -43,7 +43,6 @@ class ErgoApp(args: Seq[String]) extends Application {
 
   val readersHolderRef: ActorRef = ErgoReadersHolderRef(nodeViewHolderRef)
 
-
   val minerRef: ActorRef = ErgoMinerRef(ergoSettings, nodeViewHolderRef, readersHolderRef, timeProvider, emission)
 
   val statsCollectorRef: ActorRef = ErgoStatsCollectorRef(nodeViewHolderRef, peerManagerRef, ergoSettings, timeProvider)

src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala

@@ -17,6 +17,8 @@ class HistoryStorage(indexStore: Store, objectsStore: ObjectsStore, config: Cach
     .maximumSize(config.historyStorageCacheSize)
     .build[String, ErgoPersistentModifier]
 
+
+  // TODO remove when modifierId will be string
   private def keyById(id: ModifierId): String = Algos.encode(id)
 
   def modifierById(id: ModifierId): Option[ErgoPersistentModifier] = {

src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala

@@ -37,6 +37,7 @@ trait FullBlockProcessor extends HeadersProcessor {
     */
   protected def processFullBlock(fullBlock: ErgoFullBlock,
                                  newMod: ErgoPersistentModifier): ProgressInfo[ErgoPersistentModifier] = {
+    // TODO this is very inefficient - in most cases we do not need to calculate `bestFullChain`
     val bestFullChain = calculateBestFullChain(fullBlock)
     val newBestAfterThis = bestFullChain.last.header
     processing(ToProcess(fullBlock, newMod, newBestAfterThis, config.blocksToKeep, bestFullChain))

src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala

@@ -20,17 +20,24 @@ import scorex.crypto.hash.Digest32
 import scala.util.{Failure, Success, Try}
 
 /**
-  * Utxo set implementation.
+  * Utxo set implementation
   *
-  * @param store - database where persistent UTXO set authenticated with the help of an AVL+ tree is residing
+  * @param persistentProver - persistent prover that build authenticated AVL+ tree on top of utxo set
+  * @param store - storage of persistentProver that also keeps metadata
+  * @param version - current state version
+  * @param constants - constants, that do not change with state version changes
   */
-class UtxoState(override val version: VersionTag,
+class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32, HF],
+                override val version: VersionTag,
                 override val store: Store,
                 override val constants: StateConstants)
   extends ErgoState[UtxoState]
     with TransactionValidation[ErgoTransaction]
     with UtxoStateReader {
 
+
+  override def rootHash: ADDigest = persistentProver.digest
+
   private def onAdProofGenerated(proof: ADProofs): Unit = {
     if (constants.nodeViewHolderRef.isEmpty) log.warn("Got proof while nodeViewHolderRef is empty")
     constants.nodeViewHolderRef.foreach(h => h ! LocallyGeneratedModifier(proof))
@@ -40,17 +47,14 @@ class UtxoState(override val version: VersionTag,
 
   override val maxRollbackDepth = 10
 
-  override lazy val rootHash: ADDigest = persistentProver.digest
-
   override def rollbackTo(version: VersionTag): Try[UtxoState] = {
     val p = persistentProver
     log.info(s"Rollback UtxoState to version ${Algos.encoder.encode(version)}")
     store.get(ByteArrayWrapper(version)) match {
       case Some(hash) =>
-        val rollbackResult = p.rollback(ADDigest @@ hash.data).map { _ =>
-          new UtxoState(version, store, constants) {
-            override protected lazy val persistentProver = p
-          }
+        val rootHash: ADDigest = ADDigest @@ hash.data
+        val rollbackResult = p.rollback(rootHash).map { _ =>
+          new UtxoState(p, version, store, constants)
         }
         store.clean(constants.keepVersions)
         rollbackResult
@@ -96,14 +100,13 @@ class UtxoState(override val version: VersionTag,
     case fb: ErgoFullBlock =>
       val height = fb.header.height
 
-      log.debug(s"Trying to apply full block with header ${fb.header.encodedId} at height $height " +
-        s"to UtxoState with root hash ${Algos.encode(rootHash)}")
+      log.debug(s"Trying to apply full block with header ${fb.header.encodedId} at height $height")
+      val inRoot = rootHash
 
       val stateTry: Try[UtxoState] = applyTransactions(fb.blockTransactions.txs, fb.header.stateRoot, height).map { _: Unit =>
         val emissionBox = extractEmissionBox(fb)
         val newStateContext = stateContext.appendHeader(fb.header)
         val md = metadata(VersionTag @@ fb.id, fb.header.stateRoot, emissionBox, newStateContext)
-
         val proofBytes = persistentProver.generateProofAndUpdateStorage(md)
         val proofHash = ADProofs.proofDigest(proofBytes)
         if (fb.aDProofs.isEmpty) onAdProofGenerated(ADProofs(fb.header.id, proofBytes))
@@ -117,18 +120,18 @@ class UtxoState(override val version: VersionTag,
         }
 
         log.info(s"Valid modifier with header ${fb.header.encodedId} and emission box " +
-          s"${emissionBox.map(e => Algos.encode(e.id))} applied to UtxoState with root hash ${Algos.encode(rootHash)}")
-        new UtxoState(VersionTag @@ fb.id, store, constants)
+          s"${emissionBox.map(e => Algos.encode(e.id))} applied to UtxoState with root hash ${Algos.encode(inRoot)}")
+        new UtxoState(persistentProver, VersionTag @@ fb.id, store, constants)
       }
       stateTry.recoverWith[UtxoState] { case e =>
         log.warn(s"Error while applying full block with header ${fb.header.encodedId} to UTXOState with root" +
-          s" ${Algos.encode(rootHash)}: ", e)
-        persistentProver.rollback(rootHash).ensuring(persistentProver.digest.sameElements(rootHash))
+          s" ${Algos.encode(inRoot)}: ", e)
+        persistentProver.rollback(inRoot).ensuring(persistentProver.digest.sameElements(inRoot))
         Failure(e)
       }
 
     case h: Header =>
-      Success(new UtxoState(VersionTag @@ h.id, this.store, constants))
+      Success(new UtxoState(persistentProver, VersionTag @@ h.id, this.store, constants))
 
     case a: Any =>
       log.info(s"Unhandled modifier: $a")
@@ -143,6 +146,7 @@ class UtxoState(override val version: VersionTag,
 }
 
 object UtxoState {
+
   private lazy val bestVersionKey = Algos.hash("best state version")
   val EmissionBoxIdKey = Algos.hash("emission box id key")
 
@@ -161,8 +165,15 @@ object UtxoState {
 
   def create(dir: File, constants: StateConstants): UtxoState = {
     val store = new LSMStore(dir, keepVersions = constants.keepVersions)
-    val dbVersion = store.get(ByteArrayWrapper(bestVersionKey)).map(VersionTag @@ _.data)
-    new UtxoState(dbVersion.getOrElse(ErgoState.genesisStateVersion), store, constants)
+    val version = store.get(ByteArrayWrapper(bestVersionKey)).map(VersionTag @@ _.data)
+      .getOrElse(ErgoState.genesisStateVersion)
+    val persistentProver: PersistentBatchAVLProver[Digest32, HF] = {
+      val bp = new BatchAVLProver[Digest32, HF](keyLength = 32, valueLengthOpt = None)
+      val np = NodeParameters(keySize = 32, valueSize = None, labelSize = 32)
+      val storage: VersionedIODBAVLStorage[Digest32] = new VersionedIODBAVLStorage(store, np)(Algos.hash)
+      PersistentBatchAVLProver.create(bp, storage).get
+    }
+    new UtxoState(persistentProver, version, store, constants)
   }
 
   @SuppressWarnings(Array("OptionGet", "TryGet"))
@@ -175,17 +186,16 @@ object UtxoState {
 
     val store = new LSMStore(dir, keepVersions = constants.keepVersions)
     val defaultStateContext = ErgoStateContext(0, p.digest)
-
-    new UtxoState(ErgoState.genesisStateVersion, store, constants) {
-      override protected lazy val persistentProver =
-        PersistentBatchAVLProver.create(
-          p,
-          storage,
-          metadata(ErgoState.genesisStateVersion, p.digest, currentEmissionBoxOpt, defaultStateContext),
-          paranoidChecks = true
-        ).get
-
-      assert(persistentProver.digest.sameElements(storage.version.get))
-    }
+    val np = NodeParameters(keySize = 32, valueSize = None, labelSize = 32)
+    val storage: VersionedIODBAVLStorage[Digest32] = new VersionedIODBAVLStorage(store, np)(Algos.hash)
+    val persistentProver = PersistentBatchAVLProver.create(
+      p,
+      storage,
+      metadata(ErgoState.genesisStateVersion, p.digest, currentEmissionBoxOpt, defaultStateContext),
+      paranoidChecks = true
+    ).get
+
+    new UtxoState(persistentProver, ErgoState.genesisStateVersion, store, constants)
   }
 }
+

src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala

@@ -22,11 +22,7 @@ trait UtxoStateReader extends ErgoStateReader with TransactionValidation[ErgoTra
   private lazy val np = NodeParameters(keySize = 32, valueSize = None, labelSize = 32)
   protected lazy val storage = new VersionedIODBAVLStorage(store, np)
 
-
-  protected lazy val persistentProver: PersistentBatchAVLProver[Digest32, HF] = {
-    val bp = new BatchAVLProver[Digest32, HF](keyLength = 32, valueLengthOpt = None)
-    PersistentBatchAVLProver.create(bp, storage).get
-  }
+  protected val persistentProver: PersistentBatchAVLProver[Digest32, HF]
 
   override def validate(tx: ErgoTransaction): Try[Unit] = tx.statelessValidity
     .flatMap(_ => tx.statefulValidity(tx.inputs.flatMap(i => boxById(i.boxId)), stateContext).map(_ => Unit))

src/test/scala/org/ergoplatform/nodeView/WrappedUtxoState.scala

@@ -10,16 +10,22 @@ import org.ergoplatform.mining.emission.CoinsEmission
 import org.ergoplatform.modifiers.ErgoPersistentModifier
 import org.ergoplatform.modifiers.mempool.ErgoTransaction
 import org.ergoplatform.nodeView.state._
+import org.ergoplatform.settings.Algos
+import org.ergoplatform.settings.Algos.HF
 import scorex.core.{TransactionsCarryingPersistentNodeViewModifier, VersionTag}
+import scorex.crypto.authds.ADDigest
+import scorex.crypto.authds.avltree.batch._
+import scorex.crypto.hash.Digest32
 
 import scala.util.{Failure, Success, Try}
 
 
-class WrappedUtxoState(override val version: VersionTag,
+class WrappedUtxoState(prover: PersistentBatchAVLProver[Digest32, HF],
+                       override val version: VersionTag,
                        store: Store,
                        val versionedBoxHolder: VersionedInMemoryBoxHolder,
                        constants: StateConstants)
-  extends UtxoState(version, store, constants) {
+  extends UtxoState(prover, version, store, constants) {
 
   def size: Int = versionedBoxHolder.size
 
@@ -28,7 +34,7 @@ class WrappedUtxoState(override val version: VersionTag,
   override def rollbackTo(version: VersionTag): Try[WrappedUtxoState] = super.rollbackTo(version) match {
     case Success(us) =>
       val updHolder = versionedBoxHolder.rollback(ByteArrayWrapper(us.version))
-      Success(new WrappedUtxoState(version, us.store, updHolder, constants))
+      Success(new WrappedUtxoState(us.persistentProver, version, us.store, updHolder, constants))
     case Failure(e) => Failure(e)
   }
 
@@ -43,10 +49,10 @@ class WrappedUtxoState(override val version: VersionTag,
             ByteArrayWrapper(us.version),
             changes.toRemove.map(_.boxId).map(ByteArrayWrapper.apply),
             changes.toAppend.map(_.box))
-          Success(new WrappedUtxoState(VersionTag @@ mod.id, us.store, updHolder, constants))
+          Success(new WrappedUtxoState(us.persistentProver, VersionTag @@ mod.id, us.store, updHolder, constants))
         case _ =>
           val updHolder = versionedBoxHolder.applyChanges(ByteArrayWrapper(us.version), Seq(), Seq())
-          Success(new WrappedUtxoState(VersionTag @@ mod.id, us.store, updHolder, constants))
+          Success(new WrappedUtxoState(us.persistentProver, VersionTag @@ mod.id, us.store, updHolder, constants))
       }
     case Failure(e) => Failure(e)
   }
@@ -72,6 +78,6 @@ object WrappedUtxoState {
       IndexedSeq(version),
       Map(version -> (Seq() -> boxHolder.sortedBoxes.toSeq)))
 
-    new WrappedUtxoState(ErgoState.genesisStateVersion, us.store, vbh, constants)
+    new WrappedUtxoState(us.persistentProver, ErgoState.genesisStateVersion, us.store, vbh, constants)
   }
 }