Merge pull request #96 from ergoplatform/testnet

DigestState reopening

src/main/scala/org/ergoplatform/modifiers/ErgoFullBlock.scala

@@ -22,8 +22,7 @@ case class ErgoFullBlock(header: Header,
 
   override val parentId: ModifierId = header.parentId
 
-  //TODO is it ok to exclude ADProofs here? Otherwise we have problems in UTXOState when we can't rollback to block with proofs if we applied block withour proofs
-  override lazy val id: ModifierId = ModifierId @@ Algos.hash(header.id ++ blockTransactions.id)
+  override lazy val id: ModifierId = header.id
 
   override lazy val json: Json = Map(
     "header" -> header.json,

src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala

@@ -295,11 +295,13 @@ trait ErgoHistory
       modifier match {
         case fb: ErgoFullBlock =>
           val bestHeader = bestHeaderOpt.get
-          val ids = Seq(fb.id, fb.header.id, fb.blockTransactions.id) ++ fb.aDProofs.map(_.id)
-            .map(id => validityKey(id) -> ByteArrayWrapper(Array(1.toByte)))
+          val nonMarkedIds = (Seq(fb.header.id, fb.blockTransactions.id) ++ fb.aDProofs.map(_.id))
+            .filter(id => historyStorage.db.get(validityKey(id)).isEmpty)
 
-          historyStorage.db.update(validityKey(modifier.id), Seq(), Seq(validityKey(modifier.id) ->
-            ByteArrayWrapper(Array(1.toByte))))
+          if (nonMarkedIds.nonEmpty) {
+            historyStorage.db.update(validityKey(nonMarkedIds.head), Seq(),
+              nonMarkedIds.map(id => validityKey(id) -> ByteArrayWrapper(Array(1.toByte))))
+          }
 
           val bestFull = bestFullBlockOpt.get
           if (fb == bestFull) {

src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/adproofs/FullProofsProcessor.scala

@@ -38,7 +38,7 @@ trait FullProofsProcessor extends ADProofsProcessor with FullBlockProcessor {
       case Some(header: Header) =>
         require(header.ADProofsRoot sameElements m.digest,
           s"Header ADProofs root ${Base58.encode(header.ADProofsRoot)} differs from $m digest")
-        if(!header.isGenesis) {
+        if(!header.isGenesis && adState) {
           require(typedModifierById[Header](header.parentId).exists(h => contains(h.ADProofsId)),
             s"Trying to apply proofs ${m.encodedId} for header ${header.encodedId}, which parent proofs are empty")
         }

src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala

@@ -53,7 +53,8 @@ class DigestState protected(override val version: VersionTag,
 
     case h: Header => Success()
 
-    case a: Any => log.info(s"Modifier not validated: $a"); Try(this)
+    case a: Any =>
+      Failure(new Error(s"Modifier not validated: $a"))
   }
 
   private def update(newVersion: VersionTag, newRootHash: ADDigest): Try[DigestState] = Try {
@@ -66,7 +67,7 @@ class DigestState protected(override val version: VersionTag,
   override def applyModifier(mod: ErgoPersistentModifier): Try[DigestState] = mod match {
     case fb: ErgoFullBlock if settings.verifyTransactions =>
       log.info(s"Got new full block with id ${fb.encodedId} with root ${Algos.encoder.encode(fb.header.stateRoot)}")
-      this.validate(fb).flatMap(_ => update(VersionTag @@ fb.id, fb.header.stateRoot))
+      this.validate(fb).flatMap(_ => update(VersionTag @@ fb.header.id, fb.header.stateRoot))
 
     case fb: ErgoFullBlock if !settings.verifyTransactions =>
       //TODO should not get this messages from node view holders
@@ -90,13 +91,17 @@ class DigestState protected(override val version: VersionTag,
     log.info(s"Rollback Digest State to version ${Algos.encoder.encode(version)}")
     val wrappedVersion = ByteArrayWrapper(version)
     Try(store.rollback(wrappedVersion)).map { _ =>
+      store.clean(ErgoState.KeepVersions)
       val rootHash = ADDigest @@ store.get(wrappedVersion).get.data
       log.info(s"Rollback to version ${Algos.encoder.encode(version)} with roothash ${Algos.encoder.encode(rootHash)}")
       new DigestState(version, rootHash, store, settings)
     }
   }
 
   override def rollbackVersions: Iterable[VersionTag] = store.rollbackVersions().map(VersionTag @@ _.data)
+
+  def close(): Unit = store.close()
+
 }
 
 object DigestState {
@@ -105,7 +110,7 @@ object DigestState {
              rootHashOpt: Option[ADDigest],
              dir: File,
              settings: NodeConfigurationSettings): Try[DigestState] = Try {
-    val store = new LSMStore(dir, keepVersions = 10) //todo: read from settings
+    val store = new LSMStore(dir, keepVersions = ErgoState.KeepVersions) //todo: read from settings
 
     (versionOpt, rootHashOpt) match {
 
@@ -118,10 +123,10 @@ object DigestState {
         }.ensuring(store.lastVersionID.get.data.sameElements(version))
 
       case (None, None) =>
-        val version = ADDigest @@ store.get(store.lastVersionID.get).get.data
+        val version = VersionTag @@ store.lastVersionID.get.data
         val rootHash = store.get(ByteArrayWrapper(version)).get.data
 
-        new DigestState(VersionTag @@ version, ADDigest @@ rootHash, store, settings)
+        new DigestState(version, ADDigest @@ rootHash, store, settings)
 
       case _ => ???
     }

src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala

@@ -58,6 +58,9 @@ object ErgoState extends ScorexLogging {
 
   val BoxSize = AnyoneCanSpendNoncedBoxSerializer.Length
 
+  //TODO move to settings?
+  val KeepVersions = 200
+
   def stateDir(settings: ErgoSettings) = new File(s"${settings.directory}/state")
 
   def generateGenesisUtxoState(stateDir: File, nodeViewHolderRef: Option[ActorRef]): (UtxoState, BoxHolder) = {

src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala

@@ -96,11 +96,13 @@ class UtxoState(override val version: VersionTag, val store: Store, nodeViewHold
     log.info(s"Rollback UtxoState to version ${Algos.encoder.encode(version)}")
     store.get(ByteArrayWrapper(version)) match {
       case Some(hash) =>
-        p.rollback(ADDigest @@ hash.data).map { _ =>
+        val rollbackResult = p.rollback(ADDigest @@ hash.data).map { _ =>
           new UtxoState(version, store, nodeViewHolderRef) {
             override protected lazy val persistentProver = p
           }
         }
+        store.clean(ErgoState.KeepVersions)
+        rollbackResult
       case None =>
         Failure(new Error(s"Unable to get root hash at version ${Algos.encoder.encode(version)}"))
     }
@@ -192,7 +194,7 @@ object UtxoState {
   }
 
   def create(dir: File, nodeViewHolderRef: Option[ActorRef]): UtxoState = {
-    val store = new LSMStore(dir, keepVersions = 20) // todo: magic number, move to settings
+    val store = new LSMStore(dir, keepVersions = ErgoState.KeepVersions) // todo: magic number, move to settings
     val dbVersion = store.get(ByteArrayWrapper(bestVersionKey)).map(VersionTag @@ _.data)
     new UtxoState(dbVersion.getOrElse(ErgoState.genesisStateVersion), store, nodeViewHolderRef)
   }
@@ -201,7 +203,7 @@ object UtxoState {
     val p = new BatchAVLProver[Digest32, Blake2b256Unsafe](keyLength = 32, valueLengthOpt = Some(ErgoState.BoxSize))
     bh.sortedBoxes.foreach(b => p.performOneOperation(Insert(b.id, ADValue @@ b.bytes)).ensuring(_.isSuccess))
 
-    val store = new LSMStore(dir, keepVersions = 200) // todo: magic number, move to settings
+    val store = new LSMStore(dir, keepVersions = ErgoState.KeepVersions) // todo: magic number, move to settings
 
     new UtxoState(ErgoState.genesisStateVersion, store, nodeViewHolderRef) {
       override protected lazy val persistentProver =

src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala

@@ -1,5 +1,6 @@
 package org.ergoplatform.nodeView.state
 
+import org.ergoplatform.settings.ErgoSettings
 import org.ergoplatform.utils.{ErgoGenerators, ErgoTestHelpers}
 import org.scalatest.prop.GeneratorDrivenPropertyChecks
 import org.scalatest.{Matchers, PropSpec}
@@ -15,6 +16,23 @@ class DigestStateSpecification extends PropSpec
   private val emptyVersion: VersionTag = VersionTag @@ Array.fill(32)(0: Byte)
   private val emptyAdDigest: ADDigest = ADDigest @@ Array.fill(32)(0: Byte)
 
+  property("reopen") {
+    forAll(boxesHolderGen) { bh =>
+      val us = createUtxoState(bh)
+      bh.sortedBoxes.foreach(box => assert(us.boxById(box.id).isDefined))
+
+      val fb = validFullBlock(parentOpt = None, us, bh)
+      val dir2 = createTempDir
+      val ds = DigestState.create(Some(us.version), Some(us.rootHash), dir2, ErgoSettings.read(None).nodeSettings).get
+      ds.applyModifier(fb).isSuccess shouldBe true
+      ds.close()
+
+      val state = DigestState.create(None, None, dir2, ErgoSettings.read(None).nodeSettings).get
+      state.version shouldEqual fb.header.id
+      state.rootHash shouldEqual fb.header.stateRoot
+    }
+  }
+
   property("validate() - valid block") {
     forAll(boxesHolderGen) { bh =>
       val us = createUtxoState(bh)