Merge pull request ergoplatform#2139 from ccellado/it-tests-fix

Integration tests fix

.github/workflows/ci.yml

@@ -134,3 +134,40 @@ jobs:
       - name: Runs node tests
         run: sbt -Denv=test clean ++${{ matrix.scala }} test
 
+  test_it:
+     name: Run it node tests
+     strategy:
+       matrix:
+         os: [ ubuntu-latest ]
+         scala: [ 2.12.18 ]
+         java: [ adopt@1.8 ]
+     runs-on: ${{ matrix.os }}
+     steps:
+       - name: Checkout current branch (full)
+         uses: actions/checkout@v2
+         with:
+           path: it
+           fetch-depth: 0
+
+       - name: Setup Java and Scala
+         uses: olafurpg/setup-scala@v10
+         with:
+           java-version: ${{ matrix.java }}
+
+       - name: Cache sbt
+         uses: actions/cache@v2
+         with:
+           path: |
+             ~/.sbt
+             ~/.ivy2/cache
+             ~/.coursier/cache/v1
+             ~/.cache/coursier/v1
+             ~/AppData/Local/Coursier/Cache/v1
+             ~/Library/Caches/Coursier/v1
+           key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
+
+       - name: Runs it node tests
+         run: |
+           cd it
+           mkdir tmp
+           TMPDIR=$(pwd)/tmp sbt -Denv=test clean ++${{ matrix.scala }} it:test

README.md

@@ -94,6 +94,8 @@ This repository has modular structure, so only parts which are needed for an app
 * [ergo-core](ergo-core/README.md) - functionality needed for an SPV client (P2P messages, block section stuctures, PoW, NiPoPoW)
 * ergo-wallet - Java and Scala functionalities to sign and verify transactions
 
+Using IntelliJ IDEA be sure to set Build Tools / sbt -> `sbt shell` / `use for` / builds, to avoid compilation errors
+
 ## Contributing to Ergo
 
 Ergo is an open-source project and we welcome contributions from developers and testers! Join the discussion over [Ergo Discord](https://discord.gg/kj7s7nb) in #development channel, or Telegram: https://t.me/ErgoDevelopers. Please also check out our [Contributing documentation](https://docs.ergoplatform.com/contribute/).

build.sbt

@@ -52,7 +52,7 @@ libraryDependencies ++= Seq(
   "ch.qos.logback" % "logback-classic" % "1.3.5",
 
   // test dependencies
-  "org.scala-lang.modules" %% "scala-async" % "0.9.7" % "test",
+  "org.scala-lang.modules" %% "scala-async" % "1.0.1" % "test",
   "org.scalactic" %% "scalactic" % "3.0.3" % "test",
   "org.scalatest" %% "scalatest" % "3.2.10" % "test,it",
   "org.scalacheck" %% "scalacheck" % "1.14.+" % "test",
@@ -63,7 +63,9 @@ libraryDependencies ++= Seq(
 
   "org.asynchttpclient" % "async-http-client" % "2.6.+" % "test",
   "com.fasterxml.jackson.dataformat" % "jackson-dataformat-properties" % "2.9.2" % "test",
-  "com.spotify" % "docker-client" % "8.14.5" % "test" classifier "shaded"
+  "com.github.docker-java" % "docker-java-core" % "3.3.4" % Test,
+  "com.github.docker-java" % "docker-java-transport-httpclient5" % "3.3.4" % Test,
+
 )
 
 updateOptions := updateOptions.value.withLatestSnapshots(false)
@@ -167,6 +169,7 @@ configs(IntegrationTest extend Test)
 inConfig(IntegrationTest)(Seq(
   parallelExecution := false,
   test := (test dependsOn docker).value,
+  scalacOptions ++= Seq("-Xasync")
 ))
 
 dockerfile in docker := {
@@ -175,7 +178,7 @@ dockerfile in docker := {
   val configMainNet = (resourceDirectory in IntegrationTest).value / "mainnetTemplate.conf"
 
   new Dockerfile {
-    from("openjdk:9-jre-slim")
+    from("openjdk:11-jre-slim")
     label("ergo-integration-tests", "ergo-integration-tests")
     add(assembly.value, "/opt/ergo/ergo.jar")
     add(Seq(configDevNet), "/opt/ergo")
@@ -280,6 +283,7 @@ configs(It2Test)
 inConfig(It2Test)(Defaults.testSettings ++ Seq(
   parallelExecution := false,
   test := (test dependsOn docker).value,
+  scalacOptions ++= Seq("-Xasync")
 ))
 
 lazy val ergo = (project in file("."))

project/plugins.sbt

@@ -2,7 +2,7 @@ logLevel := Level.Warn
 
 addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.5.1")
 
-addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.4.1")
+addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.11.0")
 
 addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6")
 

src/it/resources/devnetTemplate.conf

@@ -53,6 +53,9 @@ scorex {
   }
   restApi {
     bindAddress = "0.0.0.0:9051"
-    apiKeyHash = null
+    # Hex-encoded Blake2b256 hash of an API key. Should be 64-chars long Base16 string.
+    # Below is the hash of "hello" string.
+    # Change it!
+    apiKeyHash = "324dcf027dd4a30a932c441f365a25e86b173defa4b8e58948253471b81b72cf"
   }
 }

src/it/scala/org/ergoplatform/it/DeepRollBackSpec.scala

@@ -5,16 +5,15 @@ import com.typesafe.config.Config
 import org.ergoplatform.it.container.{IntegrationSuite, Node}
 import org.ergoplatform.nodeView.history.ErgoHistoryUtils
 import org.scalatest.freespec.AnyFreeSpec
-
 import scala.async.Async
 import scala.concurrent.{Await, Future}
 import scala.concurrent.duration._
 
 class DeepRollBackSpec extends AnyFreeSpec with IntegrationSuite {
 
   val keepVersions = 350
-  val chainLength = 150
-  val delta = 50
+  val chainLength = 50
+  val delta = 150
 
   val localVolumeA = s"$localDataDir/node-rollback-spec/nodeA/data"
   val localVolumeB = s"$localDataDir/node-rollback-spec/nodeB/data"
@@ -28,40 +27,64 @@ class DeepRollBackSpec extends AnyFreeSpec with IntegrationSuite {
     .withFallback(shortInternalMinerPollingInterval)
     .withFallback(keepVersionsConfig(keepVersions))
     .withFallback(nodeSeedConfigs.head)
+    .withFallback(localOnlyConfig)
 
   val minerBConfig: Config = specialDataDirConfig(remoteVolumeB)
     .withFallback(shortInternalMinerPollingInterval)
     .withFallback(keepVersionsConfig(keepVersions))
     .withFallback(nodeSeedConfigs.last)
+    .withFallback(localOnlyConfig)
 
   val minerAConfigNonGen: Config = minerAConfig
     .withFallback(nonGeneratingPeerConfig)
+    .withFallback(localOnlyConfig)
 
   val minerBConfigNonGen: Config = minerBConfig
     .withFallback(nonGeneratingPeerConfig)
+    .withFallback(localOnlyConfig)
 
-  "Deep rollback handling" ignore {
+  "Deep rollback handling" in {
 
     val result: Future[Unit] = Async.async {
 
+      // 1. Let nodeA mine and sync nodeB
+
+      val minerAGen: Node = docker.startDevNetNode(minerAConfig,
+        specialVolumeOpt = Some((localVolumeA, remoteVolumeA))).get
+
+      val minerBGen: Node = docker.startDevNetNode(minerBConfigNonGen,
+        specialVolumeOpt = Some((localVolumeB, remoteVolumeB))).get
+
+      Async.await(minerAGen.waitForHeight(1))
+      Async.await(minerBGen.waitForHeight(1))
+
+      val genesisAGen = Async.await(minerAGen.headerIdsByHeight(ErgoHistoryUtils.GenesisHeight)).head
+      val genesisBGen = Async.await(minerBGen.headerIdsByHeight(ErgoHistoryUtils.GenesisHeight)).head
+
+      val minerAGenBestHeight = Async.await(minerAGen.fullHeight)
+      val minerBGenBestHeight = Async.await(minerBGen.fullHeight)
+
+      log.info("heightA: " + minerAGenBestHeight)
+      log.info("heightB: " + minerBGenBestHeight)
+
+      genesisAGen shouldBe genesisBGen
+
+      // 2. Stop all nodes
+      docker.stopNode(minerAGen.containerId)
+      docker.stopNode(minerBGen.containerId)
+
       val minerAIsolated: Node = docker.startDevNetNode(minerAConfig, isolatedPeersConfig,
         specialVolumeOpt = Some((localVolumeA, remoteVolumeA))).get
 
-      // 1. Let the first node mine `chainLength + delta` blocks
+      // 1. Let nodeA mine `chainLength + delta` blocks in isolation
       Async.await(minerAIsolated.waitForHeight(chainLength + delta))
 
-      val genesisA = Async.await(minerAIsolated.headerIdsByHeight(ErgoHistoryUtils.GenesisHeight)).head
-
       val minerBIsolated: Node = docker.startDevNetNode(minerBConfig, isolatedPeersConfig,
         specialVolumeOpt = Some((localVolumeB, remoteVolumeB))).get
 
-      // 2. Let another node mine `chainLength` blocks
+      // 2. Let nodeB mine `chainLength` blocks in isolation
       Async.await(minerBIsolated.waitForHeight(chainLength))
 
-      val genesisB = Async.await(minerBIsolated.headerIdsByHeight(ErgoHistoryUtils.GenesisHeight)).head
-
-      genesisA should not equal genesisB
-
       log.info("Mining phase done")
 
       val minerABestHeight = Async.await(minerAIsolated.fullHeight)
@@ -75,7 +98,7 @@ class DeepRollBackSpec extends AnyFreeSpec with IntegrationSuite {
 
       (minerABestHeight > minerBBestHeight) shouldBe true
 
-      // 3. Restart node A and node B (having shorter chain) with disabled mining
+      // 3. Restart nodeA and nodeB (having shorter chain) with disabled mining
       val minerA: Node = docker.startDevNetNode(minerAConfigNonGen,
         specialVolumeOpt = Some((localVolumeA, remoteVolumeA))).get
 
@@ -102,7 +125,7 @@ class DeepRollBackSpec extends AnyFreeSpec with IntegrationSuite {
       minerBBestBlock shouldEqual minerABestBlock
     }
 
-    Await.result(result, 25.minutes)
+    Await.result(result, 15.minutes)
   }
 
 }

src/it/scala/org/ergoplatform/it/ForkResolutionSpec.scala

@@ -30,10 +30,12 @@ class ForkResolutionSpec extends AnyFlatSpec with Matchers with IntegrationSuite
   val dirs: Seq[File] = localVolumes.map(vol => new File(vol))
   dirs.foreach(_.mkdirs())
 
-  val minerConfig: Config = nodeSeedConfigs.head
-  val onlineMiningNodesConfig: List[Config] = nodeSeedConfigs.slice(1, nodesQty)
+  val nodeConfigs: List[Config] = nodeSeedConfigs.take(4).map(_.withFallback(localOnlyConfig))
+
+  val minerConfig: Config = nodeConfigs.head
+  val onlineMiningNodesConfig: List[Config] = nodeConfigs.slice(1, nodesQty)
     .map(_.withFallback(onlineGeneratingPeerConfig))
-  val offlineMiningNodesConfig: List[Config] = nodeSeedConfigs.slice(1, nodesQty)
+  val offlineMiningNodesConfig: List[Config] = nodeConfigs.slice(1, nodesQty)
 
   def localVolume(n: Int): String = s"$localDataDir/fork-resolution-spec/node-$n/data"
 
@@ -48,7 +50,7 @@ class ForkResolutionSpec extends AnyFlatSpec with Matchers with IntegrationSuite
     implicit val patienceConfig: PatienceConfig = PatienceConfig((nodeConfigs.size * 2).seconds, 3.second)
     blocking(Thread.sleep(nodeConfigs.size * 2000))
     eventually {
-      Await.result(Future.traverse(nodes.get)(_.waitForStartup), nodeConfigs.size.seconds)
+      Await.result(Future.traverse(nodes.get)(_.waitForStartup), 180.seconds)
     }
   }
 
@@ -60,6 +62,9 @@ class ForkResolutionSpec extends AnyFlatSpec with Matchers with IntegrationSuite
   // 5. Check that nodes reached consensus on created forks;
   it should "Fork resolution after isolated mining" in {
 
+    log.info(minerConfig.toString)
+    onlineMiningNodesConfig.foreach(x => log.info(x.toString))
+
     val nodes: List[Node] = startNodesWithBinds(minerConfig +: onlineMiningNodesConfig)
 
     val result = Async.async {

src/it/scala/org/ergoplatform/it/KnownNodesSpec.scala

@@ -9,11 +9,16 @@ import scala.concurrent.duration._
 
 class KnownNodesSpec extends AnyFlatSpec with IntegrationSuite {
 
-  val nodeConfigs: List[Config] = nodeSeedConfigs.take(3).map(nonGeneratingPeerConfig.withFallback)
+  val nodeConfigs: List[Config] = nodeSeedConfigs.take(3)
+    .map(conf => nonGeneratingPeerConfig
+      .withFallback(conf)
+      .withFallback(localOnlyConfig)
+    )
+
   val nodes: List[Node] = docker.startDevNetNodes(nodeConfigs, sequentialTopologyConfig).get
 
-  // todo: https://github.com/ergoplatform/ergo/issues/653
-  it should s"The third node knows first node" ignore {
+  // All nodes should propagate sequentially, so any node knows each other
+  it should s"The third node knows first node" in {
 
     val node03 = nodes.find(_.nodeName == "node03").value
     val targetPeersCount = nodes.length - 1 /* self */

src/it/scala/org/ergoplatform/it/LongChainSyncSpec.scala

@@ -11,14 +11,19 @@ class LongChainSyncSpec extends AnyFlatSpec with IntegrationSuite {
 
   val chainLength = 300
 
-  val minerConfig: Config = shortInternalMinerPollingInterval.withFallback(nodeSeedConfigs.head)
-  val nonGeneratingConfig: Config = nonGeneratingPeerConfig.withFallback(nodeSeedConfigs(1))
+  val minerConfig: Config = shortInternalMinerPollingInterval
+    .withFallback(nodeSeedConfigs.head)
+    .withFallback(localOnlyConfig)
+
+  val nonGeneratingConfig: Config =
+    nonGeneratingPeerConfig.withFallback(nodeSeedConfigs(1)).withFallback(localOnlyConfig)
 
   val miner: Node = docker.startDevNetNode(minerConfig).get
 
   it should s"Long chain ($chainLength blocks) synchronization" in {
 
-    val result: Future[Int] = miner.waitForHeight(chainLength)
+    val result: Future[Int] = miner
+      .waitForHeight(chainLength)
       .flatMap { _ =>
         val follower = docker.startDevNetNode(nonGeneratingConfig).get
         follower.waitForHeight(chainLength)
@@ -27,4 +32,3 @@ class LongChainSyncSpec extends AnyFlatSpec with IntegrationSuite {
     Await.result(result, 10.minutes)
   }
 }
-

src/it/scala/org/ergoplatform/it/NodeRecoverySpec.scala

@@ -1,7 +1,6 @@
 package org.ergoplatform.it
 
 import java.io.File
-
 import akka.japi.Option.Some
 import com.typesafe.config.Config
 import org.ergoplatform.it.container.{IntegrationSuite, Node}
@@ -27,6 +26,7 @@ class NodeRecoverySpec
   val offlineGeneratingPeer: Config = specialDataDirConfig(remoteVolume)
     .withFallback(offlineGeneratingPeerConfig)
     .withFallback(nodeSeedConfigs.head)
+    .withFallback(localOnlyConfig)
 
   val node: Node = docker.startDevNetNode(offlineGeneratingPeer, specialVolumeOpt = Some((localVolume, remoteVolume))).get