Fixing IT tests

.github/workflows/ci.yml

@@ -57,6 +57,49 @@ jobs:
           SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
           SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
 
+  test_core:
+    name: Run ergo-core tests and publish a ergo-core snapshot
+    env:
+      HAS_SECRETS: ${{ secrets.SONATYPE_PASSWORD != '' }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        scala: [2.13.12, 2.12.18, 2.11.12]
+        java: [adopt@1.8]
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout current branch (full)
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+
+      - name: Setup Java and Scala
+        uses: olafurpg/setup-scala@v10
+        with:
+          java-version: ${{ matrix.java }}
+
+      - name: Cache sbt
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.sbt
+            ~/.ivy2/cache
+            ~/.coursier/cache/v1
+            ~/.cache/coursier/v1
+            ~/AppData/Local/Coursier/Cache/v1
+            ~/Library/Caches/Coursier/v1
+          key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
+
+      - name: Runs ergo-core tests
+        run: sbt ++${{ matrix.scala }} ergoCore/test
+
+      - name: Publish a wallet snapshot ${{ github.ref }}
+        if: env.HAS_SECRETS == 'true'
+        run: sbt ++${{ matrix.scala }} ergoCore/publish
+        env:
+          SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
+          SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
+
   test_node:
     name: Run node tests
     strategy:
@@ -91,3 +134,36 @@ jobs:
       - name: Runs node tests
         run: sbt -Denv=test clean ++${{ matrix.scala }} test
 
+  test_it:
+     name: Run it node tests
+     strategy:
+       matrix:
+         os: [ ubuntu-latest ]
+         scala: [ 2.12.18 ]
+         java: [ adopt@1.8 ]
+     runs-on: ${{ matrix.os }}
+     steps:
+       - name: Checkout current branch (full)
+         uses: actions/checkout@v2
+         with:
+           fetch-depth: 0
+
+       - name: Setup Java and Scala
+         uses: olafurpg/setup-scala@v10
+         with:
+           java-version: ${{ matrix.java }}
+
+       - name: Cache sbt
+         uses: actions/cache@v2
+         with:
+           path: |
+             ~/.sbt
+             ~/.ivy2/cache
+             ~/.coursier/cache/v1
+             ~/.cache/coursier/v1
+             ~/AppData/Local/Coursier/Cache/v1
+             ~/Library/Caches/Coursier/v1
+           key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
+
+       - name: Runs it node tests
+         run: sbt -Denv=test clean ++${{ matrix.scala }} it:test

README.md

@@ -94,6 +94,8 @@ This repository has modular structure, so only parts which are needed for an app
 * [ergo-core](ergo-core/README.md) - functionality needed for an SPV client (P2P messages, block section stuctures, PoW, NiPoPoW)
 * ergo-wallet - Java and Scala functionalities to sign and verify transactions
 
+Using IntelliJ IDEA be sure to set Build Tools / sbt -> `sbt shell` / `use for` / builds, to avoid compilation errors
+
 ## Contributing to Ergo
 
 Ergo is an open-source project and we welcome contributions from developers and testers! Join the discussion over [Ergo Discord](https://discord.gg/kj7s7nb) in #development channel, or Telegram: https://t.me/ErgoDevelopers. Please also check out our [Contributing documentation](https://docs.ergoplatform.com/contribute/).

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/benchmark/OOMTest.scala

@@ -11,9 +11,16 @@ import scorex.crypto.hash.{Blake2b256, Digest32}
 import scorex.db.{ByteArrayWrapper, LDBVersionedStore}
 
 import scala.collection.immutable.SortedMap
+import scala.math.Ordering.Implicits._
 
 object OOMTest extends App {
 
+  implicit val ordering: Ordering[Array[Byte]] =
+    new Ordering[Array[Byte]] {
+      override def compare(o1: Array[Byte], o2: Array[Byte]): Int =
+        implicitly[Ordering[Seq[Int]]].compare(o1.toSeq.map(_ & 0xFF), o2.toSeq.map(_ & 0xFF))
+    }
+
   type Box = (ADKey, ADValue)
 
   type HF = Blake2b256.type

build.sbt

@@ -52,7 +52,7 @@ libraryDependencies ++= Seq(
   "ch.qos.logback" % "logback-classic" % "1.3.5",
 
   // test dependencies
-  "org.scala-lang.modules" %% "scala-async" % "0.9.7" % "test",
+  "org.scala-lang.modules" %% "scala-async" % "1.0.1" % "test",
   "org.scalactic" %% "scalactic" % "3.0.3" % "test",
   "org.scalatest" %% "scalatest" % "3.2.10" % "test,it",
   "org.scalacheck" %% "scalacheck" % "1.14.+" % "test",
@@ -63,7 +63,10 @@ libraryDependencies ++= Seq(
 
   "org.asynchttpclient" % "async-http-client" % "2.6.+" % "test",
   "com.fasterxml.jackson.dataformat" % "jackson-dataformat-properties" % "2.9.2" % "test",
-  "com.spotify" % "docker-client" % "8.14.5" % "test" classifier "shaded"
+//  "com.spotify" % "docker-client" % "8.14.5" % "test" classifier "shaded",
+  "com.github.docker-java" % "docker-java-core" % "3.3.4" % Test,
+  "com.github.docker-java" % "docker-java-transport-httpclient5" % "3.3.4" % Test,
+
 )
 
 updateOptions := updateOptions.value.withLatestSnapshots(false)
@@ -163,10 +166,12 @@ inConfig(Linux)(
 )
 
 Defaults.itSettings
+//lazy val ItTest = config("it") extend (IntegrationTest, Test)
 configs(IntegrationTest extend Test)
 inConfig(IntegrationTest)(Seq(
   parallelExecution := false,
   test := (test dependsOn docker).value,
+  scalacOptions ++= Seq("-Xasync")
 ))
 
 dockerfile in docker := {
@@ -175,7 +180,7 @@ dockerfile in docker := {
   val configMainNet = (resourceDirectory in IntegrationTest).value / "mainnetTemplate.conf"
 
   new Dockerfile {
-    from("openjdk:9-jre-slim")
+    from("openjdk:11-jre-slim")
     label("ergo-integration-tests", "ergo-integration-tests")
     add(assembly.value, "/opt/ergo/ergo.jar")
     add(Seq(configDevNet), "/opt/ergo")
@@ -255,6 +260,7 @@ lazy val ergoCore = (project in file("ergo-core"))
     ),
     scalacOptions in(Compile, compile) ++= (if (scalaBinaryVersion.value == "2.11") Seq() else Seq("-release", "8")),
     scalacOptions in(Compile, compile) --= scalacOpts,
+    parallelExecution in Test := false,
   )
 
 lazy val ergoWallet = (project in file("ergo-wallet"))
@@ -279,6 +285,7 @@ configs(It2Test)
 inConfig(It2Test)(Defaults.testSettings ++ Seq(
   parallelExecution := false,
   test := (test dependsOn docker).value,
+  scalacOptions ++= Seq("-Xasync")
 ))
 
 lazy val ergo = (project in file("."))

ergo-core/README.md

@@ -72,7 +72,9 @@ val handshakeMessageSerialized = HandshakeSerializer.toBytes(handshakeMessage)
 Serialize the message and send it.
 If the message arrived successfully, start communicating with the peer node.
 
-All communication is wrapped with Message headers, format described [here](https://docs.ergoplatform.com/dev/p2p/network/#message-format).
+All communication is wrapped with message headers.
+Format described [here](https://docs.ergoplatform.com/dev/p2p/network/#message-format).
+[MessageBase](src/main/scala/org/ergoplatform/network/message/MessageBase.scala) interface to implement.
 
 ## Syncing with the node
 

ergo-core/build.sbt

@@ -12,6 +12,9 @@ val deps212 = Seq(
   "io.circe" %% "circe-generic" % "0.13.0",
   "io.circe" %% "circe-parser" % "0.13.0")
 
+publishMavenStyle := true
+publishArtifact in Test := false
+
 libraryDependencies ++= Seq() ++
   (if (scalaVersion.value == scala211) deps211 else deps212)
 

ergo-core/src/main/scala/org/ergoplatform/network/message/MessageBase.scala

@@ -0,0 +1,48 @@
+package org.ergoplatform.network.message
+
+import org.ergoplatform.network.message.MessageConstants._
+
+import scala.util.{Success, Try}
+
+/**
+  * Trait for a ergo network message
+  *
+  * @param spec   - message specification
+  * @param input  - message being wrapped, whether in byte-array form (if from outside),
+  *               or structured data (if formed locally)
+  * @tparam Content - message data type
+  */
+trait MessageBase[Content] {
+  val spec: MessageSpec[Content]
+  val input: Either[Array[Byte], Content]
+
+  /**
+    * Message data bytes
+    */
+  lazy val dataBytes: Array[Byte] = input match {
+    case Left(db) => db
+    case Right(d) => spec.toBytes(d)
+  }
+
+  /**
+    * Structured message content
+    */
+  lazy val data: Try[Content] = input match {
+    case Left(db) => spec.parseBytesTry(db)
+    case Right(d) => Success(d)
+  }
+
+  lazy val dataLength: Int = dataBytes.length
+
+  /**
+    * @return serialized message length in bytes
+    */
+  def messageLength: Int = {
+    if (dataLength > 0) {
+      HeaderLength + ChecksumLength + dataLength
+    } else {
+      HeaderLength
+    }
+  }
+
+}

ergo-core/src/main/scala/org/ergoplatform/settings/ChainSettingsReader.scala

@@ -0,0 +1,26 @@
+package org.ergoplatform.settings
+
+import com.typesafe.config.ConfigFactory
+import net.ceedubs.ficus.Ficus._
+import net.ceedubs.ficus.readers.ArbitraryTypeReader._
+import java.io.File
+
+object ChainSettingsReader extends PowSchemeReaders with ModifierIdReader with SettingsReaders{
+  def read(path: String): Option[ChainSettings] = {
+    val file = new File(path)
+    if (file.exists) {
+      val cfg = ConfigFactory.parseFile(file)
+      val fallback = ConfigFactory.parseFile(new File("src/main/resources/application.conf"))
+      val network = ConfigFactory.parseFile(new File("src/main/resources/testnet.conf"))
+      val fullConfig = ConfigFactory
+        .defaultOverrides()
+        .withFallback(cfg) //order
+        .withFallback(network) //matters
+        .withFallback(fallback) //here
+        .resolve()
+
+      val chainSettings = fullConfig.as[ChainSettings]("ergo.chain")
+      Some(chainSettings)
+    } else None
+  }
+}

ergo-core/src/main/scala/org/ergoplatform/settings/ModifierIdReader.scala

@@ -0,0 +1,15 @@
+package org.ergoplatform.settings
+
+import com.typesafe.config.Config
+import net.ceedubs.ficus.readers.ValueReader
+import scorex.util.ModifierId
+
+trait ModifierIdReader {
+
+  implicit val modifierIdReader: ValueReader[ModifierId] = new ValueReader[ModifierId] {
+    override def read(cfg: Config, path: String): ModifierId = {
+      ModifierId @@ cfg.getString(path)
+    }
+  }
+
+}

ergo-core/src/main/scala/org/ergoplatform/settings/PowSchemeReaders.scala

@@ -0,0 +1,26 @@
+package org.ergoplatform.settings
+
+import com.typesafe.config.{Config, ConfigException}
+import net.ceedubs.ficus.Ficus._
+import net.ceedubs.ficus.readers.ValueReader
+import org.ergoplatform.mining._
+
+trait PowSchemeReaders {
+
+  implicit val powSchemeReader: ValueReader[AutolykosPowScheme] = new ValueReader[AutolykosPowScheme] {
+    override def read(cfg: Config, path: String): AutolykosPowScheme = {
+      val schemeNameKey = s"$path.powType"
+      val schemeName = cfg.getString(schemeNameKey)
+      val n = cfg.as[Int](s"$path.n")
+      val k = cfg.as[Int](s"$path.k")
+      if (schemeName == "autolykos") {
+        new AutolykosPowScheme(k, n)
+      } else if (schemeName == "fake") {
+        new DefaultFakePowScheme(k, n)
+      } else {
+        throw new ConfigException.BadValue(schemeNameKey, schemeName)
+      }
+    }
+  }
+}
+