input-output-hk · jvdp · Apr 21, 2021 · Apr 13, 2021 · Apr 14, 2021 · Apr 14, 2021
diff --git a/.buildkite/pipeline.nix b/.buildkite/pipeline.nix
@@ -136,6 +136,13 @@ in
       command = ''
         nix-shell --run './test-ets.sh'
       '';
+      softFail = true;
+      retry.automatic = false;
+      artifactPaths = [
+        "mantis-log.txt"
+        "retesteth-GeneralStateTests-log.txt"
+        "retesteth-BlockchainTests-log.txt"
+      ];
     };
 
     coverageReport = commonAttrs // {

diff --git a/.buildkite/shell.nix b/.buildkite/shell.nix
@@ -1,7 +1,5 @@
-{ sources,  pkgs }:
-
+{ sources, pkgs }:
 let
-
   # TODO, share this code with mantis build in this project
   # sbt-protoc puts the scala plugin in /tmp/protobridge<some-random-number>.
   # it is in fact a shell script with a standard `#!/usr/bin/env sh` shebang
@@ -18,11 +16,10 @@ let
     exec ${pkgs.protobuf}/bin/protoc "$@"
   '';
 in
+with pkgs;
 
-  with pkgs;
-
-  mkShell {
-    nativeBuildInputs = [ sbt solc jdk8 protoc-wrapper ];
-    # SBT = "sbt -v -mem 2048 -J-Xmx4g -Dsbt.ivy.home=/cache/ivy2 -Dsbt.boot.directory=/cache/sbt -Dmaven.repo.local=/cache/maven -Dnix=true";
-    SBT = "sbt -v -mem 2048 -J-Xmx4g -Dnix=true";
-  }
+mkShell {
+  nativeBuildInputs = [ sbt solc lllc jdk8 protoc-wrapper retesteth netcat-gnu ];
+  # SBT = "sbt -v -mem 2048 -J-Xmx4g -Dsbt.ivy.home=/cache/ivy2 -Dsbt.boot.directory=/cache/sbt -Dmaven.repo.local=/cache/maven -Dnix=true";
+  SBT = "sbt -v -mem 2048 -J-Xmx4g -Dnix=true";
+}
diff --git a/.gitmodules b/.gitmodules
@@ -1,6 +1,7 @@
-[submodule "src/ets/resources/ets"]
-	path = src/ets/resources/ets
+[submodule "ets/tests"]
+	path = ets/tests
 	url = https://github.com/ethereum/tests
+
 [submodule "src/main/protobuf/extvm"]
 	path = src/main/protobuf/extvm
 	url = https://github.com/input-output-hk/mantis-extvm-pb
diff --git a/README.md b/README.md
@@ -116,6 +116,13 @@ In the root of the project:
 nix-build
 ```
 
+##### On a Mac
+
+This project uses Nix for CI, deployment and, optionally, local development.
+Some of the dependencies are not available for Darwin (macOS) however. To work
+with Nix on a Mac you can instead use Docker via the `nix-in-docker/run` script,
+which will start a `nix-shell` with the same environment as CI.
+
 ##### Update sbt+nix dependencies
 
 When updating project dependencies, the nix fixed-output-derivation

diff --git a/build.sbt b/build.sbt
@@ -135,8 +135,6 @@ lazy val node = {
 
   val Evm = config("evm") extend Test
 
-  val Ets = config("ets") extend Test
-
   val Rpc = config("rpcTest") extend Test
 
   val malletDeps = Seq(
@@ -182,7 +180,7 @@ lazy val node = {
 
   val node = project
     .in(file("."))
-    .configs(Integration, Benchmark, Evm, Ets, Rpc)
+    .configs(Integration, Benchmark, Evm, Rpc)
     .enablePlugins(BuildInfoPlugin)
     .dependsOn(bytes, crypto, rlp)
     .settings(
@@ -217,7 +215,6 @@ lazy val node = {
     )
     .settings(inConfig(Benchmark)(Defaults.testSettings :+ (Test / parallelExecution := false)): _*)
     .settings(inConfig(Evm)(Defaults.testSettings :+ (Test / parallelExecution := false)): _*)
-    .settings(inConfig(Ets)(Defaults.testSettings :+ (Test / parallelExecution := false)): _*)
     .settings(inConfig(Rpc)(Defaults.testSettings :+ (Test / parallelExecution := false)): _*)
     .settings(
       // protobuf compilation
@@ -266,7 +263,6 @@ addCommandAlias(
     |;test:compile
     |;evm:compile
     |;it:compile
-    |;ets:compile
     |;rpcTest:compile
     |;benchmark:compile
     |""".stripMargin

diff --git a/ci-like/Dockerfile b/ci-like/Dockerfile
diff --git a/ets/README.md b/ets/README.md
@@ -0,0 +1,51 @@
+# Ethereum Test Suite
+
+This folder contains a git submodule pointing at the Ethereum Consensus Tests,
+also known as the Ethereum Test Suite (ETS), config files for retesteth (the
+tool for running these tests) and a wrapper script to set the its command line
+options. Oh, and this readme file is in there too, of course.
+
+* ETS: https://github.com/ethereum/tests
+* retesteth: https://github.com/ethereum/retesteth
+
+## Continous integration
+
+The tests are run on CI. For more details look at `.buildkite/pipeline.nix` and
+`test-ets.sh`. Output is stored as artifacts and a summary is added as
+annotation.
+
+Two test suites are run; GeneralStateTests and BlockchainTests. These seem to
+be the only ones maintained and recommended at the moment.
+
+## Running ETS in a Nix environment
+
+Start Mantis in test mode:
+
+    sbt -Dconfig.file=./src/main/resources/conf/testmode.conf -Dlogging.logs-level=WARN run
+
+NB. raising the log level is a good idea as there will be a lot of output,
+depending on how many tests you run.
+
+Once the RPC API is up, run retesteth:
+
+    ets/retesteth -t GeneralStateTests
+
+You can also run parts of the suite; refer to `ets/retesteth --help` for details.
+
+## Running retesteth in Docker (eg. macOS)
+
+You should run Mantis outside Nix as that is probably more convenient for your
+tooling (eg. attaching a debugger.)
+
+    sbt -Dconfig.file=./src/main/resources/conf/testmode.conf -Dlogging.logs-level=WARN run
+
+Retesteth will need to be able to connect to Mantis, running on the host
+system. First, find the IP it should use:
+
+    nix-in-docker/run --command "getent hosts host.docker.internal"
+
+Edit `ets/config/mantis` and replace `0.0.0.0` with this IP.
+
+Finally, run retesteth in Nix in Docker:
+
+    nix-in-docker/run --command "ets/retesteth -t GeneralStateTests"
diff --git a/ets/config/mantis/config b/ets/config/mantis/config
@@ -0,0 +1,26 @@
+{
+    "name" : "IOHK Mantis on TCP",
+    "socketType" : "tcp",
+    "socketAddress" : [
+        "0.0.0.0:8546"
+    ],
+    "initializeTime" : "5",
+    "forks" : [
+      "Frontier",
+      "Homestead",
+      "EIP150",
+      "EIP158",
+      "Byzantium",
+      "Constantinople",
+      "ConstantinopleFix",
+      "Istanbul"
+    ],
+    "additionalForks" : [
+      "FrontierToHomesteadAt5",
+      "HomesteadToEIP150At5",
+      "EIP158ToByzantiumAt5",
+      "HomesteadToDaoAt5",
+      "ByzantiumToConstantinopleFixAt5"
+    ],
+    "exceptions" : {}
+}
diff --git a/ets/config/mantis/genesis/Berlin.json b/ets/config/mantis/genesis/Berlin.json
@@ -0,0 +1,15 @@
+{
+    "params" : {
+       "homesteadForkBlock" : "0x00",
+       "EIP150ForkBlock" : "0x00",
+       "EIP158ForkBlock" : "0x00",
+       "byzantiumForkBlock" : "0x00",
+       "constantinopleForkBlock" : "0x00",
+       "constantinopleFixForkBlock" : "0x00",
+       "istanbulForkBlock" : "0x00",
+       "berlinForkBlock" : "0x00",
+       "chainID" : "0x01"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/Byzantium.json b/ets/config/mantis/genesis/Byzantium.json
@@ -0,0 +1,10 @@
+{
+    "params" : {
+        "homesteadForkBlock" : "0x00",
+        "EIP150ForkBlock" : "0x00",
+        "EIP158ForkBlock" : "0x00",
+        "byzantiumForkBlock" : "0x00"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/ByzantiumToConstantinopleFixAt5.json b/ets/config/mantis/genesis/ByzantiumToConstantinopleFixAt5.json
@@ -0,0 +1,12 @@
+{
+    "params" : {
+      "homesteadForkBlock" : "0x00",
+      "EIP150ForkBlock" : "0x00",
+      "EIP158ForkBlock" : "0x00",
+      "byzantiumForkBlock" : "0x00",
+      "constantinopleForkBlock" : "0x05",
+      "constantinopleFixForkBlock" : "0x05"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/Constantinople.json b/ets/config/mantis/genesis/Constantinople.json
@@ -0,0 +1,11 @@
+{
+    "params" : {
+       "homesteadForkBlock" : "0x00",
+       "EIP150ForkBlock" : "0x00",
+       "EIP158ForkBlock" : "0x00",
+       "byzantiumForkBlock" : "0x00",
+       "constantinopleForkBlock" : "0x00"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/ConstantinopleFix.json b/ets/config/mantis/genesis/ConstantinopleFix.json
@@ -0,0 +1,12 @@
+{
+    "params" : {
+       "homesteadForkBlock" : "0x00",
+       "EIP150ForkBlock" : "0x00",
+       "EIP158ForkBlock" : "0x00",
+       "byzantiumForkBlock" : "0x00",
+       "constantinopleForkBlock" : "0x00",
+       "constantinopleFixForkBlock" : "0x00"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/EIP150.json b/ets/config/mantis/genesis/EIP150.json
@@ -0,0 +1,8 @@
+{
+    "params" : {
+       "homesteadForkBlock" : "0x00",
+       "EIP150ForkBlock" : "0x00"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/EIP158.json b/ets/config/mantis/genesis/EIP158.json
@@ -0,0 +1,9 @@
+{
+    "params" : {
+       "homesteadForkBlock" : "0x00",
+       "EIP150ForkBlock" : "0x00",
+       "EIP158ForkBlock" : "0x00"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/EIP158ToByzantiumAt5.json b/ets/config/mantis/genesis/EIP158ToByzantiumAt5.json
@@ -0,0 +1,10 @@
+{
+    "params" : {
+        "homesteadForkBlock" : "0x00",
+        "EIP150ForkBlock" : "0x00",
+        "EIP158ForkBlock" : "0x00",
+        "byzantiumForkBlock" : "0x05"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/Frontier.json b/ets/config/mantis/genesis/Frontier.json
@@ -0,0 +1,6 @@
+{
+    "params" : {
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/FrontierToHomesteadAt5.json b/ets/config/mantis/genesis/FrontierToHomesteadAt5.json
@@ -0,0 +1,7 @@
+{
+    "params" : {
+        "homesteadForkBlock" : "0x05"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/Homestead.json b/ets/config/mantis/genesis/Homestead.json
@@ -0,0 +1,7 @@
+{
+    "params" : {
+        "homesteadForkBlock" : "0x00"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/HomesteadToDaoAt5.json b/ets/config/mantis/genesis/HomesteadToDaoAt5.json
@@ -0,0 +1,8 @@
+{
+    "params" : {
+        "homesteadForkBlock" : "0x00",
+        "daoHardforkBlock" : "0x05"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/HomesteadToEIP150At5.json b/ets/config/mantis/genesis/HomesteadToEIP150At5.json
@@ -0,0 +1,8 @@
+{
+    "params" : {
+        "homesteadForkBlock" : "0x00",
+        "EIP150ForkBlock" : "0x05"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/Istanbul.json b/ets/config/mantis/genesis/Istanbul.json
@@ -0,0 +1,14 @@
+{
+    "params" : {
+        "homesteadForkBlock" : "0x00",
+        "EIP150ForkBlock" : "0x00",
+        "EIP158ForkBlock" : "0x00",
+        "byzantiumForkBlock" : "0x00",
+        "constantinopleForkBlock" : "0x00",
+        "constantinopleFixForkBlock" : "0x00",
+        "istanbulForkBlock" : "0x00",
+        "chainID" : "0x01"
+    },
+    "accounts" : {
+    }
+}
diff --git a/ets/config/mantis/genesis/correctMiningReward.json b/ets/config/mantis/genesis/correctMiningReward.json
@@ -0,0 +1,12 @@
+{
+    "//comment" : "State Tests does not calculate mining reward in post conditions, so when filling a blockchain test out of it, the mining reward must be set",
+    "Frontier":          "5000000000000000000",
+    "Homestead":         "5000000000000000000",
+    "EIP150":            "5000000000000000000",
+    "EIP158":            "5000000000000000000",
+    "Byzantium":         "3000000000000000000",
+    "Constantinople":    "2000000000000000000",
+    "ConstantinopleFix": "2000000000000000000",
+    "Istanbul":          "2000000000000000000",
+    "Berlin" :           "2000000000000000000"
+}
diff --git a/ets/config/version b/ets/config/version
@@ -0,0 +1 @@
+-
diff --git a/ets/retesteth b/ets/retesteth
@@ -0,0 +1,5 @@
+#!/usr/bin/env sh
+
+dir="$(cd "$(dirname "$0")" && pwd)"
+
+retesteth $@ -- --testpath $dir/tests --datadir $dir/config --clients mantis
diff --git a/ets/tests b/ets/tests
diff --git a/flake.nix b/flake.nix
@@ -49,5 +49,7 @@
       }) // (collectHydraSets [
       (mkHydraSet [ "mantis" ] [ "x86_64-linux" ])
       (mkHydraSet [ "mantis-entrypoint" ] [ "x86_64-linux" ])
+      (mkHydraSet [ "retesteth" ] [ "x86_64-linux" ])
+      (mkHydraSet [ "lllc" ] [ "x86_64-linux" ])
     ]);
 }
diff --git a/ci-like/nix.conf → nix-in-docker/nix.conf b/ci-like/nix.conf → nix-in-docker/nix.conf
@@ -1,4 +1,4 @@
-sandbox = false
+sandbox = relaxed
 experimental-features = nix-command flakes ca-references
-substituters = https://hydra.iohk.io https://cache.nixos.org https://mantis-ops.cachix.org
-trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ=
+substituters = https://hydra.iohk.io https://cache.nixos.org https://mantis-ops.cachix.org https://hydra.mantis.ist
+trusted-public-keys = hydra.iohk.io:f/Ea+s+dFdN+3Y/G+FDgSq+a5NEWhJGzdjvKNGv0/EQ= cache.nixos.org-1:6NCHdD59X431o0gWypbMrAURkbJ16ZPMQFGspcDShjY= mantis-ops.cachix.org-1:SornDcX8/9rFrpTjU+mAAb26sF8mUpnxgXNjmKGcglQ= hydra.mantis.ist-1:4LTe7Q+5pm8+HawKxvmn2Hx0E3NbkYjtf1oWv+eAmTo=