Pipe test results using JSON

This PR changes how run-tests.zkasm.js communicates results to the zkasm-result.py. Before, we used to parse log lines, now we pass a json object with a test result for each of the tests. This will allow storing more detailed information like performance counters for #90 and error messages.
near · Nov 27, 2023 · 3a45aef · 3a45aef
1 parent 6c1bb2a
commit 3a45aef
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 42 deletions.
diff --git a/ci/test-zkasm.sh b/ci/test-zkasm.sh
@@ -10,4 +10,4 @@ if [ ! -d "tests/zkasm/node_modules" ]; then
 fi
 
 TEST_PATH=${1:-"cranelift/zkasm_data"}
-(npm test --prefix tests/zkasm "../../${TEST_PATH}/generated" || true) | python3 ci/zkasm-result.py $TEST_PATH
+npm test --prefix tests/zkasm "../../${TEST_PATH}/generated" | python3 ci/zkasm-result.py $TEST_PATH
diff --git a/ci/zkasm-result.py b/ci/zkasm-result.py
@@ -2,6 +2,7 @@
 import csv
 import sys
 import argparse
+import json
 
 
 parser = argparse.ArgumentParser(description='Example script to demonstrate flag usage.')
@@ -26,11 +27,13 @@ def check_compilation_status():
 
 
 def update_status_from_stdin(status_map):
-    for line in sys.stdin:
-        if "--> fail" in line or "--> pass" in line:
-            _, _, test_path = line.partition(' ')
-            test_name, _ = os.path.splitext(os.path.basename(test_path))
-            status_map[test_name] = 'pass' if 'pass' in line else 'runtime error'
+    # Skip first 4 lines that correspond to the nodejs run command message.
+    lines = sys.stdin.readlines()[4:]
+
+    for line in lines:
+        test_result = json.loads(line)
+        test_name, _ = os.path.splitext(os.path.basename(test_result["path"]))
+        status_map[test_name] = test_result["status"]
 
 
 def write_csv(status_map):

diff --git a/tests/zkasm/package.json b/tests/zkasm/package.json
@@ -16,7 +16,6 @@
     "@0xpolygonhermez/zkevm-commonjs": "github:0xPolygonHermez/zkevm-commonjs#v2.0.0-fork.5",
     "@0xpolygonhermez/zkevm-proverjs": "github:0xPolygonHermez/zkevm-proverjs#feature/64bits",
     "chai": "^4.3.6",
-    "chalk": "^3.0.0",
     "eslint": "^8.25.0",
     "eslint-config-airbnb-base": "^15.0.0",
     "eslint-plugin-mocha": "^10.1.0"

diff --git a/tests/zkasm/run-tests-zkasm.js b/tests/zkasm/run-tests-zkasm.js
@@ -3,7 +3,6 @@
 /* eslint-disable no-use-before-define */
 const path = require('path');
 const fs = require('fs');
-const chalk = require('chalk');
 const zkasm = require('@0xpolygonhermez/zkasmcom');
 const smMain = require('@0xpolygonhermez/zkevm-proverjs/src/sm/sm_main/sm_main');
 const {
@@ -14,18 +13,12 @@ const buildPoseidon = require('@0xpolygonhermez/zkevm-commonjs').getPoseidon;
 
 const emptyInput = require('@0xpolygonhermez/zkevm-proverjs/test/inputs/empty_input.json');
 
-const {
-    argv
-} = require('yargs')
-    .alias('v', 'verbose');
-
 // Global paths to build Main PIL to fill polynomials in tests
 const pathMainPil = path.join(__dirname, 'node_modules/@0xpolygonhermez/zkevm-proverjs/pil/main.pil');
 const fileCachePil = path.join(__dirname, 'node_modules/@0xpolygonhermez/zkevm-proverjs/cache-main-pil.json');
 
 async function main() {
     // Compile pil
-    console.log(chalk.yellow('--> Compile PIL'));
     const cmPols = await compilePil();
 
     // Get all zkasm files
@@ -35,22 +28,15 @@ async function main() {
     let unexpectedFailures = 0;
     let totalTests = 0;
     // Run all zkasm files
-    // eslint-disable-next-line no-restricted-syntax
-    console.log(chalk.yellow('--> Start running zkasm files'));
     for (const file of files) {
         if (file.includes('ignore'))
             continue;
-
-        let shouldFail = file.split("/").pop().startsWith("_should_fail_");
-        let testFailed = await runTest(file, cmPols);
-        if ((testFailed && !shouldFail) || (shouldFail && !testFailed)) {
-            unexpectedFailures += 1;
-        }
-        totalTests += 1;
-    }
-    console.log('Failed', unexpectedFailures, 'tests out of ', totalTests);
-    if (unexpectedFailures > 0) {
-        process.exit(1);
+
+        const testResult = await runTest(file, cmPols);
+        const json = JSON.stringify(testResult, (key, value) =>
+            typeof value === "bigint" ? value.toString() : value
+        );
+        console.log(json);
     }
 }
 
@@ -104,27 +90,23 @@ async function runTest(pathTest, cmPols) {
         stepsN: 8388608,
         assertOutputs: false,
     };
-    let failed = false;
-    // execute zkasm tests
     try {
         const rom = await zkasm.compile(pathTest, null, configZkasm);
         const result = await smMain.execute(cmPols.Main, emptyInput, rom, config);
-        console.log(chalk.green('   --> pass'), pathTest);
-        if (argv.verbose) {
-            console.log(chalk.blue('   --> verbose'));
-            console.log(chalk.blue('        --> counters'));
-            console.log(result.counters);
-            console.log(chalk.blue('        --> outputs'));
-            console.log(result.output);
-            console.log(chalk.blue('        --> logs'));
-            console.log(result.logs);
+        return {
+            path: pathTest,
+            status: "pass",
+            counters: result.counters,
+            output: result.output,
+            logs: result.logs,
         }
     } catch (e) {
-        console.log(chalk.red('   --> fail'), pathTest);
-        console.log(e);
-        failed = true;
+        return {
+            path: pathTest,
+            status: "runtime error",
+            error: JSON.stringify(e, Object.getOwnPropertyNames(e)),
+        }
     }
-    return failed;
 }
 
 main();