Merge pull request #111 from AntelopeIO/desubmod_wasm_spec_tests

[3.2] desubmodule wasm-spec-tests

.github/workflows/build.yaml

@@ -123,8 +123,6 @@ jobs:
     container: ${{fromJSON(needs.d.outputs.p)[matrix.platform].image}}
     steps:
       - uses: actions/checkout@v3
-        with:
-          submodules: true
       - name: Download builddir
         uses: actions/download-artifact@v3
         with:

.gitmodules

@@ -22,6 +22,3 @@
 [submodule "tests/abieos"]
 	path = tests/abieos
 	url = https://github.com/AntelopeIO/abieos
-[submodule "wasm-spec-tests"]
-	path = wasm-spec-tests
-	url = https://github.com/AntelopeIO/wasm-spec-tests

CMakeLists.txt

@@ -185,10 +185,6 @@ add_subdirectory( benchmark )
 
 option(DISABLE_WASM_SPEC_TESTS "disable building of wasm spec unit tests" OFF)
 
-if (NOT DISABLE_WASM_SPEC_TESTS)
-add_subdirectory( wasm-spec-tests/generated-tests )
-endif()
-
 install(FILES testnet.template DESTINATION ${CMAKE_INSTALL_FULL_SYSCONFDIR}/eosio/launcher COMPONENT dev EXCLUDE_FROM_ALL)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testnet.template ${CMAKE_CURRENT_BINARY_DIR}/etc/eosio/launcher/testnet.template COPYONLY)
 

unittests/CMakeLists.txt

@@ -36,6 +36,10 @@ endif()
 
 add_subdirectory(contracts)
 
+if (NOT DISABLE_WASM_SPEC_TESTS)
+add_subdirectory( wasm-spec-tests/generated-tests )
+endif()
+
 add_subdirectory(snapshots)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/snapshots.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/snapshots.hpp ESCAPE_QUOTES)
 

unittests/wasm-spec-tests/README.md

@@ -0,0 +1,56 @@
+# WASM Spec Tests
+
+This directory provides a set of unit tests that can be used to check a WASM Backend's conformance to the
+WebAssembly spec.
+
+## Tests
+### How tests are generated
+1. The JSON file for a spec test suite is read.
+2. For each spec test wasm defined in the JSON:
+    - All the spec tests are created in a C++ file to match the function declaration as interpreted from the JSON.
+    - Test are categorized into `assert_trap`/`assert_exhaustion` and `assert_return`.
+    - Tests are split up into `sub_apply` functions based on the rules defined below.
+    - An `apply` function is written that switches on the third parameter to decide which `sub_apply` function to run.
+    - A map of the function name and the index in order is created to be used for merging.
+3. Unit Tests are generated based on the rules below.
+4. The generated C++ files are compiled and linked, without optimizations to prevent the empty functions from being optimized out.
+5. The generated WASM is combined with the original test wasm.
+    - The imports and apply functions (and any helper functions) from the generated wasm are combined with the test function definitions from the spec test wasm.
+    - Any necessary shifting of type/import/function/call/exports numbers is done.
+        - This is where the generated map from above is used.
+6. The newly created merged wasms and unit test C++ files are copied into the appropriate directory in the Leap repo.
+
+
+### How tests are split up
+- Within a spec test suite, each `assert_trap` and `assert_exhaustion` test case is given a unique `sub_apply` function.
+    - All tests in a suite are in the same WASM file, so the test that is run is based on the `test.name` passed in to `apply` (which calls the correct `sub_apply`).
+- Within a test suite, `assert_return` tests are grouped into sets of 100.
+    - This is due to the limit on 1024 locals and 1024 func defs built into nodeos. Some spec tests had too many functions to have a `sub_apply` per test, and some had too many variables to be put all into one `sub_apply`.
+    - 100 was found to be the number that did not exceed this maximum for all the tests.
+    - The tests also have some reliance on ordering (a store may need to be called before a load for example).
+    - 100 also works out to make sure the right ordering is achieved.
+
+- The unit tests are split into 2 groups. All of the `assert_trap` tests are grouped into one `BOOST_DATA_TEST_CASE` and all the `assert_return` tests are grouped into a second `BOOST_DATA_TEST_CASE`
+- The unit test files are grouped by test suite (all `address` tests are together, all `call` tests together, etc.)
+
+
+### How to generate tests
+- Run the `setup_eosio_tests.py` script with no options to see the help text.
+
+
+### Known Issues
+- memory_grow.3 -- Will fail if not deleted from generated tests.
+    - Unclear how to hand alter this to have memory properly zeroed where expected.
+
+- start.7 -- Will fail if not deleted from generated tests.
+    - Imports "print" from "spectest". Changing to any of the `eosio::print` functions results in "start function must be nullary" due to their requiring a parameter.
+
+- globals.2 -- Delete from generated tests or it segfaults due to missing wasm.
+    - `eosio-wasm2wast` error "mutable globals cannot be exported" when converting to wast.
+        - `wasm2wat` provided by WABT handles this correctly, implying an error in CDT.
+- globals.3 -- Delete from generated tests or it segfaults due to missing wasm.
+    - `eosio-wasm2wast` error "mutable globals cannot be exported" when converting to wast.
+        - `wasm2wat` provided by WABT handles this correctly, implying an error in CDT.
+- globals.14 -- Delete from generated tests or it segfaults due to missing wasm.
+    - Imports "global_i32" from "spectest".
+        - Unclear what an appropriate substition would be.

unittests/wasm-spec-tests/generated-tests/CMakeLists.txt

@@ -0,0 +1,39 @@
+### Build contracts with cdt if available ###
+include(ExternalProject)
+
+set(SPEC_TEST_WASM_DIR "${CMAKE_CURRENT_SOURCE_DIR}/wasms")
+configure_file(${CMAKE_CURRENT_SOURCE_DIR}/wasm_spec_tests.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/wasm_spec_tests.hpp ESCAPE_QUOTES)
+
+### BUILD UNIT TEST EXECUTABLE ###
+file(GLOB WASM_TESTS "*.cpp") # find all unit test suites
+add_executable( wasm_spec_test ${WASM_TESTS}) # build unit tests as one executable
+
+target_link_libraries( wasm_spec_test eosio_chain_wrap chainbase eosio_testing fc appbase ${PLATFORM_SPECIFIC_LIBS} )
+
+target_compile_options(wasm_spec_test PUBLIC -DDISABLE_EOSLIB_SERIALIZE)
+target_include_directories( wasm_spec_test PUBLIC
+                            ${CMAKE_SOURCE_DIR}/libraries/testing/include
+                            ${CMAKE_CURRENT_BINARY_DIR}/include )
+
+### MARK TEST SUITES FOR EXECUTION ###
+foreach(TEST_SUITE ${WASM_TESTS}) # create an independent target for each test suite
+  execute_process(COMMAND bash -c "grep -Eo 'BOOST_DATA_TEST_CASE\\(\\w*' '${TEST_SUITE}' | cut -c 22-" OUTPUT_VARIABLE SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # get the test suite name from the *.cpp file
+  if (NOT "" STREQUAL "${SUITE_NAME}") # ignore empty lines
+    string(REPLACE "\n" ";" SN_LIST ${SUITE_NAME})
+    foreach(SN ${SN_LIST})
+      foreach(RUNTIME ${EOSIO_WASM_RUNTIMES})
+          get_test_property(${SN}_unit_test_${RUNTIME} LABELS TEST_LABEL)
+          if ("NOTFOUND" STREQUAL "${TEST_LABEL}") # prevent duplicates
+            add_test(NAME ${SN}_unit_test_${RUNTIME} COMMAND wasm_spec_test --run_test=${SN} --report_level=detailed --color_output --catch_system_errors=no -- --${RUNTIME})
+            set_property(TEST ${SN}_unit_test_${RUNTIME} PROPERTY LABELS wasm_spec_tests)
+            # build list of tests to run during coverage testing
+            if(ctest_tests)
+                string(APPEND ctest_tests "|")
+            endif()
+            string(APPEND ctest_tests "${SN}_unit_test_${RUNTIME}")
+          endif()
+      endforeach()
+    endforeach(SN)
+  endif()
+endforeach(TEST_SUITE)
+set(ctest_tests "'${ctest_tests}' -j8") # surround test list string in apostrophies

unittests/wasm-spec-tests/generated-tests/address.cpp

@@ -0,0 +1,153 @@
+#include <wasm_spec_tests.hpp>
+
+const string wasm_str_address_0 = base_dir + "/address.0.wasm";
+std::vector<uint8_t> wasm_address_0= read_wasm(wasm_str_address_0.c_str());
+
+BOOST_DATA_TEST_CASE(address_0_check_throw, boost::unit_test::data::xrange(0,11), index) { try {
+   TESTER tester;
+   tester.produce_block();
+   tester.create_account( "wasmtest"_n );
+   tester.produce_block();
+   tester.set_code("wasmtest"_n, wasm_address_0);
+   tester.produce_block();
+
+   action test;
+   test.account = "wasmtest"_n;
+   test.name = account_name((uint64_t)index);
+   test.authorization = {{"wasmtest"_n, config::active_name}};
+
+   BOOST_CHECK_THROW(push_action(tester, std::move(test), "wasmtest"_n.to_uint64_t()), wasm_execution_error);
+   tester.produce_block();
+} FC_LOG_AND_RETHROW() }
+
+BOOST_DATA_TEST_CASE(address_0_pass, boost::unit_test::data::xrange(11,12), index) { try {
+   TESTER tester;
+   tester.produce_block();
+   tester.create_account( "wasmtest"_n );
+   tester.produce_block();
+   tester.set_code("wasmtest"_n, wasm_address_0);
+   tester.produce_block();
+
+   action test;
+   test.account = "wasmtest"_n;
+   test.name = account_name((uint64_t)index);
+   test.authorization = {{"wasmtest"_n, config::active_name}};
+
+   push_action(tester, std::move(test), "wasmtest"_n.to_uint64_t());
+   tester.produce_block();
+   BOOST_REQUIRE_EQUAL( tester.validate(), true );
+} FC_LOG_AND_RETHROW() }
+
+const string wasm_str_address_2 = base_dir + "/address.2.wasm";
+std::vector<uint8_t> wasm_address_2= read_wasm(wasm_str_address_2.c_str());
+
+BOOST_DATA_TEST_CASE(address_2_check_throw, boost::unit_test::data::xrange(0,15), index) { try {
+   TESTER tester;
+   tester.produce_block();
+   tester.create_account( "wasmtest"_n );
+   tester.produce_block();
+   tester.set_code("wasmtest"_n, wasm_address_2);
+   tester.produce_block();
+
+   action test;
+   test.account = "wasmtest"_n;
+   test.name = account_name((uint64_t)index);
+   test.authorization = {{"wasmtest"_n, config::active_name}};
+
+   BOOST_CHECK_THROW(push_action(tester, std::move(test), "wasmtest"_n.to_uint64_t()), wasm_execution_error);
+   tester.produce_block();
+} FC_LOG_AND_RETHROW() }
+
+BOOST_DATA_TEST_CASE(address_2_pass, boost::unit_test::data::xrange(15,17), index) { try {
+   TESTER tester;
+   tester.produce_block();
+   tester.create_account( "wasmtest"_n );
+   tester.produce_block();
+   tester.set_code("wasmtest"_n, wasm_address_2);
+   tester.produce_block();
+
+   action test;
+   test.account = "wasmtest"_n;
+   test.name = account_name((uint64_t)index);
+   test.authorization = {{"wasmtest"_n, config::active_name}};
+
+   push_action(tester, std::move(test), "wasmtest"_n.to_uint64_t());
+   tester.produce_block();
+   BOOST_REQUIRE_EQUAL( tester.validate(), true );
+} FC_LOG_AND_RETHROW() }
+
+const string wasm_str_address_3 = base_dir + "/address.3.wasm";
+std::vector<uint8_t> wasm_address_3= read_wasm(wasm_str_address_3.c_str());
+
+BOOST_DATA_TEST_CASE(address_3_check_throw, boost::unit_test::data::xrange(0,3), index) { try {
+   TESTER tester;
+   tester.produce_block();
+   tester.create_account( "wasmtest"_n );
+   tester.produce_block();
+   tester.set_code("wasmtest"_n, wasm_address_3);
+   tester.produce_block();
+
+   action test;
+   test.account = "wasmtest"_n;
+   test.name = account_name((uint64_t)index);
+   test.authorization = {{"wasmtest"_n, config::active_name}};
+
+   BOOST_CHECK_THROW(push_action(tester, std::move(test), "wasmtest"_n.to_uint64_t()), wasm_execution_error);
+   tester.produce_block();
+} FC_LOG_AND_RETHROW() }
+
+BOOST_DATA_TEST_CASE(address_3_pass, boost::unit_test::data::xrange(3,4), index) { try {
+   TESTER tester;
+   tester.produce_block();
+   tester.create_account( "wasmtest"_n );
+   tester.produce_block();
+   tester.set_code("wasmtest"_n, wasm_address_3);
+   tester.produce_block();
+
+   action test;
+   test.account = "wasmtest"_n;
+   test.name = account_name((uint64_t)index);
+   test.authorization = {{"wasmtest"_n, config::active_name}};
+
+   push_action(tester, std::move(test), "wasmtest"_n.to_uint64_t());
+   tester.produce_block();
+   BOOST_REQUIRE_EQUAL( tester.validate(), true );
+} FC_LOG_AND_RETHROW() }
+
+const string wasm_str_address_4 = base_dir + "/address.4.wasm";
+std::vector<uint8_t> wasm_address_4= read_wasm(wasm_str_address_4.c_str());
+
+BOOST_DATA_TEST_CASE(address_4_check_throw, boost::unit_test::data::xrange(0,3), index) { try {
+   TESTER tester;
+   tester.produce_block();
+   tester.create_account( "wasmtest"_n );
+   tester.produce_block();
+   tester.set_code("wasmtest"_n, wasm_address_4);
+   tester.produce_block();
+
+   action test;
+   test.account = "wasmtest"_n;
+   test.name = account_name((uint64_t)index);
+   test.authorization = {{"wasmtest"_n, config::active_name}};
+
+   BOOST_CHECK_THROW(push_action(tester, std::move(test), "wasmtest"_n.to_uint64_t()), wasm_execution_error);
+   tester.produce_block();
+} FC_LOG_AND_RETHROW() }
+
+BOOST_DATA_TEST_CASE(address_4_pass, boost::unit_test::data::xrange(3,4), index) { try {
+   TESTER tester;
+   tester.produce_block();
+   tester.create_account( "wasmtest"_n );
+   tester.produce_block();
+   tester.set_code("wasmtest"_n, wasm_address_4);
+   tester.produce_block();
+
+   action test;
+   test.account = "wasmtest"_n;
+   test.name = account_name((uint64_t)index);
+   test.authorization = {{"wasmtest"_n, config::active_name}};
+
+   push_action(tester, std::move(test), "wasmtest"_n.to_uint64_t());
+   tester.produce_block();
+   BOOST_REQUIRE_EQUAL( tester.validate(), true );
+} FC_LOG_AND_RETHROW() }