Unit Tests Now Run In Parallel

tests/CMakeLists.txt

@@ -15,7 +15,7 @@ include_directories("${CMAKE_SOURCE_DIR}/plugins/wallet_plugin/include")
 
 file(GLOB UNIT_TESTS "*.cpp")
 
-add_executable( plugin_test ${UNIT_TESTS} ${WASM_UNIT_TESTS} )
+add_executable( plugin_test ${UNIT_TESTS} )
 target_link_libraries( plugin_test eosio_testing eosio_chain chainbase chain_plugin wallet_plugin fc ${PLATFORM_SPECIFIC_LIBS} )
 
 target_include_directories( plugin_test PUBLIC

unittests/CMakeLists.txt

@@ -1,6 +1,6 @@
 find_package( Gperftools QUIET )
 
-### Build contracts with cdt if available
+### Build contracts with cdt if available ###
 include(ExternalProject)
 # if no cdt root is given use default path
 if(EOSIO_CDT_ROOT STREQUAL "" OR NOT EOSIO_CDT_ROOT)
@@ -40,13 +40,11 @@ add_subdirectory(contracts)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/config.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/config.hpp ESCAPE_QUOTES)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/contracts.hpp ESCAPE_QUOTES)
 
-file(GLOB UNIT_TESTS "*.cpp")
-
-add_executable( unit_test ${UNIT_TESTS} ${WASM_UNIT_TESTS} )
+### BUILD UNIT TEST EXECUTABLE ###
+file(GLOB UNIT_TESTS "*.cpp") # find all unit test suites
+add_executable( unit_test ${UNIT_TESTS}) # build unit tests as one executable
 target_link_libraries( unit_test eosio_chain chainbase eosio_testing fc ${PLATFORM_SPECIFIC_LIBS} )
-
 target_compile_options(unit_test PUBLIC -DDISABLE_EOSLIB_SERIALIZE)
-
 target_include_directories( unit_test PUBLIC
                             ${CMAKE_SOURCE_DIR}/libraries/testing/include
                             ${CMAKE_SOURCE_DIR}/test-contracts
@@ -55,59 +53,60 @@ target_include_directories( unit_test PUBLIC
                             ${CMAKE_CURRENT_BINARY_DIR}/contracts
                             ${CMAKE_CURRENT_BINARY_DIR}/include )
 
-#Manually run unit_test for all supported runtimes
-#To run unit_test with all log from blockchain displayed, put --verbose after --, i.e. unit_test -- --verbose
-add_test(NAME unit_test_wavm COMMAND unit_test
- -t \!wasm_tests/weighted_cpu_limit_tests
- --report_level=detailed --color_output --catch_system_errors=no -- --wavm)
- add_test(NAME unit_test_wabt COMMAND unit_test
- -t \!wasm_tests/weighted_cpu_limit_tests
- --report_level=detailed --color_output -- --wabt)
-
+### MARK TEST SUITES FOR EXECUTION ###
+foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test suite
+  execute_process(COMMAND bash -c "grep 'BOOST_AUTO_TEST_SUITE(' ${TEST_SUITE} | cut -d ')' -f 1 | cut -d '(' -f 2" OUTPUT_VARIABLE SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # get the test suite name from the *.cpp file
+  if (NOT "" STREQUAL "${SUITE_NAME}") # ignore empty lines
+    execute_process(COMMAND bash -c "echo ${SUITE_NAME} | sed -e 's/s$//' | sed -e 's/_test$//'" OUTPUT_VARIABLE TRIMMED_SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # trim "_test" or "_tests" from the end of ${SUITE_NAME}
+    # to run unit_test with all log from blockchain displayed, put "--verbose" after "--", i.e. "unit_test -- --verbose"
+    if("wasm" STREQUAL "${TRIMMED_SUITE_NAME}") # edge case: exclude some test cases from suite
+      add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test_wavm COMMAND unit_test --run_test=wasm_tests -t \!wasm_tests/weighted_cpu_limit_tests --report_level=detailed --color_output --catch_system_errors=no -- --wavm)
+      add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test_wabt COMMAND unit_test --run_test=wasm_tests -t \!wasm_tests/weighted_cpu_limit_tests --report_level=detailed --color_output -- --wabt)
+    else() # run all test cases in suite
+      add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test_wavm COMMAND unit_test --run_test=${SUITE_NAME} --report_level=detailed --color_output --catch_system_errors=no -- --wavm)
+      add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test_wabt COMMAND unit_test --run_test=${SUITE_NAME} --report_level=detailed --color_output -- --wabt)
+    endif()
+    # build list of tests to run during coverage testing
+    if(NOT "" STREQUAL "${ctest_tests}")
+      set(ctest_tests "${ctest_tests}|${TRIMMED_SUITE_NAME}_unit_test_wavm|${TRIMMED_SUITE_NAME}_unit_test_wabt")
+    else()
+      set(ctest_tests "${TRIMMED_SUITE_NAME}_unit_test_wavm|${TRIMMED_SUITE_NAME}_unit_test_wabt")
+    endif()
+  endif()
+endforeach(TEST_SUITE)
+set(ctest_tests "'${ctest_tests}' -j8") # surround test list string in apostrophies
+
+### COVERAGE TESTING ###
 if(ENABLE_COVERAGE_TESTING)
-
   set(Coverage_NAME ${PROJECT_NAME}_ut_coverage)
-
+  # check for dependencies
   if(NOT LCOV_PATH)
     message(FATAL_ERROR "lcov not found! Aborting...")
-  endif() # NOT LCOV_PATH
-
+  endif()
   if(NOT LLVMCOV_PATH)
     message(FATAL_ERROR "llvm-cov not found! Aborting...")
-  endif() # NOT LCOV_PATH
-
+  endif()
   if(NOT GENHTML_PATH)
     message(FATAL_ERROR "genhtml not found! Aborting...")
-  endif() # NOT GENHTML_PATH
-
-  # no spaces allowed within tests list
-  set(ctest_tests 'unit_test_wabt|unit_test_wavm')
-  set(ctest_exclude_tests '')
-
-  # Setup target
+  endif()
+  # tests to skip during coverage testing
+  set(ctest_exclude_tests '') # no spaces allowed within tests list
+  # setup target
   add_custom_target(${Coverage_NAME}
-
-    # Cleanup lcov
+    # cleanup lcov
     COMMAND ${LCOV_PATH} --directory . --zerocounters
-
-    # Run tests
+    # run tests
     COMMAND ./tools/ctestwrapper.sh -R ${ctest_tests} -E ${ctest_exclude_tests}
-
     COMMAND ${LCOV_PATH} --directory . --capture --gcov-tool ${CMAKE_SOURCE_DIR}/tools/llvm-gcov.sh --output-file ${Coverage_NAME}.info
-
     COMMAND ${LCOV_PATH} -remove ${Coverage_NAME}.info '*/boost/*' '/usr/lib/*' '/usr/include/*' '*/externals/*' '*/fc/*' '*/wasm-jit/*' --output-file ${Coverage_NAME}_filtered.info
-
     COMMAND ${GENHTML_PATH} -o ${Coverage_NAME} ${PROJECT_BINARY_DIR}/${Coverage_NAME}_filtered.info
-
     COMMAND if [ "$CI" != "true" ]\; then ${CMAKE_COMMAND} -E remove ${Coverage_NAME}.base ${Coverage_NAME}.info ${Coverage_NAME}_filtered.info ${Coverage_NAME}.total ${PROJECT_BINARY_DIR}/${Coverage_NAME}.info.cleaned ${PROJECT_BINARY_DIR}/${Coverage_NAME}_filtered.info.cleaned\; fi
-
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
     COMMENT "Resetting code coverage counters to zero. Processing code coverage counters and generating report. Report published in ./${Coverage_NAME}"
-    )
-
-  # Show info where to find the report
+  )
+  # show info where to find the report
   add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
     COMMAND ;
     COMMENT "Open ./${Coverage_NAME}/index.html in your browser to view the coverage report."
-    )
+  )
 endif()

unittests/wasm_tests.cpp

@@ -512,16 +512,16 @@ BOOST_FIXTURE_TEST_CASE(misaligned_tests, tester ) try {
 
 // test weighted cpu limit
 BOOST_FIXTURE_TEST_CASE(weighted_cpu_limit_tests, tester ) try {
-// TODO Increase the robustness of this test.
+   // TODO Increase the robustness of this test.
    resource_limits_manager mgr = control->get_mutable_resource_limits_manager();
    create_accounts( {N(f_tests)} );
    create_accounts( {N(acc2)} );
    bool pass = false;
 
    std::string code = R"=====(
-(module
-  (import "env" "require_auth" (func $require_auth (param i64)))
-  (import "env" "eosio_assert" (func $eosio_assert (param i32 i32)))
+   (module
+   (import "env" "require_auth" (func $require_auth (param i64)))
+   (import "env" "eosio_assert" (func $eosio_assert (param i32 i32)))
    (table 0 anyfunc)
    (memory $0 1)
    (export "apply" (func $apply))