Unit Tests Now Run In Parallel (#6716)

Unit test cases are still built as a single binary, but they are now registered individually and are run in parallel by ctest in Buildkite and during coverage testing.
EOSIO · Feb 12, 2019 · 653f616 · 653f616
1 parent dcf86f0
commit 653f616
Show file tree

Hide file tree

Showing 3 changed files with 36 additions and 105 deletions.
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
@@ -15,7 +15,7 @@ include_directories("${CMAKE_SOURCE_DIR}/plugins/wallet_plugin/include")
 
 file(GLOB UNIT_TESTS "*.cpp")
 
-add_executable( plugin_test ${UNIT_TESTS} ${WASM_UNIT_TESTS} )
+add_executable( plugin_test ${UNIT_TESTS} )
 target_link_libraries( plugin_test eosio_testing eosio_chain chainbase chain_plugin wallet_plugin fc ${PLATFORM_SPECIFIC_LIBS} )
 
 target_include_directories( plugin_test PUBLIC

diff --git a/unittests/CMakeLists.txt b/unittests/CMakeLists.txt
@@ -1,6 +1,6 @@
 find_package( Gperftools QUIET )
 
-### Build contracts with cdt if available
+### Build contracts with cdt if available ###
 include(ExternalProject)
 # if no cdt root is given use default path
 if(EOSIO_CDT_ROOT STREQUAL "" OR NOT EOSIO_CDT_ROOT)
@@ -40,13 +40,11 @@ add_subdirectory(contracts)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/include/config.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/config.hpp ESCAPE_QUOTES)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/contracts.hpp.in ${CMAKE_CURRENT_BINARY_DIR}/include/contracts.hpp ESCAPE_QUOTES)
 
-file(GLOB UNIT_TESTS "*.cpp")
-
-add_executable( unit_test ${UNIT_TESTS} ${WASM_UNIT_TESTS} )
+### BUILD UNIT TEST EXECUTABLE ###
+file(GLOB UNIT_TESTS "*.cpp") # find all unit test suites
+add_executable( unit_test ${UNIT_TESTS}) # build unit tests as one executable
 target_link_libraries( unit_test eosio_chain chainbase eosio_testing fc ${PLATFORM_SPECIFIC_LIBS} )
-
 target_compile_options(unit_test PUBLIC -DDISABLE_EOSLIB_SERIALIZE)
-
 target_include_directories( unit_test PUBLIC
                             ${CMAKE_SOURCE_DIR}/libraries/testing/include
                             ${CMAKE_SOURCE_DIR}/test-contracts
@@ -55,59 +53,55 @@ target_include_directories( unit_test PUBLIC
                             ${CMAKE_CURRENT_BINARY_DIR}/contracts
                             ${CMAKE_CURRENT_BINARY_DIR}/include )
 
-#Manually run unit_test for all supported runtimes
-#To run unit_test with all log from blockchain displayed, put --verbose after --, i.e. unit_test -- --verbose
-add_test(NAME unit_test_wavm COMMAND unit_test
- -t \!wasm_tests/weighted_cpu_limit_tests
- --report_level=detailed --color_output --catch_system_errors=no -- --wavm)
- add_test(NAME unit_test_wabt COMMAND unit_test
- -t \!wasm_tests/weighted_cpu_limit_tests
- --report_level=detailed --color_output -- --wabt)
-
+### MARK TEST SUITES FOR EXECUTION ###
+foreach(TEST_SUITE ${UNIT_TESTS}) # create an independent target for each test suite
+  execute_process(COMMAND bash -c "grep -E 'BOOST_AUTO_TEST_SUITE\\s*[(]' ${TEST_SUITE} | grep -vE '//.*BOOST_AUTO_TEST_SUITE\\s*[(]' | cut -d ')' -f 1 | cut -d '(' -f 2" OUTPUT_VARIABLE SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # get the test suite name from the *.cpp file
+  if (NOT "" STREQUAL "${SUITE_NAME}") # ignore empty lines
+    execute_process(COMMAND bash -c "echo ${SUITE_NAME} | sed -e 's/s$//' | sed -e 's/_test$//'" OUTPUT_VARIABLE TRIMMED_SUITE_NAME OUTPUT_STRIP_TRAILING_WHITESPACE) # trim "_test" or "_tests" from the end of ${SUITE_NAME}
+    # to run unit_test with all log from blockchain displayed, put "--verbose" after "--", i.e. "unit_test -- --verbose"
+    add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test_wavm COMMAND unit_test --run_test=${SUITE_NAME} --report_level=detailed --color_output --catch_system_errors=no -- --wavm)
+    add_test(NAME ${TRIMMED_SUITE_NAME}_unit_test_wabt COMMAND unit_test --run_test=${SUITE_NAME} --report_level=detailed --color_output -- --wabt)
+    # build list of tests to run during coverage testing
+    if(NOT "" STREQUAL "${ctest_tests}")
+      set(ctest_tests "${ctest_tests}|${TRIMMED_SUITE_NAME}_unit_test_wavm|${TRIMMED_SUITE_NAME}_unit_test_wabt")
+    else()
+      set(ctest_tests "${TRIMMED_SUITE_NAME}_unit_test_wavm|${TRIMMED_SUITE_NAME}_unit_test_wabt")
+    endif()
+  endif()
+endforeach(TEST_SUITE)
+set(ctest_tests "'${ctest_tests}' -j8") # surround test list string in apostrophies
+
+### COVERAGE TESTING ###
 if(ENABLE_COVERAGE_TESTING)
-
   set(Coverage_NAME ${PROJECT_NAME}_ut_coverage)
-
+  # check for dependencies
   if(NOT LCOV_PATH)
     message(FATAL_ERROR "lcov not found! Aborting...")
-  endif() # NOT LCOV_PATH
-
+  endif()
   if(NOT LLVMCOV_PATH)
     message(FATAL_ERROR "llvm-cov not found! Aborting...")
-  endif() # NOT LCOV_PATH
-
+  endif()
   if(NOT GENHTML_PATH)
     message(FATAL_ERROR "genhtml not found! Aborting...")
-  endif() # NOT GENHTML_PATH
-
-  # no spaces allowed within tests list
-  set(ctest_tests 'unit_test_wabt|unit_test_wavm')
-  set(ctest_exclude_tests '')
-
-  # Setup target
+  endif()
+  # tests to skip during coverage testing
+  set(ctest_exclude_tests '') # no spaces allowed within tests list
+  # setup target
   add_custom_target(${Coverage_NAME}
-
-    # Cleanup lcov
+    # cleanup lcov
     COMMAND ${LCOV_PATH} --directory . --zerocounters
-
-    # Run tests
+    # run tests
     COMMAND ./tools/ctestwrapper.sh -R ${ctest_tests} -E ${ctest_exclude_tests}
-
     COMMAND ${LCOV_PATH} --directory . --capture --gcov-tool ${CMAKE_SOURCE_DIR}/tools/llvm-gcov.sh --output-file ${Coverage_NAME}.info
-
     COMMAND ${LCOV_PATH} -remove ${Coverage_NAME}.info '*/boost/*' '/usr/lib/*' '/usr/include/*' '*/externals/*' '*/fc/*' '*/wasm-jit/*' --output-file ${Coverage_NAME}_filtered.info
-
     COMMAND ${GENHTML_PATH} -o ${Coverage_NAME} ${PROJECT_BINARY_DIR}/${Coverage_NAME}_filtered.info
-
     COMMAND if [ "$CI" != "true" ]\; then ${CMAKE_COMMAND} -E remove ${Coverage_NAME}.base ${Coverage_NAME}.info ${Coverage_NAME}_filtered.info ${Coverage_NAME}.total ${PROJECT_BINARY_DIR}/${Coverage_NAME}.info.cleaned ${PROJECT_BINARY_DIR}/${Coverage_NAME}_filtered.info.cleaned\; fi
-
     WORKING_DIRECTORY ${PROJECT_BINARY_DIR}
     COMMENT "Resetting code coverage counters to zero. Processing code coverage counters and generating report. Report published in ./${Coverage_NAME}"
-    )
-
-  # Show info where to find the report
+  )
+  # show info where to find the report
   add_custom_command(TARGET ${Coverage_NAME} POST_BUILD
     COMMAND ;
     COMMENT "Open ./${Coverage_NAME}/index.html in your browser to view the coverage report."
-    )
+  )
 endif()
diff --git a/unittests/wasm_tests.cpp b/unittests/wasm_tests.cpp
@@ -510,69 +510,6 @@ BOOST_FIXTURE_TEST_CASE(misaligned_tests, tester ) try {
    check_aligned(misaligned_const_ref_wast);
 } FC_LOG_AND_RETHROW()
 
-// test weighted cpu limit
-BOOST_FIXTURE_TEST_CASE(weighted_cpu_limit_tests, tester ) try {
-// TODO Increase the robustness of this test.
-   resource_limits_manager mgr = control->get_mutable_resource_limits_manager();
-   create_accounts( {N(f_tests)} );
-   create_accounts( {N(acc2)} );
-   bool pass = false;
-
-   std::string code = R"=====(
-(module
-  (import "env" "require_auth" (func $require_auth (param i64)))
-  (import "env" "eosio_assert" (func $eosio_assert (param i32 i32)))
-   (table 0 anyfunc)
-   (memory $0 1)
-   (export "apply" (func $apply))
-   (func $i64_trunc_u_f64 (param $0 f64) (result i64) (i64.trunc_u/f64 (get_local $0)))
-   (func $test (param $0 i64))
-   (func $apply (param $0 i64)(param $1 i64)(param $2 i64)
-   )=====";
-   for (int i = 0; i < 1024; ++i) {
-      code += "(call $test (call $i64_trunc_u_f64 (f64.const 1)))\n";
-   }
-   code += "))";
-
-   produce_blocks(1);
-   set_code(N(f_tests), code.c_str());
-   produce_blocks(10);
-
-   mgr.set_account_limits(N(f_tests), -1, -1, 1);
-   int count = 0;
-   while (count < 4) {
-      signed_transaction trx;
-
-      for (int i = 0; i < 2; ++i) {
-         action act;
-         act.account = N(f_tests);
-         act.name = N() + (i * 16);
-         act.authorization = vector<permission_level>{{N(f_tests),config::active_name}};
-         trx.actions.push_back(act);
-      }
-
-      set_transaction_headers(trx);
-      trx.sign(get_private_key( N(f_tests), "active" ), control->get_chain_id());
-
-      try {
-         push_transaction(trx, fc::time_point::maximum(), 0);
-         produce_block();
-         BOOST_REQUIRE_EQUAL(true, chain_has_transaction(trx.id()));
-         pass = true;
-         count++;
-      } catch( eosio::chain::leeway_deadline_exception& ) {
-         BOOST_REQUIRE_EQUAL(count, 3);
-         break;
-      }
-      BOOST_REQUIRE_EQUAL(true, validate());
-
-      if (count == 2) { // add a big weight on acc2, making f_tests out of resource
-        mgr.set_account_limits(N(acc2), -1, -1, 100000000);
-      }
-   }
-   BOOST_REQUIRE_EQUAL(count, 3);
-} FC_LOG_AND_RETHROW()
-
 /**
  * Make sure WASM "start" method is used correctly
  */