fix: don't run tests with iterations (and a simulator) with valgrind … (

#4251)
VowpalWabbit · Oct 26, 2022 · f9ded18 · f9ded18
1 parent d41c183
commit f9ded18
Show file tree

Hide file tree

Showing 5 changed files with 21 additions and 21 deletions.
diff --git a/.github/workflows/asan.yml b/.github/workflows/asan.yml
@@ -84,7 +84,7 @@ jobs:
       - name: Build
         run: cmake --build build --target vw_cli_bin spanning_tree vw-unit-test.out
       - name: Run unit tests
-        run: ./build/test/unit_test/vw-unit-test.out
+        run: ./build/test/unit_test/vw-unit-test.out --run_test='!*w_iterations*'
       - name: Run python test script
         if: ${{ success() || failure() }}
         run: python3 test/run_tests.py -f --clean_dirty -E 0.001 --include_flatbuffers
diff --git a/test/unit_test/CMakeLists.txt b/test/unit_test/CMakeLists.txt
@@ -94,7 +94,7 @@ endif()
 if(VW_UNIT_TEST_WITH_VALGRIND_INTERNAL)
   add_test(
     NAME vw_unit_test
-    COMMAND ${VALGRIND} --error-exitcode=100 --track-origins=yes --leak-check=full $<TARGET_FILE:vw-unit-test.out>
+    COMMAND ${VALGRIND} --error-exitcode=100 --track-origins=yes --leak-check=full $<TARGET_FILE:vw-unit-test.out> "--run_test=!*w_iterations*"
     WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
   )
 else()

diff --git a/test/unit_test/automl_test.cc b/test/unit_test/automl_test.cc
@@ -131,7 +131,7 @@ using aml_onediff_inclusion = VW::reductions::automl::automl<interaction_config_
 
 // Need to add save_load functionality to multiple structs in automl reduction including
 // config_manager and confidence sequence estimator.
-BOOST_AUTO_TEST_CASE(automl_save_load)
+BOOST_AUTO_TEST_CASE(automl_save_load_w_iterations)
 {
   const size_t num_iterations = 1000;
   const size_t split = 690;
@@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(automl_save_load)
   BOOST_CHECK_EQUAL_COLLECTIONS(ctr_no_save.begin(), ctr_no_save.end(), ctr_with_save.begin(), ctr_with_save.end());
 }
 
-BOOST_AUTO_TEST_CASE(automl_assert_0th_event_automl)
+BOOST_AUTO_TEST_CASE(automl_assert_0th_event_automl_w_iterations)
 {
   const size_t zero = 0;
   const size_t num_iterations = 10;
@@ -186,7 +186,7 @@ BOOST_AUTO_TEST_CASE(automl_assert_0th_event_automl)
   BOOST_CHECK_GT(ctr.back(), 0.1f);
 }
 
-BOOST_AUTO_TEST_CASE(automl_assert_0th_event_metrics)
+BOOST_AUTO_TEST_CASE(automl_assert_0th_event_metrics_w_iterations)
 {
   const auto metric_name = std::string("total_learn_calls");
   const size_t zero = 0;
@@ -218,7 +218,7 @@ BOOST_AUTO_TEST_CASE(automl_assert_0th_event_metrics)
   BOOST_CHECK_GT(ctr.back(), 0.1f);
 }
 
-BOOST_AUTO_TEST_CASE(automl_assert_live_configs_and_lease)
+BOOST_AUTO_TEST_CASE(automl_assert_live_configs_and_lease_w_iterations)
 {
   const size_t fifteen = 15;
   const size_t thirty_three = 33;
@@ -263,15 +263,15 @@ BOOST_AUTO_TEST_CASE(automl_assert_live_configs_and_lease)
 }
 
 // Note higher ctr compared to cpp_simulator_without_interaction in tutorial_test.cc
-BOOST_AUTO_TEST_CASE(automl_cpp_simulator_automl)
+BOOST_AUTO_TEST_CASE(automl_cpp_simulator_automl_w_iterations)
 {
   auto ctr = simulator::_test_helper(
       "--cb_explore_adf --quiet --epsilon 0.2 --random_seed 5 --automl 3 --priority_type "
       "favor_popular_namespaces --oracle_type rand --global_lease 10");
   BOOST_CHECK_GT(ctr.back(), 0.6f);
 }
 
-BOOST_AUTO_TEST_CASE(automl_namespace_switch)
+BOOST_AUTO_TEST_CASE(automl_namespace_switch_w_iterations)
 {
   const size_t num_iterations = 1000;
   callback_map test_hooks;
@@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(automl_namespace_switch)
   BOOST_CHECK_GT(ctr.back(), 0.65f);
 }
 
-BOOST_AUTO_TEST_CASE(automl_clear_configs)
+BOOST_AUTO_TEST_CASE(automl_clear_configs_w_iterations)
 {
   const size_t seed = 85;
   const size_t num_iterations = 1000;
@@ -366,7 +366,7 @@ BOOST_AUTO_TEST_CASE(automl_clear_configs)
   BOOST_CHECK_GT(ctr.back(), 0.4f);
 }
 
-BOOST_AUTO_TEST_CASE(automl_clear_configs_one_diff)
+BOOST_AUTO_TEST_CASE(automl_clear_configs_one_diff_w_iterations)
 {
   const size_t num_iterations = 1000;
   const std::vector<uint64_t> swap_after = {500};
@@ -416,7 +416,7 @@ BOOST_AUTO_TEST_CASE(automl_clear_configs_one_diff)
   BOOST_CHECK_GT(ctr.back(), 0.65f);
 }
 
-BOOST_AUTO_TEST_CASE(automl_q_col_consistency)
+BOOST_AUTO_TEST_CASE(automl_q_col_consistency_w_iterations)
 {
   const size_t seed = 88;
   const size_t num_iterations = 1000;
@@ -429,7 +429,7 @@ BOOST_AUTO_TEST_CASE(automl_q_col_consistency)
   BOOST_CHECK_CLOSE(ctr_q_col.back(), ctr_aml.back(), FLOAT_TOL);
 }
 
-BOOST_AUTO_TEST_CASE(one_diff_impl_unittest)
+BOOST_AUTO_TEST_CASE(one_diff_impl_unittest_w_iterations)
 {
   using namespace VW::reductions::automl;
 
@@ -587,7 +587,7 @@ BOOST_AUTO_TEST_CASE(exc_incl_unit_test)
   BOOST_CHECK_EQUAL_COLLECTIONS(interactions.begin(), interactions.end(), expected2.begin(), expected2.end());
 }
 
-BOOST_AUTO_TEST_CASE(automl_insertion_champ_change)
+BOOST_AUTO_TEST_CASE(automl_insertion_champ_change_w_iterations)
 {
   const size_t seed = 85;
   const size_t num_iterations = 4000;

diff --git a/test/unit_test/automl_weights_test.cc b/test/unit_test/automl_weights_test.cc
@@ -149,7 +149,7 @@ bool weights_offset_test(cb_sim&, VW::workspace& all, VW::multi_ex& ec)
   return true;
 }
 
-BOOST_AUTO_TEST_CASE(automl_weight_operations)
+BOOST_AUTO_TEST_CASE(automl_weight_operations_w_iterations)
 {
   const size_t seed = 10;
   const size_t num_iterations = 1331;
@@ -193,7 +193,7 @@ bool all_weights_equal_test(cb_sim&, VW::workspace& all, VW::multi_ex& ec)
   return true;
 }
 
-BOOST_AUTO_TEST_CASE(automl_noop_samechampconfig)
+BOOST_AUTO_TEST_CASE(automl_noop_samechampconfig_w_iterations)
 {
   const size_t seed = 10;
   const size_t num_iterations = 1742;
@@ -212,7 +212,7 @@ BOOST_AUTO_TEST_CASE(automl_noop_samechampconfig)
   BOOST_CHECK_GT(ctr.back(), 0.4f);
 }
 
-BOOST_AUTO_TEST_CASE(automl_learn_order)
+BOOST_AUTO_TEST_CASE(automl_learn_order_w_iterations)
 {
   callback_map test_hooks;
 
@@ -260,7 +260,7 @@ BOOST_AUTO_TEST_CASE(automl_learn_order)
   BOOST_CHECK_EQUAL(ctr1, ctr2);
 }
 
-BOOST_AUTO_TEST_CASE(automl_equal_no_automl)
+BOOST_AUTO_TEST_CASE(automl_equal_no_automl_w_iterations)
 {
   callback_map test_hooks;
 

diff --git a/test/unit_test/epsilon_decay_test.cc b/test/unit_test/epsilon_decay_test.cc
@@ -31,7 +31,7 @@ epsilon_decay_data* get_epsilon_decay_data(VW::workspace& all)
 }
 }  // namespace epsilon_decay_test
 
-BOOST_AUTO_TEST_CASE(epsilon_decay_test_init)
+BOOST_AUTO_TEST_CASE(epsilon_decay_test_init_w_iterations)
 {
   // we initialize the reduction pointing to position 0 as champ, that config is hard-coded to empty
   auto ctr = simulator::_test_helper(
@@ -40,7 +40,7 @@ BOOST_AUTO_TEST_CASE(epsilon_decay_test_init)
   float with_save = ctr.back();
 }
 
-BOOST_AUTO_TEST_CASE(epsilon_decay_test_champ_change)
+BOOST_AUTO_TEST_CASE(epsilon_decay_test_champ_change_w_iterations)
 {
   const size_t num_iterations = 8000;
   const std::vector<uint64_t> swap_after = {5000};
@@ -74,7 +74,7 @@ BOOST_AUTO_TEST_CASE(epsilon_decay_test_champ_change)
   BOOST_CHECK_GT(ctr.back(), 0.6f);
 }
 
-BOOST_AUTO_TEST_CASE(epsilon_decay_test_update_count)
+BOOST_AUTO_TEST_CASE(epsilon_decay_test_update_count_w_iterations)
 {
   const size_t num_iterations = 105;
   const size_t seed = 100;
@@ -182,7 +182,7 @@ BOOST_AUTO_TEST_CASE(epsilon_decay_test_update_count)
   BOOST_CHECK_GT(ctr.back(), 0.5f);
 }
 
-BOOST_AUTO_TEST_CASE(epsilon_decay_test_save_load)
+BOOST_AUTO_TEST_CASE(epsilon_decay_test_save_load_w_iterations)
 {
   callback_map empty_hooks;
   auto ctr = simulator::_test_helper_hook(