feat: spin off automl predict_only_model to standard cb model

test/core.vwtest.json

@@ -5375,20 +5375,6 @@
   },
   {
     "id": 417,
-    "desc": "automl with cb_adf - transform to predict only model SkipC# - Do not remove this - this test breaks test generation by requiring a dependency that is not supported",
-    "vw_command": "--aml_predict_only_model automl_predict_only.model -i models/automl.model --readable_model automl_readable.txt",
-    "diff_files": {
-      "automl_readable.txt": "train-sets/ref/cbadf_automl_readable_predictonly.txt"
-    },
-    "input_files": [
-      "models/automl.model"
-    ],
-    "depends_on": [
-      411
-    ]
-  },
-  {
-    "id": 418,
     "desc": "set up automl model to test overrides",
     "vw_command": "-d train-sets/decisionservice.json --dsjson --cb_explore_adf -f models/automl_override.model --automl 4 --global_lease 100 --priority_type favor_popular_namespaces --priority_challengers 1 --oracle_type rand --automl_significance_level .1",
     "diff_files": {
@@ -5400,7 +5386,7 @@
     ]
   },
   {
-    "id": 419,
+    "id": 418,
     "desc": "load automl model with overrides",
     "vw_command": "-d train-sets/decisionservice.json --dsjson --cb_explore_adf -i models/automl_override.model --automl 4 --global_lease 200 --priority_type none --priority_challengers 2 --oracle_type one_diff --automl_significance_level .2",
     "diff_files": {
@@ -5412,11 +5398,11 @@
       "models/automl_override.model"
     ],
     "depends_on": [
-      418
+      417
     ]
   },
   {
-    "id": 420,
+    "id": 419,
     "desc": "Save model using ksvm to test flat_example save_load",
     "vw_command": "--ksvm -d train-sets/tagged_data.dat -f models/ksvm_flat.model",
     "diff_files": {
@@ -5428,7 +5414,7 @@
     ]
   },
   {
-    "id": 421,
+    "id": 420,
     "desc": "Load model using ksvm to test flat_example save_load",
     "vw_command": "--ksvm -d train-sets/tagged_data.dat -i models/ksvm_flat.model",
     "diff_files": {
@@ -5440,11 +5426,11 @@
       "models/ksvm_flat.model"
     ],
     "depends_on": [
-      420
+      419
     ]
   },
   {
-    "id": 422,
+    "id": 421,
     "desc": "explore_eval evaluate same policy as logged policy, all examples should be used to update with default block_size",
     "vw_command": "--cb_explore_adf -d train-sets/explore_eval_egreedy.dat --coin --epsilon 0.1 -q UA --explore_eval",
     "diff_files": {
@@ -5455,7 +5441,7 @@
     ]
   },
   {
-    "id": 423,
+    "id": 422,
     "desc": "explore_eval evaluate same policy as logged policy, with block_size > 1 there should be at most ((#examples / block_size) + 1) updates",
     "vw_command": "--cb_explore_adf -d train-sets/explore_eval_egreedy.dat --coin --epsilon 0.1 -q UA --explore_eval --block_size 5",
     "diff_files": {
@@ -5465,8 +5451,74 @@
       "train-sets/explore_eval_egreedy.dat"
     ]
   },
+  {
+    "id": 423,
+    "desc": "Spin off automl model to lower bitsize to cb_explore",
+    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -f models/aml_spin_off.m -b 18 --predict_only_model --automl 2",
+    "diff_files": {
+      "stderr": "train-sets/ref/spin_off_aml_save.stderr"
+    },
+    "input_files": [
+      "train-sets/automl_spin_off.txt"
+    ]
+  },
   {
     "id": 424,
+    "desc": "Load spun off automl model and output invert hash",
+    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -i models/aml_spin_off.m -b 17 --invert_hash aml_spinoff.inv --predict_only_model",
+    "diff_files": {
+      "stderr": "train-sets/ref/spin_off_aml_load.stderr",
+      "aml_spinoff.inv": "pred-sets/ref/aml_spinoff.inv"
+    },
+    "input_files": [
+      "train-sets/automl_spin_off.txt",
+      "models/aml_spin_off.m"
+    ],
+    "depends_on": [
+      423
+    ]
+  },
+  {
+    "id": 425,
+    "desc": "Generate similar model to spun off automl using standard cb_explore_adf",
+    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -f models/aml_spin_off_cb.m -b 17 --predict_only_model --interactions \\x20\\x20 --interactions \\x20A --interactions \\x20B --interactions AA --interactions AB --interactions BB",
+    "diff_files": {
+      "stderr": "train-sets/ref/cb_similar_aml_spinoff_save.stderr"
+    },
+    "input_files": [
+      "train-sets/automl_spin_off.txt"
+    ]
+  },
+  {
+    "id": 426,
+    "desc": "Check that standard cb_explore_adf model has same invert hash as automl spinoff",
+    "vw_command": "--cb_explore_adf -d train-sets/automl_spin_off.txt --noconstant -i models/aml_spin_off_cb.m -b 17 --invert_hash aml_spinoff_cb.inv --predict_only_model",
+    "diff_files": {
+      "stderr": "train-sets/ref/spin_off_aml_load.stderr",
+      "aml_spinoff_cb.inv": "pred-sets/ref/aml_spinoff.inv"
+    },
+    "input_files": [
+      "train-sets/automl_spin_off.txt",
+      "models/aml_spin_off_cb.m"
+    ],
+    "depends_on": [
+      425
+    ]
+  },
+  {
+    "id": 427,
+    "desc": "Check that binary model files are identical between standard cb_explore_adf and automl spinoff",
+    "bash_command": "diff models/aml_spin_off.m models/aml_spin_off_cb.m",
+    "input_files": [
+      "models/aml_spin_off.m",
+      "models/aml_spin_off_cb.m"
+    ],
+    "depends_on": [
+      423, 425
+    ]
+  },
+  {
+    "id": 428,
     "desc": "creating model to be tested with -q :: and --invert_hash and hexfloat",
     "vw_command": "-d train-sets/ccb_test_interactions.dat --ccb_explore_adf -q :: -f models/288.model --invert_hash ccb_quad.inv --hexfloat_weights",
     "diff_files": {

test/pred-sets/ref/aml_spinoff.inv

@@ -0,0 +1,40 @@
+Version 9.6.0
+Id 
+Min label:0
+Max label:1
+bits:17
+lda:0
+0 ngram:
+0 skip:
+options: --cb_adf --cb_explore_adf --cb_type mtr --csoaa_ldf multiline --csoaa_rank --interactions \x20\x20 --interactions \x20A --interactions \x20B --interactions AA --interactions AB --interactions BB --noconstant
+Checksum: 1716779394
+event_sum 6
+action_sum 14
+:0
+B^b_2*B^b_2:3866:0.107581
+c_2*d_2:7150:0.108086
+B^b_2*B^d_2:17576:0.107581
+A^a_1*A^c_1:34377:0.106211
+A^a_1*A^b_1:39642:0.106211
+B^c_2*B^c_2:43272:0.107581
+A^b_1*A^c_1:43331:0.106211
+b_2*c_2:43423:0.108086
+A^b_1*A^b_1:46544:0.106211
+c_2:47372:0.108086
+B^c_2:61852:0.107581
+A^a_1*A^a_1:63924:0.106211
+A^c_1*A^c_1:64090:0.106211
+b_2*b_2:66962:0.108086
+B^b_2:67493:0.107581
+B^c_2*B^d_2:70787:0.107581
+b_2:70913:0.108086
+A^a_1:71542:0.106211
+d_2*d_2:75700:0.108086
+b_2*d_2:83609:0.108086
+B^d_2:85015:0.107581
+d_2:87562:0.108086
+A^c_1:92299:0.106211
+A^b_1:95256:0.106211
+B^d_2*B^d_2:99362:0.107581
+c_2*c_2:128232:0.108086
+B^b_2*B^c_2:129315:0.107581

test/run_tests.py

@@ -776,6 +776,7 @@ def convert_tests_for_flatbuffers(
         # (324-326) deals with corrupted data, so cannot be translated to fb
         # pdrop is not supported in fb, so 327-331 are excluded
         # 336, 337, 338 - the FB converter script seems to be affecting the invert_hash
+        # 423, 424, 425, 426 - FB converter removes feature names from invert_hash (probably the same issue as above)
         if str(test.id) in (
             "300",
             "189",
@@ -803,6 +804,10 @@ def convert_tests_for_flatbuffers(
             "407",
             "411",
             "415",
+            "423",
+            "424",
+            "425",
+            "426",
         ):
             test.skip = True
             test.skip_reason = "test skipped for automatic converted flatbuffer tests for unknown reason"

test/train-sets/automl_spin_off.txt

@@ -0,0 +1,9 @@
+1:1:1 |A a_1 b_1 c_1
+ |A a_2 b_2 c_2
+ |A a_3 b_3 c_3
+
+ |B b_1 c_1 d_1
+1:1:1 |B b_2 c_2 d_2
+
+ | b_1 c_1 d_1
+1:1:1 | b_2 c_2 d_2

test/train-sets/ref/cb_similar_aml_spinoff_save.stderr

@@ -0,0 +1,24 @@
+creating features for following interactions: x20x20 x20A x20B AA AB BB
+final_regressor = models/aml_spin_off_cb.m
+using no cache
+Reading datafile = train-sets/automl_spin_off.txt
+num sources = 1
+Num weight bits = 17
+learning rate = 0.5
+initial_t = 0
+power_t = 0.5
+cb_type = mtr
+Enabled reductions: gd, scorer-identity, csoaa_ldf-rank, cb_adf, cb_explore_adf_greedy, shared_feature_merger
+Input label = CB
+Output pred = ACTION_PROBS
+average  since         example        example        current        current  current
+loss     last          counter         weight          label        predict features
+0.333333 0.333333            1            1.0          0:1:1         0:0.33       27
+0.416667 0.500000            2            2.0          1:1:1          0:0.5       18
+
+finished run
+number of examples = 3
+weighted example sum = 3.000000
+weighted label sum = 0.000000
+average loss = 0.444444
+total feature number = 51

test/train-sets/ref/help.stdout

@@ -257,8 +257,6 @@ Weight Options:
                                             experimental)
     --debug_reversed_learn                  Debug: learn each config in reversed order (last to first). (type:
                                             bool, default: false, experimental)
-    --aml_predict_only_model arg            transform input automl model into predict only automl model (type:
-                                            str, experimental)
     --automl_significance_level arg         Set significance level for champion change (type: float, default:
                                             0.05, keep, experimental)
     --fixed_significance_level              Use fixed significance level as opposed to scaling by model count

test/train-sets/ref/spin_off_aml_load.stderr

@@ -0,0 +1,23 @@
+creating features for following interactions: \x20\x20 \x20A \x20B AA AB BB
+using no cache
+Reading datafile = train-sets/automl_spin_off.txt
+num sources = 1
+Num weight bits = 17
+learning rate = 0.5
+initial_t = 0
+power_t = 0.5
+cb_type = mtr
+Enabled reductions: gd, scorer-identity, csoaa_ldf-rank, cb_adf, cb_explore_adf_greedy, shared_feature_merger
+Input label = CB
+Output pred = ACTION_PROBS
+average  since         example        example        current        current  current
+loss     last          counter         weight          label        predict features
+0.016667 0.016667            1            1.0          0:1:1         1:0.49       27
+0.020833 0.025000            2            2.0          1:1:1         0:0.97       18
+
+finished run
+number of examples = 3
+weighted example sum = 3.000000
+weighted label sum = 0.000000
+average loss = 0.022222
+total feature number = 63

test/train-sets/ref/spin_off_aml_save.stderr

@@ -0,0 +1,23 @@
+final_regressor = models/aml_spin_off.m
+using no cache
+Reading datafile = train-sets/automl_spin_off.txt
+num sources = 1
+Num weight bits = 18
+learning rate = 0.5
+initial_t = 0
+power_t = 0.5
+cb_type = mtr
+Enabled reductions: gd, scorer-identity, csoaa_ldf-rank, cb_adf, automl, cb_explore_adf_greedy, shared_feature_merger
+Input label = CB
+Output pred = ACTION_PROBS
+average  since         example        example        current        current  current
+loss     last          counter         weight          label        predict features
+0.333333 0.333333            1            1.0          0:1:1         0:0.33       27
+0.416667 0.500000            2            2.0          1:1:1          0:0.5       18
+
+finished run
+number of examples = 3
+weighted example sum = 3.000000
+weighted label sum = 0.000000
+average loss = 0.444444
+total feature number = 63

test/unit_test/automl_weights_test.cc

@@ -6,6 +6,7 @@
 #include "test_common.h"
 #include "vw/core/array_parameters_dense.h"
 #include "vw/core/constant.h"  // FNV_PRIME
+#include "vw/core/learner.h"
 #include "vw/core/vw_math.h"
 
 #include <boost/test/test_tools.hpp>
@@ -325,3 +326,64 @@ BOOST_AUTO_TEST_CASE(automl_equal_no_automl_w_iterations)
   BOOST_CHECK(qcolcol_weights_vector == automl_champ_weights_vector);
   BOOST_CHECK(ctr1 == ctr2);
 }
+
+BOOST_AUTO_TEST_CASE(automl_equal_spin_off_model)
+{
+  callback_map test_hooks;
+
+  std::string vw_arg =
+      "--cb_explore_adf --quiet --epsilon 0.2 "
+      "--random_seed 5 --predict_only_model ";
+  std::string vw_automl_arg =
+      "--automl 4 --priority_type favor_popular_namespaces "
+      "--oracle_type one_diff --global_lease 10 ";
+  int seed = 10;
+  // a switch happens around ~1756
+  size_t num_iterations = 1700;
+
+  auto* vw_qcolcol = VW::initialize(vw_arg + "-b 17 -q ::");
+  auto* vw_automl = VW::initialize(vw_arg + vw_automl_arg + "-b 18");
+  simulator::cb_sim sim1(seed);
+  simulator::cb_sim sim2(seed);
+  auto ctr1 = sim1.run_simulation_hook(vw_qcolcol, num_iterations, test_hooks);
+  auto ctr2 = sim2.run_simulation_hook(vw_automl, num_iterations, test_hooks);
+  vw_automl->l->convert_to_single_model(*vw_automl);
+
+  auto& weights_qcolcol = vw_qcolcol->weights.dense_weights;
+  auto& weights_automl = vw_automl->weights.dense_weights;
+  auto iter_1 = weights_qcolcol.begin();
+  auto iter_2 = weights_automl.begin();
+
+  std::vector<std::tuple<float, float, float, float>> qcolcol_weights_vector;
+  std::vector<std::tuple<float, float, float, float>> automl_weights_vector;
+
+  dense_iterator<float> qcolcol_it = weights_qcolcol.begin();
+  auto qcolcol_end = weights_qcolcol.end();
+
+  if (*qcolcol_it != 0.0f)
+  { qcolcol_weights_vector.emplace_back(*qcolcol_it[0], *qcolcol_it[1], *qcolcol_it[2], *qcolcol_it[3]); }
+
+  while (qcolcol_it.next_non_zero(qcolcol_end) < qcolcol_end)
+  { qcolcol_weights_vector.emplace_back(*qcolcol_it[0], *qcolcol_it[1], *qcolcol_it[2], *qcolcol_it[3]); }
+
+  std::sort(qcolcol_weights_vector.begin(), qcolcol_weights_vector.end());
+
+  dense_iterator<float> automl_it = weights_automl.begin();
+  auto automl_end = weights_automl.end();
+
+  if (*automl_it != 0.0f)
+  { automl_weights_vector.emplace_back(*automl_it[0], *automl_it[1], *automl_it[2], *automl_it[3]); }
+
+  while (automl_it.next_non_zero(automl_end) < automl_end)
+  { automl_weights_vector.emplace_back(*automl_it[0], *automl_it[1], *automl_it[2], *automl_it[3]); }
+
+  std::sort(automl_weights_vector.begin(), automl_weights_vector.end());
+
+  VW::finish(*vw_qcolcol);
+  VW::finish(*vw_automl);
+
+  BOOST_CHECK_EQUAL(qcolcol_weights_vector.size(), 31);
+  BOOST_CHECK_EQUAL(automl_weights_vector.size(), 31);
+  BOOST_CHECK(qcolcol_weights_vector == automl_weights_vector);
+  BOOST_CHECK(ctr1 == ctr2);
+}

vowpalwabbit/config/include/vw/config/options.h

@@ -31,6 +31,7 @@ class options_i
   VW_ATTR(nodiscard) std::shared_ptr<const base_option> get_option(const std::string& key) const;
   VW_ATTR(nodiscard) std::map<std::string, std::vector<option_group_definition>> get_collection_of_options() const;
   VW_ATTR(nodiscard) const std::vector<option_group_definition>& get_all_option_group_definitions() const;
+  void remove_option(const std::string& key);
 
   template <typename T>
   VW_ATTR(nodiscard)

vowpalwabbit/config/src/options.cc

@@ -55,6 +55,11 @@ std::shared_ptr<const base_option> VW::config::options_i::get_option(const std::
   return internal_get_option(key, _options);
 }
 
+void VW::config::options_i::remove_option(const std::string& key)
+{
+  if (!_options.erase(key)) { throw std::out_of_range(key + " was not found."); }
+}
+
 void options_i::add_and_parse(const option_group_definition& group)
 {
   // Add known options before parsing so impl can make use of them.