a bit cleaner definition of test config spaces (#60)

* a bit cleaner definition of test config spaces

* update test results to match the values returned from the optimizers

* small fixes to the tunable to config space converter; roll back the cost update

* roll back the monkey patching in the optimizer

* bugfix: monkey_patch the right method in the quantization

* quantization finally works properly

* add one more test for ConfigSpace.sample_configuration()

mlos_bench/mlos_bench/optimizers/convert_configspace.py

@@ -80,6 +80,10 @@ def _tunable_to_configspace(
     meta: Dict[Hashable, TunableValue] = {"cost": cost}
     if group_name is not None:
         meta["group"] = group_name
+    if tunable.is_numerical and tunable.quantization_bins:
+        # Temporary workaround to dropped quantization support in ConfigSpace 1.0
+        # See Also: https://github.com/automl/ConfigSpace/issues/390
+        meta[QUANTIZATION_BINS_META_KEY] = tunable.quantization_bins
 
     if tunable.type == "categorical":
         return ConfigurationSpace(
@@ -140,16 +144,9 @@ def _tunable_to_configspace(
     else:
         raise TypeError(f"Invalid Parameter Type: {tunable.type}")
 
-    if tunable.is_numerical and tunable.quantization_bins:
-        # Temporary workaround to dropped quantization support in ConfigSpace 1.0
-        # See Also: https://github.com/automl/ConfigSpace/issues/390
-        new_meta = dict(range_hp.meta or {})
-        new_meta[QUANTIZATION_BINS_META_KEY] = tunable.quantization_bins
-        range_hp.meta = new_meta
-        monkey_patch_hp_quantization(range_hp)
-
+    monkey_patch_hp_quantization(range_hp)
     if not tunable.special:
-        return ConfigurationSpace({tunable.name: range_hp})
+        return ConfigurationSpace(space=[range_hp])
 
     # Compute the probabilities of switching between regular and special values.
     special_weights: Optional[List[float]] = None
@@ -162,30 +159,33 @@ def _tunable_to_configspace(
     # one for special values, and one to choose between the two.
     (special_name, type_name) = special_param_names(tunable.name)
     conf_space = ConfigurationSpace(
-        {
-            tunable.name: range_hp,
-            special_name: CategoricalHyperparameter(
+        space=[
+            range_hp,
+            CategoricalHyperparameter(
                 name=special_name,
                 choices=tunable.special,
                 weights=special_weights,
                 default_value=tunable.default if tunable.default in tunable.special else NotSet,
                 meta=meta,
             ),
-            type_name: CategoricalHyperparameter(
+            CategoricalHyperparameter(
                 name=type_name,
                 choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE],
                 weights=switch_weights,
                 default_value=TunableValueKind.SPECIAL,
             ),
-        }
-    )
-    conf_space.add(
-        EqualsCondition(conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL)
+        ]
     )
     conf_space.add(
-        EqualsCondition(conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE)
+        [
+            EqualsCondition(
+                conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL
+            ),
+            EqualsCondition(
+                conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE
+            ),
+        ]
     )
-
     return conf_space
 
 

mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py

@@ -73,7 +73,7 @@ def test_mock_optimization_loop(mock_env_no_noise: MockEnv, mock_opt: MockOptimi
         "vmSize": "Standard_B2ms",
         "idle": "halt",
         "kernel_sched_migration_cost_ns": 117026,
-        "kernel_sched_latency_ns": 149827700,
+        "kernel_sched_latency_ns": 149827706,
     }
 
 
@@ -88,7 +88,7 @@ def test_mock_optimization_loop_no_defaults(
         "vmSize": "Standard_B2s",
         "idle": "halt",
         "kernel_sched_migration_cost_ns": 49123,
-        "kernel_sched_latency_ns": 234760700,
+        "kernel_sched_latency_ns": 234760738,
     }
 
 
@@ -100,7 +100,7 @@ def test_flaml_optimization_loop(mock_env_no_noise: MockEnv, flaml_opt: MlosCore
         "vmSize": "Standard_B2s",
         "idle": "halt",
         "kernel_sched_migration_cost_ns": -1,
-        "kernel_sched_latency_ns": 13718100,
+        "kernel_sched_latency_ns": 13718105,
     }
 
 
@@ -113,7 +113,7 @@ def test_smac_optimization_loop(mock_env_no_noise: MockEnv, smac_opt: MlosCoreOp
         "vmSize": "Standard_B2s",
         "idle": "mwait",
         "kernel_sched_migration_cost_ns": 297669,
-        "kernel_sched_latency_ns": 290365100,
+        "kernel_sched_latency_ns": 290365137,
     }
     assert score == pytest.approx(expected_score, 0.01)
     assert tunables.get_param_values() == expected_tunable_values

mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py

@@ -47,20 +47,20 @@ def configuration_space() -> ConfigurationSpace:
 
     # NOTE: FLAML requires distribution to be uniform
     spaces = ConfigurationSpace(
-        {
-            "vmSize": CategoricalHyperparameter(
+        space=[
+            CategoricalHyperparameter(
                 name="vmSize",
                 choices=["Standard_B2s", "Standard_B2ms", "Standard_B4ms"],
                 default_value="Standard_B4ms",
                 meta={"group": "provision", "cost": 0},
             ),
-            "idle": CategoricalHyperparameter(
+            CategoricalHyperparameter(
                 name="idle",
                 choices=["halt", "mwait", "noidle"],
                 default_value="halt",
                 meta={"group": "boot", "cost": 0},
             ),
-            "kernel_sched_latency_ns": Integer(
+            Integer(
                 name="kernel_sched_latency_ns",
                 bounds=(0, 1000000000),
                 log=False,
@@ -71,44 +71,43 @@ def configuration_space() -> ConfigurationSpace:
                     QUANTIZATION_BINS_META_KEY: 11,
                 },
             ),
-            "kernel_sched_migration_cost_ns": Integer(
+            Integer(
                 name="kernel_sched_migration_cost_ns",
                 bounds=(0, 500000),
                 log=False,
                 default=250000,
                 meta={"group": "kernel", "cost": 0},
             ),
-            kernel_sched_migration_cost_ns_special: CategoricalHyperparameter(
+            CategoricalHyperparameter(
                 name=kernel_sched_migration_cost_ns_special,
                 choices=[-1, 0],
                 weights=[0.5, 0.5],
                 default_value=-1,
                 meta={"group": "kernel", "cost": 0},
             ),
-            kernel_sched_migration_cost_ns_type: CategoricalHyperparameter(
+            CategoricalHyperparameter(
                 name=kernel_sched_migration_cost_ns_type,
                 choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE],
                 weights=[0.5, 0.5],
                 default_value=TunableValueKind.SPECIAL,
             ),
-        }
+        ]
     )
     spaces.add(
-        EqualsCondition(
-            spaces[kernel_sched_migration_cost_ns_special],
-            spaces[kernel_sched_migration_cost_ns_type],
-            TunableValueKind.SPECIAL,
-        )
-    )
-    spaces.add(
-        EqualsCondition(
-            spaces["kernel_sched_migration_cost_ns"],
-            spaces[kernel_sched_migration_cost_ns_type],
-            TunableValueKind.RANGE,
-        )
+        [
+            EqualsCondition(
+                spaces[kernel_sched_migration_cost_ns_special],
+                spaces[kernel_sched_migration_cost_ns_type],
+                TunableValueKind.SPECIAL,
+            ),
+            EqualsCondition(
+                spaces["kernel_sched_migration_cost_ns"],
+                spaces[kernel_sched_migration_cost_ns_type],
+                TunableValueKind.RANGE,
+            ),
+        ]
     )
-    monkey_patch_cs_quantization(spaces)
-    return spaces
+    return monkey_patch_cs_quantization(spaces)
 
 
 def _cmp_tunable_hyperparameter_categorical(tunable: Tunable, space: ConfigurationSpace) -> None:

mlos_core/mlos_core/optimizers/optimizer.py

@@ -14,7 +14,6 @@
 import pandas as pd
 
 from mlos_core.spaces.adapters.adapter import BaseSpaceAdapter
-from mlos_core.spaces.converters.util import monkey_patch_cs_quantization
 from mlos_core.util import config_to_dataframe
 
 
@@ -45,9 +44,6 @@ def __init__(
         space_adapter : BaseSpaceAdapter
             The space adapter class to employ for parameter space transformations.
         """
-        # Temporary workaround to dropped quantization support in ConfigSpace 1.0
-        # See Also: https://github.com/automl/ConfigSpace/issues/390
-        monkey_patch_cs_quantization(parameter_space)
         self.parameter_space: ConfigSpace.ConfigurationSpace = parameter_space
         self.optimizer_parameter_space: ConfigSpace.ConfigurationSpace = (
             parameter_space if space_adapter is None else space_adapter.target_parameter_space

mlos_core/mlos_core/spaces/converters/util.py

@@ -11,7 +11,7 @@
 QUANTIZATION_BINS_META_KEY = "quantization_bins"
 
 
-def monkey_patch_hp_quantization(hp: Hyperparameter) -> None:
+def monkey_patch_hp_quantization(hp: Hyperparameter) -> Hyperparameter:
     """
     Monkey-patch quantization into the Hyperparameter.
 
@@ -20,22 +20,28 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> None:
 
     Parameters
     ----------
-    hp : NumericalHyperparameter
+    hp : Hyperparameter
         ConfigSpace hyperparameter to patch.
+
+    Returns
+    -------
+    hp : Hyperparameter
+        Patched hyperparameter.
     """
 
     if not isinstance(hp, NumericalHyperparameter):
-        return
+        return hp
 
     assert isinstance(hp, NumericalHyperparameter)
+    dist = hp._vector_dist  # pylint: disable=protected-access
     quantization_bins = (hp.meta or {}).get(QUANTIZATION_BINS_META_KEY)
     if quantization_bins is None:
         # No quantization requested.
         # Remove any previously applied patches.
-        if hasattr(hp, "sample_value_mlos_orig"):
-            setattr(hp, "sample_value", hp.sample_value_mlos_orig)
-            delattr(hp, "sample_value_mlos_orig")
-        return
+        if hasattr(dist, "sample_vector_mlos_orig"):
+            setattr(dist, "sample_vector", dist.sample_vector_mlos_orig)
+            delattr(dist, "sample_vector_mlos_orig")
+        return hp
 
     try:
         quantization_bins = int(quantization_bins)
@@ -45,29 +51,36 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> None:
     if quantization_bins <= 1:
         raise ValueError(f"{quantization_bins=} :: must be greater than 1.")
 
-    if not hasattr(hp, "sample_value_mlos_orig"):
-        setattr(hp, "sample_value_mlos_orig", hp.sample_value)
+    if not hasattr(dist, "sample_vector_mlos_orig"):
+        setattr(dist, "sample_vector_mlos_orig", dist.sample_vector)
 
-    assert hasattr(hp, "sample_value_mlos_orig")
+    assert hasattr(dist, "sample_vector_mlos_orig")
     setattr(
-        hp,
-        "sample_value",
-        lambda size=None, **kwargs: quantize(
-            hp.sample_value_mlos_orig(size, **kwargs),
-            bounds=(hp.lower, hp.upper),
+        dist,
+        "sample_vector",
+        lambda n, *, seed=None: quantize(
+            dist.sample_vector_mlos_orig(n, seed=seed),
+            bounds=(dist.lower_vectorized, dist.upper_vectorized),
             bins=quantization_bins,
-        ).astype(type(hp.default_value)),
+        ),
     )
+    return hp
 
 
-def monkey_patch_cs_quantization(cs: ConfigurationSpace) -> None:
+def monkey_patch_cs_quantization(cs: ConfigurationSpace) -> ConfigurationSpace:
     """
     Monkey-patch quantization into the Hyperparameters of a ConfigSpace.
 
     Parameters
     ----------
     cs : ConfigurationSpace
         ConfigSpace to patch.
+
+    Returns
+    -------
+    cs : ConfigurationSpace
+        Patched ConfigSpace.
     """
     for hp in cs.values():
         monkey_patch_hp_quantization(hp)
+    return cs

mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py

@@ -32,51 +32,38 @@ def construct_parameter_space(  # pylint: disable=too-many-arguments
     seed: int = 1234,
 ) -> CS.ConfigurationSpace:
     """Helper function for construct an instance of `ConfigSpace.ConfigurationSpace`."""
-    input_space = CS.ConfigurationSpace(seed=seed)
-
-    for idx in range(n_continuous_params):
-        input_space.add(
-            CS.UniformFloatHyperparameter(
-                name=f"cont_{idx}",
-                lower=0,
-                upper=64,
-            )
-        )
-    for idx in range(n_quantized_continuous_params):
-        input_space.add(
-            CS.UniformFloatHyperparameter(
-                name=f"cont_{idx}",
-                lower=0,
-                upper=64,
-                meta={QUANTIZATION_BINS_META_KEY: 6},
-            )
-        )
-    for idx in range(n_integer_params):
-        input_space.add(
-            CS.UniformIntegerHyperparameter(
-                name=f"int_{idx}",
-                lower=-1,
-                upper=256,
-            )
-        )
-    for idx in range(n_quantized_integer_params):
-        input_space.add(
-            CS.UniformIntegerHyperparameter(
-                name=f"int_{idx}",
-                lower=0,
-                upper=256,
-                meta={QUANTIZATION_BINS_META_KEY: 17},
-            )
-        )
-    for idx in range(n_categorical_params):
-        input_space.add(
-            CS.CategoricalHyperparameter(
-                name=f"str_{idx}", choices=[f"option_{idx}" for idx in range(5)]
-            )
-        )
-
-    monkey_patch_cs_quantization(input_space)
-    return input_space
+    input_space = CS.ConfigurationSpace(
+        seed=seed,
+        space=[
+            *(
+                CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64)
+                for idx in range(n_continuous_params)
+            ),
+            *(
+                CS.UniformFloatHyperparameter(
+                    name=f"cont_{idx}", lower=0, upper=64, meta={QUANTIZATION_BINS_META_KEY: 6}
+                )
+                for idx in range(n_quantized_continuous_params)
+            ),
+            *(
+                CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=-1, upper=256)
+                for idx in range(n_integer_params)
+            ),
+            *(
+                CS.UniformIntegerHyperparameter(
+                    name=f"int_{idx}", lower=0, upper=256, meta={QUANTIZATION_BINS_META_KEY: 17}
+                )
+                for idx in range(n_quantized_integer_params)
+            ),
+            *(
+                CS.CategoricalHyperparameter(
+                    name=f"str_{idx}", choices=[f"option_{idx}" for idx in range(5)]
+                )
+                for idx in range(n_categorical_params)
+            ),
+        ],
+    )
+    return monkey_patch_cs_quantization(input_space)
 
 
 @pytest.mark.parametrize(