Add back the quantization for LlamaTune unit tests (#59)

* move monkey_patch_quantization and its unit tests to mlos_core

* SMAC suggests slightly different config now

mlos_bench/mlos_bench/optimizers/convert_configspace.py

@@ -20,14 +20,15 @@
     Normal,
     Uniform,
 )
-from ConfigSpace.functional import quantize
 from ConfigSpace.hyperparameters import NumericalHyperparameter
 from ConfigSpace.types import NotSet
 
 from mlos_bench.tunables.tunable import Tunable, TunableValue
 from mlos_bench.tunables.tunable_groups import TunableGroups
 from mlos_bench.util import try_parse_val
 
+from mlos_core.spaces.converters.util import monkey_patch_quantization
+
 _LOG = logging.getLogger(__name__)
 
 
@@ -49,37 +50,6 @@ def _normalize_weights(weights: List[float]) -> List[float]:
     return [w / total for w in weights]
 
 
-def _monkey_patch_quantization(hp: NumericalHyperparameter, quantization_bins: int) -> None:
-    """
-    Monkey-patch quantization into the Hyperparameter.
-
-    Parameters
-    ----------
-    hp : NumericalHyperparameter
-        ConfigSpace hyperparameter to patch.
-    quantization_bins : int
-        Number of bins to quantize the hyperparameter into.
-    """
-    if quantization_bins <= 1:
-        raise ValueError(f"{quantization_bins=} :: must be greater than 1.")
-
-    # Temporary workaround to dropped quantization support in ConfigSpace 1.0
-    # See Also: https://github.com/automl/ConfigSpace/issues/390
-    if not hasattr(hp, "sample_value_mlos_orig"):
-        setattr(hp, "sample_value_mlos_orig", hp.sample_value)
-
-    assert hasattr(hp, "sample_value_mlos_orig")
-    setattr(
-        hp,
-        "sample_value",
-        lambda size=None, **kwargs: quantize(
-            hp.sample_value_mlos_orig(size, **kwargs),
-            bounds=(hp.lower, hp.upper),
-            bins=quantization_bins,
-        ).astype(type(hp.default_value)),
-    )
-
-
 def _tunable_to_configspace(
     tunable: Tunable,
     group_name: Optional[str] = None,
@@ -171,7 +141,7 @@ def _tunable_to_configspace(
     if tunable.quantization_bins:
         # Temporary workaround to dropped quantization support in ConfigSpace 1.0
         # See Also: https://github.com/automl/ConfigSpace/issues/390
-        _monkey_patch_quantization(range_hp, tunable.quantization_bins)
+        monkey_patch_quantization(range_hp, tunable.quantization_bins)
 
     if not tunable.special:
         return ConfigurationSpace({tunable.name: range_hp})

mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py

@@ -113,7 +113,7 @@ def test_smac_optimization_loop(mock_env_no_noise: MockEnv, smac_opt: MlosCoreOp
         "vmSize": "Standard_B2s",
         "idle": "mwait",
         "kernel_sched_migration_cost_ns": 297669,
-        "kernel_sched_latency_ns": 290365100,
+        "kernel_sched_latency_ns": 290365137,
     }
     assert score == pytest.approx(expected_score, 0.01)
     assert tunables.get_param_values() == expected_tunable_values

mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py

@@ -17,14 +17,15 @@
 
 from mlos_bench.optimizers.convert_configspace import (
     TunableValueKind,
-    _monkey_patch_quantization,
     _tunable_to_configspace,
     special_param_names,
     tunable_groups_to_configspace,
 )
 from mlos_bench.tunables.tunable import Tunable
 from mlos_bench.tunables.tunable_groups import TunableGroups
 
+from mlos_core.spaces.converters.util import monkey_patch_quantization
+
 # pylint: disable=redefined-outer-name
 
 
@@ -103,7 +104,7 @@ def configuration_space() -> ConfigurationSpace:
     )
     hp = spaces["kernel_sched_latency_ns"]
     assert isinstance(hp, NumericalHyperparameter)
-    _monkey_patch_quantization(hp, quantization_bins=10)
+    monkey_patch_quantization(hp, quantization_bins=11)
     return spaces
 
 

mlos_core/mlos_core/spaces/converters/util.py

@@ -0,0 +1,39 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+"""Helper functions for config space converters."""
+
+from ConfigSpace.functional import quantize
+from ConfigSpace.hyperparameters import NumericalHyperparameter
+
+
+def monkey_patch_quantization(hp: NumericalHyperparameter, quantization_bins: int) -> None:
+    """
+    Monkey-patch quantization into the Hyperparameter.
+
+    Parameters
+    ----------
+    hp : NumericalHyperparameter
+        ConfigSpace hyperparameter to patch.
+    quantization_bins : int
+        Number of bins to quantize the hyperparameter into.
+    """
+    if quantization_bins <= 1:
+        raise ValueError(f"{quantization_bins=} :: must be greater than 1.")
+
+    # Temporary workaround to dropped quantization support in ConfigSpace 1.0
+    # See Also: https://github.com/automl/ConfigSpace/issues/390
+    if not hasattr(hp, "sample_value_mlos_orig"):
+        setattr(hp, "sample_value_mlos_orig", hp.sample_value)
+
+    assert hasattr(hp, "sample_value_mlos_orig")
+    setattr(
+        hp,
+        "sample_value",
+        lambda size=None, **kwargs: quantize(
+            hp.sample_value_mlos_orig(size, **kwargs),
+            bounds=(hp.lower, hp.upper),
+            bins=quantization_bins,
+        ).astype(type(hp.default_value)),
+    )

mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py

@@ -12,6 +12,7 @@
 import pandas as pd
 import pytest
 
+from mlos_core.spaces.converters.util import monkey_patch_quantization
 from mlos_core.spaces.adapters import LlamaTuneAdapter
 
 
@@ -34,11 +35,15 @@ def construct_parameter_space(  # pylint: disable=too-many-arguments
     for idx in range(n_continuous_params):
         input_space.add(CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64))
     for idx in range(n_quantized_continuous_params):
-        input_space.add(CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64, q=12.8))
+        param_int = CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64)
+        monkey_patch_quantization(param_int, 6)
+        input_space.add(param_int)
     for idx in range(n_integer_params):
         input_space.add(CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=-1, upper=256))
     for idx in range(n_quantized_integer_params):
-        input_space.add(CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=0, upper=256, q=16))
+        param_float = CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=0, upper=256)
+        monkey_patch_quantization(param_float, 17)
+        input_space.add(param_float)
     for idx in range(n_categorical_params):
         input_space.add(
             CS.CategoricalHyperparameter(

mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_quant_test.py → mlos_core/mlos_core/tests/spaces/monkey_patch_quantization_test.py

@@ -8,8 +8,8 @@
 from ConfigSpace import UniformFloatHyperparameter, UniformIntegerHyperparameter
 from numpy.random import RandomState
 
-from mlos_bench.optimizers.convert_configspace import _monkey_patch_quantization
-from mlos_bench.tests import SEED
+from mlos_core.spaces.converters.util import monkey_patch_quantization
+from mlos_core.tests import SEED
 
 
 def test_configspace_quant_int() -> None:
@@ -20,7 +20,7 @@ def test_configspace_quant_int() -> None:
     # Before patching: expect that at least one value is not quantized.
     assert not set(hp.sample_value(100)).issubset(quantized_values)
 
-    _monkey_patch_quantization(hp, 11)
+    monkey_patch_quantization(hp, 11)
     # After patching: *all* values must belong to the set of quantized values.
     assert hp.sample_value() in quantized_values  # check scalar type
     assert set(hp.sample_value(100)).issubset(quantized_values)  # batch version
@@ -35,7 +35,7 @@ def test_configspace_quant_float() -> None:
     assert not set(hp.sample_value(100)).issubset(quantized_values)
 
     # 5 is a nice number of bins to avoid floating point errors.
-    _monkey_patch_quantization(hp, 5)
+    monkey_patch_quantization(hp, 5)
     # After patching: *all* values must belong to the set of quantized values.
     assert hp.sample_value() in quantized_values  # check scalar type
     assert set(hp.sample_value(100)).issubset(quantized_values)  # batch version
@@ -49,18 +49,18 @@ def test_configspace_quant_repatch() -> None:
     # Before patching: expect that at least one value is not quantized.
     assert not set(hp.sample_value(100)).issubset(quantized_values)
 
-    _monkey_patch_quantization(hp, 11)
+    monkey_patch_quantization(hp, 11)
     # After patching: *all* values must belong to the set of quantized values.
     samples = hp.sample_value(100, seed=RandomState(SEED))
     assert set(samples).issubset(quantized_values)
 
     # Patch the same hyperparameter again and check that the results are the same.
-    _monkey_patch_quantization(hp, 11)
+    monkey_patch_quantization(hp, 11)
     # After patching: *all* values must belong to the set of quantized values.
     assert all(samples == hp.sample_value(100, seed=RandomState(SEED)))
 
     # Repatch with the higher number of bins and make sure we get new values.
-    _monkey_patch_quantization(hp, 21)
+    monkey_patch_quantization(hp, 21)
     samples_set = set(hp.sample_value(100, seed=RandomState(SEED)))
     quantized_values_new = set(range(5, 96, 10))
     assert samples_set.issubset(set(range(0, 101, 5)))