From c2d312824a987e2fabdda6877fbd9a8ef36acb8a Mon Sep 17 00:00:00 2001 From: Sergiy Matusevych Date: Wed, 21 Aug 2024 14:44:43 -0700 Subject: [PATCH] a bit cleaner definition of test config spaces (#60) * a bit cleaner definition of test config spaces * update test results to match the values returned from the optimizers * small fixes to the tunable to config space converter; roll back the cost update * roll back the monkey patching in the optimizer * bugfix: monkey_patch the right method in the quantization * quantization finally works properly * add one more test for ConfigSpace.sample_configuration() --- .../optimizers/convert_configspace.py | 38 ++++----- .../optimizers/toy_optimization_loop_test.py | 8 +- .../tunables/tunable_to_configspace_test.py | 43 +++++------ mlos_core/mlos_core/optimizers/optimizer.py | 4 - mlos_core/mlos_core/spaces/converters/util.py | 47 +++++++---- .../tests/spaces/adapters/llamatune_test.py | 77 ++++++++----------- .../spaces/monkey_patch_quantization_test.py | 68 +++++++++++++--- 7 files changed, 162 insertions(+), 123 deletions(-) diff --git a/mlos_bench/mlos_bench/optimizers/convert_configspace.py b/mlos_bench/mlos_bench/optimizers/convert_configspace.py index e4c70a2733..755918fa99 100644 --- a/mlos_bench/mlos_bench/optimizers/convert_configspace.py +++ b/mlos_bench/mlos_bench/optimizers/convert_configspace.py @@ -80,6 +80,10 @@ def _tunable_to_configspace( meta: Dict[Hashable, TunableValue] = {"cost": cost} if group_name is not None: meta["group"] = group_name + if tunable.is_numerical and tunable.quantization_bins: + # Temporary workaround to dropped quantization support in ConfigSpace 1.0 + # See Also: https://github.com/automl/ConfigSpace/issues/390 + meta[QUANTIZATION_BINS_META_KEY] = tunable.quantization_bins if tunable.type == "categorical": return ConfigurationSpace( @@ -140,16 +144,9 @@ def _tunable_to_configspace( else: raise TypeError(f"Invalid Parameter Type: {tunable.type}") - if tunable.is_numerical and tunable.quantization_bins: - # Temporary workaround to dropped quantization support in ConfigSpace 1.0 - # See Also: https://github.com/automl/ConfigSpace/issues/390 - new_meta = dict(range_hp.meta or {}) - new_meta[QUANTIZATION_BINS_META_KEY] = tunable.quantization_bins - range_hp.meta = new_meta - monkey_patch_hp_quantization(range_hp) - + monkey_patch_hp_quantization(range_hp) if not tunable.special: - return ConfigurationSpace({tunable.name: range_hp}) + return ConfigurationSpace(space=[range_hp]) # Compute the probabilities of switching between regular and special values. special_weights: Optional[List[float]] = None @@ -162,30 +159,33 @@ def _tunable_to_configspace( # one for special values, and one to choose between the two. (special_name, type_name) = special_param_names(tunable.name) conf_space = ConfigurationSpace( - { - tunable.name: range_hp, - special_name: CategoricalHyperparameter( + space=[ + range_hp, + CategoricalHyperparameter( name=special_name, choices=tunable.special, weights=special_weights, default_value=tunable.default if tunable.default in tunable.special else NotSet, meta=meta, ), - type_name: CategoricalHyperparameter( + CategoricalHyperparameter( name=type_name, choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE], weights=switch_weights, default_value=TunableValueKind.SPECIAL, ), - } - ) - conf_space.add( - EqualsCondition(conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL) + ] ) conf_space.add( - EqualsCondition(conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE) + [ + EqualsCondition( + conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL + ), + EqualsCondition( + conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE + ), + ] ) - return conf_space diff --git a/mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py b/mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py index 377f4f1e92..db46189e44 100644 --- a/mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py +++ b/mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py @@ -73,7 +73,7 @@ def test_mock_optimization_loop(mock_env_no_noise: MockEnv, mock_opt: MockOptimi "vmSize": "Standard_B2ms", "idle": "halt", "kernel_sched_migration_cost_ns": 117026, - "kernel_sched_latency_ns": 149827700, + "kernel_sched_latency_ns": 149827706, } @@ -88,7 +88,7 @@ def test_mock_optimization_loop_no_defaults( "vmSize": "Standard_B2s", "idle": "halt", "kernel_sched_migration_cost_ns": 49123, - "kernel_sched_latency_ns": 234760700, + "kernel_sched_latency_ns": 234760738, } @@ -100,7 +100,7 @@ def test_flaml_optimization_loop(mock_env_no_noise: MockEnv, flaml_opt: MlosCore "vmSize": "Standard_B2s", "idle": "halt", "kernel_sched_migration_cost_ns": -1, - "kernel_sched_latency_ns": 13718100, + "kernel_sched_latency_ns": 13718105, } @@ -113,7 +113,7 @@ def test_smac_optimization_loop(mock_env_no_noise: MockEnv, smac_opt: MlosCoreOp "vmSize": "Standard_B2s", "idle": "mwait", "kernel_sched_migration_cost_ns": 297669, - "kernel_sched_latency_ns": 290365100, + "kernel_sched_latency_ns": 290365137, } assert score == pytest.approx(expected_score, 0.01) assert tunables.get_param_values() == expected_tunable_values diff --git a/mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py b/mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py index 3f53756090..55bc130122 100644 --- a/mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py +++ b/mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py @@ -47,20 +47,20 @@ def configuration_space() -> ConfigurationSpace: # NOTE: FLAML requires distribution to be uniform spaces = ConfigurationSpace( - { - "vmSize": CategoricalHyperparameter( + space=[ + CategoricalHyperparameter( name="vmSize", choices=["Standard_B2s", "Standard_B2ms", "Standard_B4ms"], default_value="Standard_B4ms", meta={"group": "provision", "cost": 0}, ), - "idle": CategoricalHyperparameter( + CategoricalHyperparameter( name="idle", choices=["halt", "mwait", "noidle"], default_value="halt", meta={"group": "boot", "cost": 0}, ), - "kernel_sched_latency_ns": Integer( + Integer( name="kernel_sched_latency_ns", bounds=(0, 1000000000), log=False, @@ -71,44 +71,43 @@ def configuration_space() -> ConfigurationSpace: QUANTIZATION_BINS_META_KEY: 11, }, ), - "kernel_sched_migration_cost_ns": Integer( + Integer( name="kernel_sched_migration_cost_ns", bounds=(0, 500000), log=False, default=250000, meta={"group": "kernel", "cost": 0}, ), - kernel_sched_migration_cost_ns_special: CategoricalHyperparameter( + CategoricalHyperparameter( name=kernel_sched_migration_cost_ns_special, choices=[-1, 0], weights=[0.5, 0.5], default_value=-1, meta={"group": "kernel", "cost": 0}, ), - kernel_sched_migration_cost_ns_type: CategoricalHyperparameter( + CategoricalHyperparameter( name=kernel_sched_migration_cost_ns_type, choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE], weights=[0.5, 0.5], default_value=TunableValueKind.SPECIAL, ), - } + ] ) spaces.add( - EqualsCondition( - spaces[kernel_sched_migration_cost_ns_special], - spaces[kernel_sched_migration_cost_ns_type], - TunableValueKind.SPECIAL, - ) - ) - spaces.add( - EqualsCondition( - spaces["kernel_sched_migration_cost_ns"], - spaces[kernel_sched_migration_cost_ns_type], - TunableValueKind.RANGE, - ) + [ + EqualsCondition( + spaces[kernel_sched_migration_cost_ns_special], + spaces[kernel_sched_migration_cost_ns_type], + TunableValueKind.SPECIAL, + ), + EqualsCondition( + spaces["kernel_sched_migration_cost_ns"], + spaces[kernel_sched_migration_cost_ns_type], + TunableValueKind.RANGE, + ), + ] ) - monkey_patch_cs_quantization(spaces) - return spaces + return monkey_patch_cs_quantization(spaces) def _cmp_tunable_hyperparameter_categorical(tunable: Tunable, space: ConfigurationSpace) -> None: diff --git a/mlos_core/mlos_core/optimizers/optimizer.py b/mlos_core/mlos_core/optimizers/optimizer.py index ba776463fa..d7da71ae86 100644 --- a/mlos_core/mlos_core/optimizers/optimizer.py +++ b/mlos_core/mlos_core/optimizers/optimizer.py @@ -14,7 +14,6 @@ import pandas as pd from mlos_core.spaces.adapters.adapter import BaseSpaceAdapter -from mlos_core.spaces.converters.util import monkey_patch_cs_quantization from mlos_core.util import config_to_dataframe @@ -45,9 +44,6 @@ def __init__( space_adapter : BaseSpaceAdapter The space adapter class to employ for parameter space transformations. """ - # Temporary workaround to dropped quantization support in ConfigSpace 1.0 - # See Also: https://github.com/automl/ConfigSpace/issues/390 - monkey_patch_cs_quantization(parameter_space) self.parameter_space: ConfigSpace.ConfigurationSpace = parameter_space self.optimizer_parameter_space: ConfigSpace.ConfigurationSpace = ( parameter_space if space_adapter is None else space_adapter.target_parameter_space diff --git a/mlos_core/mlos_core/spaces/converters/util.py b/mlos_core/mlos_core/spaces/converters/util.py index 978c98cd21..30d63cb92c 100644 --- a/mlos_core/mlos_core/spaces/converters/util.py +++ b/mlos_core/mlos_core/spaces/converters/util.py @@ -11,7 +11,7 @@ QUANTIZATION_BINS_META_KEY = "quantization_bins" -def monkey_patch_hp_quantization(hp: Hyperparameter) -> None: +def monkey_patch_hp_quantization(hp: Hyperparameter) -> Hyperparameter: """ Monkey-patch quantization into the Hyperparameter. @@ -20,22 +20,28 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> None: Parameters ---------- - hp : NumericalHyperparameter + hp : Hyperparameter ConfigSpace hyperparameter to patch. + + Returns + ------- + hp : Hyperparameter + Patched hyperparameter. """ if not isinstance(hp, NumericalHyperparameter): - return + return hp assert isinstance(hp, NumericalHyperparameter) + dist = hp._vector_dist # pylint: disable=protected-access quantization_bins = (hp.meta or {}).get(QUANTIZATION_BINS_META_KEY) if quantization_bins is None: # No quantization requested. # Remove any previously applied patches. - if hasattr(hp, "sample_value_mlos_orig"): - setattr(hp, "sample_value", hp.sample_value_mlos_orig) - delattr(hp, "sample_value_mlos_orig") - return + if hasattr(dist, "sample_vector_mlos_orig"): + setattr(dist, "sample_vector", dist.sample_vector_mlos_orig) + delattr(dist, "sample_vector_mlos_orig") + return hp try: quantization_bins = int(quantization_bins) @@ -45,22 +51,23 @@ def monkey_patch_hp_quantization(hp: Hyperparameter) -> None: if quantization_bins <= 1: raise ValueError(f"{quantization_bins=} :: must be greater than 1.") - if not hasattr(hp, "sample_value_mlos_orig"): - setattr(hp, "sample_value_mlos_orig", hp.sample_value) + if not hasattr(dist, "sample_vector_mlos_orig"): + setattr(dist, "sample_vector_mlos_orig", dist.sample_vector) - assert hasattr(hp, "sample_value_mlos_orig") + assert hasattr(dist, "sample_vector_mlos_orig") setattr( - hp, - "sample_value", - lambda size=None, **kwargs: quantize( - hp.sample_value_mlos_orig(size, **kwargs), - bounds=(hp.lower, hp.upper), + dist, + "sample_vector", + lambda n, *, seed=None: quantize( + dist.sample_vector_mlos_orig(n, seed=seed), + bounds=(dist.lower_vectorized, dist.upper_vectorized), bins=quantization_bins, - ).astype(type(hp.default_value)), + ), ) + return hp -def monkey_patch_cs_quantization(cs: ConfigurationSpace) -> None: +def monkey_patch_cs_quantization(cs: ConfigurationSpace) -> ConfigurationSpace: """ Monkey-patch quantization into the Hyperparameters of a ConfigSpace. @@ -68,6 +75,12 @@ def monkey_patch_cs_quantization(cs: ConfigurationSpace) -> None: ---------- cs : ConfigurationSpace ConfigSpace to patch. + + Returns + ------- + cs : ConfigurationSpace + Patched ConfigSpace. """ for hp in cs.values(): monkey_patch_hp_quantization(hp) + return cs diff --git a/mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py b/mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py index 56dae8969d..7ca3c0d6ec 100644 --- a/mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py +++ b/mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py @@ -32,51 +32,38 @@ def construct_parameter_space( # pylint: disable=too-many-arguments seed: int = 1234, ) -> CS.ConfigurationSpace: """Helper function for construct an instance of `ConfigSpace.ConfigurationSpace`.""" - input_space = CS.ConfigurationSpace(seed=seed) - - for idx in range(n_continuous_params): - input_space.add( - CS.UniformFloatHyperparameter( - name=f"cont_{idx}", - lower=0, - upper=64, - ) - ) - for idx in range(n_quantized_continuous_params): - input_space.add( - CS.UniformFloatHyperparameter( - name=f"cont_{idx}", - lower=0, - upper=64, - meta={QUANTIZATION_BINS_META_KEY: 6}, - ) - ) - for idx in range(n_integer_params): - input_space.add( - CS.UniformIntegerHyperparameter( - name=f"int_{idx}", - lower=-1, - upper=256, - ) - ) - for idx in range(n_quantized_integer_params): - input_space.add( - CS.UniformIntegerHyperparameter( - name=f"int_{idx}", - lower=0, - upper=256, - meta={QUANTIZATION_BINS_META_KEY: 17}, - ) - ) - for idx in range(n_categorical_params): - input_space.add( - CS.CategoricalHyperparameter( - name=f"str_{idx}", choices=[f"option_{idx}" for idx in range(5)] - ) - ) - - monkey_patch_cs_quantization(input_space) - return input_space + input_space = CS.ConfigurationSpace( + seed=seed, + space=[ + *( + CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64) + for idx in range(n_continuous_params) + ), + *( + CS.UniformFloatHyperparameter( + name=f"cont_{idx}", lower=0, upper=64, meta={QUANTIZATION_BINS_META_KEY: 6} + ) + for idx in range(n_quantized_continuous_params) + ), + *( + CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=-1, upper=256) + for idx in range(n_integer_params) + ), + *( + CS.UniformIntegerHyperparameter( + name=f"int_{idx}", lower=0, upper=256, meta={QUANTIZATION_BINS_META_KEY: 17} + ) + for idx in range(n_quantized_integer_params) + ), + *( + CS.CategoricalHyperparameter( + name=f"str_{idx}", choices=[f"option_{idx}" for idx in range(5)] + ) + for idx in range(n_categorical_params) + ), + ], + ) + return monkey_patch_cs_quantization(input_space) @pytest.mark.parametrize( diff --git a/mlos_core/mlos_core/tests/spaces/monkey_patch_quantization_test.py b/mlos_core/mlos_core/tests/spaces/monkey_patch_quantization_test.py index 8ef2252e0a..dd446d4f5b 100644 --- a/mlos_core/mlos_core/tests/spaces/monkey_patch_quantization_test.py +++ b/mlos_core/mlos_core/tests/spaces/monkey_patch_quantization_test.py @@ -15,6 +15,7 @@ from mlos_core.spaces.converters.util import ( QUANTIZATION_BINS_META_KEY, monkey_patch_cs_quantization, + monkey_patch_hp_quantization, ) from mlos_core.tests import SEED @@ -30,13 +31,11 @@ def test_configspace_quant_int() -> None: log=False, meta={QUANTIZATION_BINS_META_KEY: quantization_bins}, ) - cs = ConfigurationSpace() - cs.add(hp) # Before patching: expect that at least one value is not quantized. assert not set(hp.sample_value(100)).issubset(quantized_values) - monkey_patch_cs_quantization(cs) + monkey_patch_hp_quantization(hp) # After patching: *all* values must belong to the set of quantized values. assert hp.sample_value() in quantized_values # check scalar type assert set(hp.sample_value(100)).issubset(quantized_values) # batch version @@ -54,13 +53,11 @@ def test_configspace_quant_float() -> None: log=False, meta={QUANTIZATION_BINS_META_KEY: quantization_bins}, ) - cs = ConfigurationSpace() - cs.add(hp) # Before patching: expect that at least one value is not quantized. assert not set(hp.sample_value(100)).issubset(quantized_values) - monkey_patch_cs_quantization(cs) + monkey_patch_hp_quantization(hp) # After patching: *all* values must belong to the set of quantized values. assert hp.sample_value() in quantized_values # check scalar type assert set(hp.sample_value(100)).issubset(quantized_values) # batch version @@ -77,19 +74,17 @@ def test_configspace_quant_repatch() -> None: log=False, meta={QUANTIZATION_BINS_META_KEY: quantization_bins}, ) - cs = ConfigurationSpace() - cs.add(hp) # Before patching: expect that at least one value is not quantized. assert not set(hp.sample_value(100)).issubset(quantized_values) - monkey_patch_cs_quantization(cs) + monkey_patch_hp_quantization(hp) # After patching: *all* values must belong to the set of quantized values. samples = hp.sample_value(100, seed=RandomState(SEED)) assert set(samples).issubset(quantized_values) # Patch the same hyperparameter again and check that the results are the same. - monkey_patch_cs_quantization(cs) + monkey_patch_hp_quantization(hp) # After patching: *all* values must belong to the set of quantized values. assert all(samples == hp.sample_value(100, seed=RandomState(SEED))) @@ -97,7 +92,7 @@ def test_configspace_quant_repatch() -> None: new_meta = dict(hp.meta or {}) new_meta[QUANTIZATION_BINS_META_KEY] = 21 hp.meta = new_meta - monkey_patch_cs_quantization(cs) + monkey_patch_hp_quantization(hp) samples_set = set(hp.sample_value(100, seed=RandomState(SEED))) quantized_values_new = set(range(5, 96, 10)) assert samples_set.issubset(set(range(0, 101, 5))) @@ -108,7 +103,56 @@ def test_configspace_quant_repatch() -> None: del new_meta[QUANTIZATION_BINS_META_KEY] hp.meta = new_meta assert hp.meta.get(QUANTIZATION_BINS_META_KEY) is None - monkey_patch_cs_quantization(cs) + monkey_patch_hp_quantization(hp) samples_set = set(hp.sample_value(100, seed=RandomState(SEED))) assert samples_set.issubset(set(range(0, 101))) assert len(quantized_values_new) < len(quantized_values) < len(samples_set) + + +def test_configspace_quant() -> None: + """Test quantization of multiple hyperparameters in the ConfigSpace.""" + space = ConfigurationSpace( + name="cs_test", + space={ + "hp_int": (0, 100000), + "hp_int_quant": (0, 100000), + "hp_float": (0.0, 1.0), + "hp_categorical": ["a", "b", "c"], + "hp_constant": 1337, + }, + ) + space["hp_int_quant"].meta = {QUANTIZATION_BINS_META_KEY: 5} + space["hp_float"].meta = {QUANTIZATION_BINS_META_KEY: 11} + monkey_patch_cs_quantization(space) + + space.seed(SEED) + assert dict(space.sample_configuration()) == { + "hp_categorical": "c", + "hp_constant": 1337, + "hp_float": 0.6, + "hp_int": 60263, + "hp_int_quant": 0, + } + assert [dict(conf) for conf in space.sample_configuration(3)] == [ + { + "hp_categorical": "a", + "hp_constant": 1337, + "hp_float": 0.4, + "hp_int": 59150, + "hp_int_quant": 50000, + }, + { + "hp_categorical": "a", + "hp_constant": 1337, + "hp_float": 0.3, + "hp_int": 65725, + "hp_int_quant": 75000, + }, + { + "hp_categorical": "b", + "hp_constant": 1337, + "hp_float": 0.6, + "hp_int": 84654, + "hp_int_quant": 25000, + }, + ]