diff --git a/conda-envs/mlos-windows.yml b/conda-envs/mlos-windows.yml index 107b6fb2cf..b7269ee11b 100644 --- a/conda-envs/mlos-windows.yml +++ b/conda-envs/mlos-windows.yml @@ -16,7 +16,8 @@ dependencies: - jupyter - ipykernel - nb_conda_kernels - - matplotlib + - matplotlib<3.9 + - matplotlib-base<3.9 - seaborn - pandas - pyarrow diff --git a/mlos_bench/mlos_bench/optimizers/convert_configspace.py b/mlos_bench/mlos_bench/optimizers/convert_configspace.py index e4ff9897fa..6244bba7b3 100644 --- a/mlos_bench/mlos_bench/optimizers/convert_configspace.py +++ b/mlos_bench/mlos_bench/optimizers/convert_configspace.py @@ -7,10 +7,12 @@ """ import logging -from typing import Dict, List, Optional, Tuple, Union +from typing import Dict, Hashable, List, Optional, Tuple, Union from ConfigSpace import ( Beta, + BetaFloatHyperparameter, + BetaIntegerHyperparameter, CategoricalHyperparameter, Configuration, ConfigurationSpace, @@ -18,12 +20,17 @@ Float, Integer, Normal, + NormalFloatHyperparameter, + NormalIntegerHyperparameter, Uniform, + UniformFloatHyperparameter, + UniformIntegerHyperparameter, ) +from ConfigSpace.types import NotSet from mlos_bench.tunables.tunable import Tunable, TunableValue from mlos_bench.tunables.tunable_groups import TunableGroups -from mlos_bench.util import nullable, try_parse_val +from mlos_bench.util import try_parse_val _LOG = logging.getLogger(__name__) @@ -70,7 +77,9 @@ def _tunable_to_configspace( cs : ConfigurationSpace A ConfigurationSpace object that corresponds to the Tunable. """ - meta = {"group": group_name, "cost": cost} # {"scaling": ""} + meta: Dict[Hashable, TunableValue] = {"cost": cost} + if group_name is not None: + meta["group"] = group_name if tunable.type == "categorical": return ConfigurationSpace( @@ -101,12 +110,20 @@ def _tunable_to_configspace( elif tunable.distribution is not None: raise TypeError(f"Invalid Distribution Type: {tunable.distribution}") + range_hp: Union[ + BetaFloatHyperparameter, + BetaIntegerHyperparameter, + NormalFloatHyperparameter, + NormalIntegerHyperparameter, + UniformFloatHyperparameter, + UniformIntegerHyperparameter, + ] if tunable.type == "int": range_hp = Integer( name=tunable.name, bounds=(int(tunable.range[0]), int(tunable.range[1])), log=bool(tunable.is_log), - q=nullable(int, tunable.quantization), + # TODO: Restore quantization support (#803). distribution=distribution, default=( int(tunable.default) @@ -120,8 +137,8 @@ def _tunable_to_configspace( name=tunable.name, bounds=tunable.range, log=bool(tunable.is_log), - q=tunable.quantization, # type: ignore[arg-type] - distribution=distribution, # type: ignore[arg-type] + # TODO: Restore quantization support (#803). + distribution=distribution, default=( float(tunable.default) if tunable.in_range(tunable.default) and tunable.default is not None @@ -152,7 +169,7 @@ def _tunable_to_configspace( name=special_name, choices=tunable.special, weights=special_weights, - default_value=tunable.default if tunable.default in tunable.special else None, + default_value=tunable.default if tunable.default in tunable.special else NotSet, meta=meta, ), type_name: CategoricalHyperparameter( @@ -163,10 +180,10 @@ def _tunable_to_configspace( ), } ) - conf_space.add_condition( + conf_space.add( EqualsCondition(conf_space[special_name], conf_space[type_name], TunableValueKind.SPECIAL) ) - conf_space.add_condition( + conf_space.add( EqualsCondition(conf_space[tunable.name], conf_space[type_name], TunableValueKind.RANGE) ) diff --git a/mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py b/mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py index dce3e366a6..8e472a7ea5 100644 --- a/mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py +++ b/mlos_bench/mlos_bench/tests/tunables/tunable_to_configspace_test.py @@ -9,6 +9,7 @@ CategoricalHyperparameter, ConfigurationSpace, EqualsCondition, + Integer, UniformFloatHyperparameter, UniformIntegerHyperparameter, ) @@ -40,45 +41,66 @@ def configuration_space() -> ConfigurationSpace: special_param_names("kernel_sched_migration_cost_ns") ) + # TODO: Add quantization support tests (#803). + + # NOTE: FLAML requires distribution to be uniform spaces = ConfigurationSpace( - space={ - "vmSize": ["Standard_B2s", "Standard_B2ms", "Standard_B4ms"], - "idle": ["halt", "mwait", "noidle"], - "kernel_sched_migration_cost_ns": (0, 500000), - kernel_sched_migration_cost_ns_special: [-1, 0], - kernel_sched_migration_cost_ns_type: [ - TunableValueKind.SPECIAL, - TunableValueKind.RANGE, - ], - "kernel_sched_latency_ns": (0, 1000000000), + { + "vmSize": CategoricalHyperparameter( + name="vmSize", + choices=["Standard_B2s", "Standard_B2ms", "Standard_B4ms"], + default_value="Standard_B4ms", + meta={"group": "provision", "cost": 0}, + ), + "idle": CategoricalHyperparameter( + name="idle", + choices=["halt", "mwait", "noidle"], + default_value="halt", + meta={"group": "boot", "cost": 0}, + ), + "kernel_sched_latency_ns": Integer( + name="kernel_sched_latency_ns", + bounds=(0, 1000000000), + log=False, + default=2000000, + meta={"group": "kernel", "cost": 0}, + ), + "kernel_sched_migration_cost_ns": Integer( + name="kernel_sched_migration_cost_ns", + bounds=(0, 500000), + log=False, + default=250000, + meta={"group": "kernel", "cost": 0}, + ), + kernel_sched_migration_cost_ns_special: CategoricalHyperparameter( + name=kernel_sched_migration_cost_ns_special, + choices=[-1, 0], + weights=[0.5, 0.5], + default_value=-1, + meta={"group": "kernel", "cost": 0}, + ), + kernel_sched_migration_cost_ns_type: CategoricalHyperparameter( + name=kernel_sched_migration_cost_ns_type, + choices=[TunableValueKind.SPECIAL, TunableValueKind.RANGE], + weights=[0.5, 0.5], + default_value=TunableValueKind.SPECIAL, + ), } ) - - # NOTE: FLAML requires distribution to be uniform - spaces["vmSize"].default_value = "Standard_B4ms" - spaces["idle"].default_value = "halt" - spaces["kernel_sched_migration_cost_ns"].default_value = 250000 - spaces[kernel_sched_migration_cost_ns_special].default_value = -1 - spaces[kernel_sched_migration_cost_ns_special].probabilities = (0.5, 0.5) - spaces[kernel_sched_migration_cost_ns_type].default_value = TunableValueKind.SPECIAL - spaces[kernel_sched_migration_cost_ns_type].probabilities = (0.5, 0.5) - spaces["kernel_sched_latency_ns"].default_value = 2000000 - - spaces.add_condition( + spaces.add( EqualsCondition( spaces[kernel_sched_migration_cost_ns_special], spaces[kernel_sched_migration_cost_ns_type], TunableValueKind.SPECIAL, ) ) - spaces.add_condition( + spaces.add( EqualsCondition( spaces["kernel_sched_migration_cost_ns"], spaces[kernel_sched_migration_cost_ns_type], TunableValueKind.RANGE, ) ) - return spaces diff --git a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py index 611dc04044..fc801d7d05 100644 --- a/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py +++ b/mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py @@ -23,6 +23,7 @@ ) from mlos_core.spaces.adapters.adapter import BaseSpaceAdapter from mlos_core.spaces.adapters.identity_adapter import IdentityAdapter +from mlos_core.util import drop_nulls class SmacOptimizer(BaseBayesianOptimizer): @@ -350,8 +351,11 @@ def _suggest( warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning) trial: TrialInfo = self.base_optimizer.ask() - trial.config.is_valid_configuration() - self.optimizer_parameter_space.check_configuration(trial.config) + trial.config.check_valid_configuration() + ConfigSpace.Configuration( + self.optimizer_parameter_space, + values=trial.config, + ).check_valid_configuration() assert trial.config.config_space == self.optimizer_parameter_space self.trial_info_map[trial.config] = trial config_df = pd.DataFrame( @@ -441,6 +445,11 @@ def _to_configspace_configs(self, *, configs: pd.DataFrame) -> List[ConfigSpace. List of ConfigSpace configs. """ return [ - ConfigSpace.Configuration(self.optimizer_parameter_space, values=config.to_dict()) + ConfigSpace.Configuration( + self.optimizer_parameter_space, + # Remove None values for inactive parameters + values=drop_nulls(config.to_dict()), + allow_inactive_with_values=False, + ) for (_, config) in configs.astype("O").iterrows() ] diff --git a/mlos_core/mlos_core/optimizers/flaml_optimizer.py b/mlos_core/mlos_core/optimizers/flaml_optimizer.py index 50def8bc80..e5272f103e 100644 --- a/mlos_core/mlos_core/optimizers/flaml_optimizer.py +++ b/mlos_core/mlos_core/optimizers/flaml_optimizer.py @@ -13,7 +13,7 @@ from mlos_core.optimizers.optimizer import BaseOptimizer from mlos_core.spaces.adapters.adapter import BaseSpaceAdapter -from mlos_core.util import normalize_config +from mlos_core.util import drop_nulls, normalize_config class EvaluatedSample(NamedTuple): @@ -124,13 +124,16 @@ def _register( warn(f"Not Implemented: Ignoring metadata {list(metadata.columns)}", UserWarning) for (_, config), (_, score) in zip(configs.astype("O").iterrows(), scores.iterrows()): + # Remove None values for inactive config parameters + config_dict = drop_nulls(config.to_dict()) cs_config: ConfigSpace.Configuration = ConfigSpace.Configuration( - self.optimizer_parameter_space, values=config.to_dict() + self.optimizer_parameter_space, + values=config_dict, ) if cs_config in self.evaluated_samples: warn(f"Configuration {config} was already registered", UserWarning) self.evaluated_samples[cs_config] = EvaluatedSample( - config=config.to_dict(), + config=config_dict, score=float(np.average(score.astype(float), weights=self._objective_weights)), ) diff --git a/mlos_core/mlos_core/spaces/adapters/llamatune.py b/mlos_core/mlos_core/spaces/adapters/llamatune.py index abbcb5f1a0..5a39f863a5 100644 --- a/mlos_core/mlos_core/spaces/adapters/llamatune.py +++ b/mlos_core/mlos_core/spaces/adapters/llamatune.py @@ -4,7 +4,7 @@ # """Implementation of LlamaTune space adapter.""" import os -from typing import Dict, Optional +from typing import Dict, List, Optional, Union from warnings import warn import ConfigSpace @@ -16,7 +16,7 @@ from sklearn.preprocessing import MinMaxScaler from mlos_core.spaces.adapters.adapter import BaseSpaceAdapter -from mlos_core.util import normalize_config +from mlos_core.util import drop_nulls, normalize_config class LlamaTuneAdapter(BaseSpaceAdapter): # pylint: disable=too-many-instance-attributes @@ -102,7 +102,7 @@ def inverse_transform(self, configurations: pd.DataFrame) -> pd.DataFrame: for _, config in configurations.astype("O").iterrows(): configuration = ConfigSpace.Configuration( self.orig_parameter_space, - values=config.to_dict(), + values=drop_nulls(config.to_dict()), ) target_config = self._suggested_configs.get(configuration, None) @@ -222,7 +222,10 @@ def _try_inverse_transform_config( # But the inverse mapping should at least be valid in the target space. try: - self.target_parameter_space.check_configuration(target_config) + ConfigSpace.Configuration( + self.target_parameter_space, + values=target_config, + ).check_valid_configuration() except ConfigSpace.exceptions.IllegalValueError as e: raise ValueError( f"Invalid configuration {target_config} generated by " @@ -249,7 +252,10 @@ def transform(self, configuration: pd.DataFrame) -> pd.DataFrame: # Validate that the configuration is in the original space. try: - self.orig_parameter_space.check_configuration(orig_configuration) + ConfigSpace.Configuration( + self.orig_parameter_space, + values=orig_configuration, + ).check_valid_configuration() except ConfigSpace.exceptions.IllegalValueError as e: raise ValueError( f"Invalid configuration {orig_configuration} generated by " @@ -282,6 +288,9 @@ def _construct_low_dim_space( """ # Define target space parameters q_scaler = None + hyperparameters: List[ + Union[ConfigSpace.UniformFloatHyperparameter, ConfigSpace.UniformIntegerHyperparameter] + ] if max_unique_values_per_param is None: hyperparameters = [ ConfigSpace.UniformFloatHyperparameter(name=f"dim_{idx}", lower=-1, upper=1) @@ -316,7 +325,7 @@ def _construct_low_dim_space( config_space = ConfigSpace.ConfigurationSpace(name=self.orig_parameter_space.name) # use same random state as in original parameter space config_space.random = self._random_state - config_space.add_hyperparameters(hyperparameters) + config_space.add(hyperparameters) self._target_config_space = config_space def _transform(self, configuration: dict) -> dict: @@ -366,7 +375,7 @@ def _transform(self, configuration: dict) -> dict: if param.name in self._special_param_values_dict: value = self._special_param_value_scaler(param, value) - orig_value = param._transform(value) # pylint: disable=protected-access + orig_value = param.to_value(value) orig_value = np.clip(orig_value, param.lower, param.upper) else: raise NotImplementedError( @@ -379,7 +388,7 @@ def _transform(self, configuration: dict) -> dict: def _special_param_value_scaler( self, - param: ConfigSpace.UniformIntegerHyperparameter, + param: NumericalHyperparameter, input_value: float, ) -> float: """ @@ -388,7 +397,7 @@ def _special_param_value_scaler( Parameters ---------- - param: ConfigSpace.UniformIntegerHyperparameter + param: NumericalHyperparameter Parameter of the original parameter space. input_value: float @@ -403,19 +412,13 @@ def _special_param_value_scaler( # Check if input value corresponds to some special value perc_sum = 0.0 - ret: float for special_value, biasing_perc in special_values_list: perc_sum += biasing_perc if input_value < perc_sum: - ret = param._inverse_transform(special_value) # pylint: disable=protected-access - return ret + return float(param.to_vector(special_value)) # Scale input value uniformly to non-special values - # pylint: disable=protected-access - ret = param._inverse_transform( - param._transform_scalar((input_value - perc_sum) / (1 - perc_sum)) - ) - return ret + return float(param.to_vector((input_value - perc_sum) / (1 - perc_sum))) # pylint: disable=too-complex,too-many-branches def _validate_special_param_values(self, special_param_values_dict: dict) -> None: diff --git a/mlos_core/mlos_core/tests/optimizers/conftest.py b/mlos_core/mlos_core/tests/optimizers/conftest.py index fe82ff92bb..bbc10f549c 100644 --- a/mlos_core/mlos_core/tests/optimizers/conftest.py +++ b/mlos_core/mlos_core/tests/optimizers/conftest.py @@ -14,9 +14,9 @@ def configuration_space() -> CS.ConfigurationSpace: # Start defining a ConfigurationSpace for the Optimizer to search. space = CS.ConfigurationSpace(seed=1234) # Add a continuous input dimension between 0 and 1. - space.add_hyperparameter(CS.UniformFloatHyperparameter(name="x", lower=0, upper=1)) + space.add(CS.UniformFloatHyperparameter(name="x", lower=0, upper=1)) # Add a categorical hyperparameter with 3 possible values. - space.add_hyperparameter(CS.CategoricalHyperparameter(name="y", choices=["a", "b", "c"])) + space.add(CS.CategoricalHyperparameter(name="y", choices=["a", "b", "c"])) # Add a discrete input dimension between 0 and 10. - space.add_hyperparameter(CS.UniformIntegerHyperparameter(name="z", lower=0, upper=10)) + space.add(CS.UniformIntegerHyperparameter(name="z", lower=0, upper=10)) return space diff --git a/mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py b/mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py index bd04203b05..4c7edd7493 100644 --- a/mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py +++ b/mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py @@ -76,8 +76,8 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: input_space = CS.ConfigurationSpace(seed=SEED) # add a mix of numeric datatypes - input_space.add_hyperparameter(CS.UniformIntegerHyperparameter(name="x", lower=0, upper=5)) - input_space.add_hyperparameter(CS.UniformFloatHyperparameter(name="y", lower=0.0, upper=5.0)) + input_space.add(CS.UniformIntegerHyperparameter(name="x", lower=0, upper=5)) + input_space.add(CS.UniformFloatHyperparameter(name="y", lower=0.0, upper=5.0)) optimizer = optimizer_class( parameter_space=input_space, @@ -105,7 +105,7 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: optimizer.parameter_space, suggestion.astype("O").iloc[0].to_dict() ) # Raises an error if outside of configuration space - test_configuration.is_valid_configuration() + test_configuration.check_valid_configuration() # Test registering the suggested configuration with a score. observation = objective(suggestion) assert isinstance(observation, pd.DataFrame) diff --git a/mlos_core/mlos_core/tests/optimizers/optimizer_test.py b/mlos_core/mlos_core/tests/optimizers/optimizer_test.py index fce083150f..776d01c38f 100644 --- a/mlos_core/mlos_core/tests/optimizers/optimizer_test.py +++ b/mlos_core/mlos_core/tests/optimizers/optimizer_test.py @@ -112,7 +112,7 @@ def objective(x: pd.Series) -> pd.DataFrame: # check that suggestion is in the space configuration = CS.Configuration(optimizer.parameter_space, suggestion.iloc[0].to_dict()) # Raises an error if outside of configuration space - configuration.is_valid_configuration() + configuration.check_valid_configuration() observation = objective(suggestion["x"]) assert isinstance(observation, pd.DataFrame) optimizer.register(configs=suggestion, scores=observation, metadata=metadata) @@ -234,8 +234,8 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: input_space = CS.ConfigurationSpace(seed=1234) # Add two continuous inputs - input_space.add_hyperparameter(CS.UniformFloatHyperparameter(name="x", lower=0, upper=3)) - input_space.add_hyperparameter(CS.UniformFloatHyperparameter(name="y", lower=0, upper=3)) + input_space.add(CS.UniformFloatHyperparameter(name="x", lower=0, upper=3)) + input_space.add(CS.UniformFloatHyperparameter(name="y", lower=0, upper=3)) # Initialize an optimizer that uses LlamaTune space adapter space_adapter_kwargs = { @@ -387,8 +387,8 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: input_space = CS.ConfigurationSpace(seed=SEED) # add a mix of numeric datatypes - input_space.add_hyperparameter(CS.UniformIntegerHyperparameter(name="x", lower=0, upper=5)) - input_space.add_hyperparameter(CS.UniformFloatHyperparameter(name="y", lower=0.0, upper=5.0)) + input_space.add(CS.UniformIntegerHyperparameter(name="x", lower=0, upper=5)) + input_space.add(CS.UniformFloatHyperparameter(name="y", lower=0.0, upper=5.0)) if optimizer_type is None: optimizer = OptimizerFactory.create( @@ -422,7 +422,7 @@ def objective(point: pd.DataFrame) -> pd.DataFrame: optimizer.parameter_space, suggestion.astype("O").iloc[0].to_dict() ) # Raises an error if outside of configuration space - test_configuration.is_valid_configuration() + test_configuration.check_valid_configuration() # Test registering the suggested configuration with a score. observation = objective(suggestion) assert isinstance(observation, pd.DataFrame) diff --git a/mlos_core/mlos_core/tests/spaces/adapters/identity_adapter_test.py b/mlos_core/mlos_core/tests/spaces/adapters/identity_adapter_test.py index 5d394cf4e2..107ea5263b 100644 --- a/mlos_core/mlos_core/tests/spaces/adapters/identity_adapter_test.py +++ b/mlos_core/mlos_core/tests/spaces/adapters/identity_adapter_test.py @@ -15,15 +15,9 @@ def test_identity_adapter() -> None: """Tests identity adapter.""" input_space = CS.ConfigurationSpace(seed=1234) - input_space.add_hyperparameter( - CS.UniformIntegerHyperparameter(name="int_1", lower=0, upper=100) - ) - input_space.add_hyperparameter( - CS.UniformFloatHyperparameter(name="float_1", lower=0, upper=100) - ) - input_space.add_hyperparameter( - CS.CategoricalHyperparameter(name="str_1", choices=["on", "off"]) - ) + input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=0, upper=100)) + input_space.add(CS.UniformFloatHyperparameter(name="float_1", lower=0, upper=100)) + input_space.add(CS.CategoricalHyperparameter(name="str_1", choices=["on", "off"])) adapter = IdentityAdapter(orig_parameter_space=input_space) diff --git a/mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py b/mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py index 9d73b6ba8c..fc42a0234e 100644 --- a/mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py +++ b/mlos_core/mlos_core/tests/spaces/adapters/llamatune_test.py @@ -25,15 +25,11 @@ def construct_parameter_space( input_space = CS.ConfigurationSpace(seed=seed) for idx in range(n_continuous_params): - input_space.add_hyperparameter( - CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64) - ) + input_space.add(CS.UniformFloatHyperparameter(name=f"cont_{idx}", lower=0, upper=64)) for idx in range(n_integer_params): - input_space.add_hyperparameter( - CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=-1, upper=256) - ) + input_space.add(CS.UniformIntegerHyperparameter(name=f"int_{idx}", lower=-1, upper=256)) for idx in range(n_categorical_params): - input_space.add_hyperparameter( + input_space.add( CS.CategoricalHyperparameter( name=f"str_{idx}", choices=[f"option_{idx}" for idx in range(5)] ) @@ -94,7 +90,7 @@ def test_num_low_dims( # High-dim (i.e., original) config should be valid orig_config = CS.Configuration(input_space, values=orig_config_df.iloc[0].to_dict()) - input_space.check_configuration(orig_config) + orig_config.check_valid_configuration() # Transform high-dim config back to low-dim target_config_df = adapter.inverse_transform(orig_config_df) @@ -130,11 +126,11 @@ def test_special_parameter_values_validation() -> None: dictionary. """ input_space = CS.ConfigurationSpace(seed=1234) - input_space.add_hyperparameter( + input_space.add( CS.CategoricalHyperparameter(name="str", choices=[f"choice_{idx}" for idx in range(5)]) ) - input_space.add_hyperparameter(CS.UniformFloatHyperparameter(name="cont", lower=-1, upper=100)) - input_space.add_hyperparameter(CS.UniformIntegerHyperparameter(name="int", lower=0, upper=100)) + input_space.add(CS.UniformFloatHyperparameter(name="cont", lower=-1, upper=100)) + input_space.add(CS.UniformIntegerHyperparameter(name="int", lower=0, upper=100)) # Only UniformIntegerHyperparameters are currently supported with pytest.raises(NotImplementedError): @@ -224,12 +220,8 @@ def gen_random_configs(adapter: LlamaTuneAdapter, num_configs: int) -> Iterator[ def test_special_parameter_values_biasing() -> None: # pylint: disable=too-complex """Tests LlamaTune's special parameter values biasing methodology.""" input_space = CS.ConfigurationSpace(seed=1234) - input_space.add_hyperparameter( - CS.UniformIntegerHyperparameter(name="int_1", lower=0, upper=100) - ) - input_space.add_hyperparameter( - CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=100) - ) + input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=0, upper=100)) + input_space.add(CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=100)) num_configs = 400 bias_percentage = LlamaTuneAdapter.DEFAULT_SPECIAL_PARAM_VALUE_BIASING_PERCENTAGE @@ -323,22 +315,14 @@ def test_max_unique_values_per_param() -> None: """Tests LlamaTune's parameter values discretization implementation.""" # Define config space with a mix of different parameter types input_space = CS.ConfigurationSpace(seed=1234) - input_space.add_hyperparameter( + input_space.add( CS.UniformFloatHyperparameter(name="cont_1", lower=0, upper=5), ) - input_space.add_hyperparameter( - CS.UniformFloatHyperparameter(name="cont_2", lower=1, upper=100) - ) - input_space.add_hyperparameter( - CS.UniformIntegerHyperparameter(name="int_1", lower=1, upper=10) - ) - input_space.add_hyperparameter( - CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=2048) - ) - input_space.add_hyperparameter( - CS.CategoricalHyperparameter(name="str_1", choices=["on", "off"]) - ) - input_space.add_hyperparameter( + input_space.add(CS.UniformFloatHyperparameter(name="cont_2", lower=1, upper=100)) + input_space.add(CS.UniformIntegerHyperparameter(name="int_1", lower=1, upper=10)) + input_space.add(CS.UniformIntegerHyperparameter(name="int_2", lower=0, upper=2048)) + input_space.add(CS.CategoricalHyperparameter(name="str_1", choices=["on", "off"])) + input_space.add( CS.CategoricalHyperparameter(name="str_2", choices=[f"choice_{idx}" for idx in range(10)]) ) @@ -430,7 +414,7 @@ def test_approx_inverse_mapping( adapter.target_parameter_space, values=target_config_df.iloc[0].to_dict(), ) - adapter.target_parameter_space.check_configuration(target_config) + target_config.check_valid_configuration() # Test inverse transform with 100 random configs for _ in range(100): @@ -444,7 +428,7 @@ def test_approx_inverse_mapping( adapter.target_parameter_space, values=target_config_df.iloc[0].to_dict(), ) - adapter.target_parameter_space.check_configuration(target_config) + target_config.check_valid_configuration() @pytest.mark.parametrize( @@ -503,7 +487,7 @@ def test_llamatune_pipeline( orig_config_df = adapter.transform(sampled_config_df) # High-dim (i.e., original) config should be valid orig_config = CS.Configuration(input_space, values=orig_config_df.iloc[0].to_dict()) - input_space.check_configuration(orig_config) + orig_config.check_valid_configuration() # Transform high-dim config back to low-dim target_config_df = adapter.inverse_transform(orig_config_df) diff --git a/mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py b/mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py index cc5cb4f5d0..4d141bdfce 100644 --- a/mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py +++ b/mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py @@ -52,9 +52,9 @@ def test_create_space_adapter_with_factory_method( input_space = CS.ConfigurationSpace(seed=1234) # Add a single continuous input dimension between 0 and 1. - input_space.add_hyperparameter(CS.UniformFloatHyperparameter(name="x", lower=0, upper=1)) + input_space.add(CS.UniformFloatHyperparameter(name="x", lower=0, upper=1)) # Add a single continuous input dimension between 0 and 1. - input_space.add_hyperparameter(CS.UniformFloatHyperparameter(name="y", lower=0, upper=1)) + input_space.add(CS.UniformFloatHyperparameter(name="y", lower=0, upper=1)) # Adjust some kwargs for specific space adapters if space_adapter_type is SpaceAdapterType.LLAMATUNE: diff --git a/mlos_core/mlos_core/tests/spaces/spaces_test.py b/mlos_core/mlos_core/tests/spaces/spaces_test.py index 49890a7c66..479f398088 100644 --- a/mlos_core/mlos_core/tests/spaces/spaces_test.py +++ b/mlos_core/mlos_core/tests/spaces/spaces_test.py @@ -75,7 +75,7 @@ class BaseConversion(metaclass=ABCMeta): conversion_function: Callable[..., OptimizerSpace] = invalid_conversion_function @abstractmethod - def sample(self, config_space: OptimizerSpace, n_samples: int = 1) -> OptimizerParam: + def sample(self, config_space: OptimizerSpace, n_samples: int = 1) -> npt.NDArray: """ Sample from the given configuration space. @@ -115,14 +115,14 @@ def test_dimensionality(self) -> None: def test_unsupported_hyperparameter(self) -> None: input_space = CS.ConfigurationSpace() - input_space.add_hyperparameter(NormalIntegerHyperparameter("a", 2, 1)) + input_space.add(NormalIntegerHyperparameter("a", mu=50, sigma=5, lower=0, upper=99)) with pytest.raises(ValueError, match="NormalIntegerHyperparameter"): self.conversion_function(input_space) def test_continuous_bounds(self) -> None: input_space = CS.ConfigurationSpace() - input_space.add_hyperparameter(CS.UniformFloatHyperparameter("a", lower=100, upper=200)) - input_space.add_hyperparameter(CS.UniformIntegerHyperparameter("b", lower=-10, upper=-5)) + input_space.add(CS.UniformFloatHyperparameter("a", lower=100, upper=200)) + input_space.add(CS.UniformIntegerHyperparameter("b", lower=-10, upper=-5)) converted_space = self.conversion_function(input_space) assert self.get_parameter_names(converted_space) == [ # pylint: disable=unreachable @@ -134,9 +134,8 @@ def test_continuous_bounds(self) -> None: assert -10 <= point[1] <= -5 def test_uniform_samples(self) -> None: - input_space = CS.ConfigurationSpace() - input_space.add_hyperparameter(CS.UniformFloatHyperparameter("a", lower=1, upper=5)) - input_space.add_hyperparameter(CS.UniformIntegerHyperparameter("c", lower=1, upper=20)) + c = CS.UniformIntegerHyperparameter("c", lower=1, upper=20) + input_space = CS.ConfigurationSpace({"a": (1.0, 5.0), "c": c}) converted_space = self.conversion_function(input_space) np.random.seed(42) # pylint: disable=unreachable @@ -146,14 +145,14 @@ def test_uniform_samples(self) -> None: assert_is_uniform(uniform) # Check that we get both ends of the sampled range returned to us. - assert input_space["c"].lower in integer_uniform - assert input_space["c"].upper in integer_uniform + assert c.upper in integer_uniform + assert c.lower in integer_uniform # integer uniform assert_is_uniform(integer_uniform) def test_uniform_categorical(self) -> None: input_space = CS.ConfigurationSpace() - input_space.add_hyperparameter(CS.CategoricalHyperparameter("c", choices=["foo", "bar"])) + input_space.add(CS.CategoricalHyperparameter("c", choices=["foo", "bar"])) converted_space = self.conversion_function(input_space) points = self.sample(converted_space, n_samples=100) # pylint: disable=unreachable counts = self.categorical_counts(points) @@ -199,16 +198,16 @@ def categorical_counts(self, points: npt.NDArray) -> npt.NDArray: def test_dimensionality(self) -> None: input_space = CS.ConfigurationSpace() - input_space.add_hyperparameter(CS.UniformIntegerHyperparameter("a", lower=1, upper=10)) - input_space.add_hyperparameter(CS.CategoricalHyperparameter("b", choices=["bof", "bum"])) - input_space.add_hyperparameter(CS.CategoricalHyperparameter("c", choices=["foo", "bar"])) + input_space.add(CS.UniformIntegerHyperparameter("a", lower=1, upper=10)) + input_space.add(CS.CategoricalHyperparameter("b", choices=["bof", "bum"])) + input_space.add(CS.CategoricalHyperparameter("c", choices=["foo", "bar"])) output_space = configspace_to_flaml_space(input_space) assert len(output_space) == 3 def test_weighted_categorical(self) -> None: np.random.seed(42) input_space = CS.ConfigurationSpace() - input_space.add_hyperparameter( + input_space.add( CS.CategoricalHyperparameter("c", choices=["foo", "bar"], weights=[0.9, 0.1]) ) with pytest.raises(ValueError, match="non-uniform"): @@ -219,9 +218,7 @@ def test_log_int_spaces(self) -> None: np.random.seed(42) # integer is supported input_space = CS.ConfigurationSpace() - input_space.add_hyperparameter( - CS.UniformIntegerHyperparameter("d", lower=1, upper=20, log=True) - ) + input_space.add(CS.UniformIntegerHyperparameter("d", lower=1, upper=20, log=True)) converted_space = configspace_to_flaml_space(input_space) # test log integer sampling @@ -239,9 +236,7 @@ def test_log_float_spaces(self) -> None: # continuous is supported input_space = CS.ConfigurationSpace() - input_space.add_hyperparameter( - CS.UniformFloatHyperparameter("b", lower=1, upper=5, log=True) - ) + input_space.add(CS.UniformFloatHyperparameter("b", lower=1, upper=5, log=True)) converted_space = configspace_to_flaml_space(input_space) # test log integer sampling diff --git a/mlos_core/mlos_core/util.py b/mlos_core/mlos_core/util.py index 0a66c5a837..027bfd0e35 100644 --- a/mlos_core/mlos_core/util.py +++ b/mlos_core/mlos_core/util.py @@ -27,6 +27,23 @@ def config_to_dataframe(config: Configuration) -> pd.DataFrame: return pd.DataFrame([dict(config)]) +def drop_nulls(d: dict) -> dict: + """ + Remove all key-value pairs where the value is None. + + Parameters + ---------- + d : dict + The dictionary to clean. + + Returns + ------- + dict + The cleaned dictionary. + """ + return {k: v for k, v in d.items() if v is not None} + + def normalize_config( config_space: ConfigurationSpace, config: Union[Configuration, dict], diff --git a/mlos_core/notebooks/BayesianOptimization.ipynb b/mlos_core/notebooks/BayesianOptimization.ipynb index adf38869ff..d8a3118548 100644 --- a/mlos_core/notebooks/BayesianOptimization.ipynb +++ b/mlos_core/notebooks/BayesianOptimization.ipynb @@ -93,7 +93,7 @@ "input_space = CS.ConfigurationSpace(seed=1234)\n", "\n", "# Add a single continuous input dimension between 0 and 1.\n", - "input_space.add_hyperparameter(CS.UniformFloatHyperparameter(name='x', lower=0, upper=1))" + "input_space.add(CS.UniformFloatHyperparameter(name='x', lower=0, upper=1))" ] }, { diff --git a/mlos_core/setup.py b/mlos_core/setup.py index e4c792e270..87ad871dbc 100644 --- a/mlos_core/setup.py +++ b/mlos_core/setup.py @@ -69,7 +69,8 @@ def _get_long_desc_from_readme(base_url: str) -> dict: extra_requires: Dict[str, List[str]] = { # pylint: disable=consider-using-namedtuple-or-dataclass "flaml": ["flaml[blendsearch]"], - "smac": ["smac>=2.0.0"], # NOTE: Major refactoring on SMAC starting from v2.0.0 + # NOTE: Major refactoring on SMAC and ConfigSpace v1.0 starting from v2.2 + "smac": ["smac>=2.2.0"], } # construct special 'full' extra that adds requirements for all built-in @@ -97,7 +98,7 @@ def _get_long_desc_from_readme(base_url: str) -> dict: 'pandas >= 2.2.0;python_version>="3.9"', 'Bottleneck > 1.3.5;python_version>="3.9"', 'pandas >= 1.0.3;python_version<"3.9"', - "ConfigSpace==0.7.1", # Temporarily restrict ConfigSpace version. + "ConfigSpace>=1.0", ], extras_require=extra_requires, **_get_long_desc_from_readme("https://github.com/microsoft/MLOS/tree/main/mlos_core"), diff --git a/pyproject.toml b/pyproject.toml index 4380245f79..5464aa60b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,9 +30,6 @@ fail-on = [ "unused-import", ] -# Ignore pylint complaints about an upstream dependency. -ignored-modules = ["ConfigSpace.hyperparameters"] - # Help inform pylint where to find the project's source code without needing to relyon PYTHONPATH. #init-hook="from pylint.config import find_pylintrc; import os, sys; sys.path.append(os.path.dirname(find_pylintrc())); from logging import warning; warning(sys.path)" init-hook = "from logging import warning; warning(sys.path)" diff --git a/setup.cfg b/setup.cfg index 6f948f523a..cd3bdbb991 100644 --- a/setup.cfg +++ b/setup.cfg @@ -84,10 +84,6 @@ hide_error_codes = False # We also skip several vendor files that currently throw errors. exclude = (mlos_(core|bench|viz)/setup.py)|(doc/)|(/build/)|(-packages/_pytest/) -# https://github.com/automl/ConfigSpace/issues/293 -[mypy-ConfigSpace.*] -ignore_missing_imports = True - [mypy-fasteners.*] ignore_missing_imports = True