From 6bf3862ae15f47a8db04760b4b6a0a0724b15985 Mon Sep 17 00:00:00 2001 From: Sergiy Matusevych Date: Tue, 2 Jul 2024 13:47:32 -0700 Subject: [PATCH] Better naming and values for MockEnv parameters + unit tests (#769) * Use `mock_env_range`, `mock_env_seed`, and `mock_env_metrics` instead of just `range`, `seed`, and `metrics` to avoid naming conflicts. * Use integer `mocl_env_seed` values to distinguish between deterministic behavior, random seed, and user-specified seed for `MockEnv`. * Update unit tests and configs to validate the seed behavior. * Add new unit tests for different initializations of the tunable values --------- Co-authored-by: Brian Kroth --- mlos_bench/mlos_bench/DEVNOTES.md | 2 +- .../config/environments/mock/mock_env.jsonc | 6 ++-- .../environments/mock-env-subschema.json | 10 +++--- .../mlos_bench/environments/mock_env.py | 12 +++---- .../mlos_bench/optimizers/mock_optimizer.py | 2 +- .../tests/config/cli/mock-bench.jsonc | 7 ++-- .../invalid/mock_env-bad-metric-type.jsonc | 2 +- .../mock_env-const-args-list-obj.jsonc | 2 +- .../mock_env-const-args-nested-list.jsonc | 2 +- .../bad/invalid/mock_env-missing-metric.jsonc | 2 +- .../bad/invalid/mock_env-missing-name.jsonc | 2 +- .../test-cases/good/full/mock_env-full.jsonc | 6 ++-- .../good/partial/mock_env-const-args.jsonc | 2 +- .../tunable-values-example.jsonc | 6 ++-- mlos_bench/mlos_bench/tests/conftest.py | 11 +++--- .../tests/environments/composite_env_test.py | 20 +++++------ .../tests/launcher_in_process_test.py | 5 ++- .../mlos_bench/tests/launcher_run_test.py | 36 ++++++++++++++++--- 18 files changed, 84 insertions(+), 51 deletions(-) diff --git a/mlos_bench/mlos_bench/DEVNOTES.md b/mlos_bench/mlos_bench/DEVNOTES.md index 0150d25efd5..9486168e525 100644 --- a/mlos_bench/mlos_bench/DEVNOTES.md +++ b/mlos_bench/mlos_bench/DEVNOTES.md @@ -34,7 +34,7 @@ Each `Environment` config is a JSON5 file with the following structure: } // Environment constructor parameters // (specific to the Environment class being instantiated): - "seed": 42, + "mock_env_seed": 42, // ... } } diff --git a/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc b/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc index bf0021d4859..e2ec01e0afa 100644 --- a/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc +++ b/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc @@ -19,8 +19,8 @@ "linux-hugepages-2048kB", "redis" ], - "seed": 42, // Seed for the random noise generator. Omit to produce noise-free data. - "range": [60, 120], // Range of the generated output values of the benchmark. - "metrics": ["score"] // Names of fake benchmark metrics to generate. (Default is one metric, "score"). + "mock_env_seed": 42, // Seed for the random noise generator. Omit or set to 0 to produce noise-free data. + "mock_env_range": [60, 120], // Range of the generated output values of the benchmark. + "mock_env_metrics": ["score"] // Names of fake benchmark metrics to generate. (Default is one metric, "score"). } } diff --git a/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json b/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json index 35c3667ee0f..cb2de6c719f 100644 --- a/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json +++ b/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json @@ -19,12 +19,12 @@ }, { "properties": { - "seed": { + "mock_env_seed": { "type": "integer", - "description": "Seed for the random number generator", - "default": 0 + "description": "Seed for the random number generator. Set to -1 for deterministic behavior, 0 for default randomness.", + "default": -1 }, - "range": { + "mock_env_range": { "type": "array", "description": "Range of the random number generator", "items": { @@ -33,7 +33,7 @@ "minItems": 2, "maxItems": 2 }, - "metrics": { + "mock_env_metrics": { "type": "array", "description": "Names of fake benchmark metrics to be generate", "items": { diff --git a/mlos_bench/mlos_bench/environments/mock_env.py b/mlos_bench/mlos_bench/environments/mock_env.py index 00fa55014d3..d8ffe3e47d6 100644 --- a/mlos_bench/mlos_bench/environments/mock_env.py +++ b/mlos_bench/mlos_bench/environments/mock_env.py @@ -17,7 +17,6 @@ from mlos_bench.environments.status import Status from mlos_bench.environments.base_environment import Environment from mlos_bench.tunables import Tunable, TunableGroups, TunableValue -from mlos_bench.util import nullable _LOG = logging.getLogger(__name__) @@ -49,7 +48,8 @@ def __init__(self, global_config : dict Free-format dictionary of global parameters (e.g., security credentials) to be mixed in into the "const_args" section of the local config. - Optional arguments are `seed`, `range`, and `metrics`. + Optional arguments are `mock_env_seed`, `mock_env_range`, and `mock_env_metrics`. + Set `mock_env_seed` to -1 for deterministic behavior, 0 for default randomness. tunables : TunableGroups A collection of tunable parameters for *all* environments. service: Service @@ -57,10 +57,10 @@ def __init__(self, """ super().__init__(name=name, config=config, global_config=global_config, tunables=tunables, service=service) - seed = self.config.get("seed") - self._random = nullable(random.Random, seed) - self._range = self.config.get("range") - self._metrics = self.config.get("metrics", ["score"]) + seed = int(self.config.get("mock_env_seed", -1)) + self._random = random.Random(seed or None) if seed >= 0 else None + self._range = self.config.get("mock_env_range") + self._metrics = self.config.get("mock_env_metrics", ["score"]) self._is_ready = True def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]: diff --git a/mlos_bench/mlos_bench/optimizers/mock_optimizer.py b/mlos_bench/mlos_bench/optimizers/mock_optimizer.py index 44f8262b286..7d2caff8ff0 100644 --- a/mlos_bench/mlos_bench/optimizers/mock_optimizer.py +++ b/mlos_bench/mlos_bench/optimizers/mock_optimizer.py @@ -61,7 +61,7 @@ def suggest(self) -> TunableGroups: """ tunables = super().suggest() if self._start_with_defaults: - _LOG.info("Use default values for the first trial") + _LOG.info("Use default tunable values") self._start_with_defaults = False else: for (tunable, _group) in tunables: diff --git a/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc b/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc index 40ba6f29002..1deab160660 100644 --- a/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc +++ b/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc @@ -14,9 +14,10 @@ "environment": "environments/mock/mock_env.jsonc", - "tunable_values": [ - "tunable-values/tunable-values-example.jsonc" - ], + // Use default values for tunables unless otherwise specified + // "tunable_values": [ + // "tunable-values/tunable-values-example.jsonc" + // ], "globals": ["globals/global_test_config.jsonc"], diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc index b8b6d126125..85a0a4cfe20 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc @@ -2,7 +2,7 @@ "name": "mock_env-full", "class": "mlos_bench.environments.mock_env.MockEnv", "config": { - "metrics": [ + "mock_env_metrics": [ {"bad": "metric type"} ] } diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc index bfa8172b20f..86d0318f283 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc @@ -2,7 +2,7 @@ "name": "partial mock env", "class": "mlos_bench.environments.MockEnv", "config": { - "seed": 42, + "mock_env_seed": 42, "const_args": { "foo": "bar", "int": 1, diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc index f2f3be64ec7..81da08ddab7 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc @@ -2,7 +2,7 @@ "name": "partial mock env", "class": "mlos_bench.environments.MockEnv", "config": { - "seed": 42, + "mock_env_seed": 42, "const_args": { "foo": "bar", "int": 1, diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc index 03f6f1a2b31..99a61969e62 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc @@ -2,7 +2,7 @@ "name": "mock_env-full", "class": "mlos_bench.environments.mock_env.MockEnv", "config": { - "metrics": [ + "mock_env_metrics": [ // needs at least one element ] } diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc index 66f0c49eecc..1887a4de6f2 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc @@ -2,6 +2,6 @@ // "name": "missing name is invalid", "class": "mlos_bench.environments.MockEnv", "config": { - "seed": 42 + "mock_env_seed": 42 } } diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc index d196321d87b..a00f8ca60c0 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc @@ -20,9 +20,9 @@ "required_args": [ "foo" ], - "range": [0, 1], - "seed": 42, - "metrics": [ + "mock_env_range": [0, 1], + "mock_env_seed": 42, + "mock_env_metrics": [ "latency", "cost" ] diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc index 517ea318348..b71a81f57cc 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc @@ -2,7 +2,7 @@ "name": "partial mock env", "class": "mlos_bench.environments.MockEnv", "config": { - "seed": 42, + "mock_env_seed": 42, "const_args": { "foo": "bar", "int": 1, diff --git a/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc b/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc index 362ede79889..0965e39b8d6 100644 --- a/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc +++ b/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc @@ -2,8 +2,10 @@ { "$schema": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json", - "sched_migration_cost_ns": 500000, - "sched_latency_ns": 12000000, + // Values that are different from the defaults + // in order to test --tunable-values handling in OneShotOptimizer. + "sched_migration_cost_ns": 400000, + "sched_latency_ns": 10000000, "sched_child_runs_first": "0", "sched_tunable_scaling": "1" } diff --git a/mlos_bench/mlos_bench/tests/conftest.py b/mlos_bench/mlos_bench/tests/conftest.py index c632abe09da..f30fe435854 100644 --- a/mlos_bench/mlos_bench/tests/conftest.py +++ b/mlos_bench/mlos_bench/tests/conftest.py @@ -40,9 +40,9 @@ def mock_env(tunable_groups: TunableGroups) -> MockEnv: name="Test Env", config={ "tunable_params": ["provision", "boot", "kernel"], - "seed": SEED, - "range": [60, 120], - "metrics": ["score"], + "mock_env_seed": SEED, + "mock_env_range": [60, 120], + "mock_env_metrics": ["score"], }, tunables=tunable_groups ) @@ -57,8 +57,9 @@ def mock_env_no_noise(tunable_groups: TunableGroups) -> MockEnv: name="Test Env No Noise", config={ "tunable_params": ["provision", "boot", "kernel"], - "range": [60, 120], - "metrics": ["score", "other_score"], + "mock_env_seed": -1, + "mock_env_range": [60, 120], + "mock_env_metrics": ["score", "other_score"], }, tunables=tunable_groups ) diff --git a/mlos_bench/mlos_bench/tests/environments/composite_env_test.py b/mlos_bench/mlos_bench/tests/environments/composite_env_test.py index a1bfc793b4b..fd7c0229393 100644 --- a/mlos_bench/mlos_bench/tests/environments/composite_env_test.py +++ b/mlos_bench/mlos_bench/tests/environments/composite_env_test.py @@ -41,8 +41,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv: "EnvId": 1, }, "required_args": ["vmName", "someConst", "global_param"], - "range": [60, 120], - "metrics": ["score"], + "mock_env_range": [60, 120], + "mock_env_metrics": ["score"], } }, { @@ -56,8 +56,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv: "global_param": "local" }, "required_args": ["vmName"], - "range": [60, 120], - "metrics": ["score"], + "mock_env_range": [60, 120], + "mock_env_metrics": ["score"], } }, { @@ -70,8 +70,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv: "EnvId": 3, }, "required_args": ["vmName", "vm_server_name", "vm_client_name"], - "range": [60, 120], - "metrics": ["score"], + "mock_env_range": [60, 120], + "mock_env_metrics": ["score"], } } ] @@ -193,8 +193,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv: "vm_server_name", "global_param" ], - "range": [60, 120], - "metrics": ["score"], + "mock_env_range": [60, 120], + "mock_env_metrics": ["score"], } }, # ... @@ -218,8 +218,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv: "config": { "tunable_params": ["boot"], "required_args": ["vmName", "EnvId", "vm_client_name"], - "range": [60, 120], - "metrics": ["score"], + "mock_env_range": [60, 120], + "mock_env_metrics": ["score"], } }, # ... diff --git a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py index 6340923e825..90aa7e08f71 100644 --- a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py +++ b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py @@ -17,11 +17,14 @@ ("argv", "expected_score"), [ ([ "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc", - ], 65.6742), + "--trial_config_repeat_count", "5", + "--mock_env_seed", "-1", # Deterministic Mock Environment. + ], 67.40329), ([ "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc", "--trial_config_repeat_count", "3", "--max_suggestions", "3", + "--mock_env_seed", "42", # Noisy Mock Environment. ], 64.53897), ] ) diff --git a/mlos_bench/mlos_bench/tests/launcher_run_test.py b/mlos_bench/mlos_bench/tests/launcher_run_test.py index fbca259e93e..d8caf7537e2 100644 --- a/mlos_bench/mlos_bench/tests/launcher_run_test.py +++ b/mlos_bench/mlos_bench/tests/launcher_run_test.py @@ -52,7 +52,9 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService, # temp_dir = '/tmp' log_path = path_join(temp_dir, "mock-test.log") (return_code, _stdout, _stderr) = local_exec_service.local_exec( - [f"./mlos_bench/mlos_bench/run.py {cli_config} --log_file '{log_path}'"], + ["./mlos_bench/mlos_bench/run.py" + + " --config_path ./mlos_bench/mlos_bench/tests/config/" + + f" {cli_config} --log_file '{log_path}'"], cwd=root_path) assert return_code == 0 @@ -74,14 +76,35 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService, def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecService) -> None: """ Run mlos_bench command-line application with mock benchmark config - and check the results in the log. + and default tunable values and check the results in the log. + """ + _launch_main_app( + root_path, local_exec_service, + " --config cli/mock-bench.jsonc" + + " --trial_config_repeat_count 5" + + " --mock_env_seed -1", # Deterministic Mock Environment. + [ + f"^{_RE_DATE} run\\.py:\\d+ " + + r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$", + ] + ) + + +def test_launch_main_app_bench_values( + root_path: str, local_exec_service: LocalExecService) -> None: + """ + Run mlos_bench command-line application with mock benchmark config + and user-specified tunable values and check the results in the log. """ _launch_main_app( root_path, local_exec_service, - "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc", + " --config cli/mock-bench.jsonc" + + " --tunable_values tunable-values/tunable-values-example.jsonc" + + " --trial_config_repeat_count 5" + + " --mock_env_seed -1", # Deterministic Mock Environment. [ f"^{_RE_DATE} run\\.py:\\d+ " + - r"_main INFO Final score: \{'score': 65\.67\d+\}\s*$", + r"_main INFO Final score: \{'score': 67\.11\d+\}\s*$", ] ) @@ -93,7 +116,10 @@ def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecServic """ _launch_main_app( root_path, local_exec_service, - "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc --trial_config_repeat_count 3 --max_suggestions 3", + "--config cli/mock-opt.jsonc" + + " --trial_config_repeat_count 3" + + " --max_suggestions 3" + + " --mock_env_seed 42", # Noisy Mock Environment. [ # Iteration 1: Expect first value to be the baseline f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +