From 405b1b8f5fd7beffa3142fab5f2fdea8c974ed8b Mon Sep 17 00:00:00 2001
From: Sergiy Matusevych <sergiym@microsoft.com>
Date: Wed, 26 Jun 2024 15:05:26 -0700
Subject: [PATCH 1/7] enforce OneShot optimizer to always return the same
 initial config

---
 mlos_bench/mlos_bench/optimizers/mock_optimizer.py     | 2 +-
 mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/mlos_bench/mlos_bench/optimizers/mock_optimizer.py b/mlos_bench/mlos_bench/optimizers/mock_optimizer.py
index 44f8262b286..7d2caff8ff0 100644
--- a/mlos_bench/mlos_bench/optimizers/mock_optimizer.py
+++ b/mlos_bench/mlos_bench/optimizers/mock_optimizer.py
@@ -61,7 +61,7 @@ def suggest(self) -> TunableGroups:
         """
         tunables = super().suggest()
         if self._start_with_defaults:
-            _LOG.info("Use default values for the first trial")
+            _LOG.info("Use default tunable values")
             self._start_with_defaults = False
         else:
             for (tunable, _group) in tunables:
diff --git a/mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py b/mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py
index 088ed03bdf0..9cb18fab679 100644
--- a/mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py
+++ b/mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py
@@ -33,6 +33,14 @@ def __init__(self,
         _LOG.info("Run a single iteration for: %s", self._tunables)
         self._max_iter = 1  # Always run for just one iteration.
 
+    def suggest(self) -> TunableGroups:
+        """
+        Always produce the same (initial) suggestion.
+        """
+        tunables = super().suggest()
+        self._start_with_defaults = True
+        return tunables
+
     @property
     def supports_preload(self) -> bool:
         return False

From c1906e5d1de8b169a0ea654333c625d8d73f63df Mon Sep 17 00:00:00 2001
From: Sergiy Matusevych <sergiym@microsoft.com>
Date: Wed, 26 Jun 2024 15:06:51 -0700
Subject: [PATCH 2/7] minor fix in the docstring

---
 mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py b/mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py
index 9cb18fab679..314d048298f 100644
--- a/mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py
+++ b/mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py
@@ -18,7 +18,7 @@
 
 class OneShotOptimizer(MockOptimizer):
     """
-    Mock optimizer that proposes a single configuration and returns.
+    No-op optimizer that proposes a single configuration and returns.
     Explicit configs (partial or full) are possible using configuration files.
     """
 

From c8f67be5b813d60645e3b016eb498c6c5611e5ec Mon Sep 17 00:00:00 2001
From: Sergiy Matusevych <sergiym@microsoft.com>
Date: Wed, 26 Jun 2024 17:13:52 -0700
Subject: [PATCH 3/7] better naming for mock env parameters + fixes to unit
 tests

---
 mlos_bench/mlos_bench/DEVNOTES.md             |  2 +-
 .../config/environments/mock/mock_env.jsonc   |  6 +++---
 .../environments/mock-env-subschema.json      |  6 +++---
 .../mlos_bench/environments/mock_env.py       | 11 +++++-----
 .../invalid/mock_env-bad-metric-type.jsonc    |  2 +-
 .../mock_env-const-args-list-obj.jsonc        |  2 +-
 .../mock_env-const-args-nested-list.jsonc     |  2 +-
 .../bad/invalid/mock_env-missing-metric.jsonc |  2 +-
 .../bad/invalid/mock_env-missing-name.jsonc   |  2 +-
 .../test-cases/good/full/mock_env-full.jsonc  |  6 +++---
 .../good/partial/mock_env-const-args.jsonc    |  2 +-
 mlos_bench/mlos_bench/tests/conftest.py       | 11 +++++-----
 .../tests/environments/composite_env_test.py  | 20 +++++++++----------
 .../tests/launcher_in_process_test.py         |  7 +++++--
 .../mlos_bench/tests/launcher_run_test.py     | 11 +++++++---
 15 files changed, 50 insertions(+), 42 deletions(-)

diff --git a/mlos_bench/mlos_bench/DEVNOTES.md b/mlos_bench/mlos_bench/DEVNOTES.md
index 0150d25efd5..9486168e525 100644
--- a/mlos_bench/mlos_bench/DEVNOTES.md
+++ b/mlos_bench/mlos_bench/DEVNOTES.md
@@ -34,7 +34,7 @@ Each `Environment` config is a JSON5 file with the following structure:
         }
         // Environment constructor parameters
         // (specific to the Environment class being instantiated):
-        "seed": 42,
+        "mock_env_seed": 42,
         // ...
     }
 }
diff --git a/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc b/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc
index bf0021d4859..e2ec01e0afa 100644
--- a/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc
+++ b/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc
@@ -19,8 +19,8 @@
             "linux-hugepages-2048kB",
             "redis"
         ],
-        "seed": 42,            // Seed for the random noise generator. Omit to produce noise-free data.
-        "range": [60, 120],    // Range of the generated output values of the benchmark.
-        "metrics": ["score"]   // Names of fake benchmark metrics to generate. (Default is one metric, "score").
+        "mock_env_seed": 42,            // Seed for the random noise generator. Omit or set to 0 to produce noise-free data.
+        "mock_env_range": [60, 120],    // Range of the generated output values of the benchmark.
+        "mock_env_metrics": ["score"]   // Names of fake benchmark metrics to generate. (Default is one metric, "score").
     }
 }
diff --git a/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json b/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
index 35c3667ee0f..5097d6ba282 100644
--- a/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
+++ b/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
@@ -19,12 +19,12 @@
                 },
                 {
                     "properties": {
-                        "seed": {
+                        "mock_env_seed": {
                             "type": "integer",
                             "description": "Seed for the random number generator",
                             "default": 0
                         },
-                        "range": {
+                        "mock_env_range": {
                             "type": "array",
                             "description": "Range of the random number generator",
                             "items": {
@@ -33,7 +33,7 @@
                             "minItems": 2,
                             "maxItems": 2
                         },
-                        "metrics": {
+                        "mock_env_metrics": {
                             "type": "array",
                             "description": "Names of fake benchmark metrics to be generate",
                             "items": {
diff --git a/mlos_bench/mlos_bench/environments/mock_env.py b/mlos_bench/mlos_bench/environments/mock_env.py
index 00fa55014d3..030e5438e4a 100644
--- a/mlos_bench/mlos_bench/environments/mock_env.py
+++ b/mlos_bench/mlos_bench/environments/mock_env.py
@@ -17,7 +17,6 @@
 from mlos_bench.environments.status import Status
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.tunables import Tunable, TunableGroups, TunableValue
-from mlos_bench.util import nullable
 
 _LOG = logging.getLogger(__name__)
 
@@ -49,7 +48,7 @@ def __init__(self,
         global_config : dict
             Free-format dictionary of global parameters (e.g., security credentials)
             to be mixed in into the "const_args" section of the local config.
-            Optional arguments are `seed`, `range`, and `metrics`.
+            Optional arguments are `mock_env_seed`, `mock_env_range`, and `mock_env_metrics`.
         tunables : TunableGroups
             A collection of tunable parameters for *all* environments.
         service: Service
@@ -57,10 +56,10 @@ def __init__(self,
         """
         super().__init__(name=name, config=config, global_config=global_config,
                          tunables=tunables, service=service)
-        seed = self.config.get("seed")
-        self._random = nullable(random.Random, seed)
-        self._range = self.config.get("range")
-        self._metrics = self.config.get("metrics", ["score"])
+        seed = int(self.config.get("mock_env_seed", 0))
+        self._random = random.Random(seed) if seed > 0 else None
+        self._range = self.config.get("mock_env_range")
+        self._metrics = self.config.get("mock_env_metrics", ["score"])
         self._is_ready = True
 
     def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]:
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc
index b8b6d126125..85a0a4cfe20 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc
@@ -2,7 +2,7 @@
     "name": "mock_env-full",
     "class": "mlos_bench.environments.mock_env.MockEnv",
     "config": {
-        "metrics": [
+        "mock_env_metrics": [
             {"bad": "metric type"}
         ]
     }
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc
index bfa8172b20f..86d0318f283 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc
@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "seed": 42,
+        "mock_env_seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc
index f2f3be64ec7..81da08ddab7 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc
@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "seed": 42,
+        "mock_env_seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc
index 03f6f1a2b31..99a61969e62 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc
@@ -2,7 +2,7 @@
     "name": "mock_env-full",
     "class": "mlos_bench.environments.mock_env.MockEnv",
     "config": {
-        "metrics": [
+        "mock_env_metrics": [
             // needs at least one element
         ]
     }
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc
index 66f0c49eecc..1887a4de6f2 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc
@@ -2,6 +2,6 @@
     // "name": "missing name is invalid",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "seed": 42
+        "mock_env_seed": 42
     }
 }
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc
index d196321d87b..a00f8ca60c0 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc
@@ -20,9 +20,9 @@
         "required_args": [
             "foo"
         ],
-        "range": [0, 1],
-        "seed": 42,
-        "metrics": [
+        "mock_env_range": [0, 1],
+        "mock_env_seed": 42,
+        "mock_env_metrics": [
             "latency",
             "cost"
         ]
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc
index 517ea318348..b71a81f57cc 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc
@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "seed": 42,
+        "mock_env_seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,
diff --git a/mlos_bench/mlos_bench/tests/conftest.py b/mlos_bench/mlos_bench/tests/conftest.py
index c632abe09da..f5cb85e9215 100644
--- a/mlos_bench/mlos_bench/tests/conftest.py
+++ b/mlos_bench/mlos_bench/tests/conftest.py
@@ -40,9 +40,9 @@ def mock_env(tunable_groups: TunableGroups) -> MockEnv:
         name="Test Env",
         config={
             "tunable_params": ["provision", "boot", "kernel"],
-            "seed": SEED,
-            "range": [60, 120],
-            "metrics": ["score"],
+            "mock_env_seed": SEED,
+            "mock_env_range": [60, 120],
+            "mock_env_metrics": ["score"],
         },
         tunables=tunable_groups
     )
@@ -57,8 +57,9 @@ def mock_env_no_noise(tunable_groups: TunableGroups) -> MockEnv:
         name="Test Env No Noise",
         config={
             "tunable_params": ["provision", "boot", "kernel"],
-            "range": [60, 120],
-            "metrics": ["score", "other_score"],
+            "mock_env_seed": 0,
+            "mock_env_range": [60, 120],
+            "mock_env_metrics": ["score", "other_score"],
         },
         tunables=tunable_groups
     )
diff --git a/mlos_bench/mlos_bench/tests/environments/composite_env_test.py b/mlos_bench/mlos_bench/tests/environments/composite_env_test.py
index a1bfc793b4b..fd7c0229393 100644
--- a/mlos_bench/mlos_bench/tests/environments/composite_env_test.py
+++ b/mlos_bench/mlos_bench/tests/environments/composite_env_test.py
@@ -41,8 +41,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "EnvId": 1,
                         },
                         "required_args": ["vmName", "someConst", "global_param"],
-                        "range": [60, 120],
-                        "metrics": ["score"],
+                        "mock_env_range": [60, 120],
+                        "mock_env_metrics": ["score"],
                     }
                 },
                 {
@@ -56,8 +56,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "global_param": "local"
                         },
                         "required_args": ["vmName"],
-                        "range": [60, 120],
-                        "metrics": ["score"],
+                        "mock_env_range": [60, 120],
+                        "mock_env_metrics": ["score"],
                     }
                 },
                 {
@@ -70,8 +70,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "EnvId": 3,
                         },
                         "required_args": ["vmName", "vm_server_name", "vm_client_name"],
-                        "range": [60, 120],
-                        "metrics": ["score"],
+                        "mock_env_range": [60, 120],
+                        "mock_env_metrics": ["score"],
                     }
                 }
             ]
@@ -193,8 +193,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                                         "vm_server_name",
                                         "global_param"
                                     ],
-                                    "range": [60, 120],
-                                    "metrics": ["score"],
+                                    "mock_env_range": [60, 120],
+                                    "mock_env_metrics": ["score"],
                                 }
                             },
                             # ...
@@ -218,8 +218,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                                 "config": {
                                     "tunable_params": ["boot"],
                                     "required_args": ["vmName", "EnvId", "vm_client_name"],
-                                    "range": [60, 120],
-                                    "metrics": ["score"],
+                                    "mock_env_range": [60, 120],
+                                    "mock_env_metrics": ["score"],
                                 }
                             },
                             # ...
diff --git a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
index 6340923e825..747ff6a1257 100644
--- a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
@@ -17,12 +17,15 @@
     ("argv", "expected_score"), [
         ([
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
-        ], 65.6742),
+            "--trial_config_repeat_count", "5",
+            "--mock_env_seed", "0",  # Deterministic Mock Environment.
+        ], 67.40329),
         ([
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc",
             "--trial_config_repeat_count", "3",
             "--max_suggestions", "3",
-        ], 64.53897),
+            "--mock_env_seed", "0",  # Deterministic Mock Environment.
+        ], 66.61382),
     ]
 )
 def test_main_bench(argv: List[str], expected_score: float) -> None:
diff --git a/mlos_bench/mlos_bench/tests/launcher_run_test.py b/mlos_bench/mlos_bench/tests/launcher_run_test.py
index fbca259e93e..47946f13efc 100644
--- a/mlos_bench/mlos_bench/tests/launcher_run_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_run_test.py
@@ -78,10 +78,12 @@ def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecServ
     """
     _launch_main_app(
         root_path, local_exec_service,
-        "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
+        "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc" +
+        " --trial_config_repeat_count 5" +
+        " --mock_env_seed 0",  # Deterministic Mock Environment.
         [
             f"^{_RE_DATE} run\\.py:\\d+ " +
-            r"_main INFO Final score: \{'score': 65\.67\d+\}\s*$",
+            r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$",
         ]
     )
 
@@ -93,7 +95,10 @@ def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecServic
     """
     _launch_main_app(
         root_path, local_exec_service,
-        "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc --trial_config_repeat_count 3 --max_suggestions 3",
+        "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc" +
+        " --trial_config_repeat_count 3" +
+        " --max_suggestions 3" +
+        " --mock_env_seed 42",  # Noisy Mock Environment.
         [
             # Iteration 1: Expect first value to be the baseline
             f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +

From 3e628967a02ceabac88db57f4a5101c39f4863ac Mon Sep 17 00:00:00 2001
From: Sergiy Matusevych <sergiym@microsoft.com>
Date: Wed, 26 Jun 2024 17:40:09 -0700
Subject: [PATCH 4/7] use noisy mock env for optimziations

---
 mlos_bench/mlos_bench/tests/launcher_in_process_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
index 747ff6a1257..0f22793caba 100644
--- a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
@@ -24,8 +24,8 @@
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc",
             "--trial_config_repeat_count", "3",
             "--max_suggestions", "3",
-            "--mock_env_seed", "0",  # Deterministic Mock Environment.
-        ], 66.61382),
+            "--mock_env_seed", "42",  # Noisy Mock Environment.
+        ], 64.53897),
     ]
 )
 def test_main_bench(argv: List[str], expected_score: float) -> None:

From 58aa2b1c67cd4aaf5a757bd59da788e483468d5a Mon Sep 17 00:00:00 2001
From: Sergiy Matusevych <sergiym@microsoft.com>
Date: Fri, 28 Jun 2024 15:49:07 -0700
Subject: [PATCH 5/7] better values for mock env seed

---
 .../environments/mock-env-subschema.json      |  4 +--
 .../mlos_bench/environments/mock_env.py       |  5 ++--
 mlos_bench/mlos_bench/tests/conftest.py       |  2 +-
 .../tests/launcher_in_process_test.py         |  2 +-
 .../mlos_bench/tests/launcher_run_test.py     | 28 ++++++++++++++++---
 5 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json b/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
index 5097d6ba282..cb2de6c719f 100644
--- a/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
+++ b/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
@@ -21,8 +21,8 @@
                     "properties": {
                         "mock_env_seed": {
                             "type": "integer",
-                            "description": "Seed for the random number generator",
-                            "default": 0
+                            "description": "Seed for the random number generator. Set to -1 for deterministic behavior, 0 for default randomness.",
+                            "default": -1
                         },
                         "mock_env_range": {
                             "type": "array",
diff --git a/mlos_bench/mlos_bench/environments/mock_env.py b/mlos_bench/mlos_bench/environments/mock_env.py
index 030e5438e4a..d8ffe3e47d6 100644
--- a/mlos_bench/mlos_bench/environments/mock_env.py
+++ b/mlos_bench/mlos_bench/environments/mock_env.py
@@ -49,6 +49,7 @@ def __init__(self,
             Free-format dictionary of global parameters (e.g., security credentials)
             to be mixed in into the "const_args" section of the local config.
             Optional arguments are `mock_env_seed`, `mock_env_range`, and `mock_env_metrics`.
+            Set `mock_env_seed` to -1 for deterministic behavior, 0 for default randomness.
         tunables : TunableGroups
             A collection of tunable parameters for *all* environments.
         service: Service
@@ -56,8 +57,8 @@ def __init__(self,
         """
         super().__init__(name=name, config=config, global_config=global_config,
                          tunables=tunables, service=service)
-        seed = int(self.config.get("mock_env_seed", 0))
-        self._random = random.Random(seed) if seed > 0 else None
+        seed = int(self.config.get("mock_env_seed", -1))
+        self._random = random.Random(seed or None) if seed >= 0 else None
         self._range = self.config.get("mock_env_range")
         self._metrics = self.config.get("mock_env_metrics", ["score"])
         self._is_ready = True
diff --git a/mlos_bench/mlos_bench/tests/conftest.py b/mlos_bench/mlos_bench/tests/conftest.py
index f5cb85e9215..f30fe435854 100644
--- a/mlos_bench/mlos_bench/tests/conftest.py
+++ b/mlos_bench/mlos_bench/tests/conftest.py
@@ -57,7 +57,7 @@ def mock_env_no_noise(tunable_groups: TunableGroups) -> MockEnv:
         name="Test Env No Noise",
         config={
             "tunable_params": ["provision", "boot", "kernel"],
-            "mock_env_seed": 0,
+            "mock_env_seed": -1,
             "mock_env_range": [60, 120],
             "mock_env_metrics": ["score", "other_score"],
         },
diff --git a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
index 0f22793caba..90aa7e08f71 100644
--- a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
@@ -18,7 +18,7 @@
         ([
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
             "--trial_config_repeat_count", "5",
-            "--mock_env_seed", "0",  # Deterministic Mock Environment.
+            "--mock_env_seed", "-1",  # Deterministic Mock Environment.
         ], 67.40329),
         ([
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc",
diff --git a/mlos_bench/mlos_bench/tests/launcher_run_test.py b/mlos_bench/mlos_bench/tests/launcher_run_test.py
index 47946f13efc..fa601d93a36 100644
--- a/mlos_bench/mlos_bench/tests/launcher_run_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_run_test.py
@@ -34,6 +34,7 @@ def local_exec_service() -> LocalExecService:
     return LocalExecService(parent=ConfigPersistenceService({
         "config_path": [
             "mlos_bench/config",
+            "mlos_bench/tests/config",
             "mlos_bench/examples",
         ]
     }))
@@ -74,13 +75,32 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
 def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecService) -> None:
     """
     Run mlos_bench command-line application with mock benchmark config
-    and check the results in the log.
+    and default tunable values and check the results in the log.
+    """
+    _launch_main_app(
+        root_path, local_exec_service,
+        "--config cli/mock-bench.jsonc" +
+        " --trial_config_repeat_count 5" +
+        " --mock_env_seed -1",  # Deterministic Mock Environment.
+        [
+            f"^{_RE_DATE} run\\.py:\\d+ " +
+            r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$",
+        ]
+    )
+
+
+def test_launch_main_app_bench_values(
+        root_path: str, local_exec_service: LocalExecService) -> None:
+    """
+    Run mlos_bench command-line application with mock benchmark config
+    and user-specified tunable values and check the results in the log.
     """
     _launch_main_app(
         root_path, local_exec_service,
-        "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc" +
+        " --config cli/mock-bench.jsonc" +
+        " --tunable_values tunable-values/tunable-values-example.jsonc" +
         " --trial_config_repeat_count 5" +
-        " --mock_env_seed 0",  # Deterministic Mock Environment.
+        " --mock_env_seed -1",  # Deterministic Mock Environment.
         [
             f"^{_RE_DATE} run\\.py:\\d+ " +
             r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$",
@@ -95,7 +115,7 @@ def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecServic
     """
     _launch_main_app(
         root_path, local_exec_service,
-        "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc" +
+        "--config cli/mock-opt.jsonc" +
         " --trial_config_repeat_count 3" +
         " --max_suggestions 3" +
         " --mock_env_seed 42",  # Noisy Mock Environment.

From 46ee026edd95354487fd98f6aff3a5a2e71398c6 Mon Sep 17 00:00:00 2001
From: Sergiy Matusevych <sergiym@microsoft.com>
Date: Fri, 28 Jun 2024 16:05:10 -0700
Subject: [PATCH 6/7] better unit tests for tunables intialization

---
 mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc  | 7 ++++---
 .../config/tunable-values/tunable-values-example.jsonc   | 4 ++--
 mlos_bench/mlos_bench/tests/launcher_run_test.py         | 9 +++++----
 3 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc b/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
index 40ba6f29002..1deab160660 100644
--- a/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
@@ -14,9 +14,10 @@
 
     "environment": "environments/mock/mock_env.jsonc",
 
-    "tunable_values": [
-        "tunable-values/tunable-values-example.jsonc"
-    ],
+    // Use default values for tunables unless otherwise specified
+    // "tunable_values": [
+    //     "tunable-values/tunable-values-example.jsonc"
+    // ],
 
     "globals": ["globals/global_test_config.jsonc"],
 
diff --git a/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc b/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc
index 362ede79889..707f9d6424d 100644
--- a/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc
@@ -2,8 +2,8 @@
 {
     "$schema": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json",
 
-    "sched_migration_cost_ns": 500000,
-    "sched_latency_ns": 12000000,
+    "sched_migration_cost_ns": 400000,
+    "sched_latency_ns": 10000000,
     "sched_child_runs_first": "0",
     "sched_tunable_scaling": "1"
 }
diff --git a/mlos_bench/mlos_bench/tests/launcher_run_test.py b/mlos_bench/mlos_bench/tests/launcher_run_test.py
index fa601d93a36..d8caf7537e2 100644
--- a/mlos_bench/mlos_bench/tests/launcher_run_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_run_test.py
@@ -34,7 +34,6 @@ def local_exec_service() -> LocalExecService:
     return LocalExecService(parent=ConfigPersistenceService({
         "config_path": [
             "mlos_bench/config",
-            "mlos_bench/tests/config",
             "mlos_bench/examples",
         ]
     }))
@@ -53,7 +52,9 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
         # temp_dir = '/tmp'
         log_path = path_join(temp_dir, "mock-test.log")
         (return_code, _stdout, _stderr) = local_exec_service.local_exec(
-            [f"./mlos_bench/mlos_bench/run.py {cli_config} --log_file '{log_path}'"],
+            ["./mlos_bench/mlos_bench/run.py" +
+             " --config_path ./mlos_bench/mlos_bench/tests/config/" +
+             f" {cli_config} --log_file '{log_path}'"],
             cwd=root_path)
         assert return_code == 0
 
@@ -79,7 +80,7 @@ def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecServ
     """
     _launch_main_app(
         root_path, local_exec_service,
-        "--config cli/mock-bench.jsonc" +
+        " --config cli/mock-bench.jsonc" +
         " --trial_config_repeat_count 5" +
         " --mock_env_seed -1",  # Deterministic Mock Environment.
         [
@@ -103,7 +104,7 @@ def test_launch_main_app_bench_values(
         " --mock_env_seed -1",  # Deterministic Mock Environment.
         [
             f"^{_RE_DATE} run\\.py:\\d+ " +
-            r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$",
+            r"_main INFO Final score: \{'score': 67\.11\d+\}\s*$",
         ]
     )
 

From f5ab2b0f9cf358ee55a7d725cd01153552d760db Mon Sep 17 00:00:00 2001
From: Sergiy Matusevych <sergiym@microsoft.com>
Date: Fri, 28 Jun 2024 16:38:45 -0700
Subject: [PATCH 7/7] roll back all changes not related to OneShotOptimizer

---
 CONTRIBUTING.md                               | 26 +++++++++++---
 mlos_bench/mlos_bench/DEVNOTES.md             |  2 +-
 .../config/environments/mock/mock_env.jsonc   |  6 ++--
 .../environments/mock-env-subschema.json      | 10 +++---
 .../mlos_bench/environments/mock_env.py       | 12 +++----
 .../mlos_bench/optimizers/mock_optimizer.py   |  2 +-
 .../tests/config/cli/mock-bench.jsonc         |  7 ++--
 .../invalid/mock_env-bad-metric-type.jsonc    |  2 +-
 .../mock_env-const-args-list-obj.jsonc        |  2 +-
 .../mock_env-const-args-nested-list.jsonc     |  2 +-
 .../bad/invalid/mock_env-missing-metric.jsonc |  2 +-
 .../bad/invalid/mock_env-missing-name.jsonc   |  2 +-
 .../test-cases/good/full/mock_env-full.jsonc  |  6 ++--
 .../good/partial/mock_env-const-args.jsonc    |  2 +-
 .../tunable-values-example.jsonc              |  4 +--
 mlos_bench/mlos_bench/tests/conftest.py       | 11 +++---
 .../tests/environments/composite_env_test.py  | 20 +++++------
 .../tests/launcher_in_process_test.py         |  5 +--
 .../mlos_bench/tests/launcher_run_test.py     | 36 +++----------------
 19 files changed, 73 insertions(+), 86 deletions(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d6f0377a01e..2dec110876d 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -62,7 +62,7 @@ Simply open the project in VSCode and follow the prompts to build and open the d
 We expect development to follow a typical "forking" style workflow:
 
 1. Fork a copy of the [MLOS repo in Github](https://github.com/microsoft/MLOS).
-2. Create a development (a.k.a. topic) branch off of `main` to work on changes.
+1. Create a development (a.k.a. topic) branch off of `main` to work on changes.
 
     For instance:
 
@@ -70,9 +70,27 @@ We expect development to follow a typical "forking" style workflow:
     git checkout -b YourDevName/some-topic-description main
     ```
 
-3. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls).
-4. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass).
-5. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history.
+1. Ensure all of the lint checks and tests pass.
+
+    The easiest way to do this is to run the `make` commands that are also used in the CI pipeline:
+
+    ```shell
+    # All at once.
+    make all
+
+    # Or individually (for easier debugging)
+    make check
+    make test
+    make dist-test
+    make doc-test
+    ```
+
+1. Submit changes for inclusion as a [Pull Request on Github](https://github.com/microsoft/MLOS/pulls).
+
+    > Please try to keep PRs small whenver possible and don't include unnecessaary formatting changes.
+
+1. PRs are associated with [Github Issues](https://github.com/microsoft/MLOS/issues) and need [MLOS-committers](https://github.com/orgs/microsoft/teams/MLOS-committers) to sign-off (in addition to other CI pipeline checks like tests and lint checks to pass).
+1. Once approved, the PR can be completed using a squash merge in order to keep a nice linear history.
 
 ## Distributing
 
diff --git a/mlos_bench/mlos_bench/DEVNOTES.md b/mlos_bench/mlos_bench/DEVNOTES.md
index 9486168e525..0150d25efd5 100644
--- a/mlos_bench/mlos_bench/DEVNOTES.md
+++ b/mlos_bench/mlos_bench/DEVNOTES.md
@@ -34,7 +34,7 @@ Each `Environment` config is a JSON5 file with the following structure:
         }
         // Environment constructor parameters
         // (specific to the Environment class being instantiated):
-        "mock_env_seed": 42,
+        "seed": 42,
         // ...
     }
 }
diff --git a/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc b/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc
index e2ec01e0afa..bf0021d4859 100644
--- a/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc
+++ b/mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc
@@ -19,8 +19,8 @@
             "linux-hugepages-2048kB",
             "redis"
         ],
-        "mock_env_seed": 42,            // Seed for the random noise generator. Omit or set to 0 to produce noise-free data.
-        "mock_env_range": [60, 120],    // Range of the generated output values of the benchmark.
-        "mock_env_metrics": ["score"]   // Names of fake benchmark metrics to generate. (Default is one metric, "score").
+        "seed": 42,            // Seed for the random noise generator. Omit to produce noise-free data.
+        "range": [60, 120],    // Range of the generated output values of the benchmark.
+        "metrics": ["score"]   // Names of fake benchmark metrics to generate. (Default is one metric, "score").
     }
 }
diff --git a/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json b/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
index cb2de6c719f..35c3667ee0f 100644
--- a/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
+++ b/mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json
@@ -19,12 +19,12 @@
                 },
                 {
                     "properties": {
-                        "mock_env_seed": {
+                        "seed": {
                             "type": "integer",
-                            "description": "Seed for the random number generator. Set to -1 for deterministic behavior, 0 for default randomness.",
-                            "default": -1
+                            "description": "Seed for the random number generator",
+                            "default": 0
                         },
-                        "mock_env_range": {
+                        "range": {
                             "type": "array",
                             "description": "Range of the random number generator",
                             "items": {
@@ -33,7 +33,7 @@
                             "minItems": 2,
                             "maxItems": 2
                         },
-                        "mock_env_metrics": {
+                        "metrics": {
                             "type": "array",
                             "description": "Names of fake benchmark metrics to be generate",
                             "items": {
diff --git a/mlos_bench/mlos_bench/environments/mock_env.py b/mlos_bench/mlos_bench/environments/mock_env.py
index d8ffe3e47d6..00fa55014d3 100644
--- a/mlos_bench/mlos_bench/environments/mock_env.py
+++ b/mlos_bench/mlos_bench/environments/mock_env.py
@@ -17,6 +17,7 @@
 from mlos_bench.environments.status import Status
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.tunables import Tunable, TunableGroups, TunableValue
+from mlos_bench.util import nullable
 
 _LOG = logging.getLogger(__name__)
 
@@ -48,8 +49,7 @@ def __init__(self,
         global_config : dict
             Free-format dictionary of global parameters (e.g., security credentials)
             to be mixed in into the "const_args" section of the local config.
-            Optional arguments are `mock_env_seed`, `mock_env_range`, and `mock_env_metrics`.
-            Set `mock_env_seed` to -1 for deterministic behavior, 0 for default randomness.
+            Optional arguments are `seed`, `range`, and `metrics`.
         tunables : TunableGroups
             A collection of tunable parameters for *all* environments.
         service: Service
@@ -57,10 +57,10 @@ def __init__(self,
         """
         super().__init__(name=name, config=config, global_config=global_config,
                          tunables=tunables, service=service)
-        seed = int(self.config.get("mock_env_seed", -1))
-        self._random = random.Random(seed or None) if seed >= 0 else None
-        self._range = self.config.get("mock_env_range")
-        self._metrics = self.config.get("mock_env_metrics", ["score"])
+        seed = self.config.get("seed")
+        self._random = nullable(random.Random, seed)
+        self._range = self.config.get("range")
+        self._metrics = self.config.get("metrics", ["score"])
         self._is_ready = True
 
     def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]:
diff --git a/mlos_bench/mlos_bench/optimizers/mock_optimizer.py b/mlos_bench/mlos_bench/optimizers/mock_optimizer.py
index 7d2caff8ff0..44f8262b286 100644
--- a/mlos_bench/mlos_bench/optimizers/mock_optimizer.py
+++ b/mlos_bench/mlos_bench/optimizers/mock_optimizer.py
@@ -61,7 +61,7 @@ def suggest(self) -> TunableGroups:
         """
         tunables = super().suggest()
         if self._start_with_defaults:
-            _LOG.info("Use default tunable values")
+            _LOG.info("Use default values for the first trial")
             self._start_with_defaults = False
         else:
             for (tunable, _group) in tunables:
diff --git a/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc b/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
index 1deab160660..40ba6f29002 100644
--- a/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc
@@ -14,10 +14,9 @@
 
     "environment": "environments/mock/mock_env.jsonc",
 
-    // Use default values for tunables unless otherwise specified
-    // "tunable_values": [
-    //     "tunable-values/tunable-values-example.jsonc"
-    // ],
+    "tunable_values": [
+        "tunable-values/tunable-values-example.jsonc"
+    ],
 
     "globals": ["globals/global_test_config.jsonc"],
 
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc
index 85a0a4cfe20..b8b6d126125 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc
@@ -2,7 +2,7 @@
     "name": "mock_env-full",
     "class": "mlos_bench.environments.mock_env.MockEnv",
     "config": {
-        "mock_env_metrics": [
+        "metrics": [
             {"bad": "metric type"}
         ]
     }
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc
index 86d0318f283..bfa8172b20f 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc
@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "mock_env_seed": 42,
+        "seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc
index 81da08ddab7..f2f3be64ec7 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc
@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "mock_env_seed": 42,
+        "seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc
index 99a61969e62..03f6f1a2b31 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc
@@ -2,7 +2,7 @@
     "name": "mock_env-full",
     "class": "mlos_bench.environments.mock_env.MockEnv",
     "config": {
-        "mock_env_metrics": [
+        "metrics": [
             // needs at least one element
         ]
     }
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc
index 1887a4de6f2..66f0c49eecc 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc
@@ -2,6 +2,6 @@
     // "name": "missing name is invalid",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "mock_env_seed": 42
+        "seed": 42
     }
 }
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc
index a00f8ca60c0..d196321d87b 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc
@@ -20,9 +20,9 @@
         "required_args": [
             "foo"
         ],
-        "mock_env_range": [0, 1],
-        "mock_env_seed": 42,
-        "mock_env_metrics": [
+        "range": [0, 1],
+        "seed": 42,
+        "metrics": [
             "latency",
             "cost"
         ]
diff --git a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc
index b71a81f57cc..517ea318348 100644
--- a/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc
@@ -2,7 +2,7 @@
     "name": "partial mock env",
     "class": "mlos_bench.environments.MockEnv",
     "config": {
-        "mock_env_seed": 42,
+        "seed": 42,
         "const_args": {
             "foo": "bar",
             "int": 1,
diff --git a/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc b/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc
index 707f9d6424d..362ede79889 100644
--- a/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc
+++ b/mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc
@@ -2,8 +2,8 @@
 {
     "$schema": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json",
 
-    "sched_migration_cost_ns": 400000,
-    "sched_latency_ns": 10000000,
+    "sched_migration_cost_ns": 500000,
+    "sched_latency_ns": 12000000,
     "sched_child_runs_first": "0",
     "sched_tunable_scaling": "1"
 }
diff --git a/mlos_bench/mlos_bench/tests/conftest.py b/mlos_bench/mlos_bench/tests/conftest.py
index f30fe435854..c632abe09da 100644
--- a/mlos_bench/mlos_bench/tests/conftest.py
+++ b/mlos_bench/mlos_bench/tests/conftest.py
@@ -40,9 +40,9 @@ def mock_env(tunable_groups: TunableGroups) -> MockEnv:
         name="Test Env",
         config={
             "tunable_params": ["provision", "boot", "kernel"],
-            "mock_env_seed": SEED,
-            "mock_env_range": [60, 120],
-            "mock_env_metrics": ["score"],
+            "seed": SEED,
+            "range": [60, 120],
+            "metrics": ["score"],
         },
         tunables=tunable_groups
     )
@@ -57,9 +57,8 @@ def mock_env_no_noise(tunable_groups: TunableGroups) -> MockEnv:
         name="Test Env No Noise",
         config={
             "tunable_params": ["provision", "boot", "kernel"],
-            "mock_env_seed": -1,
-            "mock_env_range": [60, 120],
-            "mock_env_metrics": ["score", "other_score"],
+            "range": [60, 120],
+            "metrics": ["score", "other_score"],
         },
         tunables=tunable_groups
     )
diff --git a/mlos_bench/mlos_bench/tests/environments/composite_env_test.py b/mlos_bench/mlos_bench/tests/environments/composite_env_test.py
index fd7c0229393..a1bfc793b4b 100644
--- a/mlos_bench/mlos_bench/tests/environments/composite_env_test.py
+++ b/mlos_bench/mlos_bench/tests/environments/composite_env_test.py
@@ -41,8 +41,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "EnvId": 1,
                         },
                         "required_args": ["vmName", "someConst", "global_param"],
-                        "mock_env_range": [60, 120],
-                        "mock_env_metrics": ["score"],
+                        "range": [60, 120],
+                        "metrics": ["score"],
                     }
                 },
                 {
@@ -56,8 +56,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "global_param": "local"
                         },
                         "required_args": ["vmName"],
-                        "mock_env_range": [60, 120],
-                        "mock_env_metrics": ["score"],
+                        "range": [60, 120],
+                        "metrics": ["score"],
                     }
                 },
                 {
@@ -70,8 +70,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                             "EnvId": 3,
                         },
                         "required_args": ["vmName", "vm_server_name", "vm_client_name"],
-                        "mock_env_range": [60, 120],
-                        "mock_env_metrics": ["score"],
+                        "range": [60, 120],
+                        "metrics": ["score"],
                     }
                 }
             ]
@@ -193,8 +193,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                                         "vm_server_name",
                                         "global_param"
                                     ],
-                                    "mock_env_range": [60, 120],
-                                    "mock_env_metrics": ["score"],
+                                    "range": [60, 120],
+                                    "metrics": ["score"],
                                 }
                             },
                             # ...
@@ -218,8 +218,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
                                 "config": {
                                     "tunable_params": ["boot"],
                                     "required_args": ["vmName", "EnvId", "vm_client_name"],
-                                    "mock_env_range": [60, 120],
-                                    "mock_env_metrics": ["score"],
+                                    "range": [60, 120],
+                                    "metrics": ["score"],
                                 }
                             },
                             # ...
diff --git a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
index 90aa7e08f71..6340923e825 100644
--- a/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_in_process_test.py
@@ -17,14 +17,11 @@
     ("argv", "expected_score"), [
         ([
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
-            "--trial_config_repeat_count", "5",
-            "--mock_env_seed", "-1",  # Deterministic Mock Environment.
-        ], 67.40329),
+        ], 65.6742),
         ([
             "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc",
             "--trial_config_repeat_count", "3",
             "--max_suggestions", "3",
-            "--mock_env_seed", "42",  # Noisy Mock Environment.
         ], 64.53897),
     ]
 )
diff --git a/mlos_bench/mlos_bench/tests/launcher_run_test.py b/mlos_bench/mlos_bench/tests/launcher_run_test.py
index d8caf7537e2..fbca259e93e 100644
--- a/mlos_bench/mlos_bench/tests/launcher_run_test.py
+++ b/mlos_bench/mlos_bench/tests/launcher_run_test.py
@@ -52,9 +52,7 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
         # temp_dir = '/tmp'
         log_path = path_join(temp_dir, "mock-test.log")
         (return_code, _stdout, _stderr) = local_exec_service.local_exec(
-            ["./mlos_bench/mlos_bench/run.py" +
-             " --config_path ./mlos_bench/mlos_bench/tests/config/" +
-             f" {cli_config} --log_file '{log_path}'"],
+            [f"./mlos_bench/mlos_bench/run.py {cli_config} --log_file '{log_path}'"],
             cwd=root_path)
         assert return_code == 0
 
@@ -76,35 +74,14 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
 def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecService) -> None:
     """
     Run mlos_bench command-line application with mock benchmark config
-    and default tunable values and check the results in the log.
-    """
-    _launch_main_app(
-        root_path, local_exec_service,
-        " --config cli/mock-bench.jsonc" +
-        " --trial_config_repeat_count 5" +
-        " --mock_env_seed -1",  # Deterministic Mock Environment.
-        [
-            f"^{_RE_DATE} run\\.py:\\d+ " +
-            r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$",
-        ]
-    )
-
-
-def test_launch_main_app_bench_values(
-        root_path: str, local_exec_service: LocalExecService) -> None:
-    """
-    Run mlos_bench command-line application with mock benchmark config
-    and user-specified tunable values and check the results in the log.
+    and check the results in the log.
     """
     _launch_main_app(
         root_path, local_exec_service,
-        " --config cli/mock-bench.jsonc" +
-        " --tunable_values tunable-values/tunable-values-example.jsonc" +
-        " --trial_config_repeat_count 5" +
-        " --mock_env_seed -1",  # Deterministic Mock Environment.
+        "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
         [
             f"^{_RE_DATE} run\\.py:\\d+ " +
-            r"_main INFO Final score: \{'score': 67\.11\d+\}\s*$",
+            r"_main INFO Final score: \{'score': 65\.67\d+\}\s*$",
         ]
     )
 
@@ -116,10 +93,7 @@ def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecServic
     """
     _launch_main_app(
         root_path, local_exec_service,
-        "--config cli/mock-opt.jsonc" +
-        " --trial_config_repeat_count 3" +
-        " --max_suggestions 3" +
-        " --mock_env_seed 42",  # Noisy Mock Environment.
+        "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc --trial_config_repeat_count 3 --max_suggestions 3",
         [
             # Iteration 1: Expect first value to be the baseline
             f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +