From dbaccea50329e66d9d2e5a58d2263a5cac9bc9c4 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Fri, 4 Oct 2024 16:47:06 -0500 Subject: [PATCH] Allow empty tunable values to represent the defaults (#868) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit # Pull Request ## Title Allows an empty dictionary (object) to represent the default tunable values for tunable params. --- ## Description This should make it easier to maintain configs that loop over a chosen set of tunable values, where some of the tunable values are the default values without needing to copy/paste those values from the tunable params definitions. - See Also: [discussion in #855](https://github.com/microsoft/MLOS/pull/855#discussion_r1786896189) --- ## Type of Change - ✨ New feature --- ## Testing Extends existing testing infrastructure to check that this works. --- --- .../tunables/tunable-values-schema.json | 1 - .../bare-tunable-values-with-schema.jsonc | 2 ++ .../empty-tunable-values-with-schema.jsonc | 4 +++ .../empty-tunable-values-without-schema.jsonc | 3 +++ .../tests/tunables/tunables_assign_test.py | 26 +++++++++++++++++++ .../mlos_bench/tunables/tunable_groups.py | 12 +++++++++ 6 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/full/empty-tunable-values-with-schema.jsonc create mode 100644 mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/partial/empty-tunable-values-without-schema.jsonc diff --git a/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json b/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json index 186007a449..e14130dbca 100644 --- a/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json +++ b/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json @@ -12,7 +12,6 @@ "type": ["string", "number", "boolean"] } }, - "minProperties": 1, "not": { "required": ["tunable_values"] } diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/full/bare-tunable-values-with-schema.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/full/bare-tunable-values-with-schema.jsonc index a91c629b61..2e5379d0df 100644 --- a/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/full/bare-tunable-values-with-schema.jsonc +++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/full/bare-tunable-values-with-schema.jsonc @@ -1,4 +1,6 @@ { + "$schema": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json", + "foo": "bar", "int": 1, "float": 1.1, diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/full/empty-tunable-values-with-schema.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/full/empty-tunable-values-with-schema.jsonc new file mode 100644 index 0000000000..fc43969b4d --- /dev/null +++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/full/empty-tunable-values-with-schema.jsonc @@ -0,0 +1,4 @@ +{ + "$schema": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json" + // empty tunable values represents the defaults +} diff --git a/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/partial/empty-tunable-values-without-schema.jsonc b/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/partial/empty-tunable-values-without-schema.jsonc new file mode 100644 index 0000000000..c6bd77b952 --- /dev/null +++ b/mlos_bench/mlos_bench/tests/config/schemas/tunable-values/test-cases/good/partial/empty-tunable-values-without-schema.jsonc @@ -0,0 +1,3 @@ +{ + // empty tunable values represents the defaults +} diff --git a/mlos_bench/mlos_bench/tests/tunables/tunables_assign_test.py b/mlos_bench/mlos_bench/tests/tunables/tunables_assign_test.py index 05f29a9064..e4ddbed28b 100644 --- a/mlos_bench/mlos_bench/tests/tunables/tunables_assign_test.py +++ b/mlos_bench/mlos_bench/tests/tunables/tunables_assign_test.py @@ -28,6 +28,32 @@ def test_tunables_assign_unknown_param(tunable_groups: TunableGroups) -> None: ) +def test_tunables_assign_defaults(tunable_groups: TunableGroups) -> None: + """Make sure we can assign the default values using an empty dictionary.""" + tunable_groups_defaults = tunable_groups.copy() + assert tunable_groups.is_defaults() + # Assign the default values. + # Also reset the _is_updated flag to avoid considering that in the comparison. + tunable_groups.assign({}).reset() + assert tunable_groups_defaults == tunable_groups + assert tunable_groups.is_defaults() + new_vm_size = "Standard_B2s" + assert tunable_groups["vmSize"] != new_vm_size + # Change one value. + tunable_groups.assign({"vmSize": new_vm_size}).reset() + assert tunable_groups["vmSize"] == new_vm_size + # Check that the other values are still defaults. + idle_tunable, _ = tunable_groups.get_tunable("idle") + assert idle_tunable.is_default() + assert tunable_groups_defaults != tunable_groups + assert not tunable_groups.is_defaults() + # Reassign defaults. + tunable_groups.assign({}).reset() + assert tunable_groups["vmSize"] != new_vm_size + assert tunable_groups.is_defaults() + assert tunable_groups_defaults == tunable_groups + + def test_tunables_assign_categorical(tunable_categorical: Tunable) -> None: """Regular assignment for categorical tunable.""" # Must be one of: {"Standard_B2s", "Standard_B2ms", "Standard_B4ms"} diff --git a/mlos_bench/mlos_bench/tunables/tunable_groups.py b/mlos_bench/mlos_bench/tunables/tunable_groups.py index da56eb79ac..45d8a02f9c 100644 --- a/mlos_bench/mlos_bench/tunables/tunable_groups.py +++ b/mlos_bench/mlos_bench/tunables/tunable_groups.py @@ -4,12 +4,15 @@ # """TunableGroups definition.""" import copy +import logging from typing import Dict, Generator, Iterable, Mapping, Optional, Tuple, Union from mlos_bench.config.schemas import ConfigSchema from mlos_bench.tunables.covariant_group import CovariantTunableGroup from mlos_bench.tunables.tunable import Tunable, TunableValue +_LOG = logging.getLogger(__name__) + class TunableGroups: """A collection of covariant groups of tunable parameters.""" @@ -346,11 +349,20 @@ def assign(self, param_values: Mapping[str, TunableValue]) -> "TunableGroups": param_values : Mapping[str, TunableValue] Dictionary mapping Tunable parameter names to new values. + As a special behavior when the mapping is empty the method will restore + the default values rather than no-op. + This allows an empty dictionary in json configs to be used to reset the + tunables to defaults without having to copy the original values from the + tunable_params definition. + Returns ------- self : TunableGroups Self-reference for chaining. """ + if not param_values: + _LOG.info("Empty tunable values set provided. Resetting all tunables to defaults.") + return self.restore_defaults() for key, value in param_values.items(): self[key] = value return self