Add log scale and quantization to the tunables

mlos_bench/mlos_bench/config/schemas/tunables/tunable-params-schema.json

@@ -5,14 +5,55 @@
 
     "$defs": {
         "tunable_param_meta": {
+            "description": "A dictionary of metadata about the tunable parameter.  Can be used by scripts for additional info when generating configs from the suggested values.",
             "type": "object",
             "additionalProperties": {
+                "$comment": "Only flat dictionaries are allowed.",
                 "type": ["array", "string", "boolean", "null", "number"],
                 "items": {
                     "type": ["string", "boolean", "null", "number"]
                 }
             }
         },
+        "numeric_range": {
+            "description": "Two element array representing the lower and upper bounds of the range.",
+            "type": "array",
+            "$comment": "items type left unspecified here",
+            "minItems": 2,
+            "maxItems": 2
+        },
+        "quantization": {
+            "description": "The number of buckets to quantize the range into.\nSee Also:\nhttps://automl.github.io/ConfigSpace/main/api/hyperparameters.html#module-ConfigSpace.api.types.float,\nhttps://automl.github.io/ConfigSpace/main/api/hyperparameters.html#module-ConfigSpace.api.types.integer",
+            "$comment": "type left unspecified here"
+        },
+        "log_scale": {
+            "description": "Whether to use log instead of linear scale for the range search.",
+            "type": "boolean"
+        },
+        "special_values": {
+            "description": "An array of values that may have special meaning for the target system and could be outside the usual search range.",
+            "type": "array",
+            "items": {
+                "description": "Some special values may have a different type than the numeric parameter (e.g., keyword \"AUTO\").",
+                "type": ["number", "string", "boolean", "null"]
+            },
+            "minItems": 1,
+            "uniqueItems": true
+        },
+        "weights": {
+            "description": "An array of weights to be associated with the values in order to influence their search priorities.",
+            "type": "array",
+            "items": {
+                "type": "number",
+                "minimum": 0
+            },
+            "minItems": 1
+        },
+        "range_weight": {
+            "description": "The weight to be associated with the range in order to influence its search priority relative to specials values.",
+            "type": "number",
+            "minimum": 0
+        },
         "tunable_param_categorical": {
             "type": "object",
             "properties": {
@@ -23,12 +64,14 @@
                     "$ref": "#/$defs/tunable_param_meta"
                 },
                 "type": {
+                    "description": "A categorical type tunable.",
                     "const": "categorical"
                 },
                 "default": {
                     "type": ["string", "number", "boolean"]
                 },
                 "values": {
+                    "description": "List of values for this categorical type tunable",
                     "type": "array",
                     "items": {
                         "type": ["string", "number", "boolean"]
@@ -37,10 +80,7 @@
                     "uniqueItems": true
                 },
                 "values_weights": {
-                    "type": "array",
-                    "items": {
-                        "type": "number"
-                    }
+                    "$ref": "#/$defs/weights"
                 }
             },
             "required": ["type", "default", "values"],
@@ -60,34 +100,34 @@
                     "$ref": "#/$defs/tunable_param_meta"
                 },
                 "type": {
+                    "description": "An integer type tunable.",
                     "const": "int"
                 },
                 "default": {
                     "type": "integer"
                 },
                 "range": {
-                    "type": "array",
+                    "$ref": "#/$defs/numeric_range",
                     "items": {
                         "type": "integer"
-                    },
-                    "minItems": 2,
-                    "maxItems": 2
+                    }
+                },
+                "quantization": {
+                    "$ref": "#/$defs/quantization",
+                    "type": "integer",
+                    "exclusiveMinimum": 1
+                },
+                "log": {
+                    "$ref": "#/$defs/log_scale"
                 },
                 "special": {
-                    "type": "array",
-                    "items": {
-                        "type": "integer"
-                    },
-                    "uniqueItems": true
+                    "$ref": "#/$defs/special_values"
                 },
                 "special_weights": {
-                    "type": "array",
-                    "items": {
-                        "type": "number"
-                    }
+                    "$ref": "#/$defs/weights"
                 },
                 "range_weight": {
-                    "type": "number"
+                    "$ref": "#/$defs/range_weight"
                 }
             },
             "required": ["type", "default", "range"],
@@ -107,34 +147,34 @@
                     "$ref": "#/$defs/tunable_param_meta"
                 },
                 "type": {
+                    "description": "A continuous numerical type tunable.",
                     "const": "float"
                 },
                 "default": {
                     "type": "number"
                 },
                 "range": {
-                    "type": "array",
+                    "$ref": "#/$defs/numeric_range",
                     "items": {
                         "type": "number"
-                    },
-                    "minItems": 2,
-                    "maxItems": 2
+                    }
+                },
+                "quantization": {
+                    "$ref": "#/$defs/quantization",
+                    "type": "number",
+                    "exclusiveMinimum": 0
+                },
+                "log": {
+                    "$ref": "#/$defs/log_scale"
                 },
                 "special": {
-                    "type": "array",
-                    "items": {
-                        "type": "number"
-                    },
-                    "uniqueItems": true
+                    "$ref": "#/$defs/special_values"
                 },
                 "special_weights": {
-                    "type": "array",
-                    "items": {
-                        "type": "number"
-                    }
+                    "$ref": "#/$defs/weights"
                 },
                 "range_weight": {
-                    "type": "number"
+                    "$ref": "#/$defs/range_weight"
                 }
             },
             "required": ["type", "default", "range"],

mlos_bench/mlos_bench/optimizers/convert_configspace.py

@@ -89,6 +89,8 @@ def _tunable_to_configspace(
                 name=tunable.name,
                 lower=tunable.range[0],
                 upper=tunable.range[1],
+                log=tunable.is_log,
+                q=tunable.quantization,
                 default_value=tunable.default if tunable.in_range(tunable.default) else None,
                 meta=meta)
         })
@@ -108,6 +110,8 @@ def _tunable_to_configspace(
             name=tunable.name,
             lower=tunable.range[0],
             upper=tunable.range[1],
+            log=tunable.is_log,
+            q=tunable.quantization,
             default_value=tunable.default if tunable.in_range(tunable.default) else None,
             meta=meta
         ),

mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-float-bad-log.jsonc

@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "float": {
+                "type": "float",
+                "default": 10,
+                "range": [0, 10],
+                "log": "yes"  // <-- this is invalid
+            }
+        }
+    }
+}

mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-float-bad-quantization-type.jsonc

@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "float": {
+                "type": "float",
+                "default": 10,
+                "range": [0, 10],
+                "quantization": true  // <-- this is invalid
+            }
+        }
+    }
+}

mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-float-quantization.jsonc

@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "float": {
+                "type": "float",
+                "default": 10,
+                "range": [1, 500],
+                "quantization": 0   // <-- should be greater than 0
+            }
+        }
+    }
+}

mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-int-quantization.jsonc

@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "int": {
+                "type": "int",
+                "default": 10,
+                "range": [1, 500],
+                "quantization": 1   // <-- should be greater than 1
+            }
+        }
+    }
+}

mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-bad-log.jsonc

@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "int": {
+                "type": "int",
+                "default": 10,
+                "range": [1, 500],
+                "log": 1 // <-- this is invalid
+            }
+        }
+    }
+}

mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/bad/invalid/tunable-params-int-wrong-quantization-type.jsonc

@@ -0,0 +1,13 @@
+{
+    "covariant_group_name-1": {
+        "cost": 1,
+        "params": {
+            "int": {
+                "type": "int",
+                "default": 10,
+                "range": [1, 500],
+                "quantization": "yes"  // <-- this is invalid
+            }
+        }
+    }
+}

mlos_bench/mlos_bench/tests/config/schemas/tunable-params/test-cases/good/full/full-tunable-params-test.jsonc

@@ -11,15 +11,19 @@
                 "meta": {"suffix": "MB"},
                 "special": [-1],
                 "special_weights": [0.1],
-                "range_weight": 0.9
+                "range_weight": 0.9,
+                "quantization": 50,
+                "log": true
             },
             "float": {
                 "description": "Float",
                 "type": "float",
                 "default": 10.1,
                 "meta": {"scale": 1000, "prefix": "/proc/var/random/", "base": 2.71828},
                 "range": [1.1, 111.1],
-                "special": [-1.1]
+                "special": [-1.1],
+                "quantization": 10,
+                "log": false
             },
             "cat": {
                 "description": "Cat",

mlos_bench/mlos_bench/tests/tunable_groups_fixtures.py

@@ -55,13 +55,15 @@
                 // FLAML requires uniform weights, separately for
                 // specials and switching between specials and range.
                 "special_weights": [0.25, 0.25],
-                "range_weight": 0.5
+                "range_weight": 0.5,
+                "log": false
             },
             "kernel_sched_latency_ns": {
                 "description": "Initial value for the scheduler period",
                 "type": "int",
                 "default": 2000000,
-                "range": [0, 1000000000]
+                "range": [0, 1000000000],
+                "log": false
             }
         }
     }

mlos_bench/mlos_bench/tests/tunables/tunable_definition_test.py

@@ -247,6 +247,43 @@ def test_numerical_weights(tunable_type: str) -> None:
     assert tunable.range_weight == 0.9
 
 
+@pytest.mark.parametrize("tunable_type", ["int", "float"])
+def test_numerical_quantization(tunable_type: str) -> None:
+    """
+    Instantiate a numerical tunable with quantization.
+    """
+    json_config = f"""
+    {{
+        "type": "{tunable_type}",
+        "range": [0, 100],
+        "quantization": 10,
+        "default": 0
+    }}
+    """
+    config = json.loads(json_config)
+    tunable = Tunable(name='test', config=config)
+    assert tunable.quantization == 10
+    assert not tunable.is_log
+
+
+@pytest.mark.parametrize("tunable_type", ["int", "float"])
+def test_numerical_log(tunable_type: str) -> None:
+    """
+    Instantiate a numerical tunable with log scale.
+    """
+    json_config = f"""
+    {{
+        "type": "{tunable_type}",
+        "range": [0, 100],
+        "log": true,
+        "default": 0
+    }}
+    """
+    config = json.loads(json_config)
+    tunable = Tunable(name='test', config=config)
+    assert tunable.is_log
+
+
 @pytest.mark.parametrize("tunable_type", ["int", "float"])
 def test_numerical_weights_no_specials(tunable_type: str) -> None:
     """
@@ -384,6 +421,24 @@ def test_numerical_weights_wrong_values(tunable_type: str) -> None:
         Tunable(name='test', config=config)
 
 
+@pytest.mark.parametrize("tunable_type", ["int", "float"])
+def test_numerical_quantization_wrong(tunable_type: str) -> None:
+    """
+    Instantiate a numerical tunable with invalid number of quantization points.
+    """
+    json_config = f"""
+    {{
+        "type": "{tunable_type}",
+        "range": [0, 100],
+        "quantization": 0,
+        "default": 0
+    }}
+    """
+    config = json.loads(json_config)
+    with pytest.raises(ValueError):
+        Tunable(name='test', config=config)
+
+
 def test_bad_type() -> None:
     """
     Disallow bad types.