From 6a14f7a82cfd3be47c87663462de02f4f2f76dbe Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 31 Jul 2023 11:36:01 +0100 Subject: [PATCH 1/8] Allow registering of custom resolvers to OCL Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 13 ++++++++++++- tests/config/test_omegaconf_config.py | 18 ++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index d7d9bd245b..b0aacf7b9c 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -7,7 +7,7 @@ import logging import mimetypes from pathlib import Path -from typing import Any, Iterable +from typing import Any, Callable, Iterable import fsspec from omegaconf import OmegaConf @@ -82,6 +82,7 @@ def __init__( # noqa: too-many-arguments config_patterns: dict[str, list[str]] = None, base_env: str = "base", default_run_env: str = "local", + custom_resolvers: dict[str, Callable] = None, ): """Instantiates a ``OmegaConfigLoader``. @@ -97,6 +98,7 @@ def __init__( # noqa: too-many-arguments the configuration paths. default_run_env: Name of the default run environment. Defaults to `"local"`. Can be overridden by supplying the `env` argument. + custom_resolvers: """ self.base_env = base_env self.default_run_env = default_run_env @@ -111,6 +113,9 @@ def __init__( # noqa: too-many-arguments # Deactivate oc.env built-in resolver for OmegaConf OmegaConf.clear_resolver("oc.env") + # Register user provided custom resolvers + if custom_resolvers: + self._register_new_resolvers(custom_resolvers) file_mimetype, _ = mimetypes.guess_type(conf_source) if file_mimetype == "application/x-tar": @@ -302,6 +307,12 @@ def _is_valid_config_path(self, path): ".json", ] + @staticmethod + def _register_new_resolvers(resolvers: dict[str, Callable]): + """Register custom resolvers""" + for name, resolver in resolvers.items(): + OmegaConf.register_new_resolver(name, resolver, replace=True) + @staticmethod def _check_duplicates(seen_files_to_keys: dict[Path, set[Any]]): duplicates = [] diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index dd49292019..df617ee161 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -649,3 +649,21 @@ def test_variable_interpolation_in_catalog_with_separate_templates_file( conf = OmegaConfigLoader(str(tmp_path)) conf.default_run_env = "" assert conf["catalog"]["companies"]["type"] == "pandas.CSVDataSet" + + def test_custom_resolvers(self, tmp_path): + base_params = tmp_path / _BASE_ENV / "parameters.yml" + param_config = { + "model_options": { + "test_size": "${add: 3, 4}", + "random_state": "${plus_2: 1}", + } + } + _write_yaml(base_params, param_config) + custom_resolvers = { + "add": lambda *x: sum(x), + "plus_2": lambda x: x + 2, + } + conf = OmegaConfigLoader(str(tmp_path), custom_resolvers=custom_resolvers) + conf.default_run_env = "" + assert conf["parameters"]["model_options"]["test_size"] == 7 + assert conf["parameters"]["model_options"]["random_state"] == 3 From 89ebb1231478c1ef6b08da353a406acc35c59446 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 31 Jul 2023 16:21:38 +0100 Subject: [PATCH 2/8] Complete doc string Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index b0aacf7b9c..6c08baad82 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -98,7 +98,8 @@ def __init__( # noqa: too-many-arguments the configuration paths. default_run_env: Name of the default run environment. Defaults to `"local"`. Can be overridden by supplying the `env` argument. - custom_resolvers: + custom_resolvers: A dictionary of custom resolvers to be registered. For more information, + see here: https://omegaconf.readthedocs.io/en/2.3_branch/custom_resolvers.html#custom-resolvers """ self.base_env = base_env self.default_run_env = default_run_env From 3163c5e5bcee608f4299e458880b9b839e955703 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Mon, 31 Jul 2023 16:43:40 +0100 Subject: [PATCH 3/8] Add test for overwritten resolvers Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 2 +- tests/config/test_omegaconf_config.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 6c08baad82..9091402ef9 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -312,7 +312,7 @@ def _is_valid_config_path(self, path): def _register_new_resolvers(resolvers: dict[str, Callable]): """Register custom resolvers""" for name, resolver in resolvers.items(): - OmegaConf.register_new_resolver(name, resolver, replace=True) + OmegaConf.register_new_resolver(name=name, resolver=resolver, replace=True) @staticmethod def _check_duplicates(seen_files_to_keys: dict[Path, set[Any]]): diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index df617ee161..9bed0298ca 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -667,3 +667,22 @@ def test_custom_resolvers(self, tmp_path): conf.default_run_env = "" assert conf["parameters"]["model_options"]["test_size"] == 7 assert conf["parameters"]["model_options"]["random_state"] == 3 + + def test_overwrite_resolvers(self, tmp_path): + base_params = tmp_path / _BASE_ENV / "parameters.yml" + # OmegaConf is a singleton, register a resolver to be overwritten + OmegaConf.register_new_resolver("custom", lambda x: x + 10) + + param_config = { + "model_options": { + "test_size": "${custom: 10}", + } + } + _write_yaml(base_params, param_config) + custom_resolvers = { + "custom": lambda x: x + 20, + } + conf = OmegaConfigLoader(str(tmp_path), custom_resolvers=custom_resolvers) + conf.default_run_env = "" + # test_size should be calculated using overwritten custom resolver (x + 20) + assert conf["parameters"]["model_options"]["test_size"] == 30 From b03b1e4a5b25fdfb29c6987a6d5f3fe477fac52b Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Tue, 1 Aug 2023 13:27:45 +0100 Subject: [PATCH 4/8] Update test for overwritten resolvers Signed-off-by: Ankita Katiyar --- tests/config/test_omegaconf_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 9bed0298ca..98c81be7d3 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -679,6 +679,9 @@ def test_overwrite_resolvers(self, tmp_path): } } _write_yaml(base_params, param_config) + conf_original = OmegaConf.load(base_params) + # test_size should be calculated using custom resolver (x + 10) + assert conf_original["model_options"]["test_size"] == 20 custom_resolvers = { "custom": lambda x: x + 20, } From 7420b28c9a3155a6f91495d4b7633e0145072243 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Tue, 1 Aug 2023 17:47:38 +0100 Subject: [PATCH 5/8] Remove replace=True by default Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 3 ++- tests/config/test_omegaconf_config.py | 36 +++++++-------------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 9091402ef9..7f167ae63e 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -312,7 +312,8 @@ def _is_valid_config_path(self, path): def _register_new_resolvers(resolvers: dict[str, Callable]): """Register custom resolvers""" for name, resolver in resolvers.items(): - OmegaConf.register_new_resolver(name=name, resolver=resolver, replace=True) + if not OmegaConf.has_resolver("name"): + OmegaConf.register_new_resolver(name=name, resolver=resolver) @staticmethod def _check_duplicates(seen_files_to_keys: dict[Path, set[Any]]): diff --git a/tests/config/test_omegaconf_config.py b/tests/config/test_omegaconf_config.py index 98c81be7d3..af57b52224 100644 --- a/tests/config/test_omegaconf_config.py +++ b/tests/config/test_omegaconf_config.py @@ -654,38 +654,20 @@ def test_custom_resolvers(self, tmp_path): base_params = tmp_path / _BASE_ENV / "parameters.yml" param_config = { "model_options": { - "test_size": "${add: 3, 4}", - "random_state": "${plus_2: 1}", + "param1": "${add: 3, 4}", + "param2": "${plus_2: 1}", + "param3": "${oc.env: VAR}", } } _write_yaml(base_params, param_config) custom_resolvers = { "add": lambda *x: sum(x), "plus_2": lambda x: x + 2, + "oc.env": oc.env, } - conf = OmegaConfigLoader(str(tmp_path), custom_resolvers=custom_resolvers) + os.environ["VAR"] = "my_env_variable" + conf = OmegaConfigLoader(tmp_path, custom_resolvers=custom_resolvers) conf.default_run_env = "" - assert conf["parameters"]["model_options"]["test_size"] == 7 - assert conf["parameters"]["model_options"]["random_state"] == 3 - - def test_overwrite_resolvers(self, tmp_path): - base_params = tmp_path / _BASE_ENV / "parameters.yml" - # OmegaConf is a singleton, register a resolver to be overwritten - OmegaConf.register_new_resolver("custom", lambda x: x + 10) - - param_config = { - "model_options": { - "test_size": "${custom: 10}", - } - } - _write_yaml(base_params, param_config) - conf_original = OmegaConf.load(base_params) - # test_size should be calculated using custom resolver (x + 10) - assert conf_original["model_options"]["test_size"] == 20 - custom_resolvers = { - "custom": lambda x: x + 20, - } - conf = OmegaConfigLoader(str(tmp_path), custom_resolvers=custom_resolvers) - conf.default_run_env = "" - # test_size should be calculated using overwritten custom resolver (x + 20) - assert conf["parameters"]["model_options"]["test_size"] == 30 + assert conf["parameters"]["model_options"]["param1"] == 7 + assert conf["parameters"]["model_options"]["param2"] == 3 + assert conf["parameters"]["model_options"]["param3"] == "my_env_variable" From 557325ea38fcdc4f0fa32b4007ad3d0b94c4bdba Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 2 Aug 2023 12:33:45 +0100 Subject: [PATCH 6/8] Update release notes Signed-off-by: Ankita Katiyar --- RELEASE.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 2fcf553bcf..9cbddc1fbc 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -11,7 +11,8 @@ # Upcoming Release 0.18.13 ## Major features and improvements - +* Allowed registering of custom resolvers to `OmegaConfigLoader` through `CONFIG_LOADER_ARGS`. +* ## Bug fixes and other changes ## Documentation changes From 5b0b86315ab210744b0dcdd2f59b5a64c0c40f1a Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 2 Aug 2023 12:34:19 +0100 Subject: [PATCH 7/8] Update release notes Signed-off-by: Ankita Katiyar --- RELEASE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/RELEASE.md b/RELEASE.md index 9cbddc1fbc..ae0714b8d9 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -12,7 +12,7 @@ ## Major features and improvements * Allowed registering of custom resolvers to `OmegaConfigLoader` through `CONFIG_LOADER_ARGS`. -* + ## Bug fixes and other changes ## Documentation changes From c609179cdfc28edb1a8564bab2432fc56eb24cf6 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar Date: Wed, 2 Aug 2023 13:32:06 +0100 Subject: [PATCH 8/8] Add debug level log for registering new resolver Signed-off-by: Ankita Katiyar --- kedro/config/omegaconf_config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kedro/config/omegaconf_config.py b/kedro/config/omegaconf_config.py index 7f167ae63e..4d2ace59d4 100644 --- a/kedro/config/omegaconf_config.py +++ b/kedro/config/omegaconf_config.py @@ -312,7 +312,9 @@ def _is_valid_config_path(self, path): def _register_new_resolvers(resolvers: dict[str, Callable]): """Register custom resolvers""" for name, resolver in resolvers.items(): - if not OmegaConf.has_resolver("name"): + if not OmegaConf.has_resolver(name): + msg = f"Registering new custom resolver: {name}" + _config_logger.debug(msg) OmegaConf.register_new_resolver(name=name, resolver=resolver) @staticmethod