From 309fceba4082826fb564d3f9bd3b1bba60ed85d8 Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Fri, 15 Nov 2024 13:35:13 +0700 Subject: [PATCH 1/9] refactor data converter for databricks topic --- taipy/core/data/_data_converter.py | 38 ++++++++++++++++++------------ 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/taipy/core/data/_data_converter.py b/taipy/core/data/_data_converter.py index 10442e626e..d934d42497 100644 --- a/taipy/core/data/_data_converter.py +++ b/taipy/core/data/_data_converter.py @@ -12,6 +12,7 @@ from copy import copy from datetime import datetime, timedelta from pydoc import locate +from typing import Dict from .._repository._abstract_converter import _AbstractConverter from ..common._utils import _load_fct @@ -120,14 +121,13 @@ def __serialize_exposed_type(properties: dict, exposed_type_key: str, valid_str_ for v in properties[exposed_type_key] ] else: - properties[ - exposed_type_key - ] = f"{properties[exposed_type_key].__module__}.{properties[exposed_type_key].__qualname__}" + properties[exposed_type_key] = ( + f"{properties[exposed_type_key].__module__}.{properties[exposed_type_key].__qualname__}" + ) return properties @classmethod - def _entity_to_model(cls, data_node: DataNode) -> _DataNodeModel: - properties = data_node._properties.data.copy() + def _serialize_properties(cls, data_node: DataNode, properties: Dict) -> Dict: if data_node.storage_type() == GenericDataNode.storage_type(): properties = cls.__serialize_generic_dn_properties(properties) @@ -144,6 +144,11 @@ def _entity_to_model(cls, data_node: DataNode) -> _DataNodeModel: properties = cls.__serialize_exposed_type( properties, cls._EXPOSED_TYPE_KEY, cls._VALID_STRING_EXPOSED_TYPES ) + return properties + + @classmethod + def _entity_to_model(cls, data_node: DataNode) -> _DataNodeModel: + properties = cls._serialize_properties(data_node, data_node._properties.data.copy()) return _DataNodeModel( data_node.id, @@ -273,25 +278,28 @@ def __deserialize_exposed_type(properties: dict, exposed_type_key: str, valid_st return properties @classmethod - def _model_to_entity(cls, model: _DataNodeModel) -> DataNode: - data_node_properties = model.data_node_properties.copy() - + def _deserialize_properties(cls, model: _DataNodeModel, properties: Dict) -> Dict: if model.storage_type == GenericDataNode.storage_type(): - data_node_properties = cls.__deserialize_generic_dn_properties(data_node_properties) + properties = cls.__deserialize_generic_dn_properties(properties) if model.storage_type == JSONDataNode.storage_type(): - data_node_properties = cls.__deserialize_json_dn_properties(data_node_properties) + properties = cls.__deserialize_json_dn_properties(properties) if model.storage_type == SQLDataNode.storage_type(): - data_node_properties = cls.__deserialize_sql_dn_model_properties(data_node_properties) + properties = cls.__deserialize_sql_dn_model_properties(properties) if model.storage_type == MongoCollectionDataNode.storage_type(): - data_node_properties = cls.__deserialize_mongo_collection_dn_model_properties(data_node_properties) + properties = cls.__deserialize_mongo_collection_dn_model_properties(properties) - if cls._EXPOSED_TYPE_KEY in data_node_properties.keys(): - data_node_properties = cls.__deserialize_exposed_type( - data_node_properties, cls._EXPOSED_TYPE_KEY, cls._VALID_STRING_EXPOSED_TYPES + if cls._EXPOSED_TYPE_KEY in properties.keys(): + properties = cls.__deserialize_exposed_type( + properties, cls._EXPOSED_TYPE_KEY, cls._VALID_STRING_EXPOSED_TYPES ) + return properties + + @classmethod + def _model_to_entity(cls, model: _DataNodeModel) -> DataNode: + data_node_properties = cls._deserialize_properties(model, model.data_node_properties.copy()) validity_period = None if model.validity_seconds is not None and model.validity_days is not None: From a8c83965f6aefc395ff6ac41205a790df58c3d12 Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Wed, 27 Nov 2024 15:22:28 +0700 Subject: [PATCH 2/9] refactor checker code --- .../checkers/_data_node_config_checker.py | 51 ++++++++++++------- .../checkers/test_data_node_config_checker.py | 12 ++--- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/taipy/core/config/checkers/_data_node_config_checker.py b/taipy/core/config/checkers/_data_node_config_checker.py index f81b65dae0..aa93a452e5 100644 --- a/taipy/core/config/checkers/_data_node_config_checker.py +++ b/taipy/core/config/checkers/_data_node_config_checker.py @@ -10,7 +10,7 @@ # specific language governing permissions and limitations under the License. from datetime import timedelta -from typing import Dict, List, cast +from typing import Any, Callable, Dict, List, Tuple, cast from taipy.common.config._config import _Config from taipy.common.config.checker._checkers._config_checker import _ConfigChecker @@ -46,7 +46,7 @@ def _check(self) -> IssueCollector: self._check_scope(data_node_config_id, data_node_config) self._check_validity_period(data_node_config_id, data_node_config) self._check_required_properties(data_node_config_id, data_node_config) - self._check_callable(data_node_config_id, data_node_config) + self._check_class_type(data_node_config_id, data_node_config) self._check_generic_read_write_fct_and_args(data_node_config_id, data_node_config) self._check_exposed_type(data_node_config_id, data_node_config) return self._collector @@ -196,28 +196,43 @@ def _check_generic_read_write_fct_and_args(self, data_node_config_id: str, data_ f"DataNodeConfig `{data_node_config_id}` must be populated with a Callable function.", ) - def _check_callable(self, data_node_config_id: str, data_node_config: DataNodeConfig): - properties_to_check = { + @staticmethod + def _get_class_type_and_properties() -> Dict[str, List[Tuple[Any, List[str]]]]: + return { DataNodeConfig._STORAGE_TYPE_VALUE_GENERIC: [ - DataNodeConfig._OPTIONAL_READ_FUNCTION_GENERIC_PROPERTY, - DataNodeConfig._OPTIONAL_WRITE_FUNCTION_GENERIC_PROPERTY, + ( + Callable, + [ + DataNodeConfig._OPTIONAL_READ_FUNCTION_GENERIC_PROPERTY, + DataNodeConfig._OPTIONAL_WRITE_FUNCTION_GENERIC_PROPERTY, + ], + ) ], DataNodeConfig._STORAGE_TYPE_VALUE_SQL: [ - DataNodeConfig._REQUIRED_WRITE_QUERY_BUILDER_SQL_PROPERTY, - DataNodeConfig._OPTIONAL_APPEND_QUERY_BUILDER_SQL_PROPERTY, + ( + Callable, + [ + DataNodeConfig._REQUIRED_WRITE_QUERY_BUILDER_SQL_PROPERTY, + DataNodeConfig._OPTIONAL_APPEND_QUERY_BUILDER_SQL_PROPERTY, + ], + ), ], } - if data_node_config.storage_type in properties_to_check.keys(): - for prop_key in properties_to_check[data_node_config.storage_type]: - prop_value = data_node_config.properties.get(prop_key) if data_node_config.properties else None - if prop_value and not callable(prop_value): - self._error( - prop_key, - prop_value, - f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be" - f" populated with a Callable function.", - ) + def _check_class_type(self, data_node_config_id: str, data_node_config: DataNodeConfig): + properties_to_check = self._get_class_type_and_properties() + + for storage_type in properties_to_check.keys(): + for class_type, prop_keys in properties_to_check[storage_type]: + for prop_key in prop_keys: + prop_value = data_node_config.properties.get(prop_key) if data_node_config.properties else None + if prop_value and not isinstance(prop_value, class_type): + self._error( + prop_key, + prop_value, + f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be" + f" populated with a {class_type.__name__}.", + ) def _check_exposed_type(self, data_node_config_id: str, data_node_config: DataNodeConfig): if not isinstance(data_node_config.exposed_type, str): diff --git a/tests/core/config/checkers/test_data_node_config_checker.py b/tests/core/config/checkers/test_data_node_config_checker.py index 9a4037b6b0..ca75449c0b 100644 --- a/tests/core/config/checkers/test_data_node_config_checker.py +++ b/tests/core/config/checkers/test_data_node_config_checker.py @@ -513,12 +513,12 @@ def test_check_callable_properties(self, caplog): Config.check() assert len(Config._collector.errors) == 2 expected_error_message_1 = ( - "`write_query_builder` of DataNodeConfig `new` must be populated with a Callable function." + "`write_query_builder` of DataNodeConfig `new` must be populated with a Callable." " Current value of property `write_query_builder` is 1." ) assert expected_error_message_1 in caplog.text expected_error_message_2 = ( - "`append_query_builder` of DataNodeConfig `new` must be populated with a Callable function." + "`append_query_builder` of DataNodeConfig `new` must be populated with a Callable." " Current value of property `append_query_builder` is 2." ) assert expected_error_message_2 in caplog.text @@ -530,7 +530,7 @@ def test_check_callable_properties(self, caplog): Config.check() assert len(Config._collector.errors) == 1 expected_error_messages = [ - "`write_fct` of DataNodeConfig `new` must be populated with a Callable function. Current value" + "`write_fct` of DataNodeConfig `new` must be populated with a Callable. Current value" " of property `write_fct` is 12.", ] assert all(message in caplog.text for message in expected_error_messages) @@ -542,7 +542,7 @@ def test_check_callable_properties(self, caplog): Config.check() assert len(Config._collector.errors) == 1 expected_error_messages = [ - "`read_fct` of DataNodeConfig `new` must be populated with a Callable function. Current value" + "`read_fct` of DataNodeConfig `new` must be populated with a Callable. Current value" " of property `read_fct` is 5.", ] assert all(message in caplog.text for message in expected_error_messages) @@ -554,9 +554,9 @@ def test_check_callable_properties(self, caplog): Config.check() assert len(Config._collector.errors) == 2 expected_error_messages = [ - "`write_fct` of DataNodeConfig `new` must be populated with a Callable function. Current value" + "`write_fct` of DataNodeConfig `new` must be populated with a Callable. Current value" " of property `write_fct` is 9.", - "`read_fct` of DataNodeConfig `new` must be populated with a Callable function. Current value" + "`read_fct` of DataNodeConfig `new` must be populated with a Callable. Current value" " of property `read_fct` is 5.", ] assert all(message in caplog.text for message in expected_error_messages) From 7996e5fe7ec389ff98742f265fa705d748fe5f8a Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Wed, 27 Nov 2024 15:50:08 +0700 Subject: [PATCH 3/9] minor fix --- taipy/core/config/checkers/_data_node_config_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taipy/core/config/checkers/_data_node_config_checker.py b/taipy/core/config/checkers/_data_node_config_checker.py index aa93a452e5..26fa8c345a 100644 --- a/taipy/core/config/checkers/_data_node_config_checker.py +++ b/taipy/core/config/checkers/_data_node_config_checker.py @@ -220,7 +220,7 @@ def _get_class_type_and_properties() -> Dict[str, List[Tuple[Any, List[str]]]]: } def _check_class_type(self, data_node_config_id: str, data_node_config: DataNodeConfig): - properties_to_check = self._get_class_type_and_properties() + properties_to_check = _DataNodeConfigChecker._get_class_type_and_properties() for storage_type in properties_to_check.keys(): for class_type, prop_keys in properties_to_check[storage_type]: From f986964361e82d839c466c932a7d3898a2d1d1da Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Wed, 27 Nov 2024 16:08:48 +0700 Subject: [PATCH 4/9] minor fix --- taipy/core/config/checkers/_data_node_config_checker.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/taipy/core/config/checkers/_data_node_config_checker.py b/taipy/core/config/checkers/_data_node_config_checker.py index 26fa8c345a..7cc3f17dcd 100644 --- a/taipy/core/config/checkers/_data_node_config_checker.py +++ b/taipy/core/config/checkers/_data_node_config_checker.py @@ -222,8 +222,8 @@ def _get_class_type_and_properties() -> Dict[str, List[Tuple[Any, List[str]]]]: def _check_class_type(self, data_node_config_id: str, data_node_config: DataNodeConfig): properties_to_check = _DataNodeConfigChecker._get_class_type_and_properties() - for storage_type in properties_to_check.keys(): - for class_type, prop_keys in properties_to_check[storage_type]: + if data_node_config.storage_type in properties_to_check.keys(): + for class_type, prop_keys in properties_to_check[data_node_config.storage_type]: for prop_key in prop_keys: prop_value = data_node_config.properties.get(prop_key) if data_node_config.properties else None if prop_value and not isinstance(prop_value, class_type): From 3611990f1c2857fcbd62d85887e0b90ef60b96e4 Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Thu, 28 Nov 2024 10:38:11 +0700 Subject: [PATCH 5/9] refactor check properties type --- .../checkers/_data_node_config_checker.py | 50 +++++++++---------- 1 file changed, 23 insertions(+), 27 deletions(-) diff --git a/taipy/core/config/checkers/_data_node_config_checker.py b/taipy/core/config/checkers/_data_node_config_checker.py index 7cc3f17dcd..910ee88376 100644 --- a/taipy/core/config/checkers/_data_node_config_checker.py +++ b/taipy/core/config/checkers/_data_node_config_checker.py @@ -23,6 +23,27 @@ class _DataNodeConfigChecker(_ConfigChecker): + _PROPERTIES_TYPES: Dict[str, List[Tuple[Any, List[str]]]] = { + DataNodeConfig._STORAGE_TYPE_VALUE_GENERIC: [ + ( + Callable, + [ + DataNodeConfig._OPTIONAL_READ_FUNCTION_GENERIC_PROPERTY, + DataNodeConfig._OPTIONAL_WRITE_FUNCTION_GENERIC_PROPERTY, + ], + ) + ], + DataNodeConfig._STORAGE_TYPE_VALUE_SQL: [ + ( + Callable, + [ + DataNodeConfig._REQUIRED_WRITE_QUERY_BUILDER_SQL_PROPERTY, + DataNodeConfig._OPTIONAL_APPEND_QUERY_BUILDER_SQL_PROPERTY, + ], + ), + ], + } + def __init__(self, config: _Config, collector: IssueCollector): super().__init__(config, collector) @@ -196,34 +217,9 @@ def _check_generic_read_write_fct_and_args(self, data_node_config_id: str, data_ f"DataNodeConfig `{data_node_config_id}` must be populated with a Callable function.", ) - @staticmethod - def _get_class_type_and_properties() -> Dict[str, List[Tuple[Any, List[str]]]]: - return { - DataNodeConfig._STORAGE_TYPE_VALUE_GENERIC: [ - ( - Callable, - [ - DataNodeConfig._OPTIONAL_READ_FUNCTION_GENERIC_PROPERTY, - DataNodeConfig._OPTIONAL_WRITE_FUNCTION_GENERIC_PROPERTY, - ], - ) - ], - DataNodeConfig._STORAGE_TYPE_VALUE_SQL: [ - ( - Callable, - [ - DataNodeConfig._REQUIRED_WRITE_QUERY_BUILDER_SQL_PROPERTY, - DataNodeConfig._OPTIONAL_APPEND_QUERY_BUILDER_SQL_PROPERTY, - ], - ), - ], - } - def _check_class_type(self, data_node_config_id: str, data_node_config: DataNodeConfig): - properties_to_check = _DataNodeConfigChecker._get_class_type_and_properties() - - if data_node_config.storage_type in properties_to_check.keys(): - for class_type, prop_keys in properties_to_check[data_node_config.storage_type]: + if data_node_config.storage_type in self._PROPERTIES_TYPES.keys(): + for class_type, prop_keys in self._PROPERTIES_TYPES[data_node_config.storage_type]: for prop_key in prop_keys: prop_value = data_node_config.properties.get(prop_key) if data_node_config.properties else None if prop_value and not isinstance(prop_value, class_type): From 41a7a6012ceae03022471262b0c5780f541b2f31 Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Thu, 28 Nov 2024 11:13:28 +0700 Subject: [PATCH 6/9] attempt to fix failing tests on python 3.9 --- taipy/core/config/checkers/_data_node_config_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taipy/core/config/checkers/_data_node_config_checker.py b/taipy/core/config/checkers/_data_node_config_checker.py index 910ee88376..0abe781533 100644 --- a/taipy/core/config/checkers/_data_node_config_checker.py +++ b/taipy/core/config/checkers/_data_node_config_checker.py @@ -227,7 +227,7 @@ def _check_class_type(self, data_node_config_id: str, data_node_config: DataNode prop_key, prop_value, f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be" - f" populated with a {class_type.__name__}.", + f" populated with a {class_type.__qualname__}.", ) def _check_exposed_type(self, data_node_config_id: str, data_node_config: DataNodeConfig): From 03fe9a6c8650d1f4f87ceac8e9332509e114b3de Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Thu, 28 Nov 2024 12:56:54 +0700 Subject: [PATCH 7/9] attempt to fix failing tests on python 3.9 --- .../config/checkers/_data_node_config_checker.py | 9 ++++++++- .../checkers/test_data_node_config_checker.py | 16 ++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/taipy/core/config/checkers/_data_node_config_checker.py b/taipy/core/config/checkers/_data_node_config_checker.py index 0abe781533..a15291198d 100644 --- a/taipy/core/config/checkers/_data_node_config_checker.py +++ b/taipy/core/config/checkers/_data_node_config_checker.py @@ -227,7 +227,14 @@ def _check_class_type(self, data_node_config_id: str, data_node_config: DataNode prop_key, prop_value, f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be" - f" populated with a {class_type.__qualname__}.", + f" populated with a {'Callable' if class_type == Callable else class_type.__name__}.", + ) + if class_type == Callable and callable(prop_value) and prop_value.__name__ == "": + self._error( + prop_key, + prop_value, + f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be" + f" populated with a Callable and not a lambda.", ) def _check_exposed_type(self, data_node_config_id: str, data_node_config: DataNodeConfig): diff --git a/tests/core/config/checkers/test_data_node_config_checker.py b/tests/core/config/checkers/test_data_node_config_checker.py index ca75449c0b..32f0ba81be 100644 --- a/tests/core/config/checkers/test_data_node_config_checker.py +++ b/tests/core/config/checkers/test_data_node_config_checker.py @@ -581,6 +581,22 @@ def test_check_callable_properties(self, caplog): Config.check() assert len(Config._collector.errors) == 0 + config._sections[DataNodeConfig.name]["new"].storage_type = "generic" + config._sections[DataNodeConfig.name]["new"].properties = {"write_fct": lambda x: x, "read_fct": lambda y: y} + with pytest.raises(SystemExit): + Config._collector = IssueCollector() + Config.check() + assert len(Config._collector.errors) == 2 + expected_error_messages = [ + "`write_fct` of DataNodeConfig `new` must be populated with a Callable and not a lambda." + " Current value of property `write_fct` is .", + "`read_fct` of DataNodeConfig `new` must be populated with a Callable and not a lambda." + " Current value of property `read_fct` is .", + ] + assert all(message in caplog.text for message in expected_error_messages) + def test_check_read_write_fct_args(self, caplog): config = Config._applied_config Config._compile_configs() From be1d2638e7ad37992fc8ee30bbfb2d036de303c5 Mon Sep 17 00:00:00 2001 From: Toan Quach <93168955+toan-quach@users.noreply.github.com> Date: Tue, 3 Dec 2024 14:24:52 +0700 Subject: [PATCH 8/9] Update taipy/core/config/checkers/_data_node_config_checker.py Co-authored-by: Jean-Robin --- taipy/core/config/checkers/_data_node_config_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/taipy/core/config/checkers/_data_node_config_checker.py b/taipy/core/config/checkers/_data_node_config_checker.py index a15291198d..27eb270ba4 100644 --- a/taipy/core/config/checkers/_data_node_config_checker.py +++ b/taipy/core/config/checkers/_data_node_config_checker.py @@ -234,7 +234,7 @@ def _check_class_type(self, data_node_config_id: str, data_node_config: DataNode prop_key, prop_value, f"`{prop_key}` of DataNodeConfig `{data_node_config_id}` must be" - f" populated with a Callable and not a lambda.", + f" populated with a serializable Callable function but not a lambda.", ) def _check_exposed_type(self, data_node_config_id: str, data_node_config: DataNodeConfig): From 0e445b930d5deb0b0fcf889c2820bbc9125f3ffa Mon Sep 17 00:00:00 2001 From: Toan Quach Date: Tue, 3 Dec 2024 14:42:52 +0700 Subject: [PATCH 9/9] fixed failed test --- .../core/config/checkers/test_data_node_config_checker.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/core/config/checkers/test_data_node_config_checker.py b/tests/core/config/checkers/test_data_node_config_checker.py index 32f0ba81be..7c9c3f93b5 100644 --- a/tests/core/config/checkers/test_data_node_config_checker.py +++ b/tests/core/config/checkers/test_data_node_config_checker.py @@ -588,11 +588,11 @@ def test_check_callable_properties(self, caplog): Config.check() assert len(Config._collector.errors) == 2 expected_error_messages = [ - "`write_fct` of DataNodeConfig `new` must be populated with a Callable and not a lambda." - " Current value of property `write_fct` is .", - "`read_fct` of DataNodeConfig `new` must be populated with a Callable and not a lambda." - " Current value of property `read_fct` is .", ] assert all(message in caplog.text for message in expected_error_messages)