From cab700306f27392ab05bc14c6262d421665437ce Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Thu, 15 Sep 2022 10:24:37 +0300 Subject: [PATCH] CDK: TypeTransformer - warning message more informative (#16695) Signed-off-by: Sergey Chvalyuk --- airbyte-cdk/python/CHANGELOG.md | 3 +++ .../python/airbyte_cdk/sources/utils/transform.py | 14 ++++++++++++-- airbyte-cdk/python/setup.py | 2 +- .../unit_tests/sources/utils/test_transform.py | 14 +++++++------- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 42dc59ff18a2..09c6d7ca7582 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.86 +- TypeTransformer make warning message more informative + ## 0.1.85 - Make TypeTransformer more robust to incorrect incoming records diff --git a/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py b/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py index 46e3f0e8f46f..bb899cb89c04 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py @@ -7,7 +7,10 @@ from enum import Flag, auto from typing import Any, Callable, Dict, Mapping, Optional -from jsonschema import Draft7Validator, validators +from jsonschema import Draft7Validator, ValidationError, validators + +json_to_python = {"string": str, "number": float, "integer": int, "boolean": bool, "null": type(None), "object": dict, "array": list} +python_to_json = {v: k for k, v in json_to_python.items()} logger = logging.getLogger("airbyte") @@ -178,4 +181,11 @@ def transform(self, record: Dict[str, Any], schema: Mapping[str, Any]): just calling normalizer.validate() would throw an exception on first validation occurences and stop processing rest of schema. """ - logger.warning(e.message) + logger.warning(self.get_error_message(e)) + + def get_error_message(self, e: ValidationError) -> str: + instance_json_type = python_to_json[type(e.instance)] + key_path = "." + ".".join(e.path) + return ( + f"Failed to transform value {repr(e.instance)} of type '{instance_json_type}' to '{e.validator_value}', key path: '{key_path}'" + ) diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 4798a04f06af..90ce2d1697f8 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.85", + version="0.1.86", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown", diff --git a/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py b/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py index 9a219dd05b22..26e7b39839de 100644 --- a/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py +++ b/airbyte-cdk/python/unit_tests/sources/utils/test_transform.py @@ -81,14 +81,14 @@ COMPLEX_SCHEMA, {"prop": 12, "number_prop": "aa12", "array": [12]}, {"prop": "12", "number_prop": "aa12", "array": ["12"]}, - "'aa12' is not of type 'number'", + "Failed to transform value 'aa12' of type 'string' to 'number', key path: '.number_prop'", ), # Field too_many_types have ambigious type, skip formatting ( COMPLEX_SCHEMA, {"prop": 12, "too_many_types": 1212, "array": [12]}, {"prop": "12", "too_many_types": 1212, "array": ["12"]}, - "1212 is not of type 'boolean', 'null', 'string'", + "Failed to transform value 1212 of type 'integer' to '['boolean', 'null', 'string']', key path: '.too_many_types'", ), # Test null field (COMPLEX_SCHEMA, {"prop": None, "array": [12]}, {"prop": "None", "array": ["12"]}, None), @@ -115,21 +115,21 @@ {"type": "object", "properties": {"value": {"type": "array"}}}, {"value": "12"}, {"value": "12"}, - "'12' is not of type 'array'", + "Failed to transform value '12' of type 'string' to 'array', key path: '.value'", ), ( # Schema root object is not an object, no convertion should happen {"type": "integer"}, {"value": "12"}, {"value": "12"}, - "{'value': '12'} is not of type 'integer'", + "Failed to transform value {'value': '12'} of type 'object' to 'integer', key path: '.'", ), ( # More than one type except null, no conversion should happen {"type": "object", "properties": {"value": {"type": ["string", "boolean", "null"]}}}, {"value": 12}, {"value": 12}, - "12 is not of type 'string', 'boolean', 'null'", + "Failed to transform value 12 of type 'integer' to '['string', 'boolean', 'null']', key path: '.value'", ), ( # Oneof not suported, no conversion for one_of_value should happen @@ -153,13 +153,13 @@ {"type": "object", "properties": {"value": {"type": "array", "items": {"type": "string"}}}}, {"value": {"key": "value"}}, {"value": {"key": "value"}}, - "{'key': 'value'} is not of type 'array'", + "Failed to transform value {'key': 'value'} of type 'object' to 'array', key path: '.value'", ), ( {"type": "object", "properties": {"value1": {"type": "object", "properties": {"value2": {"type": "string"}}}}}, {"value1": "value2"}, {"value1": "value2"}, - "'value2' is not of type 'object'", + "Failed to transform value 'value2' of type 'string' to 'object', key path: '.value1'", ), ], )