CDK: TypeTransformer - warning message more informative (airbytehq#16695

)

Signed-off-by: Sergey Chvalyuk <grubberr@gmail.com>

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.86
+- TypeTransformer make warning message more informative
+
 ## 0.1.85
 - Make TypeTransformer more robust to incorrect incoming records
 

airbyte-cdk/python/airbyte_cdk/sources/utils/transform.py

@@ -7,7 +7,10 @@
 from enum import Flag, auto
 from typing import Any, Callable, Dict, Mapping, Optional
 
-from jsonschema import Draft7Validator, validators
+from jsonschema import Draft7Validator, ValidationError, validators
+
+json_to_python = {"string": str, "number": float, "integer": int, "boolean": bool, "null": type(None), "object": dict, "array": list}
+python_to_json = {v: k for k, v in json_to_python.items()}
 
 logger = logging.getLogger("airbyte")
 
@@ -178,4 +181,11 @@ def transform(self, record: Dict[str, Any], schema: Mapping[str, Any]):
             just calling normalizer.validate() would throw an exception on
             first validation occurences and stop processing rest of schema.
             """
-            logger.warning(e.message)
+            logger.warning(self.get_error_message(e))
+
+    def get_error_message(self, e: ValidationError) -> str:
+        instance_json_type = python_to_json[type(e.instance)]
+        key_path = "." + ".".join(e.path)
+        return (
+            f"Failed to transform value {repr(e.instance)} of type '{instance_json_type}' to '{e.validator_value}', key path: '{key_path}'"
+        )

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.85",
+    version="0.1.86",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/sources/utils/test_transform.py

@@ -81,14 +81,14 @@
             COMPLEX_SCHEMA,
             {"prop": 12, "number_prop": "aa12", "array": [12]},
             {"prop": "12", "number_prop": "aa12", "array": ["12"]},
-            "'aa12' is not of type 'number'",
+            "Failed to transform value 'aa12' of type 'string' to 'number', key path: '.number_prop'",
         ),
         # Field too_many_types have ambigious type, skip formatting
         (
             COMPLEX_SCHEMA,
             {"prop": 12, "too_many_types": 1212, "array": [12]},
             {"prop": "12", "too_many_types": 1212, "array": ["12"]},
-            "1212 is not of type 'boolean', 'null', 'string'",
+            "Failed to transform value 1212 of type 'integer' to '['boolean', 'null', 'string']', key path: '.too_many_types'",
         ),
         # Test null field
         (COMPLEX_SCHEMA, {"prop": None, "array": [12]}, {"prop": "None", "array": ["12"]}, None),
@@ -115,21 +115,21 @@
             {"type": "object", "properties": {"value": {"type": "array"}}},
             {"value": "12"},
             {"value": "12"},
-            "'12' is not of type 'array'",
+            "Failed to transform value '12' of type 'string' to 'array', key path: '.value'",
         ),
         (
             # Schema root object is not an object, no convertion should happen
             {"type": "integer"},
             {"value": "12"},
             {"value": "12"},
-            "{'value': '12'} is not of type 'integer'",
+            "Failed to transform value {'value': '12'} of type 'object' to 'integer', key path: '.'",
         ),
         (
             # More than one type except null, no conversion should happen
             {"type": "object", "properties": {"value": {"type": ["string", "boolean", "null"]}}},
             {"value": 12},
             {"value": 12},
-            "12 is not of type 'string', 'boolean', 'null'",
+            "Failed to transform value 12 of type 'integer' to '['string', 'boolean', 'null']', key path: '.value'",
         ),
         (
             # Oneof not suported, no conversion for one_of_value should happen
@@ -153,13 +153,13 @@
             {"type": "object", "properties": {"value": {"type": "array", "items": {"type": "string"}}}},
             {"value": {"key": "value"}},
             {"value": {"key": "value"}},
-            "{'key': 'value'} is not of type 'array'",
+            "Failed to transform value {'key': 'value'} of type 'object' to 'array', key path: '.value'",
         ),
         (
             {"type": "object", "properties": {"value1": {"type": "object", "properties": {"value2": {"type": "string"}}}}},
             {"value1": "value2"},
             {"value1": "value2"},
-            "'value2' is not of type 'object'",
+            "Failed to transform value 'value2' of type 'string' to 'object', key path: '.value1'",
         ),
     ],
 )