get module name from sys.modules (#17779)

* get module name from sys.modules

* bump

* fix comment

* throw exception

* fix unittests

* Add missing files

* remove debug prints

* indent

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.1.99
+
+- Low-code: Fix default stream schema loader
+
 ## 0.1.98
 
 - Low-code: Expose WaitUntilTimeFromHeader strategy and WaitTimeFromHeader as component type

airbyte-cdk/python/airbyte_cdk/sources/declarative/schema/json_schema.py

@@ -4,19 +4,25 @@
 
 import json
 import pkgutil
+import sys
 from dataclasses import InitVar, dataclass, field
 from typing import Any, Mapping, Union
 
-import __main__
 from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.schema.schema_loader import SchemaLoader
 from airbyte_cdk.sources.declarative.types import Config
 from dataclasses_jsonschema import JsonSchemaMixin
 
 
 def _default_file_path() -> str:
-    main_file = __main__.__file__
-    module = main_file.split("/")[-2].replace("-", "_")
+    # schema files are always in "source_<connector_name>/schemas/<stream_name>.json
+    # the connector's module name can be inferred by looking at the modules loaded and look for the one starting with source_
+    source_modules = [
+        k for k, v in sys.modules.items() if "source_" in k  # example: ['source_exchange_rates', 'source_exchange_rates.source']
+    ]
+    if not source_modules:
+        raise RuntimeError("Expected at least one module starting with 'source_'")
+    module = source_modules[0].split(".")[0]
     return f"./{module}/schemas/{{{{options['name']}}}}.json"
 
 
@@ -34,9 +40,11 @@ class JsonSchema(SchemaLoader, JsonSchemaMixin):
 
     config: Config
     options: InitVar[Mapping[str, Any]]
-    file_path: Union[InterpolatedString, str] = field(default=_default_file_path())
+    file_path: Union[InterpolatedString, str] = field(default=None)
 
     def __post_init__(self, options: Mapping[str, Any]):
+        if not self.file_path:
+            self.file_path = _default_file_path()
         self.file_path = InterpolatedString.create(self.file_path, options=options)
 
     def get_json_schema(self) -> Mapping[str, Any]:

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.98",
+    version="0.1.99",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",

airbyte-cdk/python/unit_tests/sources/declarative/schema/__init__.py

@@ -0,0 +1,6 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+from .source_test.SourceTest import SourceTest
+
+__all__ = ["SourceTest"]

airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/SourceTest.py

@@ -0,0 +1,8 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+
+class SourceTest:
+    def __init__(self):
+        pass

airbyte-cdk/python/unit_tests/sources/declarative/schema/source_test/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#

airbyte-cdk/python/unit_tests/sources/declarative/test_declarative_stream.py

@@ -9,6 +9,8 @@
 from airbyte_cdk.sources.declarative.declarative_stream import DeclarativeStream
 from airbyte_cdk.sources.declarative.transformations import RecordTransformation
 
+from .schema.source_test import SourceTest  # noqa #pylint: disable=unused-import
+
 
 def test_declarative_stream():
     name = "stream"

airbyte-cdk/python/unit_tests/sources/declarative/test_yaml_declarative_source.py

@@ -4,9 +4,10 @@
 
 import json
 
+from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
+
 # import pytest
 # from airbyte_cdk.sources.declarative.exceptions import InvalidConnectorDefinitionException
-from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
 
 # import os
 # import tempfile