From a407b3849a0b32071b6cec7accb60acc2df5f0f3 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Thu, 14 Apr 2022 19:00:23 -0400 Subject: [PATCH 1/7] support loading spec from yaml file --- airbyte-cdk/python/airbyte_cdk/connector.py | 28 ++++++-- .../python/unit_tests/test_connector.py | 64 +++++++++++++++++++ 2 files changed, 88 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/connector.py b/airbyte-cdk/python/airbyte_cdk/connector.py index f17c76ab5754..28fdc918ac41 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector.py +++ b/airbyte-cdk/python/airbyte_cdk/connector.py @@ -4,6 +4,7 @@ import json +import yaml import logging import os import pkgutil @@ -13,6 +14,13 @@ from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification +def load_optional_package_file(package: str, filename: str) -> Optional[bytes]: + try: + return pkgutil.get_data(package, filename) + except FileNotFoundError: + return None + + class AirbyteSpec(object): @staticmethod def from_file(file_name: str): @@ -53,10 +61,22 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) required to run this integration. """ - raw_spec: Optional[bytes] = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") - if not raw_spec: - raise ValueError("Unable to find spec.json.") - return ConnectorSpecification.parse_obj(json.loads(raw_spec)) + + package = self.__class__.__module__.split(".")[0] + yaml_spec = load_optional_package_file(package, "spec.yaml") + json_spec = load_optional_package_file(package, "spec.json") + + if yaml_spec and json_spec: + raise ValueError("Found multiple spec files in the package. Only one of spec.yaml or spec.json should be provided.") + + if yaml_spec: + spec_obj = yaml.load(yaml_spec, Loader=yaml.SafeLoader) + elif json_spec: + spec_obj = json.loads(json_spec) + else: + raise ValueError("Unable to find spec.yaml or spec.json in the package.") + + return ConnectorSpecification.parse_obj(spec_obj) @abstractmethod def check(self, logger: logging.Logger, config: Mapping[str, Any]) -> AirbyteConnectionStatus: diff --git a/airbyte-cdk/python/unit_tests/test_connector.py b/airbyte-cdk/python/unit_tests/test_connector.py index 463b58290cba..1fa1647c91bd 100644 --- a/airbyte-cdk/python/unit_tests/test_connector.py +++ b/airbyte-cdk/python/unit_tests/test_connector.py @@ -3,6 +3,8 @@ # +import sys +import os import json import logging import tempfile @@ -10,10 +12,19 @@ from typing import Any, Mapping import pytest +import yaml + from airbyte_cdk import AirbyteSpec, Connector from airbyte_cdk.models import AirbyteConnectionStatus +logger = logging.getLogger('airbyte') + +MODULE = sys.modules[__name__] +MODULE_PATH = os.path.abspath(MODULE.__file__) +SPEC_ROOT = os.path.dirname(MODULE_PATH) + + class TestAirbyteSpec: VALID_SPEC = { "documentationUrl": "https://google.com", @@ -71,3 +82,56 @@ def test_write_config(integration, mock_config): integration.write_config(mock_config, str(config_path)) with open(config_path, "r") as actual: assert mock_config == json.loads(actual.read()) + + +class TestConnectorSpec: + CONNECTION_SPECIFICATION = { + "type": "object", + "required": ["api_token"], + "additionalProperties": False, + "properties": {"api_token": {"type": "string"}}, + } + + @pytest.fixture + def use_json_spec(self): + spec = { + "documentationUrl": "https://airbyte.com/#json", + "connectionSpecification": self.CONNECTION_SPECIFICATION, + } + + json_path = os.path.join(SPEC_ROOT, "spec.json") + with open(json_path, "w") as f: + f.write(json.dumps(spec)) + yield + os.remove(json_path) + + @pytest.fixture + def use_yaml_spec(self): + spec = { + "documentationUrl": "https://airbyte.com/#yaml", + "connectionSpecification": self.CONNECTION_SPECIFICATION + } + + yaml_path = os.path.join(SPEC_ROOT, "spec.yaml") + with open(yaml_path, "w") as f: + f.write(yaml.dump(spec)) + yield + os.remove(yaml_path) + + def test_spec_from_json_file(self, integration, use_json_spec): + connector_spec = integration.spec(logger) + assert connector_spec.documentationUrl == "https://airbyte.com/#json" + assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION + + def test_spec_from_yaml_file(self, integration, use_yaml_spec): + connector_spec = integration.spec(logger) + assert connector_spec.documentationUrl == "https://airbyte.com/#yaml" + assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION + + def test_multiple_spec_files_raises_exception(self, integration, use_yaml_spec, use_json_spec): + with pytest.raises(ValueError, match="spec.yaml or spec.json"): + integration.spec(logger) + + def test_no_spec_file_raises_exception(self, integration): + with pytest.raises(ValueError, match="Unable to find spec."): + integration.spec(logger) From 84ecf8c003df81d8f094ae9d0170cb08b7e07c04 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Tue, 19 Apr 2022 10:52:47 -0400 Subject: [PATCH 2/7] formatting --- airbyte-cdk/python/airbyte_cdk/connector.py | 27 ++++++++++++++++++- .../airbyte_cdk/utils/resource_utils.py | 0 .../python/unit_tests/test_connector.py | 13 +++------ 3 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 airbyte-cdk/python/airbyte_cdk/utils/resource_utils.py diff --git a/airbyte-cdk/python/airbyte_cdk/connector.py b/airbyte-cdk/python/airbyte_cdk/connector.py index 28fdc918ac41..0e5d4d533b3b 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector.py +++ b/airbyte-cdk/python/airbyte_cdk/connector.py @@ -4,17 +4,18 @@ import json -import yaml import logging import os import pkgutil from abc import ABC, abstractmethod from typing import Any, Mapping, Optional +import yaml from airbyte_cdk.models import AirbyteConnectionStatus, ConnectorSpecification def load_optional_package_file(package: str, filename: str) -> Optional[bytes]: + """Gets a resource from a package, returning None if it does not exist""" try: return pkgutil.get_data(package, filename) except FileNotFoundError: @@ -63,6 +64,30 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: """ package = self.__class__.__module__.split(".")[0] + + # formats = [ + # {"extension": "yaml", "loader": lambda data: yaml.load(data, yaml.SafeLoader)}, + # {"extension": "yml", "loader": lambda data: yaml.load(data, yaml.SafeLoader)}, + # {"extension": "json", "loader": lambda data: json.loads(data)} + # ] + # + # spec_obj = None + # for spec_format in formats: + # try: + # raw_spec = pkgutil.get_data(package, f"spec.{spec_format['extension']}") + # if not raw_spec: + # continue + # if spec_obj: + # raise ValueError("Found multiple spec files in the package. Only one of spec.yaml or spec.json should be provided.") + # spec_obj = spec_format["loader"](raw_spec) + # except FileNotFoundError: + # continue + # + # if not spec_obj: + # raise ValueError("Unable to find spec.") + # + # return ConnectorSpecification.parse_obj(spec_obj) + yaml_spec = load_optional_package_file(package, "spec.yaml") json_spec = load_optional_package_file(package, "spec.json") diff --git a/airbyte-cdk/python/airbyte_cdk/utils/resource_utils.py b/airbyte-cdk/python/airbyte_cdk/utils/resource_utils.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-cdk/python/unit_tests/test_connector.py b/airbyte-cdk/python/unit_tests/test_connector.py index 1fa1647c91bd..31e0fc239295 100644 --- a/airbyte-cdk/python/unit_tests/test_connector.py +++ b/airbyte-cdk/python/unit_tests/test_connector.py @@ -3,22 +3,20 @@ # -import sys -import os import json import logging +import os +import sys import tempfile from pathlib import Path from typing import Any, Mapping import pytest import yaml - from airbyte_cdk import AirbyteSpec, Connector from airbyte_cdk.models import AirbyteConnectionStatus - -logger = logging.getLogger('airbyte') +logger = logging.getLogger("airbyte") MODULE = sys.modules[__name__] MODULE_PATH = os.path.abspath(MODULE.__file__) @@ -107,10 +105,7 @@ def use_json_spec(self): @pytest.fixture def use_yaml_spec(self): - spec = { - "documentationUrl": "https://airbyte.com/#yaml", - "connectionSpecification": self.CONNECTION_SPECIFICATION - } + spec = {"documentationUrl": "https://airbyte.com/#yaml", "connectionSpecification": self.CONNECTION_SPECIFICATION} yaml_path = os.path.join(SPEC_ROOT, "spec.yaml") with open(yaml_path, "w") as f: From e4b20032c4c4b43a057e3385cef617ff7337f34f Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Tue, 19 Apr 2022 10:53:18 -0400 Subject: [PATCH 3/7] remove commented code --- airbyte-cdk/python/airbyte_cdk/connector.py | 23 --------------------- 1 file changed, 23 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/connector.py b/airbyte-cdk/python/airbyte_cdk/connector.py index 0e5d4d533b3b..a2cc97886dbf 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector.py +++ b/airbyte-cdk/python/airbyte_cdk/connector.py @@ -65,29 +65,6 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: package = self.__class__.__module__.split(".")[0] - # formats = [ - # {"extension": "yaml", "loader": lambda data: yaml.load(data, yaml.SafeLoader)}, - # {"extension": "yml", "loader": lambda data: yaml.load(data, yaml.SafeLoader)}, - # {"extension": "json", "loader": lambda data: json.loads(data)} - # ] - # - # spec_obj = None - # for spec_format in formats: - # try: - # raw_spec = pkgutil.get_data(package, f"spec.{spec_format['extension']}") - # if not raw_spec: - # continue - # if spec_obj: - # raise ValueError("Found multiple spec files in the package. Only one of spec.yaml or spec.json should be provided.") - # spec_obj = spec_format["loader"](raw_spec) - # except FileNotFoundError: - # continue - # - # if not spec_obj: - # raise ValueError("Unable to find spec.") - # - # return ConnectorSpecification.parse_obj(spec_obj) - yaml_spec = load_optional_package_file(package, "spec.yaml") json_spec = load_optional_package_file(package, "spec.json") From a46688b79c20a8109e2960885b7f3633a7b8f2dc Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Tue, 19 Apr 2022 10:54:52 -0400 Subject: [PATCH 4/7] update comment --- airbyte-cdk/python/airbyte_cdk/connector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-cdk/python/airbyte_cdk/connector.py b/airbyte-cdk/python/airbyte_cdk/connector.py index a2cc97886dbf..efa169d6532b 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector.py +++ b/airbyte-cdk/python/airbyte_cdk/connector.py @@ -60,7 +60,7 @@ def write_config(config: Mapping[str, Any], config_path: str): def spec(self, logger: logging.Logger) -> ConnectorSpecification: """ Returns the spec for this integration. The spec is a JSON-Schema object describing the required configurations (e.g: username and password) - required to run this integration. + required to run this integration. By default, this will be loaded from a "spec.yaml" or a "spec.json" in the package root. """ package = self.__class__.__module__.split(".")[0] From c9a75e8fea41c4634964d818b7272795ae7233a9 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Tue, 19 Apr 2022 10:56:38 -0400 Subject: [PATCH 5/7] remove unused file --- airbyte-cdk/python/airbyte_cdk/utils/resource_utils.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 airbyte-cdk/python/airbyte_cdk/utils/resource_utils.py diff --git a/airbyte-cdk/python/airbyte_cdk/utils/resource_utils.py b/airbyte-cdk/python/airbyte_cdk/utils/resource_utils.py deleted file mode 100644 index e69de29bb2d1..000000000000 From da65656392f85a3ecd5ccd8358b5828ba29daa30 Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 08:37:34 -0400 Subject: [PATCH 6/7] raise correct exception types --- airbyte-cdk/python/airbyte_cdk/connector.py | 4 ++-- airbyte-cdk/python/unit_tests/test_connector.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-cdk/python/airbyte_cdk/connector.py b/airbyte-cdk/python/airbyte_cdk/connector.py index efa169d6532b..f75bf0c717a7 100644 --- a/airbyte-cdk/python/airbyte_cdk/connector.py +++ b/airbyte-cdk/python/airbyte_cdk/connector.py @@ -69,14 +69,14 @@ def spec(self, logger: logging.Logger) -> ConnectorSpecification: json_spec = load_optional_package_file(package, "spec.json") if yaml_spec and json_spec: - raise ValueError("Found multiple spec files in the package. Only one of spec.yaml or spec.json should be provided.") + raise RuntimeError("Found multiple spec files in the package. Only one of spec.yaml or spec.json should be provided.") if yaml_spec: spec_obj = yaml.load(yaml_spec, Loader=yaml.SafeLoader) elif json_spec: spec_obj = json.loads(json_spec) else: - raise ValueError("Unable to find spec.yaml or spec.json in the package.") + raise FileNotFoundError("Unable to find spec.yaml or spec.json in the package.") return ConnectorSpecification.parse_obj(spec_obj) diff --git a/airbyte-cdk/python/unit_tests/test_connector.py b/airbyte-cdk/python/unit_tests/test_connector.py index 31e0fc239295..b1840ef08d52 100644 --- a/airbyte-cdk/python/unit_tests/test_connector.py +++ b/airbyte-cdk/python/unit_tests/test_connector.py @@ -124,9 +124,9 @@ def test_spec_from_yaml_file(self, integration, use_yaml_spec): assert connector_spec.connectionSpecification == self.CONNECTION_SPECIFICATION def test_multiple_spec_files_raises_exception(self, integration, use_yaml_spec, use_json_spec): - with pytest.raises(ValueError, match="spec.yaml or spec.json"): + with pytest.raises(RuntimeError, match="spec.yaml or spec.json"): integration.spec(logger) def test_no_spec_file_raises_exception(self, integration): - with pytest.raises(ValueError, match="Unable to find spec."): + with pytest.raises(FileNotFoundError, match="Unable to find spec."): integration.spec(logger) From 0ca80ce77d9eb12683a5e1a95067204958d336de Mon Sep 17 00:00:00 2001 From: Pedro Lopez Date: Wed, 20 Apr 2022 08:52:09 -0400 Subject: [PATCH 7/7] bump version, update changelog --- airbyte-cdk/python/CHANGELOG.md | 3 +++ airbyte-cdk/python/setup.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-cdk/python/CHANGELOG.md b/airbyte-cdk/python/CHANGELOG.md index 729660e35351..145ef289cc7c 100644 --- a/airbyte-cdk/python/CHANGELOG.md +++ b/airbyte-cdk/python/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +## 0.1.55 +Add support for reading the spec from a YAML file (`spec.yaml`) + ## 0.1.54 - Add ability to import `IncrementalMixin` from `airbyte_cdk.sources.streams`. - Bumped minimum supported Python version to 3.9. diff --git a/airbyte-cdk/python/setup.py b/airbyte-cdk/python/setup.py index 5be56e0ee84d..df9eaacc46fb 100644 --- a/airbyte-cdk/python/setup.py +++ b/airbyte-cdk/python/setup.py @@ -15,7 +15,7 @@ setup( name="airbyte-cdk", - version="0.1.54", + version="0.1.55", description="A framework for writing Airbyte Connectors.", long_description=README, long_description_content_type="text/markdown",