diff --git a/airbyte-integrations/bases/base-python/base_python/client.py b/airbyte-integrations/bases/base-python/base_python/client.py index a17a46ad3798..5c215d4fcd90 100644 --- a/airbyte-integrations/bases/base-python/base_python/client.py +++ b/airbyte-integrations/bases/base-python/base_python/client.py @@ -97,7 +97,7 @@ def streams(self) -> Generator[AirbyteStream, None, None]: supported_sync_modes = [SyncMode.full_refresh] source_defined_cursor = False if self.stream_has_state(name): - supported_sync_modes = [SyncMode.incremental] + supported_sync_modes += [SyncMode.incremental] source_defined_cursor = True yield AirbyteStream( diff --git a/airbyte-integrations/bases/source-acceptance-test/.dockerignore b/airbyte-integrations/bases/source-acceptance-test/.dockerignore new file mode 100644 index 000000000000..6cf5797a1a31 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/.dockerignore @@ -0,0 +1,4 @@ +* +!Dockerfile +!standard_test +!setup.py diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile new file mode 100644 index 000000000000..21b9a205aae0 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -0,0 +1,13 @@ +FROM airbyte/integration-base-python:dev + +ENV CODE_PATH="source_acceptance_test" + +WORKDIR /airbyte/source_acceptance_test +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install . + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-acceptance-test + +ENTRYPOINT ["python", "-m", "pytest"] diff --git a/airbyte-integrations/bases/source-acceptance-test/README.md b/airbyte-integrations/bases/source-acceptance-test/README.md new file mode 100644 index 000000000000..be0e43a88abf --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/README.md @@ -0,0 +1,59 @@ +# Standard tests +This package uses pytest to discover, configure and execute the tests. +It implemented as a pytest plugin. + +It adds new configuration option `--acceptance-test-config` - path to configuration file (by default is current folder). +Configuration stored in YaML format and validated by pydantic. + +Example configuration can be found in `sample_files/` folder: +```yaml +connector_image: +tests: + spec: + - spec_path: "/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "sample_files/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: true + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + state_path: "sample_files/abnormal_state.json" + cursor_paths: + subscription_changes: ["timestamp"] + email_events: ["timestamp"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" +``` +Required steps to test connector are the following: +* Build docker image for connector +* Create `acceptance-test-config.yml` file with test settings, Note: all paths in this files are relative to its location +* Use one of the following ways to run tests: + +## Running +Using python +```bash +cd ../../base/source-acceptance-test +python -m pytest source_acceptance_test/tests --acceptance-test-config= -vvv +``` +_Note: this will assume that docker image for connector is already built_ + +Using Gradle +```bash +./gradlew :airbyte-integrations:connectors:source-:standardTest +``` +_Note: this way will also build docker image for the connector_ + +Using Bash +```bash +./source-acceptance-test.sh -vv +``` +_Note: you can append any arguments to this command, they will be forwarded to pytest diff --git a/airbyte-integrations/bases/source-acceptance-test/build.gradle b/airbyte-integrations/bases/source-acceptance-test/build.gradle new file mode 100644 index 000000000000..c6d043311980 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/build.gradle @@ -0,0 +1,15 @@ +plugins { + id 'airbyte-docker' + id 'airbyte-python' +} + +airbytePython { + moduleDirectory 'source_acceptance_test' +} + + +dependencies { + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} + +installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") diff --git a/airbyte-integrations/bases/source-acceptance-test/pytest.ini b/airbyte-integrations/bases/source-acceptance-test/pytest.ini new file mode 100644 index 000000000000..5cad9bd2139f --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/pytest.ini @@ -0,0 +1,5 @@ +[pytest] + +addopts = -r a --capture=no -vv --log-level=INFO +testpaths = + source_acceptance_test/tests diff --git a/airbyte-integrations/bases/source-acceptance-test/requirements.txt b/airbyte-integrations/bases/source-acceptance-test/requirements.txt new file mode 100644 index 000000000000..58a824426a32 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/requirements.txt @@ -0,0 +1,2 @@ +-e ../airbyte-protocol +-e . diff --git a/airbyte-integrations/bases/source-acceptance-test/sample_files/acceptance-test-config.yml b/airbyte-integrations/bases/source-acceptance-test/sample_files/acceptance-test-config.yml new file mode 100644 index 000000000000..aabee4acc155 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/sample_files/acceptance-test-config.yml @@ -0,0 +1,25 @@ +connector_image: airbyte/source-hubspot:dev +tests: + spec: + - spec_path: "source_hubspot/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "sample_files/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + state_path: "sample_files/abnormal_state.json" + cursor_paths: + subscription_changes: ["timestamp"] + email_events: ["timestamp"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" diff --git a/airbyte-integrations/bases/source-acceptance-test/setup.py b/airbyte-integrations/bases/source-acceptance-test/setup.py new file mode 100644 index 000000000000..7d8d8a6547d2 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/setup.py @@ -0,0 +1,49 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import setuptools + +MAIN_REQUIREMENTS = [ + "airbyte-protocol", + "docker==4.4.4", + "PyYAML==5.3.1", + "inflection==0.5.1", + "icdiff==1.9.1", + "pendulum==1.2.0", + "pydantic==1.6.1", + "pytest==6.1.2", + "pytest-timeout==1.4.2", + "pprintpp==0.4.0", +] + +setuptools.setup( + name="source-acceptance-test", + description="Contains classes for running integration tests.", + author="Airbyte", + author_email="contact@airbyte.io", + url="https://github.com/airbytehq/airbyte", + packages=setuptools.find_packages(), + install_requires=MAIN_REQUIREMENTS, + entry_points={"pytest11": ["pytest-airbyte = source_acceptance_test.plugin"]}, +) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py new file mode 100644 index 000000000000..f7fbc3fb16f7 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .base import BaseTest + +__all__ = ["BaseTest"] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py new file mode 100644 index 000000000000..2b22fb93f6fa --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py @@ -0,0 +1,37 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import inflection +import pytest + + +@pytest.mark.usefixtures("inputs") +class BaseTest: + @classmethod + def config_key(cls): + """Name of the test in configuration file, used to override test inputs,""" + class_name = cls.__name__ + if class_name.startswith("Test"): + class_name = class_name[len("Test") :] + return inflection.underscore(class_name) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py new file mode 100644 index 000000000000..2c2157ca9ce6 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py @@ -0,0 +1,94 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from enum import Enum +from typing import List, Mapping, Optional + +from pydantic import BaseModel, Field + +config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration") +invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration") +spec_path: str = Field( + default="secrets/spec.json", description="Path to a JSON object representing the spec expected to be output by this connector" +) +configured_catalog_path: str = Field(default="sample_files/configured_catalog.json", description="Path to configured catalog") + + +class BaseConfig(BaseModel): + class Config: + extra = "forbid" + + +class SpecTestConfig(BaseConfig): + spec_path: str = spec_path + + +class ConnectionTestConfig(BaseConfig): + class Status(Enum): + Succeed = "succeed" + Failed = "failed" + Exception = "exception" + + config_path: str = config_path + status: Status = Field(Status.Succeed, description="Indicate if connection check should succeed with provided config") + + +class DiscoveryTestConfig(BaseConfig): + config_path: str = config_path + configured_catalog_path: Optional[str] = configured_catalog_path + + +class BasicReadTestConfig(BaseConfig): + config_path: str = config_path + configured_catalog_path: Optional[str] = configured_catalog_path + validate_output_from_all_streams: bool = Field(False, description="Verify that all streams have records") + + +class FullRefreshConfig(BaseConfig): + config_path: str = config_path + configured_catalog_path: str = configured_catalog_path + + +class IncrementalConfig(BaseConfig): + config_path: str = config_path + configured_catalog_path: str = configured_catalog_path + cursor_paths: Optional[Mapping[str, List[str]]] = Field( + description="For each stream, the path of its cursor field in the output state messages." + ) + state_path: Optional[str] = Field(description="Path to state file") + + +class TestConfig(BaseConfig): + spec: Optional[List[SpecTestConfig]] = Field(description="TODO") + connection: Optional[List[ConnectionTestConfig]] = Field(description="TODO") + discovery: Optional[List[DiscoveryTestConfig]] = Field(description="TODO") + basic_read: Optional[List[BasicReadTestConfig]] = Field(description="TODO") + full_refresh: Optional[List[FullRefreshConfig]] = Field(description="TODO") + incremental: Optional[List[IncrementalConfig]] = Field(description="TODO") + + +class Config(BaseConfig): + connector_image: str = Field(description="Docker image to test, for example 'airbyte/source-hubspot:dev'") + base_path: Optional[str] = Field(description="Base path for all relative paths") + tests: TestConfig = Field(description="List of the tests with their configs") diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py new file mode 100644 index 000000000000..9912eade9460 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py @@ -0,0 +1,142 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import copy +import json +from pathlib import Path +from typing import Any, List, MutableMapping, Optional + +import pytest +from airbyte_protocol import AirbyteCatalog, AirbyteMessage, ConfiguredAirbyteCatalog, ConnectorSpecification +from source_acceptance_test.config import Config +from source_acceptance_test.utils import ConnectorRunner, SecretDict, load_config + + +@pytest.fixture(name="base_path") +def base_path_fixture(pytestconfig, standard_test_config) -> Path: + """Fixture to define base path for every path-like fixture""" + if standard_test_config.base_path: + return Path(standard_test_config.base_path).absolute() + return Path(pytestconfig.getoption("--acceptance-test-config")).absolute() + + +@pytest.fixture(name="standard_test_config", scope="session") +def standard_test_config_fixture(pytestconfig) -> Config: + """Fixture with test's config""" + return load_config(pytestconfig.getoption("--acceptance-test-config")) + + +@pytest.fixture(name="connector_config_path") +def connector_config_path_fixture(inputs, base_path) -> Path: + """Fixture with connector's config path""" + return Path(base_path) / getattr(inputs, "config_path") + + +@pytest.fixture(name="invalid_connector_config_path") +def invalid_connector_config_path_fixture(inputs, base_path) -> Path: + """Fixture with connector's config path""" + return Path(base_path) / getattr(inputs, "invalid_config_path") + + +@pytest.fixture(name="connector_spec_path") +def connector_spec_path_fixture(inputs, base_path) -> Path: + """Fixture with connector's specification path""" + return Path(base_path) / getattr(inputs, "spec_path") + + +@pytest.fixture(name="configured_catalog_path") +def configured_catalog_path_fixture(inputs, base_path) -> Optional[str]: + """Fixture with connector's configured_catalog path""" + if getattr(inputs, "configured_catalog_path"): + return Path(base_path) / getattr(inputs, "configured_catalog_path") + return None + + +@pytest.fixture(name="configured_catalog") +def configured_catalog_fixture(configured_catalog_path) -> Optional[ConfiguredAirbyteCatalog]: + if configured_catalog_path: + return ConfiguredAirbyteCatalog.parse_file(configured_catalog_path) + return None + + +@pytest.fixture(name="catalog") +def catalog_fixture(configured_catalog: ConfiguredAirbyteCatalog) -> Optional[AirbyteCatalog]: + if configured_catalog: + return AirbyteCatalog(streams=[stream.stream for stream in configured_catalog.streams]) + return None + + +@pytest.fixture(name="image_tag") +def image_tag_fixture(standard_test_config) -> str: + return standard_test_config.connector_image + + +@pytest.fixture(name="connector_config") +def connector_config_fixture(base_path, connector_config_path) -> SecretDict: + with open(str(connector_config_path), "r") as file: + contents = file.read() + return SecretDict(json.loads(contents)) + + +@pytest.fixture(name="invalid_connector_config") +def invalid_connector_config_fixture(base_path, invalid_connector_config_path) -> MutableMapping[str, Any]: + """TODO: implement default value - generate from valid config""" + with open(str(invalid_connector_config_path), "r") as file: + contents = file.read() + return json.loads(contents) + + +@pytest.fixture(name="malformed_connector_config") +def malformed_connector_config_fixture(connector_config) -> MutableMapping[str, Any]: + """TODO: drop required field, add extra""" + malformed_config = copy.deepcopy(connector_config) + return malformed_config + + +@pytest.fixture(name="connector_spec") +def connector_spec_fixture(connector_spec_path) -> ConnectorSpecification: + return ConnectorSpecification.parse_file(connector_spec_path) + + +@pytest.fixture(name="docker_runner") +def docker_runner_fixture(image_tag, tmp_path) -> ConnectorRunner: + return ConnectorRunner(image_tag, volume=tmp_path) + + +@pytest.fixture(scope="session", autouse=True) +def pull_docker_image(standard_test_config) -> None: + """Startup fixture to pull docker image""" + print("Pulling docker image", standard_test_config.connector_image) + ConnectorRunner(image_name=standard_test_config.connector_image, volume=Path(".")) + print("Pulling completed") + + +@pytest.fixture(name="expected_records") +def expected_records_fixture(inputs, base_path) -> List[AirbyteMessage]: + path = getattr(inputs, "expected_records_path") + if not path: + return [] + + with open(str(base_path / path)) as f: + return [AirbyteMessage.parse_raw(line) for line in f] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py new file mode 100644 index 000000000000..d709c72d8bd5 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/plugin.py @@ -0,0 +1,73 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import pytest +from source_acceptance_test.utils import diff_dicts, load_config + + +def pytest_addoption(parser): + """Hook function to add CLI option `standard_test_config`""" + parser.addoption( + "--acceptance-test-config", action="store", default=".", help="Folder with standard test config - acceptance_test_config.yml" + ) + + +def pytest_generate_tests(metafunc): + """Hook function to customize test discovery and parametrization. + It does two things: + 1. skip test class if its name omitted in the config file (or it has no inputs defined) + 2. parametrize each test with inputs from config file. + + For example config file contains this: + tests: + test_suite1: + - input1: value1 + input2: value2 + - input1: value3 + input2: value4 + test_suite2: [] + + Hook function will skip test_suite2 and test_suite3, but parametrize test_suite1 with two sets of inputs. + """ + + if "inputs" in metafunc.fixturenames: + config_key = metafunc.cls.config_key() + test_name = f"{metafunc.cls.__name__}.{metafunc.function.__name__}" + config = load_config(metafunc.config.getoption("--acceptance-test-config")) + if not hasattr(config.tests, config_key) or not getattr(config.tests, config_key): + pytest.skip(f"Skipping {test_name} because not found in the config") + else: + test_inputs = getattr(config.tests, config_key) + if not test_inputs: + pytest.skip(f"Skipping {test_name} because no inputs provided") + + metafunc.parametrize("inputs", test_inputs) + + +def pytest_assertrepr_compare(config, op, left, right): + if op != "==": + return + + use_markup = config.get_terminal_writer().hasmarkup + return diff_dicts(left, right, use_markup=use_markup) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/__init__.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/__init__.py new file mode 100644 index 000000000000..449fd7833c09 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/__init__.py @@ -0,0 +1,29 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .test_core import TestBasicRead, TestConnection, TestDiscovery, TestSpec +from .test_full_refresh import TestFullRefresh +from .test_incremental import TestIncremental + +__all__ = ["TestSpec", "TestBasicRead", "TestConnection", "TestDiscovery", "TestFullRefresh", "TestIncremental"] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py new file mode 100644 index 000000000000..8fb60f850850 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -0,0 +1,100 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from collections import Counter + +import pytest +from airbyte_protocol import ConnectorSpecification, Status, Type +from docker.errors import ContainerError +from source_acceptance_test.base import BaseTest +from source_acceptance_test.config import BasicReadTestConfig, ConnectionTestConfig +from source_acceptance_test.utils import ConnectorRunner + + +@pytest.mark.timeout(10) +class TestSpec(BaseTest): + def test_spec(self, connector_spec: ConnectorSpecification, docker_runner: ConnectorRunner): + output = docker_runner.call_spec() + spec_messages = [message for message in output if message.type == Type.SPEC] + + assert len(spec_messages) == 1, "Spec message should be emitted exactly once" + if connector_spec: + assert spec_messages[0].spec == connector_spec, "Spec should be equal to the one in spec.json file" + + +@pytest.mark.timeout(30) +class TestConnection(BaseTest): + def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runner: ConnectorRunner): + if inputs.status == ConnectionTestConfig.Status.Succeed: + output = docker_runner.call_check(config=connector_config) + con_messages = [message for message in output if message.type == Type.CONNECTION_STATUS] + + assert len(con_messages) == 1, "Connection status message should be emitted exactly once" + assert con_messages[0].connectionStatus.status == Status.SUCCEEDED + elif inputs.status == ConnectionTestConfig.Status.Failed: + output = docker_runner.call_check(config=connector_config) + con_messages = [message for message in output if message.type == Type.CONNECTION_STATUS] + + assert len(con_messages) == 1, "Connection status message should be emitted exactly once" + assert con_messages[0].connectionStatus.status == Status.FAILED + elif inputs.status == ConnectionTestConfig.Status.Exception: + with pytest.raises(ContainerError) as err: + docker_runner.call_check(config=connector_config) + + assert err.value.exit_status != 0, "Connector should exit with error code" + assert "Traceback" in err.value.stderr.decode("utf-8"), "Connector should print exception" + + +@pytest.mark.timeout(30) +class TestDiscovery(BaseTest): + def test_discover(self, connector_config, catalog, docker_runner: ConnectorRunner): + output = docker_runner.call_discover(config=connector_config) + catalog_messages = [message for message in output if message.type == Type.CATALOG] + + assert len(catalog_messages) == 1, "Catalog message should be emitted exactly once" + if catalog: + for stream1, stream2 in zip(catalog_messages[0].catalog.streams, catalog.streams): + assert stream1.json_schema == stream2.json_schema, f"Streams: {stream1.name} vs {stream2.name}, stream schemas should match" + stream1.json_schema = None + stream2.json_schema = None + assert stream1.dict() == stream2.dict(), f"Streams {stream1.name} and {stream2.name}, stream configs should match" + + +@pytest.mark.timeout(300) +class TestBasicRead(BaseTest): + def test_read(self, connector_config, configured_catalog, inputs: BasicReadTestConfig, docker_runner: ConnectorRunner): + output = docker_runner.call_read(connector_config, configured_catalog) + records = [message.record for message in output if message.type == Type.RECORD] + counter = Counter(record.stream for record in records) + + all_streams = set(stream.stream.name for stream in configured_catalog.streams) + streams_with_records = set(counter.keys()) + streams_without_records = all_streams - streams_with_records + + assert records, "At least one record should be read using provided catalog" + + if inputs.validate_output_from_all_streams: + assert ( + not streams_without_records + ), f"All streams should return some records, streams without records: {streams_without_records}" diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py new file mode 100644 index 000000000000..339dcc6dbad0 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py @@ -0,0 +1,47 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +from functools import partial + +import pytest +from airbyte_protocol import Type +from source_acceptance_test.base import BaseTest +from source_acceptance_test.utils import ConnectorRunner, full_refresh_only_catalog + + +@pytest.mark.timeout(20 * 60) +class TestFullRefresh(BaseTest): + def test_sequential_reads(self, connector_config, configured_catalog, docker_runner: ConnectorRunner): + configured_catalog = full_refresh_only_catalog(configured_catalog) + output = docker_runner.call_read(connector_config, configured_catalog) + records_1 = [message.record.data for message in output if message.type == Type.RECORD] + + output = docker_runner.call_read(connector_config, configured_catalog) + records_2 = [message.record.data for message in output if message.type == Type.RECORD] + serialize = partial(json.dumps, sort_keys=True) + + assert not ( + set(map(serialize, records_1)) - set(map(serialize, records_2)) + ), "The two sequential reads should produce either equal set of records or one of them is a strict subset of the other" diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py new file mode 100644 index 000000000000..b9a58cbf48d7 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_incremental.py @@ -0,0 +1,116 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +from pathlib import Path +from typing import Any, Iterable, Mapping, Tuple + +import pytest +from airbyte_protocol import ConfiguredAirbyteCatalog, Type +from source_acceptance_test import BaseTest +from source_acceptance_test.utils import ConnectorRunner, JsonSchemaHelper, filter_output, incremental_only_catalog + + +@pytest.fixture(name="future_state_path") +def future_state_path_fixture(inputs, base_path) -> Path: + """Fixture with connector's future state path (relative to base_path)""" + if getattr(inputs, "state_path"): + return Path(base_path) / getattr(inputs, "state_path") + pytest.skip("`state_path` not specified, skipping") + + +@pytest.fixture(name="future_state") +def future_state_fixture(future_state_path) -> Path: + """""" + with open(str(future_state_path), "r") as file: + contents = file.read() + return json.loads(contents) + + +@pytest.fixture(name="cursor_paths") +def cursor_paths_fixture(inputs, configured_catalog_for_incremental) -> Mapping[str, Any]: + cursor_paths = getattr(inputs, "cursor_paths") + result = {} + + for stream in configured_catalog_for_incremental.streams: + path = cursor_paths.get(stream.stream.name, [stream.cursor_field[-1]]) + result[stream.stream.name] = path + + return result + + +@pytest.fixture(name="configured_catalog_for_incremental") +def configured_catalog_for_incremental_fixture(configured_catalog) -> ConfiguredAirbyteCatalog: + catalog = incremental_only_catalog(configured_catalog) + for stream in catalog.streams: + if not stream.cursor_field: + pytest.fail("Configured catalog should have cursor_field specified for all incremental streams") + return catalog + + +def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any]]: + """Iterate over records and return cursor value with corresponding cursor value from state""" + for record in records: + stream_name = record.record.stream + stream = stream_mapping[stream_name] + helper = JsonSchemaHelper(schema=stream.stream.json_schema) + record_value = helper.get_cursor_value(record=record.record.data, cursor_path=stream.cursor_field) + state_value = helper.get_state_value(state=state[stream_name], cursor_path=state_cursor_paths[stream_name]) + yield record_value, state_value + + +@pytest.mark.timeout(20 * 60) +class TestIncremental(BaseTest): + def test_two_sequential_reads(self, connector_config, configured_catalog_for_incremental, cursor_paths, docker_runner: ConnectorRunner): + stream_mapping = {stream.stream.name: stream for stream in configured_catalog_for_incremental.streams} + + output = docker_runner.call_read(connector_config, configured_catalog_for_incremental) + records_1 = filter_output(output, type_=Type.RECORD) + states_1 = filter_output(output, type_=Type.STATE) + + assert states_1, "Should produce at least one state" + assert records_1, "Should produce at least one record" + + latest_state = states_1[-1].state.data + for record_value, state_value in records_with_state(records_1, latest_state, stream_mapping, cursor_paths): + assert ( + record_value <= state_value + ), "First incremental sync should produce records younger or equal to cursor value from the state" + + output = docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=latest_state) + records_2 = filter_output(output, type_=Type.RECORD) + + for record_value, state_value in records_with_state(records_2, latest_state, stream_mapping, cursor_paths): + assert ( + record_value >= state_value + ), "Second incremental sync should produce records older or equal to cursor value from the state" + + def test_state_with_abnormally_large_values(self, connector_config, configured_catalog, future_state, docker_runner: ConnectorRunner): + configured_catalog = incremental_only_catalog(configured_catalog) + output = docker_runner.call_read_with_state(config=connector_config, catalog=configured_catalog, state=future_state) + records = filter_output(output, type_=Type.RECORD) + states = filter_output(output, type_=Type.STATE) + + assert not records, "The sync should produce no records when run with the state with abnormally large values" + assert states, "The sync should produce at least one STATE message" diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/__init__.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/__init__.py new file mode 100644 index 000000000000..9e78ec189cf7 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/__init__.py @@ -0,0 +1,39 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .common import SecretDict, filter_output, full_refresh_only_catalog, incremental_only_catalog, load_config +from .compare import diff_dicts +from .connector_runner import ConnectorRunner +from .json_schema_helper import JsonSchemaHelper + +__all__ = [ + "JsonSchemaHelper", + "load_config", + "filter_output", + "full_refresh_only_catalog", + "incremental_only_catalog", + "SecretDict", + "ConnectorRunner", + "diff_dicts", +] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py new file mode 100644 index 000000000000..3f22c64c4d6d --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/common.py @@ -0,0 +1,86 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from collections import UserDict +from pathlib import Path +from typing import Iterable, List + +import pytest +from yaml import load + +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +from airbyte_protocol import AirbyteMessage, ConfiguredAirbyteCatalog, SyncMode +from source_acceptance_test.config import Config + + +def load_config(path: str) -> Config: + """Function to load test config, avoid duplication of code in places where we can't use fixture""" + path = Path(path) / "acceptance-test-config.yml" + if not path.exists(): + pytest.fail(f"config file {path.absolute()} does not exist") + + with open(str(path), "r") as file: + data = load(file, Loader=Loader) + return Config.parse_obj(data) + + +def full_refresh_only_catalog(configured_catalog: ConfiguredAirbyteCatalog) -> ConfiguredAirbyteCatalog: + """Transform provided catalog to catalog with all streams configured to use Full Refresh sync (when possible)""" + streams = [] + for stream in configured_catalog.streams: + if SyncMode.full_refresh in stream.stream.supported_sync_modes: + stream.sync_mode = SyncMode.full_refresh + streams.append(stream) + + configured_catalog.streams = streams + return configured_catalog + + +def incremental_only_catalog(configured_catalog: ConfiguredAirbyteCatalog) -> ConfiguredAirbyteCatalog: + """Transform provided catalog to catalog with all streams configured to use Incremental sync (when possible)""" + streams = [] + for stream in configured_catalog.streams: + if SyncMode.incremental in stream.stream.supported_sync_modes: + stream.sync_mode = SyncMode.incremental + streams.append(stream) + + configured_catalog.streams = streams + return configured_catalog + + +def filter_output(records: Iterable[AirbyteMessage], type_) -> List[AirbyteMessage]: + """Filter messages to match specific type""" + return list(filter(lambda x: x.type == type_, records)) + + +class SecretDict(UserDict): + def __str__(self) -> str: + return f"{self.__class__.__name__}(******)" + + def __repr__(self) -> str: + return str(self) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/compare.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/compare.py new file mode 100644 index 000000000000..b325a05a1e16 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/compare.py @@ -0,0 +1,68 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from typing import List, Optional + +import icdiff +import py +from pprintpp import pformat + +MAX_COLS = py.io.TerminalWriter().fullwidth +MARGIN_LEFT = 20 +GUTTER = 3 +MARGINS = MARGIN_LEFT + GUTTER + 1 + + +def diff_dicts(left, right, use_markup) -> Optional[List[str]]: + half_cols = MAX_COLS / 2 - MARGINS + + pretty_left = pformat(left, indent=1, width=half_cols).splitlines() + pretty_right = pformat(right, indent=1, width=half_cols).splitlines() + diff_cols = MAX_COLS - MARGINS + + if len(pretty_left) < 3 or len(pretty_right) < 3: + # avoid small diffs far apart by smooshing them up to the left + smallest_left = pformat(left, indent=2, width=1).splitlines() + smallest_right = pformat(right, indent=2, width=1).splitlines() + max_side = max(len(line) + 1 for line in smallest_left + smallest_right) + if (max_side * 2 + MARGIN_LEFT) < MAX_COLS: + diff_cols = max_side * 2 + GUTTER + pretty_left = pformat(left, indent=2, width=max_side).splitlines() + pretty_right = pformat(right, indent=2, width=max_side).splitlines() + + differ = icdiff.ConsoleDiff(cols=diff_cols, tabsize=2) + + if not use_markup: + # colorization is disabled in Pytest - either due to the terminal not + # supporting it or the user disabling it. We should obey, but there is + # no option in icdiff to disable it, so we replace its colorization + # function with a no-op + differ.colorize = lambda string: string + color_off = "" + else: + color_off = icdiff.color_codes["none"] + + icdiff_lines = list(differ.make_table(pretty_left, pretty_right, context=True)) + + return ["equals failed"] + [color_off + line for line in icdiff_lines] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/connector_runner.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/connector_runner.py new file mode 100644 index 000000000000..f0b06e5bf215 --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/connector_runner.py @@ -0,0 +1,114 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +import logging +from pathlib import Path +from typing import Iterable, List, Mapping, Optional + +import docker +from airbyte_protocol import AirbyteMessage, ConfiguredAirbyteCatalog + + +class ConnectorRunner: + def __init__(self, image_name: str, volume: Path): + self._client = docker.from_env() + self._image = self._client.images.pull(image_name) + self._runs = 0 + self._volume_base = volume + + @property + def output_folder(self) -> Path: + return self._volume_base / f"run_{self._runs}" / "output" + + @property + def input_folder(self) -> Path: + return self._volume_base / f"run_{self._runs}" / "input" + + def _prepare_volumes(self, config: Optional[Mapping], state: Optional[Mapping], catalog: Optional[ConfiguredAirbyteCatalog]): + self.input_folder.mkdir(parents=True) + self.output_folder.mkdir(parents=True) + + if config: + with open(str(self.input_folder / "tap_config.json"), "w") as outfile: + json.dump(dict(config), outfile) + + if state: + with open(str(self.input_folder / "state.json"), "w") as outfile: + json.dump(dict(state), outfile) + + if catalog: + with open(str(self.input_folder / "catalog.json"), "w") as outfile: + outfile.write(catalog.json()) + + volumes = { + str(self.input_folder): { + "bind": "/data", + # "mode": "ro", + }, + str(self.output_folder): { + "bind": "/local", + "mode": "rw", + }, + } + return volumes + + def call_spec(self, **kwargs) -> List[AirbyteMessage]: + cmd = "spec" + output = list(self.run(cmd=cmd, **kwargs)) + return output + + def call_check(self, config, **kwargs) -> List[AirbyteMessage]: + cmd = "check --config tap_config.json" + output = list(self.run(cmd=cmd, config=config, **kwargs)) + return output + + def call_discover(self, config, **kwargs) -> List[AirbyteMessage]: + cmd = "discover --config tap_config.json" + output = list(self.run(cmd=cmd, config=config, **kwargs)) + return output + + def call_read(self, config, catalog, **kwargs) -> List[AirbyteMessage]: + cmd = "read --config tap_config.json --catalog catalog.json" + output = list(self.run(cmd=cmd, config=config, catalog=catalog, **kwargs)) + return output + + def call_read_with_state(self, config, catalog, state, **kwargs) -> List[AirbyteMessage]: + cmd = "read --config tap_config.json --catalog catalog.json --state state.json" + output = list(self.run(cmd=cmd, config=config, catalog=catalog, state=state, **kwargs)) + return output + + def run(self, cmd, config=None, state=None, catalog=None, **kwargs) -> Iterable[AirbyteMessage]: + self._runs += 1 + volumes = self._prepare_volumes(config, state, catalog) + logs = self._client.containers.run( + image=self._image, command=cmd, working_dir="/data", volumes=volumes, network="host", stdout=True, stderr=True, **kwargs + ) + logging.info("Docker run: \n%s\ninput: %s\noutput: %s", cmd, self.input_folder, self.output_folder) + + with open(str(self.output_folder / "raw"), "wb+") as f: + f.write(logs) + + for line in logs.decode("utf-8").splitlines(): + yield AirbyteMessage.parse_raw(line) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py new file mode 100644 index 000000000000..03b1a0559add --- /dev/null +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/utils/json_schema_helper.py @@ -0,0 +1,69 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from functools import reduce +from typing import List + +import pendulum + + +class JsonSchemaHelper: + def __init__(self, schema): + self._schema = schema + + def get_ref(self, path): + node = self._schema + for segment in path.split("/")[1:]: + node = node[segment] + return node + + def get_property(self, path: List[str]): + node = self._schema + for segment in path: + if "$ref" in node: + node = self.get_ref(node["$ref"]) + node = node["properties"][segment] + return node + + def get_type_for_key_path(self, path: List[str]): + try: + return self.get_property(path)["type"] + except KeyError: + return None + + def get_cursor_value(self, record, cursor_path): + type_ = self.get_type_for_key_path(path=cursor_path) + value = reduce(lambda data, key: data[key], cursor_path, record) + return self.parse_value(value, type_) + + @staticmethod + def parse_value(value, type_): + if type_ in ("datetime", "date-time"): + return pendulum.parse(value) + return value + + def get_state_value(self, state, cursor_path): + type_ = self.get_type_for_key_path(path=cursor_path) + value = state[cursor_path[-1]] + return self.parse_value(value, type_) diff --git a/airbyte-integrations/connectors/source-hubspot/.dockerignore b/airbyte-integrations/connectors/source-hubspot/.dockerignore index 0113a1636f95..461b1bb7ee9e 100644 --- a/airbyte-integrations/connectors/source-hubspot/.dockerignore +++ b/airbyte-integrations/connectors/source-hubspot/.dockerignore @@ -4,3 +4,5 @@ !source_hubspot !setup.py !secrets +!acceptance-test-config.yml +!acceptance-test.sh diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 37da0c7f1fb2..59c04c46b077 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceHubspot" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml new file mode 100644 index 000000000000..9ccc34da09c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test-config.yml @@ -0,0 +1,25 @@ +connector_image: airbyte/source-hubspot:0.1.1 +tests: + spec: + - spec_path: "source_hubspot/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "sample_files/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + state_path: "sample_files/abnormal_state.json" + cursor_paths: + subscription_changes: ["timestamp"] + email_events: ["timestamp"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-hubspot/acceptance-test.sh b/airbyte-integrations/connectors/source-hubspot/acceptance-test.sh new file mode 100755 index 000000000000..495c419ab8aa --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/acceptance-test.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env sh +CWD=$(pwd) +cd ../../bases/source-acceptance-test +pip install -e . +python -m pytest --acceptance-test-config=${CWD} "$@" diff --git a/airbyte-integrations/connectors/source-hubspot/build.gradle b/airbyte-integrations/connectors/source-hubspot/build.gradle index 5e625a30dc66..6b0f40c192a1 100644 --- a/airbyte-integrations/connectors/source-hubspot/build.gradle +++ b/airbyte-integrations/connectors/source-hubspot/build.gradle @@ -9,19 +9,8 @@ airbytePython { } airbyteStandardSourceTestFile { - // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors - - // All these input paths must live inside this connector's directory (or subdirectories) - // TODO update the spec JSON file specPath = "source_hubspot/spec.json" - - // configPath points to a config file which matches the spec.json supplied above. secrets/ is gitignored by default, so place your config file - // there (in case it contains any credentials) - // TODO update the config file to contain actual credentials configPath = "secrets/config.json" - // TODO update the sample configured_catalog JSON for use in testing - // Note: If your source supports incremental syncing, then make sure that the catalog that is returned in the get_catalog method is configured - // for incremental syncing (e.g. include cursor fields, etc). configuredCatalogPath = "sample_files/configured_catalog.json" } diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/abnormal_state.json b/airbyte-integrations/connectors/source-hubspot/sample_files/abnormal_state.json new file mode 100644 index 000000000000..1f6f5a46a9e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/abnormal_state.json @@ -0,0 +1,8 @@ +{ + "email_events": { + "timestamp": "2121-03-19T17:00:45.743000+00:00" + }, + "subscription_changes": { + "timestamp": "2121-03-19T16:58:54.301000+00:00" + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json index f208ef37d43f..7c5c2f482a6f 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json @@ -97,7 +97,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["lastUpdatedTime"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -314,6 +314,9 @@ "hs_total_deal_value": { "type": "number" }, + "hs_unique_creation_key": { + "type": "string" + }, "hs_updated_by_user_id": { "type": "number" }, @@ -515,7 +518,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -622,7 +625,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -3551,7 +3554,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -3632,7 +3635,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -3992,7 +3995,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -4119,7 +4122,7 @@ } } }, - "supported_sync_modes": ["incremental", "full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["created"] }, @@ -4771,7 +4774,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["lastUpdated"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5060,7 +5063,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5220,7 +5223,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5304,7 +5307,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5425,7 +5428,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5684,7 +5687,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5748,7 +5751,7 @@ } } }, - "supported_sync_modes": ["incremental", "full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["timestamp"] }, @@ -6003,7 +6006,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -6061,7 +6064,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-hubspot/sample_files/invalid_config.json new file mode 100644 index 000000000000..4982a5cb2b34 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/invalid_config.json @@ -0,0 +1,5 @@ +{ + "credentials": { + "api_key": "1234567" + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index e6099aae2618..c3a9b1790751 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -32,10 +32,7 @@ "requests==2.25.1", ] -TEST_REQUIREMENTS = [ - "pytest", - "requests_mock==1.8.0", -] +TEST_REQUIREMENTS = ["pytest==6.1.2", "requests_mock==1.8.0"] setup( name="source_hubspot", diff --git a/buildSrc/src/main/groovy/airbyte-source-acceptance-test.gradle b/buildSrc/src/main/groovy/airbyte-source-acceptance-test.gradle new file mode 100644 index 000000000000..e8f68461a865 --- /dev/null +++ b/buildSrc/src/main/groovy/airbyte-source-acceptance-test.gradle @@ -0,0 +1,44 @@ +import org.gradle.api.Plugin +import org.gradle.api.Project + +class AirbyteStandardTestPlugin implements Plugin { + void apply(Project project) { + project.task('SourceAcceptanceTest') { + doFirst { + project.exec { + def targetMountDirectory = "/test_input" + def args = [ + 'docker', 'run', '--rm', '-i', + // provide access to the docker daemon + '-v', "/var/run/docker.sock:/var/run/docker.sock", + // A container within a container mounts from the host filesystem, not the parent container. + // this forces /tmp to be the same directory for host, parent container, and child container. + '-v', "/tmp:/tmp", + // mount the project dir. all provided input paths must be relative to that dir. + '-v', "${project.projectDir.absolutePath}:${targetMountDirectory}", + 'airbyte/source-acceptance-test:dev', + '--standard_test_config', "$targetMountDirectory", + ] + commandLine args + } + } + + outputs.upToDateWhen { false } + } + + project.standardTest.dependsOn(':airbyte-integrations:bases:source-acceptance-test:airbyteDocker') + project.standardTest.dependsOn(project.build) + project.standardTest.dependsOn(project.airbyteDocker) + if (project.hasProperty('airbyteDockerTest')){ + project.standardTest.dependsOn(project.airbyteDockerTest) + } + + // make sure we create the integrationTest task once in case a java integration test was already initialized + if (!project.hasProperty('integrationTest')) { + project.task('integrationTest') + } + + project.integrationTest.dependsOn(project.standardTest) + } +} + diff --git a/settings.gradle b/settings.gradle index 23fa9c8f73a3..b1bbc8b9d4d4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -32,6 +32,7 @@ include ':airbyte-integrations:bases:base-singer' include ':airbyte-integrations:bases:base-standard-source-test-file' include ':airbyte-integrations:bases:standard-destination-test' include ':airbyte-integrations:bases:standard-source-test' +include ':airbyte-integrations:bases:standard-test' include ':airbyte-integrations:connector-templates:generator' include ':airbyte-json-validation' include ':airbyte-migration'