From a825b2596805bf674cff25137cdbe96df8356525 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Sun, 18 Apr 2021 20:54:52 +0300 Subject: [PATCH 01/10] fixes and address comments from @sherifnada --- .../bases/base-python/base_python/client.py | 2 +- .../bases/standard-test/Dockerfile | 6 +- .../bases/standard-test/README.md | 47 +++++++++- .../bases/standard-test/pytest.ini | 5 + .../sample_files/standard_test_config.yml | 26 ++++-- .../bases/standard-test/setup.py | 4 + .../bases/standard-test/standard_test/base.py | 24 +++++ .../standard-test/standard_test/compare.py | 68 ++++++++++++++ .../standard-test/standard_test/config.py | 61 ++++++------- .../standard-test/standard_test/conftest.py | 44 ++++----- .../standard_test/connector_runner.py | 25 ++--- .../standard_test/json_schema_helper.py | 69 ++++++++++++++ .../standard-test/standard_test/plugin.py | 37 +++++++- .../standard_test/tests/__init__.py | 6 ++ .../standard_test/tests/test_core.py | 44 +++++---- .../standard_test/tests/test_full_refresh.py | 19 +++- .../standard_test/tests/test_incremental.py | 91 ++++++++++++++++++- .../standard-test/standard_test/utils.py | 13 ++- .../connectors/source-hubspot/Dockerfile | 2 +- .../sample_files/abnormal_state.json | 8 ++ .../sample_files/configured_catalog.json | 35 +++---- .../sample_files/invalid_config.json | 5 + .../connectors/source-hubspot/setup.py | 5 +- .../source-hubspot/standard_test_config.yml | 25 +++++ 24 files changed, 539 insertions(+), 132 deletions(-) create mode 100644 airbyte-integrations/bases/standard-test/pytest.ini create mode 100644 airbyte-integrations/bases/standard-test/standard_test/compare.py create mode 100644 airbyte-integrations/bases/standard-test/standard_test/json_schema_helper.py create mode 100644 airbyte-integrations/connectors/source-hubspot/sample_files/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-hubspot/sample_files/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-hubspot/standard_test_config.yml diff --git a/airbyte-integrations/bases/base-python/base_python/client.py b/airbyte-integrations/bases/base-python/base_python/client.py index 3e1d32ea8672..09e8d93da263 100644 --- a/airbyte-integrations/bases/base-python/base_python/client.py +++ b/airbyte-integrations/bases/base-python/base_python/client.py @@ -173,7 +173,7 @@ def streams(self) -> Generator[AirbyteStream, None, None]: supported_sync_modes = [SyncMode.full_refresh] source_defined_cursor = False if self.stream_has_state(name): - supported_sync_modes = [SyncMode.incremental] + supported_sync_modes += [SyncMode.incremental] source_defined_cursor = True yield AirbyteStream( diff --git a/airbyte-integrations/bases/standard-test/Dockerfile b/airbyte-integrations/bases/standard-test/Dockerfile index dfd095c28c2e..6dacbdcb54af 100644 --- a/airbyte-integrations/bases/standard-test/Dockerfile +++ b/airbyte-integrations/bases/standard-test/Dockerfile @@ -2,12 +2,12 @@ FROM airbyte/integration-base-python:dev ENV CODE_PATH="standard_test" -WORKDIR /airbyte/base_python_test_code +WORKDIR /airbyte/standard_test_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ RUN pip install . -ENTRYPOINT ["airbyte-python-test"] - LABEL io.airbyte.version=0.1.0 LABEL io.airbyte.name=airbyte/standard-test + +ENTRYPOINT ["python", "-m", "pytest"] diff --git a/airbyte-integrations/bases/standard-test/README.md b/airbyte-integrations/bases/standard-test/README.md index 88a370005ced..4407f8beab1f 100644 --- a/airbyte-integrations/bases/standard-test/README.md +++ b/airbyte-integrations/bases/standard-test/README.md @@ -1,7 +1,46 @@ -# Running Standard Source Tests +# Standard tests +This package uses pytest to discover, configure and execute the tests. +It implemented as a pytest plugin. + +It adds new configuration option `--standard_test_config` - path to configuration file (by default is current folder). +Configuration stored in YaML format and validated by pydantic. + +Example configuration can be found in `sample_files/` folder: +```yaml +connector_image: +tests: + spec: + - spec_path: "/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "sample_files/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: true + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + state_path: "sample_files/abnormal_state.json" + cursor_paths: + subscription_changes: ["timestamp"] + email_events: ["timestamp"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" +``` +# Running ```bash -PYTHONPATH=. pytest standard_test.py --standard_test_config=/connectors/source-hubspot/ -vvv +python -m pytest standard_test/tests --standard_test_config= -vvv ``` +_Note: this will assume that docker image for connector is already built_ -## How to -TODO +Using Gradle +```bash +./gradlew :airbyte-integrations:connectors:source-:standardTest +``` +_Note: this will also build docker image for connector_ diff --git a/airbyte-integrations/bases/standard-test/pytest.ini b/airbyte-integrations/bases/standard-test/pytest.ini new file mode 100644 index 000000000000..e79a1ff56102 --- /dev/null +++ b/airbyte-integrations/bases/standard-test/pytest.ini @@ -0,0 +1,5 @@ +[pytest] + +addopts = -r a --capture=no -vv +testpaths = + standard_test/tests diff --git a/airbyte-integrations/bases/standard-test/sample_files/standard_test_config.yml b/airbyte-integrations/bases/standard-test/sample_files/standard_test_config.yml index 541306b59ef8..6321d2fc88b4 100644 --- a/airbyte-integrations/bases/standard-test/sample_files/standard_test_config.yml +++ b/airbyte-integrations/bases/standard-test/sample_files/standard_test_config.yml @@ -1,13 +1,25 @@ connector_image: airbyte/source-hubspot:dev tests: - core: + spec: + - spec_path: "source_hubspot/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "sample_files/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: true + incremental: - config_path: "secrets/config.json" - invalid_config_path: "secrets/invalid_config.json" - spec_path: "source_hubspot/spec.json" configured_catalog_path: "sample_files/configured_catalog.json" + state_path: "sample_files/abnormal_state.json" + cursor_paths: + subscription_changes: ["timestamp"] + email_events: ["timestamp"] + full_refresh: - config_path: "secrets/config.json" - invalid_config_path: "secrets/invalid_config.json" - spec_path: "source_hubspot/spec.json" configured_catalog_path: "sample_files/configured_catalog.json" - incremental: [] - full_refresh: [] diff --git a/airbyte-integrations/bases/standard-test/setup.py b/airbyte-integrations/bases/standard-test/setup.py index 763ad7c7d088..5f5e5947ec97 100644 --- a/airbyte-integrations/bases/standard-test/setup.py +++ b/airbyte-integrations/bases/standard-test/setup.py @@ -29,8 +29,12 @@ "docker==4.4.4", "PyYAML==5.3.1", "inflection==0.5.1", + "icdiff==1.9.1", + "pendulum==1.2.0", "pydantic==1.6.1", "pytest==6.1.2", + "pytest-timeout==1.4.2", + "pprintpp==0.4.0", ] setuptools.setup( diff --git a/airbyte-integrations/bases/standard-test/standard_test/base.py b/airbyte-integrations/bases/standard-test/standard_test/base.py index f59e676168b5..2b22fb93f6fa 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/base.py +++ b/airbyte-integrations/bases/standard-test/standard_test/base.py @@ -1,3 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + import inflection import pytest diff --git a/airbyte-integrations/bases/standard-test/standard_test/compare.py b/airbyte-integrations/bases/standard-test/standard_test/compare.py new file mode 100644 index 000000000000..3a1de5b9c8d0 --- /dev/null +++ b/airbyte-integrations/bases/standard-test/standard_test/compare.py @@ -0,0 +1,68 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import py +from typing import List, Optional + +import icdiff +from pprintpp import pformat + +MAX_COLS = py.io.TerminalWriter().fullwidth +MARGIN_LEFT = 20 +GUTTER = 3 +MARGINS = MARGIN_LEFT + GUTTER + 1 + + +def diff_dicts(left, right, use_markup) -> Optional[List[str]]: + half_cols = MAX_COLS / 2 - MARGINS + + pretty_left = pformat(left, indent=1, width=half_cols).splitlines() + pretty_right = pformat(right, indent=1, width=half_cols).splitlines() + diff_cols = MAX_COLS - MARGINS + + if len(pretty_left) < 3 or len(pretty_right) < 3: + # avoid small diffs far apart by smooshing them up to the left + smallest_left = pformat(left, indent=2, width=1).splitlines() + smallest_right = pformat(right, indent=2, width=1).splitlines() + max_side = max(len(line) + 1 for line in smallest_left + smallest_right) + if (max_side * 2 + MARGIN_LEFT) < MAX_COLS: + diff_cols = max_side * 2 + GUTTER + pretty_left = pformat(left, indent=2, width=max_side).splitlines() + pretty_right = pformat(right, indent=2, width=max_side).splitlines() + + differ = icdiff.ConsoleDiff(cols=diff_cols, tabsize=2) + + if not use_markup: + # colorization is disabled in Pytest - either due to the terminal not + # supporting it or the user disabling it. We should obey, but there is + # no option in icdiff to disable it, so we replace its colorization + # function with a no-op + differ.colorize = lambda string: string + color_off = "" + else: + color_off = icdiff.color_codes["none"] + + icdiff_lines = list(differ.make_table(pretty_left, pretty_right, context=True)) + + return ["equals failed"] + [color_off + line for line in icdiff_lines] diff --git a/airbyte-integrations/bases/standard-test/standard_test/config.py b/airbyte-integrations/bases/standard-test/standard_test/config.py index 8ffb781a0b64..36d23fa15f8b 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/config.py +++ b/airbyte-integrations/bases/standard-test/standard_test/config.py @@ -22,68 +22,64 @@ SOFTWARE. """ -from typing import List, Optional +from typing import List, Mapping, Optional +from enum import Enum from pydantic import BaseModel, Field -config_path: str = Field(description="Path to a JSON object representing a valid connector configuration") +config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration") invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration") -spec_path: str = Field(description="Path to a JSON object representing the spec expected to be output by this connector") -configured_catalog_path: str = Field(description="Path to configured catalog") +spec_path: str = Field( + default="secrets/spec.json", description="Path to a JSON object representing the spec expected to be output by this connector" +) +configured_catalog_path: str = Field(default="sample_files/configured_catalog.json", description="Path to configured catalog") -class SpecTestConfig(BaseModel): +class BaseConfig(BaseModel): class Config: extra = "forbid" + +class SpecTestConfig(BaseConfig): spec_path: str = spec_path -class ConnectionTestConfig(BaseModel): - class Config: - extra = "forbid" +class ConnectionTestConfig(BaseConfig): + class Status(Enum): + Succeed = 'succeed' + Failed = 'failed' + Exception = 'exception' config_path: str = config_path - invalid_config_path: str = invalid_config_path + status: Status = Field(Status.Succeed, description="Indicate if connection check should succeed with provided config") -class DiscoveryTestConfig(BaseModel): - class Config: - extra = "forbid" - +class DiscoveryTestConfig(BaseConfig): config_path: str = config_path configured_catalog_path: Optional[str] = configured_catalog_path -class BasicReadTestConfig(BaseModel): - class Config: - extra = "forbid" - +class BasicReadTestConfig(BaseConfig): config_path: str = config_path configured_catalog_path: Optional[str] = configured_catalog_path validate_output_from_all_streams: bool = Field(False, description="Verify that all streams have records") -class FullRefreshConfig(BaseModel): - class Config: - extra = "forbid" - +class FullRefreshConfig(BaseConfig): config_path: str = config_path configured_catalog_path: str = configured_catalog_path -class IncrementalConfig(BaseModel): - class Config: - extra = "forbid" - +class IncrementalConfig(BaseConfig): config_path: str = config_path configured_catalog_path: str = configured_catalog_path + cursor_paths: Optional[Mapping[str, List[str]]] = Field( + description="For each stream, the path of its cursor field in the output state messages." + ) + state_path: Optional[str] = Field(description="Path to state file") -class TestConfig(BaseModel): - class Config: - extra = "forbid" - +class TestConfig(BaseConfig): spec: Optional[List[SpecTestConfig]] = Field(description="TODO") connection: Optional[List[ConnectionTestConfig]] = Field(description="TODO") discovery: Optional[List[DiscoveryTestConfig]] = Field(description="TODO") @@ -92,10 +88,7 @@ class Config: incremental: Optional[List[IncrementalConfig]] = Field(description="TODO") -class Config(BaseModel): - class Config: - extra = "forbid" - +class Config(BaseConfig): connector_image: str = Field(description="Docker image to test, for example 'airbyte/source-hubspot:dev'") base_path: Optional[str] = Field(description="Base path for all relative paths") - tests: TestConfig = Field(description="TODO") + tests: TestConfig = Field(description="List of the tests with their configs") diff --git a/airbyte-integrations/bases/standard-test/standard_test/conftest.py b/airbyte-integrations/bases/standard-test/standard_test/conftest.py index f7e4ca7cae54..0856faac9da3 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/conftest.py +++ b/airbyte-integrations/bases/standard-test/standard_test/conftest.py @@ -25,11 +25,11 @@ import copy import json from pathlib import Path -from typing import Optional +from typing import Optional, MutableMapping, Any import pytest from airbyte_protocol import AirbyteCatalog, ConfiguredAirbyteCatalog, ConnectorSpecification - +from standard_test.config import Config from standard_test.connector_runner import ConnectorRunner from standard_test.utils import load_config @@ -42,33 +42,33 @@ def base_path_fixture(pytestconfig, standard_test_config): return Path(pytestconfig.getoption("--standard_test_config")).absolute() -@pytest.fixture(name="standard_test_config") -def standard_test_config_fixture(pytestconfig): +@pytest.fixture(name="standard_test_config", scope="session") +def standard_test_config_fixture(pytestconfig) -> Config: """Fixture with test's config""" return load_config(pytestconfig.getoption("--standard_test_config")) @pytest.fixture(name="connector_config_path") -def connector_config_path_fixture(inputs, base_path): - """Fixture with connector's config path (relative to base_path)""" +def connector_config_path_fixture(inputs, base_path) -> Path: + """Fixture with connector's config path""" return Path(base_path) / getattr(inputs, "config_path") @pytest.fixture(name="invalid_connector_config_path") -def invalid_connector_config_path_fixture(inputs, base_path): - """Fixture with connector's config path (relative to base_path)""" +def invalid_connector_config_path_fixture(inputs, base_path) -> Path: + """Fixture with connector's config path""" return Path(base_path) / getattr(inputs, "invalid_config_path") @pytest.fixture(name="connector_spec_path") -def connector_spec_path_fixture(inputs, base_path): - """Fixture with connector's specification path (relative to base_path)""" +def connector_spec_path_fixture(inputs, base_path) -> Path: + """Fixture with connector's specification path""" return Path(base_path) / getattr(inputs, "spec_path") @pytest.fixture(name="configured_catalog_path") -def configured_catalog_path_fixture(inputs, base_path): - """Fixture with connector's configured_catalog path (relative to base_path)""" +def configured_catalog_path_fixture(inputs, base_path) -> Optional[str]: + """Fixture with connector's configured_catalog path""" if getattr(inputs, "configured_catalog_path"): return Path(base_path) / getattr(inputs, "configured_catalog_path") return None @@ -89,27 +89,27 @@ def catalog_fixture(configured_catalog: ConfiguredAirbyteCatalog) -> Optional[Ai @pytest.fixture(name="image_tag") -def image_tag_fixture(standard_test_config): +def image_tag_fixture(standard_test_config) -> str: return standard_test_config.connector_image @pytest.fixture(name="connector_config") -def connector_config_fixture(base_path, connector_config_path): - with open(str(Path(base_path) / connector_config_path), "r") as file: +def connector_config_fixture(base_path, connector_config_path) -> MutableMapping[str, Any]: + with open(str(connector_config_path), "r") as file: contents = file.read() return json.loads(contents) @pytest.fixture(name="invalid_connector_config") -def invalid_connector_config_fixture(base_path, invalid_connector_config_path): +def invalid_connector_config_fixture(base_path, invalid_connector_config_path) -> MutableMapping[str, Any]: """TODO: implement default value - generate from valid config""" - with open(str(Path(base_path) / invalid_connector_config_path), "r") as file: + with open(str(invalid_connector_config_path), "r") as file: contents = file.read() return json.loads(contents) @pytest.fixture(name="malformed_connector_config") -def malformed_connector_config_fixture(connector_config): +def malformed_connector_config_fixture(connector_config) -> MutableMapping[str, Any]: """TODO: drop required field, add extra""" malformed_config = copy.deepcopy(connector_config) return malformed_config @@ -125,7 +125,7 @@ def docker_runner_fixture(image_tag, tmp_path) -> ConnectorRunner: return ConnectorRunner(image_tag, volume=tmp_path) -@pytest.fixture(name="validate_output_from_all_streams") -def validate_output_from_all_streams_fixture(inputs): - """Fixture to provide value of `validate output from all streams` flag""" - return getattr(inputs, "validate_output_from_all_streams") +@pytest.fixture(scope="session", autouse=True) +def pull_docker_image(standard_test_config) -> None: + """Startup fixture to pull docker image""" + ConnectorRunner(image_tag=standard_test_config.connector_image) diff --git a/airbyte-integrations/bases/standard-test/standard_test/connector_runner.py b/airbyte-integrations/bases/standard-test/standard_test/connector_runner.py index 91bc191de0f2..5552c8cef586 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/connector_runner.py +++ b/airbyte-integrations/bases/standard-test/standard_test/connector_runner.py @@ -25,21 +25,16 @@ import json import logging from pathlib import Path -from typing import Iterable, Mapping, Optional - -try: - from yaml import CLoader as Loader -except ImportError: - from yaml import Loader +from typing import Iterable, Mapping, Optional, List import docker from airbyte_protocol import AirbyteMessage, ConfiguredAirbyteCatalog class ConnectorRunner: - def __init__(self, name: str, volume: Path): - self._name = name + def __init__(self, image_name: str, volume: Path): self._client = docker.from_env() + self._image = self._client.images.pull(image_name) self._runs = 0 self._volume_base = volume @@ -48,7 +43,6 @@ def _prepare_volumes(self, config: Optional[Mapping], state: Optional[Mapping], output_path = self._volume_base / f"run_{self._runs}" / "output" input_path.mkdir(parents=True) output_path.mkdir(parents=True) - # print(input_path, output_path) if config: with open(str(input_path / "tap_config.json"), "w") as outfile: @@ -74,27 +68,27 @@ def _prepare_volumes(self, config: Optional[Mapping], state: Optional[Mapping], } return volumes - def call_spec(self, **kwargs): + def call_spec(self, **kwargs) -> List[AirbyteMessage]: cmd = "spec" output = list(self.run(cmd=cmd, **kwargs)) return output - def call_check(self, config, **kwargs): + def call_check(self, config, **kwargs) -> List[AirbyteMessage]: cmd = "check --config tap_config.json" output = list(self.run(cmd=cmd, config=config, **kwargs)) return output - def call_discover(self, config, **kwargs): + def call_discover(self, config, **kwargs) -> List[AirbyteMessage]: cmd = "discover --config tap_config.json" output = list(self.run(cmd=cmd, config=config, **kwargs)) return output - def call_read(self, config, catalog, **kwargs): + def call_read(self, config, catalog, **kwargs) -> List[AirbyteMessage]: cmd = "read --config tap_config.json --catalog catalog.json" output = list(self.run(cmd=cmd, config=config, catalog=catalog, **kwargs)) return output - def call_read_with_state(self, config, catalog, state, **kwargs): + def call_read_with_state(self, config, catalog, state, **kwargs) -> List[AirbyteMessage]: cmd = "read --config tap_config.json --catalog catalog.json --state state.json" output = list(self.run(cmd=cmd, config=config, catalog=catalog, state=state, **kwargs)) return output @@ -103,8 +97,9 @@ def run(self, cmd, config=None, state=None, catalog=None, **kwargs) -> Iterable[ self._runs += 1 volumes = self._prepare_volumes(config, state, catalog) logs = self._client.containers.run( - image=self._name, command=cmd, working_dir="/data", volumes=volumes, network="host", stdout=True, stderr=True, **kwargs + image=self._image, command=cmd, working_dir="/data", volumes=volumes, network="host", stdout=True, stderr=True, **kwargs ) + logging.info("Running docker, folders: %s", volumes) for line in logs.decode("utf-8").splitlines(): logging.info(AirbyteMessage.parse_raw(line).type) yield AirbyteMessage.parse_raw(line) diff --git a/airbyte-integrations/bases/standard-test/standard_test/json_schema_helper.py b/airbyte-integrations/bases/standard-test/standard_test/json_schema_helper.py new file mode 100644 index 000000000000..03b1a0559add --- /dev/null +++ b/airbyte-integrations/bases/standard-test/standard_test/json_schema_helper.py @@ -0,0 +1,69 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from functools import reduce +from typing import List + +import pendulum + + +class JsonSchemaHelper: + def __init__(self, schema): + self._schema = schema + + def get_ref(self, path): + node = self._schema + for segment in path.split("/")[1:]: + node = node[segment] + return node + + def get_property(self, path: List[str]): + node = self._schema + for segment in path: + if "$ref" in node: + node = self.get_ref(node["$ref"]) + node = node["properties"][segment] + return node + + def get_type_for_key_path(self, path: List[str]): + try: + return self.get_property(path)["type"] + except KeyError: + return None + + def get_cursor_value(self, record, cursor_path): + type_ = self.get_type_for_key_path(path=cursor_path) + value = reduce(lambda data, key: data[key], cursor_path, record) + return self.parse_value(value, type_) + + @staticmethod + def parse_value(value, type_): + if type_ in ("datetime", "date-time"): + return pendulum.parse(value) + return value + + def get_state_value(self, state, cursor_path): + type_ = self.get_type_for_key_path(path=cursor_path) + value = state[cursor_path[-1]] + return self.parse_value(value, type_) diff --git a/airbyte-integrations/bases/standard-test/standard_test/plugin.py b/airbyte-integrations/bases/standard-test/standard_test/plugin.py index 82d80b0e5682..0a5c42715528 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/plugin.py +++ b/airbyte-integrations/bases/standard-test/standard_test/plugin.py @@ -1,6 +1,30 @@ -import pytest +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -from .utils import load_config +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import pytest +from standard_test.compare import diff_dicts +from standard_test.utils import load_config def pytest_addoption(parser): @@ -32,7 +56,6 @@ def pytest_generate_tests(metafunc): config_key = metafunc.cls.config_key() test_name = f"{metafunc.cls.__name__}.{metafunc.function.__name__}" config = load_config(metafunc.config.getoption("--standard_test_config")) - # print(config.dict()) if not hasattr(config.tests, config_key) or not getattr(config.tests, config_key): pytest.skip(f"Skipping {test_name} because not found in the config") else: @@ -41,3 +64,11 @@ def pytest_generate_tests(metafunc): pytest.skip(f"Skipping {test_name} because no inputs provided") metafunc.parametrize("inputs", test_inputs) + + +def pytest_assertrepr_compare(config, op, left, right): + if op != "==": + return + + use_markup = config.get_terminal_writer().hasmarkup + return diff_dicts(left, right, use_markup=use_markup) diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/__init__.py b/airbyte-integrations/bases/standard-test/standard_test/tests/__init__.py index ee23a18fc66f..449fd7833c09 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/__init__.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/__init__.py @@ -21,3 +21,9 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + +from .test_core import TestBasicRead, TestConnection, TestDiscovery, TestSpec +from .test_full_refresh import TestFullRefresh +from .test_incremental import TestIncremental + +__all__ = ["TestSpec", "TestBasicRead", "TestConnection", "TestDiscovery", "TestFullRefresh", "TestIncremental"] diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/test_core.py b/airbyte-integrations/bases/standard-test/standard_test/tests/test_core.py index 788216605d35..039cfdbf9f74 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/test_core.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/test_core.py @@ -27,12 +27,12 @@ import pytest from airbyte_protocol import ConnectorSpecification, Status, Type from docker.errors import ContainerError - from standard_test.base import BaseTest +from standard_test.config import BasicReadTestConfig, ConnectionTestConfig from standard_test.connector_runner import ConnectorRunner -from standard_test.utils import full_refresh_only_catalog +@pytest.mark.timeout(10) class TestSpec(BaseTest): def test_spec(self, connector_spec: ConnectorSpecification, docker_runner: ConnectorRunner): output = docker_runner.call_spec() @@ -43,22 +43,30 @@ def test_spec(self, connector_spec: ConnectorSpecification, docker_runner: Conne assert spec_messages[0].spec == connector_spec, "Spec should be equal to the one in spec.json file" +@pytest.mark.timeout(30) class TestConnection(BaseTest): - def test_check(self, connector_config, docker_runner: ConnectorRunner): - output = docker_runner.call_check(config=connector_config) - con_messages = [message for message in output if message.type == Type.CONNECTION_STATUS] + def test_check(self, connector_config, inputs: ConnectionTestConfig, docker_runner: ConnectorRunner): + if inputs.status == ConnectionTestConfig.Status.Succeed: + output = docker_runner.call_check(config=connector_config) + con_messages = [message for message in output if message.type == Type.CONNECTION_STATUS] - assert len(con_messages) == 1, "Connection status message should be emitted exactly once" - assert con_messages[0].connectionStatus.status == Status.SUCCEEDED + assert len(con_messages) == 1, "Connection status message should be emitted exactly once" + assert con_messages[0].connectionStatus.status == Status.SUCCEEDED + elif inputs.status == ConnectionTestConfig.Status.Failed: + output = docker_runner.call_check(config=connector_config) + con_messages = [message for message in output if message.type == Type.CONNECTION_STATUS] - def test_check_with_invalid_config(self, invalid_connector_config, docker_runner: ConnectorRunner): - with pytest.raises(ContainerError) as err: - docker_runner.call_check(config=invalid_connector_config) + assert len(con_messages) == 1, "Connection status message should be emitted exactly once" + assert con_messages[0].connectionStatus.status == Status.FAILED + elif inputs.status == ConnectionTestConfig.Status.Exception: + with pytest.raises(ContainerError) as err: + docker_runner.call_check(config=connector_config) - assert err.value.exit_status != 0, "Connector should exit with error code" - assert "Traceback" in err.value.stderr.decode("utf-8"), "Connector should print exception" + assert err.value.exit_status != 0, "Connector should exit with error code" + assert "Traceback" in err.value.stderr.decode("utf-8"), "Connector should print exception" +@pytest.mark.timeout(30) class TestDiscovery(BaseTest): def test_discover(self, connector_config, catalog, docker_runner: ConnectorRunner): output = docker_runner.call_discover(config=connector_config) @@ -66,12 +74,16 @@ def test_discover(self, connector_config, catalog, docker_runner: ConnectorRunne assert len(catalog_messages) == 1, "Catalog message should be emitted exactly once" if catalog: - assert catalog_messages[0].catalog == catalog, "Catalog should match the one that was provided" + for stream1, stream2 in zip(catalog_messages[0].catalog.streams, catalog.streams): + assert stream1.json_schema == stream2.json_schema, f"Streams: {stream1.name} vs {stream2.name}, stream schemas should match" + stream1.json_schema = None + stream2.json_schema = None + assert stream1.dict() == stream2.dict(), f"Streams {stream1.name} and {stream2.name}, stream configs should match" +@pytest.mark.timeout(300) class TestBasicRead(BaseTest): - def test_read(self, connector_config, configured_catalog, validate_output_from_all_streams, docker_runner: ConnectorRunner): - configured_catalog = full_refresh_only_catalog(configured_catalog) + def test_read(self, connector_config, configured_catalog, inputs: BasicReadTestConfig, docker_runner: ConnectorRunner): output = docker_runner.call_read(connector_config, configured_catalog) records = [message.record for message in output if message.type == Type.RECORD] counter = Counter(record.stream for record in records) @@ -82,7 +94,7 @@ def test_read(self, connector_config, configured_catalog, validate_output_from_a assert records, "At least one record should be read using provided catalog" - if validate_output_from_all_streams: + if inputs.validate_output_from_all_streams: assert ( not streams_without_records ), f"All streams should return some records, streams without records: {streams_without_records}" diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py b/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py index 06975bacf394..aec3b56e73a9 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py @@ -22,8 +22,25 @@ SOFTWARE. """ +import json + +import pytest +from airbyte_protocol import Type from standard_test.base import BaseTest +from standard_test.connector_runner import ConnectorRunner +from standard_test.utils import full_refresh_only_catalog +@pytest.mark.timeout(20 * 60) class TestFullRefresh(BaseTest): - pass + def test_sequential_reads(self, connector_config, configured_catalog, docker_runner: ConnectorRunner): + configured_catalog = full_refresh_only_catalog(configured_catalog) + output = docker_runner.call_read(connector_config, configured_catalog) + records_1 = [message.record.data for message in output if message.type == Type.RECORD] + + output = docker_runner.call_read(connector_config, configured_catalog) + records_2 = [message.record.data for message in output if message.type == Type.RECORD] + + assert not ( + set(map(json.dumps, records_1)) - set(map(json.dumps, records_2)) + ), "The two sequential reads should produce either equal set of records or one of them is a strict subset of the other" diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/test_incremental.py b/airbyte-integrations/bases/standard-test/standard_test/tests/test_incremental.py index 7121e0d648af..c92dd3613dfb 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/test_incremental.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/test_incremental.py @@ -22,8 +22,97 @@ SOFTWARE. """ +import json +from pathlib import Path +from typing import Mapping, Any, Tuple, Iterable + +import pytest +from airbyte_protocol import ConfiguredAirbyteCatalog, Type from standard_test import BaseTest +from standard_test.connector_runner import ConnectorRunner +from standard_test.json_schema_helper import JsonSchemaHelper +from standard_test.utils import filter_output, incremental_only_catalog + + +@pytest.fixture(name="future_state_path") +def future_state_path_fixture(inputs, base_path) -> Path: + """Fixture with connector's future state path (relative to base_path)""" + if getattr(inputs, "state_path"): + return Path(base_path) / getattr(inputs, "state_path") + pytest.skip("`state_path` not specified, skipping") + + +@pytest.fixture(name="future_state") +def future_state_fixture(future_state_path) -> Path: + """""" + with open(str(future_state_path), "r") as file: + contents = file.read() + return json.loads(contents) + + +@pytest.fixture(name="cursor_paths") +def cursor_paths_fixture(inputs, configured_catalog_for_incremental) -> Mapping[str, Any]: + cursor_paths = getattr(inputs, "cursor_paths") + result = {} + + for stream in configured_catalog_for_incremental.streams: + path = cursor_paths.get(stream.stream.name, [stream.cursor_field[-1]]) + result[stream.stream.name] = path + + return result + +@pytest.fixture(name="configured_catalog_for_incremental") +def configured_catalog_for_incremental_fixture(configured_catalog) -> ConfiguredAirbyteCatalog: + catalog = incremental_only_catalog(configured_catalog) + for stream in catalog.streams: + if not stream.cursor_field: + pytest.fail("Configured catalog should have cursor_field specified for all incremental streams") + return catalog + +def records_with_state(records, state, stream_mapping, state_cursor_paths) -> Iterable[Tuple[Any, Any]]: + """Iterate over records and return cursor value with corresponding cursor value from state""" + for record in records: + stream_name = record.record.stream + stream = stream_mapping[stream_name] + helper = JsonSchemaHelper(schema=stream.stream.json_schema) + record_value = helper.get_cursor_value(record=record.record.data, cursor_path=stream.cursor_field) + state_value = helper.get_state_value(state=state[stream_name], cursor_path=state_cursor_paths[stream_name]) + yield record_value, state_value + + +@pytest.mark.timeout(20 * 60) class TestIncremental(BaseTest): - pass + def test_two_sequential_reads(self, connector_config, configured_catalog_for_incremental, cursor_paths, docker_runner: ConnectorRunner): + stream_mapping = {stream.stream.name: stream for stream in configured_catalog_for_incremental.streams} + + output = docker_runner.call_read(connector_config, configured_catalog_for_incremental) + records_1 = filter_output(output, type_=Type.RECORD) + states_1 = filter_output(output, type_=Type.STATE) + + assert states_1, "Should produce at least one state" + assert records_1, "Should produce at least one record" + + latest_state = states_1[-1].state.data + for record_value, state_value in records_with_state(records_1, latest_state, stream_mapping, cursor_paths): + assert ( + record_value <= state_value + ), "First incremental sync should produce records younger or equal to cursor value from the state" + + output = docker_runner.call_read_with_state(connector_config, configured_catalog_for_incremental, state=latest_state) + records_2 = filter_output(output, type_=Type.RECORD) + + for record_value, state_value in records_with_state(records_2, latest_state, stream_mapping, cursor_paths): + assert ( + record_value >= state_value + ), "Second incremental sync should produce records older or equal to cursor value from the state" + + def test_state_with_abnormally_large_values(self, connector_config, configured_catalog, future_state, docker_runner: ConnectorRunner): + configured_catalog = incremental_only_catalog(configured_catalog) + output = docker_runner.call_read_with_state(config=connector_config, catalog=configured_catalog, state=future_state) + records = filter_output(output, type_=Type.RECORD) + states = filter_output(output, type_=Type.STATE) + + assert not records, "The sync should produce no records when run with the state with abnormally large values" + assert states, "The sync should produce at least one STATE message" diff --git a/airbyte-integrations/bases/standard-test/standard_test/utils.py b/airbyte-integrations/bases/standard-test/standard_test/utils.py index 2d867d0bc70e..72951e88fdad 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/utils.py +++ b/airbyte-integrations/bases/standard-test/standard_test/utils.py @@ -23,6 +23,7 @@ """ from pathlib import Path +from typing import Iterable, List import pytest from yaml import load @@ -32,8 +33,7 @@ except ImportError: from yaml import Loader -from airbyte_protocol import ConfiguredAirbyteCatalog, SyncMode - +from airbyte_protocol import AirbyteMessage, ConfiguredAirbyteCatalog, SyncMode from standard_test.config import Config @@ -48,7 +48,7 @@ def load_config(path: str) -> Config: return Config.parse_obj(data) -def full_refresh_only_catalog(configured_catalog: ConfiguredAirbyteCatalog): +def full_refresh_only_catalog(configured_catalog: ConfiguredAirbyteCatalog) -> ConfiguredAirbyteCatalog: """Transform provided catalog to catalog with all streams configured to use Full Refresh sync (when possible)""" streams = [] for stream in configured_catalog.streams: @@ -60,7 +60,7 @@ def full_refresh_only_catalog(configured_catalog: ConfiguredAirbyteCatalog): return configured_catalog -def incremental_only_catalog(configured_catalog: ConfiguredAirbyteCatalog): +def incremental_only_catalog(configured_catalog: ConfiguredAirbyteCatalog) -> ConfiguredAirbyteCatalog: """Transform provided catalog to catalog with all streams configured to use Incremental sync (when possible)""" streams = [] for stream in configured_catalog.streams: @@ -70,3 +70,8 @@ def incremental_only_catalog(configured_catalog: ConfiguredAirbyteCatalog): configured_catalog.streams = streams return configured_catalog + + +def filter_output(records: Iterable[AirbyteMessage], type_) -> List[AirbyteMessage]: + """Filter messages to match specific type""" + return list(filter(lambda x: x.type == type_, records)) diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index 10ee0569532b..55c3b242e764 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -10,7 +10,7 @@ ENV AIRBYTE_IMPL_PATH="SourceHubspot" WORKDIR /airbyte/integration_code COPY $CODE_PATH ./$CODE_PATH COPY setup.py ./ -RUN pip install ".[main]" +RUN pip install . LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/abnormal_state.json b/airbyte-integrations/connectors/source-hubspot/sample_files/abnormal_state.json new file mode 100644 index 000000000000..1f6f5a46a9e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/abnormal_state.json @@ -0,0 +1,8 @@ +{ + "email_events": { + "timestamp": "2121-03-19T17:00:45.743000+00:00" + }, + "subscription_changes": { + "timestamp": "2121-03-19T16:58:54.301000+00:00" + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json index f208ef37d43f..7c5c2f482a6f 100644 --- a/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/configured_catalog.json @@ -97,7 +97,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["lastUpdatedTime"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -314,6 +314,9 @@ "hs_total_deal_value": { "type": "number" }, + "hs_unique_creation_key": { + "type": "string" + }, "hs_updated_by_user_id": { "type": "number" }, @@ -515,7 +518,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -622,7 +625,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -3551,7 +3554,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -3632,7 +3635,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -3992,7 +3995,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -4119,7 +4122,7 @@ } } }, - "supported_sync_modes": ["incremental", "full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["created"] }, @@ -4771,7 +4774,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["lastUpdated"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5060,7 +5063,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5220,7 +5223,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5304,7 +5307,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5425,7 +5428,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5684,7 +5687,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -5748,7 +5751,7 @@ } } }, - "supported_sync_modes": ["incremental", "full_refresh"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["timestamp"] }, @@ -6003,7 +6006,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, @@ -6061,7 +6064,7 @@ }, "supported_sync_modes": ["full_refresh"], "source_defined_cursor": false, - "default_cursor_field": null + "default_cursor_field": ["updatedAt"] }, "sync_mode": "full_refresh", "cursor_field": null, diff --git a/airbyte-integrations/connectors/source-hubspot/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-hubspot/sample_files/invalid_config.json new file mode 100644 index 000000000000..4982a5cb2b34 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/sample_files/invalid_config.json @@ -0,0 +1,5 @@ +{ + "credentials": { + "api_key": "1234567" + } +} diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py index e6099aae2618..c3a9b1790751 100644 --- a/airbyte-integrations/connectors/source-hubspot/setup.py +++ b/airbyte-integrations/connectors/source-hubspot/setup.py @@ -32,10 +32,7 @@ "requests==2.25.1", ] -TEST_REQUIREMENTS = [ - "pytest", - "requests_mock==1.8.0", -] +TEST_REQUIREMENTS = ["pytest==6.1.2", "requests_mock==1.8.0"] setup( name="source_hubspot", diff --git a/airbyte-integrations/connectors/source-hubspot/standard_test_config.yml b/airbyte-integrations/connectors/source-hubspot/standard_test_config.yml new file mode 100644 index 000000000000..aabee4acc155 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/standard_test_config.yml @@ -0,0 +1,25 @@ +connector_image: airbyte/source-hubspot:dev +tests: + spec: + - spec_path: "source_hubspot/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "sample_files/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + state_path: "sample_files/abnormal_state.json" + cursor_paths: + subscription_changes: ["timestamp"] + email_events: ["timestamp"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" From a0a2c67c932780e9660a2a176159d8cedc68f6c7 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Sun, 18 Apr 2021 22:10:25 +0300 Subject: [PATCH 02/10] fix tests --- .../bases/standard-test/standard_test/conftest.py | 2 +- airbyte-integrations/connectors/source-hubspot/build.gradle | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/standard-test/standard_test/conftest.py b/airbyte-integrations/bases/standard-test/standard_test/conftest.py index 0856faac9da3..47c8a1e9bf7e 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/conftest.py +++ b/airbyte-integrations/bases/standard-test/standard_test/conftest.py @@ -128,4 +128,4 @@ def docker_runner_fixture(image_tag, tmp_path) -> ConnectorRunner: @pytest.fixture(scope="session", autouse=True) def pull_docker_image(standard_test_config) -> None: """Startup fixture to pull docker image""" - ConnectorRunner(image_tag=standard_test_config.connector_image) + ConnectorRunner(image_name=standard_test_config.connector_image) diff --git a/airbyte-integrations/connectors/source-hubspot/build.gradle b/airbyte-integrations/connectors/source-hubspot/build.gradle index 5e625a30dc66..024f2bc12ce2 100644 --- a/airbyte-integrations/connectors/source-hubspot/build.gradle +++ b/airbyte-integrations/connectors/source-hubspot/build.gradle @@ -28,6 +28,7 @@ airbyteStandardSourceTestFile { dependencies { implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:standard-test').airbyteDocker.outputs) } task("pythonIntegrationTests", type: PythonTask, dependsOn: installTestReqs) { From 4f048c0830221eb2cbf2116c38fb7816c9d3be16 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Sun, 18 Apr 2021 22:43:25 +0300 Subject: [PATCH 03/10] fix leaking of secrets in the output --- .../standard-test/standard_test/conftest.py | 8 ++--- .../standard_test/tests/utils/__init__.py | 30 +++++++++++++++++++ .../{utils.py => tests/utils/common.py} | 7 ++++- .../{ => tests/utils}/connector_runner.py | 0 .../{ => tests/utils}/json_schema_helper.py | 0 .../source-hubspot/standard_test_config.yml | 2 +- 6 files changed, 41 insertions(+), 6 deletions(-) create mode 100644 airbyte-integrations/bases/standard-test/standard_test/tests/utils/__init__.py rename airbyte-integrations/bases/standard-test/standard_test/{utils.py => tests/utils/common.py} (95%) rename airbyte-integrations/bases/standard-test/standard_test/{ => tests/utils}/connector_runner.py (100%) rename airbyte-integrations/bases/standard-test/standard_test/{ => tests/utils}/json_schema_helper.py (100%) diff --git a/airbyte-integrations/bases/standard-test/standard_test/conftest.py b/airbyte-integrations/bases/standard-test/standard_test/conftest.py index 47c8a1e9bf7e..9b6093c6503f 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/conftest.py +++ b/airbyte-integrations/bases/standard-test/standard_test/conftest.py @@ -31,7 +31,7 @@ from airbyte_protocol import AirbyteCatalog, ConfiguredAirbyteCatalog, ConnectorSpecification from standard_test.config import Config from standard_test.connector_runner import ConnectorRunner -from standard_test.utils import load_config +from standard_test.utils import load_config, SecretDict @pytest.fixture(name="base_path") @@ -94,10 +94,10 @@ def image_tag_fixture(standard_test_config) -> str: @pytest.fixture(name="connector_config") -def connector_config_fixture(base_path, connector_config_path) -> MutableMapping[str, Any]: +def connector_config_fixture(base_path, connector_config_path) -> SecretDict: with open(str(connector_config_path), "r") as file: contents = file.read() - return json.loads(contents) + return SecretDict(json.loads(contents)) @pytest.fixture(name="invalid_connector_config") @@ -128,4 +128,4 @@ def docker_runner_fixture(image_tag, tmp_path) -> ConnectorRunner: @pytest.fixture(scope="session", autouse=True) def pull_docker_image(standard_test_config) -> None: """Startup fixture to pull docker image""" - ConnectorRunner(image_name=standard_test_config.connector_image) + ConnectorRunner(image_name=standard_test_config.connector_image, volume=Path(".")) diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/utils/__init__.py b/airbyte-integrations/bases/standard-test/standard_test/tests/utils/__init__.py new file mode 100644 index 000000000000..d013b8fd47ae --- /dev/null +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/utils/__init__.py @@ -0,0 +1,30 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .json_schema_helper import JsonSchemaHelper +from .common import load_config, full_refresh_only_catalog, incremental_only_catalog, filter_output, SecretDict +from .connector_runner import ConnectorRunner + +__all__ = ["JsonSchemaHelper", "load_config", "filter_output", "full_refresh_only_catalog", "incremental_only_catalog", "SecretDict", + "ConnectorRunner"] diff --git a/airbyte-integrations/bases/standard-test/standard_test/utils.py b/airbyte-integrations/bases/standard-test/standard_test/tests/utils/common.py similarity index 95% rename from airbyte-integrations/bases/standard-test/standard_test/utils.py rename to airbyte-integrations/bases/standard-test/standard_test/tests/utils/common.py index 72951e88fdad..dd652e2b0e72 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/utils.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/utils/common.py @@ -21,7 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ - +from collections import UserDict from pathlib import Path from typing import Iterable, List @@ -75,3 +75,8 @@ def incremental_only_catalog(configured_catalog: ConfiguredAirbyteCatalog) -> Co def filter_output(records: Iterable[AirbyteMessage], type_) -> List[AirbyteMessage]: """Filter messages to match specific type""" return list(filter(lambda x: x.type == type_, records)) + + +class SecretDict(UserDict): + def __str__(self) -> str: + return str(self.__class__) diff --git a/airbyte-integrations/bases/standard-test/standard_test/connector_runner.py b/airbyte-integrations/bases/standard-test/standard_test/tests/utils/connector_runner.py similarity index 100% rename from airbyte-integrations/bases/standard-test/standard_test/connector_runner.py rename to airbyte-integrations/bases/standard-test/standard_test/tests/utils/connector_runner.py diff --git a/airbyte-integrations/bases/standard-test/standard_test/json_schema_helper.py b/airbyte-integrations/bases/standard-test/standard_test/tests/utils/json_schema_helper.py similarity index 100% rename from airbyte-integrations/bases/standard-test/standard_test/json_schema_helper.py rename to airbyte-integrations/bases/standard-test/standard_test/tests/utils/json_schema_helper.py diff --git a/airbyte-integrations/connectors/source-hubspot/standard_test_config.yml b/airbyte-integrations/connectors/source-hubspot/standard_test_config.yml index aabee4acc155..9ccc34da09c5 100644 --- a/airbyte-integrations/connectors/source-hubspot/standard_test_config.yml +++ b/airbyte-integrations/connectors/source-hubspot/standard_test_config.yml @@ -1,4 +1,4 @@ -connector_image: airbyte/source-hubspot:dev +connector_image: airbyte/source-hubspot:0.1.1 tests: spec: - spec_path: "source_hubspot/spec.json" From a3799bc68a40dbde8a6ef734de348ab40c0c0f4b Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Sun, 18 Apr 2021 22:47:04 +0300 Subject: [PATCH 04/10] fix package path --- .../standard-test/standard_test/{tests => }/utils/__init__.py | 0 .../standard-test/standard_test/{tests => }/utils/common.py | 2 +- .../standard_test/{tests => }/utils/connector_runner.py | 0 .../standard_test/{tests => }/utils/json_schema_helper.py | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename airbyte-integrations/bases/standard-test/standard_test/{tests => }/utils/__init__.py (100%) rename airbyte-integrations/bases/standard-test/standard_test/{tests => }/utils/common.py (98%) rename airbyte-integrations/bases/standard-test/standard_test/{tests => }/utils/connector_runner.py (100%) rename airbyte-integrations/bases/standard-test/standard_test/{tests => }/utils/json_schema_helper.py (100%) diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/utils/__init__.py b/airbyte-integrations/bases/standard-test/standard_test/utils/__init__.py similarity index 100% rename from airbyte-integrations/bases/standard-test/standard_test/tests/utils/__init__.py rename to airbyte-integrations/bases/standard-test/standard_test/utils/__init__.py diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/utils/common.py b/airbyte-integrations/bases/standard-test/standard_test/utils/common.py similarity index 98% rename from airbyte-integrations/bases/standard-test/standard_test/tests/utils/common.py rename to airbyte-integrations/bases/standard-test/standard_test/utils/common.py index dd652e2b0e72..4d44e7ef934a 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/utils/common.py +++ b/airbyte-integrations/bases/standard-test/standard_test/utils/common.py @@ -79,4 +79,4 @@ def filter_output(records: Iterable[AirbyteMessage], type_) -> List[AirbyteMessa class SecretDict(UserDict): def __str__(self) -> str: - return str(self.__class__) + return f"{self.__class__}(******)" diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/utils/connector_runner.py b/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py similarity index 100% rename from airbyte-integrations/bases/standard-test/standard_test/tests/utils/connector_runner.py rename to airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/utils/json_schema_helper.py b/airbyte-integrations/bases/standard-test/standard_test/utils/json_schema_helper.py similarity index 100% rename from airbyte-integrations/bases/standard-test/standard_test/tests/utils/json_schema_helper.py rename to airbyte-integrations/bases/standard-test/standard_test/utils/json_schema_helper.py From f62a666765aaeeba3f2ae7f6d169a616c9e331d8 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Sun, 18 Apr 2021 22:49:55 +0300 Subject: [PATCH 05/10] fix imports --- .../bases/standard-test/standard_test/conftest.py | 3 +-- .../bases/standard-test/standard_test/plugin.py | 3 +-- .../bases/standard-test/standard_test/utils/__init__.py | 3 ++- .../bases/standard-test/standard_test/{ => utils}/compare.py | 0 4 files changed, 4 insertions(+), 5 deletions(-) rename airbyte-integrations/bases/standard-test/standard_test/{ => utils}/compare.py (100%) diff --git a/airbyte-integrations/bases/standard-test/standard_test/conftest.py b/airbyte-integrations/bases/standard-test/standard_test/conftest.py index 9b6093c6503f..aa56c40dcf72 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/conftest.py +++ b/airbyte-integrations/bases/standard-test/standard_test/conftest.py @@ -30,8 +30,7 @@ import pytest from airbyte_protocol import AirbyteCatalog, ConfiguredAirbyteCatalog, ConnectorSpecification from standard_test.config import Config -from standard_test.connector_runner import ConnectorRunner -from standard_test.utils import load_config, SecretDict +from standard_test.utils import load_config, SecretDict, ConnectorRunner @pytest.fixture(name="base_path") diff --git a/airbyte-integrations/bases/standard-test/standard_test/plugin.py b/airbyte-integrations/bases/standard-test/standard_test/plugin.py index 0a5c42715528..ff0dc8d09016 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/plugin.py +++ b/airbyte-integrations/bases/standard-test/standard_test/plugin.py @@ -23,8 +23,7 @@ """ import pytest -from standard_test.compare import diff_dicts -from standard_test.utils import load_config +from standard_test.utils import load_config, diff_dicts def pytest_addoption(parser): diff --git a/airbyte-integrations/bases/standard-test/standard_test/utils/__init__.py b/airbyte-integrations/bases/standard-test/standard_test/utils/__init__.py index d013b8fd47ae..b240d646b4b4 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/utils/__init__.py +++ b/airbyte-integrations/bases/standard-test/standard_test/utils/__init__.py @@ -25,6 +25,7 @@ from .json_schema_helper import JsonSchemaHelper from .common import load_config, full_refresh_only_catalog, incremental_only_catalog, filter_output, SecretDict from .connector_runner import ConnectorRunner +from .compare import diff_dicts __all__ = ["JsonSchemaHelper", "load_config", "filter_output", "full_refresh_only_catalog", "incremental_only_catalog", "SecretDict", - "ConnectorRunner"] + "ConnectorRunner", "diff_dicts"] diff --git a/airbyte-integrations/bases/standard-test/standard_test/compare.py b/airbyte-integrations/bases/standard-test/standard_test/utils/compare.py similarity index 100% rename from airbyte-integrations/bases/standard-test/standard_test/compare.py rename to airbyte-integrations/bases/standard-test/standard_test/utils/compare.py From f524ef2d7c7f985b20354892c915da4dccb151a3 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Sun, 18 Apr 2021 22:59:29 +0300 Subject: [PATCH 06/10] hide secrets --- .../bases/standard-test/standard_test/tests/test_core.py | 2 +- .../standard-test/standard_test/tests/test_full_refresh.py | 3 +-- .../standard-test/standard_test/tests/test_incremental.py | 4 +--- .../bases/standard-test/standard_test/utils/common.py | 5 ++++- .../standard-test/standard_test/utils/connector_runner.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/test_core.py b/airbyte-integrations/bases/standard-test/standard_test/tests/test_core.py index 039cfdbf9f74..24a49bf0922a 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/test_core.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/test_core.py @@ -29,7 +29,7 @@ from docker.errors import ContainerError from standard_test.base import BaseTest from standard_test.config import BasicReadTestConfig, ConnectionTestConfig -from standard_test.connector_runner import ConnectorRunner +from standard_test.utils import ConnectorRunner @pytest.mark.timeout(10) diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py b/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py index aec3b56e73a9..8841093d1708 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py @@ -27,8 +27,7 @@ import pytest from airbyte_protocol import Type from standard_test.base import BaseTest -from standard_test.connector_runner import ConnectorRunner -from standard_test.utils import full_refresh_only_catalog +from standard_test.utils import full_refresh_only_catalog, ConnectorRunner @pytest.mark.timeout(20 * 60) diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/test_incremental.py b/airbyte-integrations/bases/standard-test/standard_test/tests/test_incremental.py index c92dd3613dfb..7e479991f708 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/test_incremental.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/test_incremental.py @@ -29,9 +29,7 @@ import pytest from airbyte_protocol import ConfiguredAirbyteCatalog, Type from standard_test import BaseTest -from standard_test.connector_runner import ConnectorRunner -from standard_test.json_schema_helper import JsonSchemaHelper -from standard_test.utils import filter_output, incremental_only_catalog +from standard_test.utils import filter_output, incremental_only_catalog, JsonSchemaHelper, ConnectorRunner @pytest.fixture(name="future_state_path") diff --git a/airbyte-integrations/bases/standard-test/standard_test/utils/common.py b/airbyte-integrations/bases/standard-test/standard_test/utils/common.py index 4d44e7ef934a..3548979febea 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/utils/common.py +++ b/airbyte-integrations/bases/standard-test/standard_test/utils/common.py @@ -79,4 +79,7 @@ def filter_output(records: Iterable[AirbyteMessage], type_) -> List[AirbyteMessa class SecretDict(UserDict): def __str__(self) -> str: - return f"{self.__class__}(******)" + return f"{self.__class__.__name__}(******)" + + def __repr__(self) -> str: + return str(self) diff --git a/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py b/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py index 5552c8cef586..c870c064a001 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py @@ -46,11 +46,11 @@ def _prepare_volumes(self, config: Optional[Mapping], state: Optional[Mapping], if config: with open(str(input_path / "tap_config.json"), "w") as outfile: - json.dump(config, outfile) + json.dump(dict(config), outfile) if state: with open(str(input_path / "state.json"), "w") as outfile: - json.dump(state, outfile) + json.dump(dict(state), outfile) if catalog: with open(str(input_path / "catalog.json"), "w") as outfile: From 885fd492b3dd13a4714a082d793e2e6e87edc99c Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Sun, 18 Apr 2021 23:43:38 +0300 Subject: [PATCH 07/10] fix logs --- airbyte-integrations/bases/standard-test/pytest.ini | 2 +- .../standard-test/standard_test/utils/connector_runner.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/bases/standard-test/pytest.ini b/airbyte-integrations/bases/standard-test/pytest.ini index e79a1ff56102..a39244c14f86 100644 --- a/airbyte-integrations/bases/standard-test/pytest.ini +++ b/airbyte-integrations/bases/standard-test/pytest.ini @@ -1,5 +1,5 @@ [pytest] -addopts = -r a --capture=no -vv +addopts = -r a --capture=no -vv --log-level=INFO testpaths = standard_test/tests diff --git a/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py b/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py index c870c064a001..dd60cef52772 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py +++ b/airbyte-integrations/bases/standard-test/standard_test/utils/connector_runner.py @@ -101,5 +101,6 @@ def run(self, cmd, config=None, state=None, catalog=None, **kwargs) -> Iterable[ ) logging.info("Running docker, folders: %s", volumes) for line in logs.decode("utf-8").splitlines(): - logging.info(AirbyteMessage.parse_raw(line).type) - yield AirbyteMessage.parse_raw(line) + message = AirbyteMessage.parse_raw(line) + logging.info(message.type) + yield message From 4f2787c6b7b930ace03841445423f136680bdd06 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 19 Apr 2021 00:04:05 +0300 Subject: [PATCH 08/10] fix sequential read test --- .../standard-test/standard_test/tests/test_full_refresh.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py b/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py index 8841093d1708..afbe1676ceac 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py +++ b/airbyte-integrations/bases/standard-test/standard_test/tests/test_full_refresh.py @@ -23,6 +23,7 @@ """ import json +from functools import partial import pytest from airbyte_protocol import Type @@ -39,7 +40,8 @@ def test_sequential_reads(self, connector_config, configured_catalog, docker_run output = docker_runner.call_read(connector_config, configured_catalog) records_2 = [message.record.data for message in output if message.type == Type.RECORD] + serialize = partial(json.dumps, sort_keys=True) assert not ( - set(map(json.dumps, records_1)) - set(map(json.dumps, records_2)) + set(map(serialize, records_1)) - set(map(serialize, records_2)) ), "The two sequential reads should produce either equal set of records or one of them is a strict subset of the other" From 43f80809801d91a601ff0a41c24a05e47eb1ffaa Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 19 Apr 2021 00:06:49 +0300 Subject: [PATCH 09/10] fix build --- airbyte-integrations/connectors/source-hubspot/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-hubspot/build.gradle b/airbyte-integrations/connectors/source-hubspot/build.gradle index 024f2bc12ce2..5e625a30dc66 100644 --- a/airbyte-integrations/connectors/source-hubspot/build.gradle +++ b/airbyte-integrations/connectors/source-hubspot/build.gradle @@ -28,7 +28,6 @@ airbyteStandardSourceTestFile { dependencies { implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) - implementation files(project(':airbyte-integrations:bases:standard-test').airbyteDocker.outputs) } task("pythonIntegrationTests", type: PythonTask, dependsOn: installTestReqs) { From 95257a2f7fa00730519e30088bbe2b18d11126c6 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Mon, 19 Apr 2021 00:12:04 +0300 Subject: [PATCH 10/10] format --- .../bases/standard-test/standard_test/utils/common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/bases/standard-test/standard_test/utils/common.py b/airbyte-integrations/bases/standard-test/standard_test/utils/common.py index 3548979febea..1a1eefd8f17c 100644 --- a/airbyte-integrations/bases/standard-test/standard_test/utils/common.py +++ b/airbyte-integrations/bases/standard-test/standard_test/utils/common.py @@ -21,6 +21,7 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ + from collections import UserDict from pathlib import Path from typing import Iterable, List