From 8dea9f2f673b56f6166f09c57727e699fd245f1d Mon Sep 17 00:00:00 2001 From: Augustin Date: Tue, 2 Aug 2022 09:22:22 +0200 Subject: [PATCH] SAT: retrieve previous connector spec and create test to run checks against it (#14954) --- .../bases/source-acceptance-test/CHANGELOG.md | 5 ++- .../bases/source-acceptance-test/Dockerfile | 2 +- .../source_acceptance_test/config.py | 12 ++++++ .../source_acceptance_test/conftest.py | 24 +++++++++++ .../source_acceptance_test/tests/test_core.py | 43 +++++++++++++++++-- .../source-acceptance-tests-reference.md | 2 + 6 files changed, 83 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md index 908be06a69d3..43a344cac6df 100644 --- a/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md +++ b/airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md @@ -1,6 +1,9 @@ # Changelog -## 0.1.56 +## 0.1.58 +Bootstrap spec backward compatibility tests. Add fixtures to retrieve a previous connector version spec [#14954](https://github.com/airbytehq/airbyte/pull/14954/). + +## 0.1.57 Run connector from its image `working_dir` instead of from `/data`. ## 0.1.56 diff --git a/airbyte-integrations/bases/source-acceptance-test/Dockerfile b/airbyte-integrations/bases/source-acceptance-test/Dockerfile index 3cdd9c60d292..f1b78978db01 100644 --- a/airbyte-integrations/bases/source-acceptance-test/Dockerfile +++ b/airbyte-integrations/bases/source-acceptance-test/Dockerfile @@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./ COPY source_acceptance_test ./source_acceptance_test RUN pip install . -LABEL io.airbyte.version=0.1.57 +LABEL io.airbyte.version=0.1.58 LABEL io.airbyte.name=airbyte/source-acceptance-test ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"] diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py index 1fe2b3629c28..b974d03e0b7f 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py @@ -23,10 +23,22 @@ class Config: extra = "forbid" +class BackwardCompatibilityTestsConfig(BaseConfig): + previous_connector_version: str = Field( + default="latest", description="Previous connector version to use for backward compatibility tests." + ) + disable_backward_compatibility_tests_for_version: Optional[str] = Field( + default=None, description="Disable backward compatibility tests for a specific connector version." + ) + + class SpecTestConfig(BaseConfig): spec_path: str = spec_path config_path: str = config_path timeout_seconds: int = timeout_seconds + backward_compatibility_tests_config: BackwardCompatibilityTestsConfig = Field( + description="Configuration for the backward compatibility tests.", default=BackwardCompatibilityTestsConfig() + ) class ConnectionTestConfig(BaseConfig): diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py index c092929ab9ec..6fc61ff76b70 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py @@ -126,6 +126,30 @@ def docker_runner_fixture(image_tag, tmp_path) -> ConnectorRunner: return ConnectorRunner(image_tag, volume=tmp_path) +@pytest.fixture(name="previous_connector_image_name") +def previous_connector_image_name_fixture(image_tag, inputs) -> str: + """Fixture with previous connector image name to use for backward compatibility tests""" + return f"{image_tag.split(':')[0]}:{inputs.backward_compatibility_tests_config.previous_connector_version}" + + +@pytest.fixture(name="previous_connector_docker_runner") +def previous_connector_docker_runner_fixture(previous_connector_image_name, tmp_path) -> ConnectorRunner: + """Fixture to create a connector runner with the previous connector docker image. + Returns None if the latest image was not found, to skip downstream tests if the current connector is not yet published to the docker registry. + Raise not found error if the previous connector image is not latest and expected to be published. + """ + try: + return ConnectorRunner(previous_connector_image_name, volume=tmp_path / "previous_connector") + except (errors.NotFound, errors.ImageNotFound) as e: + if previous_connector_image_name.endswith("latest"): + logging.warning( + f"\n We did not find the {previous_connector_image_name} image for this connector. This probably means this version has not yet been published to an accessible docker registry like DockerHub." + ) + return None + else: + raise e + + @pytest.fixture(scope="session", autouse=True) def pull_docker_image(acceptance_test_config) -> None: """Startup fixture to pull docker image""" diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py index 836301dbe2e0..d5986b53561a 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py @@ -8,7 +8,7 @@ from collections import Counter, defaultdict from functools import reduce from logging import Logger -from typing import Any, Dict, List, Mapping, MutableMapping, Set +from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Set import dpath.util import jsonschema @@ -27,7 +27,7 @@ from docker.errors import ContainerError from jsonschema._utils import flatten from source_acceptance_test.base import BaseTest -from source_acceptance_test.config import BasicReadTestConfig, ConnectionTestConfig +from source_acceptance_test.config import BasicReadTestConfig, ConnectionTestConfig, SpecTestConfig from source_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, make_hashable, verify_records_schema from source_acceptance_test.utils.common import find_all_values_for_key_in_schema, find_keyword_schema from source_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure @@ -39,7 +39,7 @@ def connector_spec_dict_fixture(actual_connector_spec): @pytest.fixture(name="actual_connector_spec") -def actual_connector_spec_fixture(request: BaseTest, docker_runner): +def actual_connector_spec_fixture(request: BaseTest, docker_runner: ConnectorRunner) -> ConnectorSpecification: if not request.instance.spec_cache: output = docker_runner.call_spec() spec_messages = filter_output(output, Type.SPEC) @@ -49,10 +49,29 @@ def actual_connector_spec_fixture(request: BaseTest, docker_runner): return request.spec_cache +@pytest.fixture(name="previous_connector_spec") +def previous_connector_spec_fixture( + request: BaseTest, previous_connector_docker_runner: ConnectorRunner +) -> Optional[ConnectorSpecification]: + if previous_connector_docker_runner is None: + logging.warning( + "\n We could not retrieve the previous connector spec as a connector runner for the previous connector version could not be instantiated." + ) + return None + if not request.instance.previous_spec_cache: + output = previous_connector_docker_runner.call_spec() + spec_messages = filter_output(output, Type.SPEC) + assert len(spec_messages) == 1, "Spec message should be emitted exactly once" + spec = spec_messages[0].spec + request.instance.previous_spec_cache = spec + return request.instance.previous_spec_cache + + @pytest.mark.default_timeout(10) class TestSpec(BaseTest): spec_cache: ConnectorSpecification = None + previous_spec_cache: ConnectorSpecification = None def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, connector_config: SecretDict): """Check that config matches the actual schema from the spec call""" @@ -177,6 +196,24 @@ def test_additional_properties_is_true(self, actual_connector_spec): [additional_properties_value is True for additional_properties_value in additional_properties_values] ), "When set, additionalProperties field value must be true for backward compatibility." + @pytest.mark.default_timeout(60) # Pulling the previous connector image can take more than 10 sec. + @pytest.mark.spec_backward_compatibility + def test_backward_compatibility( + self, inputs: SpecTestConfig, actual_connector_spec: ConnectorSpecification, previous_connector_spec: ConnectorSpecification + ): + """Run multiple checks to make sure the actual_connector_spec is backward compatible with the previous_connector_spec""" + if ( + inputs.backward_compatibility_tests_config.disable_backward_compatibility_tests_for_version + == inputs.backward_compatibility_tests_config.previous_connector_version + ): + pytest.skip( + f"Backward compatibility tests are disabled for version {inputs.backward_compatibility_tests_config.disable_backward_compatibility_tests_for_version}." + ) + if previous_connector_spec is None: + pytest.skip("The previous connector spec could not be retrieved.") + assert isinstance(actual_connector_spec, ConnectorSpecification) and isinstance(previous_connector_spec, ConnectorSpecification) + # TODO alafanechere: add the actual tests for backward compatibility below or in a dedicated module. + @pytest.mark.default_timeout(30) class TestConnection(BaseTest): diff --git a/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md b/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md index d1f2f11c130f..ffaadea30ee0 100644 --- a/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md +++ b/docs/connector-development/testing-connectors/source-acceptance-tests-reference.md @@ -98,6 +98,8 @@ Verify that a spec operation issued to the connector returns a valid spec. | Input | Type | Default | Note | | :--- | :--- | :--- |:-------------------------------------------------------------------------------------------------| | `spec_path` | string | `secrets/spec.json` | Path to a YAML or JSON file representing the spec expected to be output by this connector | +| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests. | +| `backward_compatibility_tests_config.run_backward_compatibility_tests` | boolean | True | Flag to run or skip backward compatibility tests. | | `timeout_seconds` | int | 10 | Test execution timeout in seconds | ## Test Connection