SAT: retrieve previous connector spec and create test to run checks a…

…gainst it (airbytehq#14954)

airbyte-integrations/bases/source-acceptance-test/CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 
-## 0.1.56
+## 0.1.58
+Bootstrap spec backward compatibility tests. Add fixtures to retrieve a previous connector version spec [#14954](https://github.com/airbytehq/airbyte/pull/14954/).
+
+## 0.1.57
 Run connector from its image `working_dir` instead of from `/data`.
 
 ## 0.1.56

airbyte-integrations/bases/source-acceptance-test/Dockerfile

@@ -33,7 +33,7 @@ COPY pytest.ini setup.py ./
 COPY source_acceptance_test ./source_acceptance_test
 RUN pip install .
 
-LABEL io.airbyte.version=0.1.57
+LABEL io.airbyte.version=0.1.58
 LABEL io.airbyte.name=airbyte/source-acceptance-test
 
 ENTRYPOINT ["python", "-m", "pytest", "-p", "source_acceptance_test.plugin", "-r", "fEsx"]

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py

@@ -23,10 +23,22 @@ class Config:
         extra = "forbid"
 
 
+class BackwardCompatibilityTestsConfig(BaseConfig):
+    previous_connector_version: str = Field(
+        default="latest", description="Previous connector version to use for backward compatibility tests."
+    )
+    disable_backward_compatibility_tests_for_version: Optional[str] = Field(
+        default=None, description="Disable backward compatibility tests for a specific connector version."
+    )
+
+
 class SpecTestConfig(BaseConfig):
     spec_path: str = spec_path
     config_path: str = config_path
     timeout_seconds: int = timeout_seconds
+    backward_compatibility_tests_config: BackwardCompatibilityTestsConfig = Field(
+        description="Configuration for the backward compatibility tests.", default=BackwardCompatibilityTestsConfig()
+    )
 
 
 class ConnectionTestConfig(BaseConfig):

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/conftest.py

@@ -126,6 +126,30 @@ def docker_runner_fixture(image_tag, tmp_path) -> ConnectorRunner:
     return ConnectorRunner(image_tag, volume=tmp_path)
 
 
+@pytest.fixture(name="previous_connector_image_name")
+def previous_connector_image_name_fixture(image_tag, inputs) -> str:
+    """Fixture with previous connector image name to use for backward compatibility tests"""
+    return f"{image_tag.split(':')[0]}:{inputs.backward_compatibility_tests_config.previous_connector_version}"
+
+
+@pytest.fixture(name="previous_connector_docker_runner")
+def previous_connector_docker_runner_fixture(previous_connector_image_name, tmp_path) -> ConnectorRunner:
+    """Fixture to create a connector runner with the previous connector docker image.
+    Returns None if the latest image was not found, to skip downstream tests if the current connector is not yet published to the docker registry.
+    Raise not found error if the previous connector image is not latest and expected to be published.
+    """
+    try:
+        return ConnectorRunner(previous_connector_image_name, volume=tmp_path / "previous_connector")
+    except (errors.NotFound, errors.ImageNotFound) as e:
+        if previous_connector_image_name.endswith("latest"):
+            logging.warning(
+                f"\n We did not find the {previous_connector_image_name} image for this connector. This probably means this version has not yet been published to an accessible docker registry like DockerHub."
+            )
+            return None
+        else:
+            raise e
+
+
 @pytest.fixture(scope="session", autouse=True)
 def pull_docker_image(acceptance_test_config) -> None:
     """Startup fixture to pull docker image"""

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_core.py

@@ -8,7 +8,7 @@
 from collections import Counter, defaultdict
 from functools import reduce
 from logging import Logger
-from typing import Any, Dict, List, Mapping, MutableMapping, Set
+from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Set
 
 import dpath.util
 import jsonschema
@@ -27,7 +27,7 @@
 from docker.errors import ContainerError
 from jsonschema._utils import flatten
 from source_acceptance_test.base import BaseTest
-from source_acceptance_test.config import BasicReadTestConfig, ConnectionTestConfig
+from source_acceptance_test.config import BasicReadTestConfig, ConnectionTestConfig, SpecTestConfig
 from source_acceptance_test.utils import ConnectorRunner, SecretDict, filter_output, make_hashable, verify_records_schema
 from source_acceptance_test.utils.common import find_all_values_for_key_in_schema, find_keyword_schema
 from source_acceptance_test.utils.json_schema_helper import JsonSchemaHelper, get_expected_schema_structure, get_object_structure
@@ -39,7 +39,7 @@ def connector_spec_dict_fixture(actual_connector_spec):
 
 
 @pytest.fixture(name="actual_connector_spec")
-def actual_connector_spec_fixture(request: BaseTest, docker_runner):
+def actual_connector_spec_fixture(request: BaseTest, docker_runner: ConnectorRunner) -> ConnectorSpecification:
     if not request.instance.spec_cache:
         output = docker_runner.call_spec()
         spec_messages = filter_output(output, Type.SPEC)
@@ -49,10 +49,29 @@ def actual_connector_spec_fixture(request: BaseTest, docker_runner):
     return request.spec_cache
 
 
+@pytest.fixture(name="previous_connector_spec")
+def previous_connector_spec_fixture(
+    request: BaseTest, previous_connector_docker_runner: ConnectorRunner
+) -> Optional[ConnectorSpecification]:
+    if previous_connector_docker_runner is None:
+        logging.warning(
+            "\n We could not retrieve the previous connector spec as a connector runner for the previous connector version could not be instantiated."
+        )
+        return None
+    if not request.instance.previous_spec_cache:
+        output = previous_connector_docker_runner.call_spec()
+        spec_messages = filter_output(output, Type.SPEC)
+        assert len(spec_messages) == 1, "Spec message should be emitted exactly once"
+        spec = spec_messages[0].spec
+        request.instance.previous_spec_cache = spec
+    return request.instance.previous_spec_cache
+
+
 @pytest.mark.default_timeout(10)
 class TestSpec(BaseTest):
 
     spec_cache: ConnectorSpecification = None
+    previous_spec_cache: ConnectorSpecification = None
 
     def test_config_match_spec(self, actual_connector_spec: ConnectorSpecification, connector_config: SecretDict):
         """Check that config matches the actual schema from the spec call"""
@@ -177,6 +196,24 @@ def test_additional_properties_is_true(self, actual_connector_spec):
                 [additional_properties_value is True for additional_properties_value in additional_properties_values]
             ), "When set, additionalProperties field value must be true for backward compatibility."
 
+    @pytest.mark.default_timeout(60)  # Pulling the previous connector image can take more than 10 sec.
+    @pytest.mark.spec_backward_compatibility
+    def test_backward_compatibility(
+        self, inputs: SpecTestConfig, actual_connector_spec: ConnectorSpecification, previous_connector_spec: ConnectorSpecification
+    ):
+        """Run multiple checks to make sure the actual_connector_spec is backward compatible with the previous_connector_spec"""
+        if (
+            inputs.backward_compatibility_tests_config.disable_backward_compatibility_tests_for_version
+            == inputs.backward_compatibility_tests_config.previous_connector_version
+        ):
+            pytest.skip(
+                f"Backward compatibility tests are disabled for version {inputs.backward_compatibility_tests_config.disable_backward_compatibility_tests_for_version}."
+            )
+        if previous_connector_spec is None:
+            pytest.skip("The previous connector spec could not be retrieved.")
+        assert isinstance(actual_connector_spec, ConnectorSpecification) and isinstance(previous_connector_spec, ConnectorSpecification)
+        # TODO alafanechere: add the actual tests for backward compatibility below or in a dedicated module.
+
 
 @pytest.mark.default_timeout(30)
 class TestConnection(BaseTest):

docs/connector-development/testing-connectors/source-acceptance-tests-reference.md

@@ -98,6 +98,8 @@ Verify that a spec operation issued to the connector returns a valid spec.
 | Input | Type | Default | Note                                                                                             |
 | :--- | :--- | :--- |:-------------------------------------------------------------------------------------------------|
 | `spec_path` | string | `secrets/spec.json` | Path to a YAML or JSON file representing the spec expected to be output by this connector |
+| `backward_compatibility_tests_config.previous_connector_version` | string | `latest` | Previous connector version to use for backward compatibility tests. |
+| `backward_compatibility_tests_config.run_backward_compatibility_tests` | boolean | True | Flag to run or skip backward compatibility tests. |
 | `timeout_seconds` | int | 10 | Test execution timeout in seconds                                                                |
 
 ## Test Connection