Standard Tests: create modular standard test suite v2 scaffolding (#2702

)

* Standard Tests: Implement v2 test suites #2181 (#2792)
* rename standard tests (v2) to source_acceptance_test

airbyte-integrations/bases/base-python/base_python/client.py

@@ -97,7 +97,7 @@ def streams(self) -> Generator[AirbyteStream, None, None]:
             supported_sync_modes = [SyncMode.full_refresh]
             source_defined_cursor = False
             if self.stream_has_state(name):
-                supported_sync_modes = [SyncMode.incremental]
+                supported_sync_modes += [SyncMode.incremental]
                 source_defined_cursor = True
 
             yield AirbyteStream(

airbyte-integrations/bases/source-acceptance-test/.dockerignore

@@ -0,0 +1,4 @@
+*
+!Dockerfile
+!standard_test
+!setup.py

airbyte-integrations/bases/source-acceptance-test/Dockerfile

@@ -0,0 +1,13 @@
+FROM airbyte/integration-base-python:dev
+
+ENV CODE_PATH="source_acceptance_test"
+
+WORKDIR /airbyte/source_acceptance_test
+COPY $CODE_PATH ./$CODE_PATH
+COPY setup.py ./
+RUN pip install .
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-acceptance-test
+
+ENTRYPOINT ["python", "-m", "pytest"]

airbyte-integrations/bases/source-acceptance-test/README.md

@@ -0,0 +1,59 @@
+# Standard tests
+This package uses pytest to discover, configure and execute the tests.
+It implemented as a pytest plugin.
+
+It adds new configuration option `--acceptance-test-config` - path to configuration file (by default is current folder). 
+Configuration stored in YaML format and validated by pydantic.
+
+Example configuration can be found in `sample_files/` folder:
+```yaml
+connector_image: <your_image>
+tests:
+  spec:
+    - spec_path: "<connector_folder>/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "sample_files/invalid_config.json"
+      status: "exception"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      validate_output_from_all_streams: true
+  incremental:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      state_path: "sample_files/abnormal_state.json"
+      cursor_paths:
+        subscription_changes: ["timestamp"]
+        email_events: ["timestamp"]
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+```
+Required steps to test connector are the following:
+* Build docker image for connector
+* Create `acceptance-test-config.yml` file with test settings, Note: all paths in this files are relative to its location
+* Use one of the following ways to run tests:
+
+## Running
+Using python
+```bash
+cd ../../base/source-acceptance-test
+python -m pytest source_acceptance_test/tests --acceptance-test-config=<path_to_your_connector> -vvv
+```
+_Note: this will assume that docker image for connector is already built_
+
+Using Gradle
+```bash
+./gradlew :airbyte-integrations:connectors:source-<name>:standardTest
+```
+_Note: this way will also build docker image for the connector_
+
+Using Bash
+```bash
+./source-acceptance-test.sh -vv
+```
+_Note: you can append any arguments to this command, they will be forwarded to pytest

airbyte-integrations/bases/source-acceptance-test/build.gradle

@@ -0,0 +1,15 @@
+plugins {
+    id 'airbyte-docker'
+    id 'airbyte-python'
+}
+
+airbytePython {
+    moduleDirectory 'source_acceptance_test'
+}
+
+
+dependencies {
+    implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs)
+}
+
+installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs")

airbyte-integrations/bases/source-acceptance-test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+
+addopts = -r a --capture=no -vv --log-level=INFO
+testpaths =
+    source_acceptance_test/tests

airbyte-integrations/bases/source-acceptance-test/requirements.txt

@@ -0,0 +1,2 @@
+-e ../airbyte-protocol
+-e .

airbyte-integrations/bases/source-acceptance-test/sample_files/acceptance-test-config.yml

@@ -0,0 +1,25 @@
+connector_image: airbyte/source-hubspot:dev
+tests:
+  spec:
+    - spec_path: "source_hubspot/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "sample_files/invalid_config.json"
+      status: "exception"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      validate_output_from_all_streams: yes
+  incremental:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      state_path: "sample_files/abnormal_state.json"
+      cursor_paths:
+        subscription_changes: ["timestamp"]
+        email_events: ["timestamp"]
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"

airbyte-integrations/bases/source-acceptance-test/setup.py

@@ -0,0 +1,49 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+import setuptools
+
+MAIN_REQUIREMENTS = [
+    "airbyte-protocol",
+    "docker==4.4.4",
+    "PyYAML==5.3.1",
+    "inflection==0.5.1",
+    "icdiff==1.9.1",
+    "pendulum==1.2.0",
+    "pydantic==1.6.1",
+    "pytest==6.1.2",
+    "pytest-timeout==1.4.2",
+    "pprintpp==0.4.0",
+]
+
+setuptools.setup(
+    name="source-acceptance-test",
+    description="Contains classes for running integration tests.",
+    author="Airbyte",
+    author_email="contact@airbyte.io",
+    url="https://github.com/airbytehq/airbyte",
+    packages=setuptools.find_packages(),
+    install_requires=MAIN_REQUIREMENTS,
+    entry_points={"pytest11": ["pytest-airbyte = source_acceptance_test.plugin"]},
+)

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py

@@ -0,0 +1,27 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from .base import BaseTest
+
+__all__ = ["BaseTest"]

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py

@@ -0,0 +1,37 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+import inflection
+import pytest
+
+
+@pytest.mark.usefixtures("inputs")
+class BaseTest:
+    @classmethod
+    def config_key(cls):
+        """Name of the test in configuration file, used to override test inputs,"""
+        class_name = cls.__name__
+        if class_name.startswith("Test"):
+            class_name = class_name[len("Test") :]
+        return inflection.underscore(class_name)

airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py

@@ -0,0 +1,94 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from enum import Enum
+from typing import List, Mapping, Optional
+
+from pydantic import BaseModel, Field
+
+config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration")
+invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration")
+spec_path: str = Field(
+    default="secrets/spec.json", description="Path to a JSON object representing the spec expected to be output by this connector"
+)
+configured_catalog_path: str = Field(default="sample_files/configured_catalog.json", description="Path to configured catalog")
+
+
+class BaseConfig(BaseModel):
+    class Config:
+        extra = "forbid"
+
+
+class SpecTestConfig(BaseConfig):
+    spec_path: str = spec_path
+
+
+class ConnectionTestConfig(BaseConfig):
+    class Status(Enum):
+        Succeed = "succeed"
+        Failed = "failed"
+        Exception = "exception"
+
+    config_path: str = config_path
+    status: Status = Field(Status.Succeed, description="Indicate if connection check should succeed with provided config")
+
+
+class DiscoveryTestConfig(BaseConfig):
+    config_path: str = config_path
+    configured_catalog_path: Optional[str] = configured_catalog_path
+
+
+class BasicReadTestConfig(BaseConfig):
+    config_path: str = config_path
+    configured_catalog_path: Optional[str] = configured_catalog_path
+    validate_output_from_all_streams: bool = Field(False, description="Verify that all streams have records")
+
+
+class FullRefreshConfig(BaseConfig):
+    config_path: str = config_path
+    configured_catalog_path: str = configured_catalog_path
+
+
+class IncrementalConfig(BaseConfig):
+    config_path: str = config_path
+    configured_catalog_path: str = configured_catalog_path
+    cursor_paths: Optional[Mapping[str, List[str]]] = Field(
+        description="For each stream, the path of its cursor field in the output state messages."
+    )
+    state_path: Optional[str] = Field(description="Path to state file")
+
+
+class TestConfig(BaseConfig):
+    spec: Optional[List[SpecTestConfig]] = Field(description="TODO")
+    connection: Optional[List[ConnectionTestConfig]] = Field(description="TODO")
+    discovery: Optional[List[DiscoveryTestConfig]] = Field(description="TODO")
+    basic_read: Optional[List[BasicReadTestConfig]] = Field(description="TODO")
+    full_refresh: Optional[List[FullRefreshConfig]] = Field(description="TODO")
+    incremental: Optional[List[IncrementalConfig]] = Field(description="TODO")
+
+
+class Config(BaseConfig):
+    connector_image: str = Field(description="Docker image to test, for example 'airbyte/source-hubspot:dev'")
+    base_path: Optional[str] = Field(description="Base path for all relative paths")
+    tests: TestConfig = Field(description="List of the tests with their configs")