Standard Tests: gradle command for running tests

airbyte-integrations/bases/base-python/base_python/client.py

@@ -173,7 +173,7 @@ def streams(self) -> Generator[AirbyteStream, None, None]:
             supported_sync_modes = [SyncMode.full_refresh]
             source_defined_cursor = False
             if self.stream_has_state(name):
-                supported_sync_modes = [SyncMode.incremental]
+                supported_sync_modes += [SyncMode.incremental]
                 source_defined_cursor = True
 
             yield AirbyteStream(

airbyte-integrations/bases/standard-test/Dockerfile

@@ -10,4 +10,4 @@ RUN pip install .
 LABEL io.airbyte.version=0.1.0
 LABEL io.airbyte.name=airbyte/standard-test
 
-ENTRYPOINT ["python", "-m", "pytest", "standard_test/tests", "-rsx", "-vvv"]
+ENTRYPOINT ["python", "-m", "pytest"]

airbyte-integrations/bases/standard-test/README.md

@@ -1,7 +1,46 @@
-# Running Standard Source Tests
+# Standard tests
+This package uses pytest to discover, configure and execute the tests.
+It implemented as a pytest plugin.
+
+It adds new configuration option `--standard_test_config` - path to configuration file (by default is current folder). 
+Configuration stored in YaML format and validated by pydantic.
+
+Example configuration can be found in `sample_files/` folder:
+```yaml
+connector_image: <your_image>
+tests:
+  spec:
+    - spec_path: "<connector_folder>/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "sample_files/invalid_config.json"
+      status: "exception"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      validate_output_from_all_streams: true
+  incremental:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      state_path: "sample_files/abnormal_state.json"
+      cursor_paths:
+        subscription_changes: ["timestamp"]
+        email_events: ["timestamp"]
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+```
+# Running
 ```bash
-python -m pytest standard_test/tests --standard_test_config=../../connectors/source-hubspot/ -vvv
+python -m pytest standard_test/tests --standard_test_config=<path_to_your_connector> -vvv
 ```
+_Note: this will assume that docker image for connector is already built_
 
-## How to
-TODO
+Using Gradle
+```bash
+./gradlew :airbyte-integrations:connectors:source-<name>:standardTest
+```
+_Note: this will also build docker image for connector_

airbyte-integrations/bases/standard-test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+
+addopts = -r a --capture=no -vv
+testpaths =
+    standard_test/tests

airbyte-integrations/bases/standard-test/sample_files/standard_test_config.yml

@@ -4,7 +4,9 @@ tests:
     - spec_path: "source_hubspot/spec.json"
   connection:
     - config_path: "secrets/config.json"
-      invalid_config_path: "sample_files/invalid_config.json"
+      status: "succeed"
+    - config_path: "sample_files/invalid_config.json"
+      status: "exception"
   discovery:
     - config_path: "secrets/config.json"
   basic_read:

airbyte-integrations/bases/standard-test/setup.py

@@ -29,10 +29,12 @@
     "docker==4.4.4",
     "PyYAML==5.3.1",
     "inflection==0.5.1",
+    "icdiff==1.9.1",
     "pendulum==1.2.0",
     "pydantic==1.6.1",
     "pytest==6.1.2",
     "pytest-timeout==1.4.2",
+    "pprintpp==0.4.0",
 ]
 
 setuptools.setup(

airbyte-integrations/bases/standard-test/standard_test/compare.py

@@ -0,0 +1,68 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+import py
+from typing import List, Optional
+
+import icdiff
+from pprintpp import pformat
+
+MAX_COLS = py.io.TerminalWriter().fullwidth
+MARGIN_LEFT = 20
+GUTTER = 3
+MARGINS = MARGIN_LEFT + GUTTER + 1
+
+
+def diff_dicts(left, right, use_markup) -> Optional[List[str]]:
+    half_cols = MAX_COLS / 2 - MARGINS
+
+    pretty_left = pformat(left, indent=1, width=half_cols).splitlines()
+    pretty_right = pformat(right, indent=1, width=half_cols).splitlines()
+    diff_cols = MAX_COLS - MARGINS
+
+    if len(pretty_left) < 3 or len(pretty_right) < 3:
+        # avoid small diffs far apart by smooshing them up to the left
+        smallest_left = pformat(left, indent=2, width=1).splitlines()
+        smallest_right = pformat(right, indent=2, width=1).splitlines()
+        max_side = max(len(line) + 1 for line in smallest_left + smallest_right)
+        if (max_side * 2 + MARGIN_LEFT) < MAX_COLS:
+            diff_cols = max_side * 2 + GUTTER
+            pretty_left = pformat(left, indent=2, width=max_side).splitlines()
+            pretty_right = pformat(right, indent=2, width=max_side).splitlines()
+
+    differ = icdiff.ConsoleDiff(cols=diff_cols, tabsize=2)
+
+    if not use_markup:
+        # colorization is disabled in Pytest - either due to the terminal not
+        # supporting it or the user disabling it. We should obey, but there is
+        # no option in icdiff to disable it, so we replace its colorization
+        # function with a no-op
+        differ.colorize = lambda string: string
+        color_off = ""
+    else:
+        color_off = icdiff.color_codes["none"]
+
+    icdiff_lines = list(differ.make_table(pretty_left, pretty_right, context=True))
+
+    return ["equals failed"] + [color_off + line for line in icdiff_lines]

airbyte-integrations/bases/standard-test/standard_test/config.py

@@ -22,13 +22,16 @@
 SOFTWARE.
 """
 
-from typing import List, Optional, Mapping
+from typing import List, Mapping, Optional
 
+from enum import Enum
 from pydantic import BaseModel, Field
 
 config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration")
 invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration")
-spec_path: str = Field(default="secrets/spec.json", description="Path to a JSON object representing the spec expected to be output by this connector")
+spec_path: str = Field(
+    default="secrets/spec.json", description="Path to a JSON object representing the spec expected to be output by this connector"
+)
 configured_catalog_path: str = Field(default="sample_files/configured_catalog.json", description="Path to configured catalog")
 
 
@@ -42,8 +45,13 @@ class SpecTestConfig(BaseConfig):
 
 
 class ConnectionTestConfig(BaseConfig):
+    class Status(Enum):
+        Succeed = 'succeed'
+        Failed = 'failed'
+        Exception = 'exception'
+
     config_path: str = config_path
-    invalid_config_path: str = invalid_config_path
+    status: Status = Field(Status.Succeed, description="Indicate if connection check should succeed with provided config")
 
 
 class DiscoveryTestConfig(BaseConfig):
@@ -65,7 +73,9 @@ class FullRefreshConfig(BaseConfig):
 class IncrementalConfig(BaseConfig):
     config_path: str = config_path
     configured_catalog_path: str = configured_catalog_path
-    cursor_paths: Optional[Mapping[str, List[str]]] = Field(description="For each stream, the path of its cursor field in the output state messages.")
+    cursor_paths: Optional[Mapping[str, List[str]]] = Field(
+        description="For each stream, the path of its cursor field in the output state messages."
+    )
     state_path: Optional[str] = Field(description="Path to state file")
 
 

airbyte-integrations/bases/standard-test/standard_test/conftest.py

@@ -25,12 +25,11 @@
 import copy
 import json
 from pathlib import Path
-from typing import Optional
+from typing import Optional, MutableMapping, Any
 
 import pytest
-from airbyte_protocol import (AirbyteCatalog, ConfiguredAirbyteCatalog,
-                              ConnectorSpecification)
-
+from airbyte_protocol import AirbyteCatalog, ConfiguredAirbyteCatalog, ConnectorSpecification
+from standard_test.config import Config
 from standard_test.connector_runner import ConnectorRunner
 from standard_test.utils import load_config
 
@@ -43,32 +42,32 @@ def base_path_fixture(pytestconfig, standard_test_config):
     return Path(pytestconfig.getoption("--standard_test_config")).absolute()
 
 
-@pytest.fixture(name="standard_test_config")
-def standard_test_config_fixture(pytestconfig):
+@pytest.fixture(name="standard_test_config", scope="session")
+def standard_test_config_fixture(pytestconfig) -> Config:
     """Fixture with test's config"""
     return load_config(pytestconfig.getoption("--standard_test_config"))
 
 
 @pytest.fixture(name="connector_config_path")
-def connector_config_path_fixture(inputs, base_path):
+def connector_config_path_fixture(inputs, base_path) -> Path:
     """Fixture with connector's config path"""
     return Path(base_path) / getattr(inputs, "config_path")
 
 
 @pytest.fixture(name="invalid_connector_config_path")
-def invalid_connector_config_path_fixture(inputs, base_path):
+def invalid_connector_config_path_fixture(inputs, base_path) -> Path:
     """Fixture with connector's config path"""
     return Path(base_path) / getattr(inputs, "invalid_config_path")
 
 
 @pytest.fixture(name="connector_spec_path")
-def connector_spec_path_fixture(inputs, base_path):
+def connector_spec_path_fixture(inputs, base_path) -> Path:
     """Fixture with connector's specification path"""
     return Path(base_path) / getattr(inputs, "spec_path")
 
 
 @pytest.fixture(name="configured_catalog_path")
-def configured_catalog_path_fixture(inputs, base_path):
+def configured_catalog_path_fixture(inputs, base_path) -> Optional[str]:
     """Fixture with connector's configured_catalog path"""
     if getattr(inputs, "configured_catalog_path"):
         return Path(base_path) / getattr(inputs, "configured_catalog_path")
@@ -90,27 +89,27 @@ def catalog_fixture(configured_catalog: ConfiguredAirbyteCatalog) -> Optional[Ai
 
 
 @pytest.fixture(name="image_tag")
-def image_tag_fixture(standard_test_config):
+def image_tag_fixture(standard_test_config) -> str:
     return standard_test_config.connector_image
 
 
 @pytest.fixture(name="connector_config")
-def connector_config_fixture(base_path, connector_config_path):
+def connector_config_fixture(base_path, connector_config_path) -> MutableMapping[str, Any]:
     with open(str(connector_config_path), "r") as file:
         contents = file.read()
     return json.loads(contents)
 
 
 @pytest.fixture(name="invalid_connector_config")
-def invalid_connector_config_fixture(base_path, invalid_connector_config_path):
+def invalid_connector_config_fixture(base_path, invalid_connector_config_path) -> MutableMapping[str, Any]:
     """TODO: implement default value - generate from valid config"""
     with open(str(invalid_connector_config_path), "r") as file:
         contents = file.read()
     return json.loads(contents)
 
 
 @pytest.fixture(name="malformed_connector_config")
-def malformed_connector_config_fixture(connector_config):
+def malformed_connector_config_fixture(connector_config) -> MutableMapping[str, Any]:
     """TODO: drop required field, add extra"""
     malformed_config = copy.deepcopy(connector_config)
     return malformed_config
@@ -126,7 +125,7 @@ def docker_runner_fixture(image_tag, tmp_path) -> ConnectorRunner:
     return ConnectorRunner(image_tag, volume=tmp_path)
 
 
-@pytest.fixture(name="validate_output_from_all_streams")
-def validate_output_from_all_streams_fixture(inputs):
-    """Fixture to provide value of `validate output from all streams` flag"""
-    return getattr(inputs, "validate_output_from_all_streams")
+@pytest.fixture(scope="session", autouse=True)
+def pull_docker_image(standard_test_config) -> None:
+    """Startup fixture to pull docker image"""
+    ConnectorRunner(image_tag=standard_test_config.connector_image)

airbyte-integrations/bases/standard-test/standard_test/connector_runner.py

@@ -25,16 +25,16 @@
 import json
 import logging
 from pathlib import Path
-from typing import Iterable, Mapping, Optional
+from typing import Iterable, Mapping, Optional, List
 
 import docker
 from airbyte_protocol import AirbyteMessage, ConfiguredAirbyteCatalog
 
 
 class ConnectorRunner:
-    def __init__(self, name: str, volume: Path):
-        self._name = name
+    def __init__(self, image_name: str, volume: Path):
         self._client = docker.from_env()
+        self._image = self._client.images.pull(image_name)
         self._runs = 0
         self._volume_base = volume
 
@@ -68,27 +68,27 @@ def _prepare_volumes(self, config: Optional[Mapping], state: Optional[Mapping],
         }
         return volumes
 
-    def call_spec(self, **kwargs):
+    def call_spec(self, **kwargs) -> List[AirbyteMessage]:
         cmd = "spec"
         output = list(self.run(cmd=cmd, **kwargs))
         return output
 
-    def call_check(self, config, **kwargs):
+    def call_check(self, config, **kwargs) -> List[AirbyteMessage]:
         cmd = "check --config tap_config.json"
         output = list(self.run(cmd=cmd, config=config, **kwargs))
         return output
 
-    def call_discover(self, config, **kwargs):
+    def call_discover(self, config, **kwargs) -> List[AirbyteMessage]:
         cmd = "discover --config tap_config.json"
         output = list(self.run(cmd=cmd, config=config, **kwargs))
         return output
 
-    def call_read(self, config, catalog, **kwargs):
+    def call_read(self, config, catalog, **kwargs) -> List[AirbyteMessage]:
         cmd = "read --config tap_config.json --catalog catalog.json"
         output = list(self.run(cmd=cmd, config=config, catalog=catalog, **kwargs))
         return output
 
-    def call_read_with_state(self, config, catalog, state, **kwargs):
+    def call_read_with_state(self, config, catalog, state, **kwargs) -> List[AirbyteMessage]:
         cmd = "read --config tap_config.json --catalog catalog.json --state state.json"
         output = list(self.run(cmd=cmd, config=config, catalog=catalog, state=state, **kwargs))
         return output
@@ -97,8 +97,9 @@ def run(self, cmd, config=None, state=None, catalog=None, **kwargs) -> Iterable[
         self._runs += 1
         volumes = self._prepare_volumes(config, state, catalog)
         logs = self._client.containers.run(
-            image=self._name, command=cmd, working_dir="/data", volumes=volumes, network="host", stdout=True, stderr=True, **kwargs
+            image=self._image, command=cmd, working_dir="/data", volumes=volumes, network="host", stdout=True, stderr=True, **kwargs
         )
+        logging.info("Running docker, folders: %s", volumes)
         for line in logs.decode("utf-8").splitlines():
             logging.info(AirbyteMessage.parse_raw(line).type)
             yield AirbyteMessage.parse_raw(line)