Standard Tests: Implement v2 test suites #2181

airbyte-integrations/bases/base-python/base_python/client.py

@@ -173,7 +173,7 @@ def streams(self) -> Generator[AirbyteStream, None, None]:
             supported_sync_modes = [SyncMode.full_refresh]
             source_defined_cursor = False
             if self.stream_has_state(name):
-                supported_sync_modes = [SyncMode.incremental]
+                supported_sync_modes += [SyncMode.incremental]
                 source_defined_cursor = True
 
             yield AirbyteStream(

airbyte-integrations/bases/standard-test/Dockerfile

@@ -2,12 +2,12 @@ FROM airbyte/integration-base-python:dev
 
 ENV CODE_PATH="standard_test"
 
-WORKDIR /airbyte/base_python_test_code
+WORKDIR /airbyte/standard_test_code
 COPY $CODE_PATH ./$CODE_PATH
 COPY setup.py ./
 RUN pip install .
 
-ENTRYPOINT ["airbyte-python-test"]
-
 LABEL io.airbyte.version=0.1.0
 LABEL io.airbyte.name=airbyte/standard-test
+
+ENTRYPOINT ["python", "-m", "pytest"]

airbyte-integrations/bases/standard-test/README.md

@@ -1,7 +1,46 @@
-# Running Standard Source Tests
+# Standard tests
+This package uses pytest to discover, configure and execute the tests.
+It implemented as a pytest plugin.
+
+It adds new configuration option `--standard_test_config` - path to configuration file (by default is current folder). 
+Configuration stored in YaML format and validated by pydantic.
+
+Example configuration can be found in `sample_files/` folder:
+```yaml
+connector_image: <your_image>
+tests:
+  spec:
+    - spec_path: "<connector_folder>/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "sample_files/invalid_config.json"
+      status: "exception"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      validate_output_from_all_streams: true
+  incremental:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      state_path: "sample_files/abnormal_state.json"
+      cursor_paths:
+        subscription_changes: ["timestamp"]
+        email_events: ["timestamp"]
+  full_refresh:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+```
+# Running
 ```bash
-PYTHONPATH=. pytest standard_test.py --standard_test_config=/connectors/source-hubspot/ -vvv
+python -m pytest standard_test/tests --standard_test_config=<path_to_your_connector> -vvv
 ```
+_Note: this will assume that docker image for connector is already built_
 
-## How to
-TODO
+Using Gradle
+```bash
+./gradlew :airbyte-integrations:connectors:source-<name>:standardTest
+```
+_Note: this will also build docker image for connector_

airbyte-integrations/bases/standard-test/pytest.ini

@@ -0,0 +1,5 @@
+[pytest]
+
+addopts = -r a --capture=no -vv
+testpaths =
+    standard_test/tests

airbyte-integrations/bases/standard-test/sample_files/standard_test_config.yml

@@ -1,13 +1,25 @@
 connector_image: airbyte/source-hubspot:dev
 tests:
-  core:
+  spec:
+    - spec_path: "source_hubspot/spec.json"
+  connection:
+    - config_path: "secrets/config.json"
+      status: "succeed"
+    - config_path: "sample_files/invalid_config.json"
+      status: "exception"
+  discovery:
+    - config_path: "secrets/config.json"
+  basic_read:
+    - config_path: "secrets/config.json"
+      configured_catalog_path: "sample_files/configured_catalog.json"
+      validate_output_from_all_streams: true
+  incremental:
     - config_path: "secrets/config.json"
-      invalid_config_path: "secrets/invalid_config.json"
-      spec_path: "source_hubspot/spec.json"
       configured_catalog_path: "sample_files/configured_catalog.json"
+      state_path: "sample_files/abnormal_state.json"
+      cursor_paths:
+        subscription_changes: ["timestamp"]
+        email_events: ["timestamp"]
+  full_refresh:
     - config_path: "secrets/config.json"
-      invalid_config_path: "secrets/invalid_config.json"
-      spec_path: "source_hubspot/spec.json"
       configured_catalog_path: "sample_files/configured_catalog.json"
-  incremental: []
-  full_refresh: []

airbyte-integrations/bases/standard-test/setup.py

@@ -29,8 +29,12 @@
     "docker==4.4.4",
     "PyYAML==5.3.1",
     "inflection==0.5.1",
+    "icdiff==1.9.1",
+    "pendulum==1.2.0",
     "pydantic==1.6.1",
     "pytest==6.1.2",
+    "pytest-timeout==1.4.2",
+    "pprintpp==0.4.0",
 ]
 
 setuptools.setup(

airbyte-integrations/bases/standard-test/standard_test/base.py

@@ -1,3 +1,27 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
 import inflection
 import pytest
 

airbyte-integrations/bases/standard-test/standard_test/compare.py

@@ -0,0 +1,68 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+import py
+from typing import List, Optional
+
+import icdiff
+from pprintpp import pformat
+
+MAX_COLS = py.io.TerminalWriter().fullwidth
+MARGIN_LEFT = 20
+GUTTER = 3
+MARGINS = MARGIN_LEFT + GUTTER + 1
+
+
+def diff_dicts(left, right, use_markup) -> Optional[List[str]]:
+    half_cols = MAX_COLS / 2 - MARGINS
+
+    pretty_left = pformat(left, indent=1, width=half_cols).splitlines()
+    pretty_right = pformat(right, indent=1, width=half_cols).splitlines()
+    diff_cols = MAX_COLS - MARGINS
+
+    if len(pretty_left) < 3 or len(pretty_right) < 3:
+        # avoid small diffs far apart by smooshing them up to the left
+        smallest_left = pformat(left, indent=2, width=1).splitlines()
+        smallest_right = pformat(right, indent=2, width=1).splitlines()
+        max_side = max(len(line) + 1 for line in smallest_left + smallest_right)
+        if (max_side * 2 + MARGIN_LEFT) < MAX_COLS:
+            diff_cols = max_side * 2 + GUTTER
+            pretty_left = pformat(left, indent=2, width=max_side).splitlines()
+            pretty_right = pformat(right, indent=2, width=max_side).splitlines()
+
+    differ = icdiff.ConsoleDiff(cols=diff_cols, tabsize=2)
+
+    if not use_markup:
+        # colorization is disabled in Pytest - either due to the terminal not
+        # supporting it or the user disabling it. We should obey, but there is
+        # no option in icdiff to disable it, so we replace its colorization
+        # function with a no-op
+        differ.colorize = lambda string: string
+        color_off = ""
+    else:
+        color_off = icdiff.color_codes["none"]
+
+    icdiff_lines = list(differ.make_table(pretty_left, pretty_right, context=True))
+
+    return ["equals failed"] + [color_off + line for line in icdiff_lines]

airbyte-integrations/bases/standard-test/standard_test/config.py

@@ -22,68 +22,64 @@
 SOFTWARE.
 """
 
-from typing import List, Optional
+from typing import List, Mapping, Optional
 
+from enum import Enum
 from pydantic import BaseModel, Field
 
-config_path: str = Field(description="Path to a JSON object representing a valid connector configuration")
+config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration")
 invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration")
-spec_path: str = Field(description="Path to a JSON object representing the spec expected to be output by this connector")
-configured_catalog_path: str = Field(description="Path to configured catalog")
+spec_path: str = Field(
+    default="secrets/spec.json", description="Path to a JSON object representing the spec expected to be output by this connector"
+)
+configured_catalog_path: str = Field(default="sample_files/configured_catalog.json", description="Path to configured catalog")
 
 
-class SpecTestConfig(BaseModel):
+class BaseConfig(BaseModel):
     class Config:
         extra = "forbid"
 
+
+class SpecTestConfig(BaseConfig):
     spec_path: str = spec_path
 
 
-class ConnectionTestConfig(BaseModel):
-    class Config:
-        extra = "forbid"
+class ConnectionTestConfig(BaseConfig):
+    class Status(Enum):
+        Succeed = 'succeed'
+        Failed = 'failed'
+        Exception = 'exception'
 
     config_path: str = config_path
-    invalid_config_path: str = invalid_config_path
+    status: Status = Field(Status.Succeed, description="Indicate if connection check should succeed with provided config")
 
 
-class DiscoveryTestConfig(BaseModel):
-    class Config:
-        extra = "forbid"
-
+class DiscoveryTestConfig(BaseConfig):
     config_path: str = config_path
     configured_catalog_path: Optional[str] = configured_catalog_path
 
 
-class BasicReadTestConfig(BaseModel):
-    class Config:
-        extra = "forbid"
-
+class BasicReadTestConfig(BaseConfig):
     config_path: str = config_path
     configured_catalog_path: Optional[str] = configured_catalog_path
     validate_output_from_all_streams: bool = Field(False, description="Verify that all streams have records")
 
 
-class FullRefreshConfig(BaseModel):
-    class Config:
-        extra = "forbid"
-
+class FullRefreshConfig(BaseConfig):
     config_path: str = config_path
     configured_catalog_path: str = configured_catalog_path
 
 
-class IncrementalConfig(BaseModel):
-    class Config:
-        extra = "forbid"
-
+class IncrementalConfig(BaseConfig):
     config_path: str = config_path
     configured_catalog_path: str = configured_catalog_path
+    cursor_paths: Optional[Mapping[str, List[str]]] = Field(
+        description="For each stream, the path of its cursor field in the output state messages."
+    )
+    state_path: Optional[str] = Field(description="Path to state file")
 
 
-class TestConfig(BaseModel):
-    class Config:
-        extra = "forbid"
-
+class TestConfig(BaseConfig):
     spec: Optional[List[SpecTestConfig]] = Field(description="TODO")
     connection: Optional[List[ConnectionTestConfig]] = Field(description="TODO")
     discovery: Optional[List[DiscoveryTestConfig]] = Field(description="TODO")
@@ -92,10 +88,7 @@ class Config:
     incremental: Optional[List[IncrementalConfig]] = Field(description="TODO")
 
 
-class Config(BaseModel):
-    class Config:
-        extra = "forbid"
-
+class Config(BaseConfig):
     connector_image: str = Field(description="Docker image to test, for example 'airbyte/source-hubspot:dev'")
     base_path: Optional[str] = Field(description="Base path for all relative paths")
-    tests: TestConfig = Field(description="TODO")
+    tests: TestConfig = Field(description="List of the tests with their configs")