-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Standard Tests: create modular standard test suite v2 scaffolding (#2702
) * Standard Tests: Implement v2 test suites #2181 (#2792) * rename standard tests (v2) to source_acceptance_test
- Loading branch information
Showing
34 changed files
with
1,325 additions
and
33 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
4 changes: 4 additions & 0 deletions
4
airbyte-integrations/bases/source-acceptance-test/.dockerignore
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
* | ||
!Dockerfile | ||
!standard_test | ||
!setup.py |
13 changes: 13 additions & 0 deletions
13
airbyte-integrations/bases/source-acceptance-test/Dockerfile
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,13 @@ | ||
FROM airbyte/integration-base-python:dev | ||
|
||
ENV CODE_PATH="source_acceptance_test" | ||
|
||
WORKDIR /airbyte/source_acceptance_test | ||
COPY $CODE_PATH ./$CODE_PATH | ||
COPY setup.py ./ | ||
RUN pip install . | ||
|
||
LABEL io.airbyte.version=0.1.0 | ||
LABEL io.airbyte.name=airbyte/source-acceptance-test | ||
|
||
ENTRYPOINT ["python", "-m", "pytest"] |
59 changes: 59 additions & 0 deletions
59
airbyte-integrations/bases/source-acceptance-test/README.md
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
# Standard tests | ||
This package uses pytest to discover, configure and execute the tests. | ||
It implemented as a pytest plugin. | ||
|
||
It adds new configuration option `--acceptance-test-config` - path to configuration file (by default is current folder). | ||
Configuration stored in YaML format and validated by pydantic. | ||
|
||
Example configuration can be found in `sample_files/` folder: | ||
```yaml | ||
connector_image: <your_image> | ||
tests: | ||
spec: | ||
- spec_path: "<connector_folder>/spec.json" | ||
connection: | ||
- config_path: "secrets/config.json" | ||
status: "succeed" | ||
- config_path: "sample_files/invalid_config.json" | ||
status: "exception" | ||
discovery: | ||
- config_path: "secrets/config.json" | ||
basic_read: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "sample_files/configured_catalog.json" | ||
validate_output_from_all_streams: true | ||
incremental: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "sample_files/configured_catalog.json" | ||
state_path: "sample_files/abnormal_state.json" | ||
cursor_paths: | ||
subscription_changes: ["timestamp"] | ||
email_events: ["timestamp"] | ||
full_refresh: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "sample_files/configured_catalog.json" | ||
``` | ||
Required steps to test connector are the following: | ||
* Build docker image for connector | ||
* Create `acceptance-test-config.yml` file with test settings, Note: all paths in this files are relative to its location | ||
* Use one of the following ways to run tests: | ||
|
||
## Running | ||
Using python | ||
```bash | ||
cd ../../base/source-acceptance-test | ||
python -m pytest source_acceptance_test/tests --acceptance-test-config=<path_to_your_connector> -vvv | ||
``` | ||
_Note: this will assume that docker image for connector is already built_ | ||
|
||
Using Gradle | ||
```bash | ||
./gradlew :airbyte-integrations:connectors:source-<name>:standardTest | ||
``` | ||
_Note: this way will also build docker image for the connector_ | ||
|
||
Using Bash | ||
```bash | ||
./source-acceptance-test.sh -vv | ||
``` | ||
_Note: you can append any arguments to this command, they will be forwarded to pytest |
15 changes: 15 additions & 0 deletions
15
airbyte-integrations/bases/source-acceptance-test/build.gradle
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
plugins { | ||
id 'airbyte-docker' | ||
id 'airbyte-python' | ||
} | ||
|
||
airbytePython { | ||
moduleDirectory 'source_acceptance_test' | ||
} | ||
|
||
|
||
dependencies { | ||
implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) | ||
} | ||
|
||
installReqs.dependsOn(":airbyte-integrations:bases:airbyte-protocol:installReqs") |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
[pytest] | ||
|
||
addopts = -r a --capture=no -vv --log-level=INFO | ||
testpaths = | ||
source_acceptance_test/tests |
2 changes: 2 additions & 0 deletions
2
airbyte-integrations/bases/source-acceptance-test/requirements.txt
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
-e ../airbyte-protocol | ||
-e . |
25 changes: 25 additions & 0 deletions
25
airbyte-integrations/bases/source-acceptance-test/sample_files/acceptance-test-config.yml
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,25 @@ | ||
connector_image: airbyte/source-hubspot:dev | ||
tests: | ||
spec: | ||
- spec_path: "source_hubspot/spec.json" | ||
connection: | ||
- config_path: "secrets/config.json" | ||
status: "succeed" | ||
- config_path: "sample_files/invalid_config.json" | ||
status: "exception" | ||
discovery: | ||
- config_path: "secrets/config.json" | ||
basic_read: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "sample_files/configured_catalog.json" | ||
validate_output_from_all_streams: yes | ||
incremental: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "sample_files/configured_catalog.json" | ||
state_path: "sample_files/abnormal_state.json" | ||
cursor_paths: | ||
subscription_changes: ["timestamp"] | ||
email_events: ["timestamp"] | ||
full_refresh: | ||
- config_path: "secrets/config.json" | ||
configured_catalog_path: "sample_files/configured_catalog.json" |
49 changes: 49 additions & 0 deletions
49
airbyte-integrations/bases/source-acceptance-test/setup.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
""" | ||
MIT License | ||
Copyright (c) 2020 Airbyte | ||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
The above copyright notice and this permission notice shall be included in all | ||
copies or substantial portions of the Software. | ||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
SOFTWARE. | ||
""" | ||
|
||
import setuptools | ||
|
||
MAIN_REQUIREMENTS = [ | ||
"airbyte-protocol", | ||
"docker==4.4.4", | ||
"PyYAML==5.3.1", | ||
"inflection==0.5.1", | ||
"icdiff==1.9.1", | ||
"pendulum==1.2.0", | ||
"pydantic==1.6.1", | ||
"pytest==6.1.2", | ||
"pytest-timeout==1.4.2", | ||
"pprintpp==0.4.0", | ||
] | ||
|
||
setuptools.setup( | ||
name="source-acceptance-test", | ||
description="Contains classes for running integration tests.", | ||
author="Airbyte", | ||
author_email="contact@airbyte.io", | ||
url="https://github.com/airbytehq/airbyte", | ||
packages=setuptools.find_packages(), | ||
install_requires=MAIN_REQUIREMENTS, | ||
entry_points={"pytest11": ["pytest-airbyte = source_acceptance_test.plugin"]}, | ||
) |
27 changes: 27 additions & 0 deletions
27
airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
""" | ||
MIT License | ||
Copyright (c) 2020 Airbyte | ||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
The above copyright notice and this permission notice shall be included in all | ||
copies or substantial portions of the Software. | ||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
SOFTWARE. | ||
""" | ||
|
||
from .base import BaseTest | ||
|
||
__all__ = ["BaseTest"] |
37 changes: 37 additions & 0 deletions
37
airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/base.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
""" | ||
MIT License | ||
Copyright (c) 2020 Airbyte | ||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
The above copyright notice and this permission notice shall be included in all | ||
copies or substantial portions of the Software. | ||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
SOFTWARE. | ||
""" | ||
|
||
import inflection | ||
import pytest | ||
|
||
|
||
@pytest.mark.usefixtures("inputs") | ||
class BaseTest: | ||
@classmethod | ||
def config_key(cls): | ||
"""Name of the test in configuration file, used to override test inputs,""" | ||
class_name = cls.__name__ | ||
if class_name.startswith("Test"): | ||
class_name = class_name[len("Test") :] | ||
return inflection.underscore(class_name) |
94 changes: 94 additions & 0 deletions
94
airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/config.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,94 @@ | ||
""" | ||
MIT License | ||
Copyright (c) 2020 Airbyte | ||
Permission is hereby granted, free of charge, to any person obtaining a copy | ||
of this software and associated documentation files (the "Software"), to deal | ||
in the Software without restriction, including without limitation the rights | ||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
copies of the Software, and to permit persons to whom the Software is | ||
furnished to do so, subject to the following conditions: | ||
The above copyright notice and this permission notice shall be included in all | ||
copies or substantial portions of the Software. | ||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
SOFTWARE. | ||
""" | ||
|
||
from enum import Enum | ||
from typing import List, Mapping, Optional | ||
|
||
from pydantic import BaseModel, Field | ||
|
||
config_path: str = Field(default="secrets/config.json", description="Path to a JSON object representing a valid connector configuration") | ||
invalid_config_path: str = Field(description="Path to a JSON object representing an invalid connector configuration") | ||
spec_path: str = Field( | ||
default="secrets/spec.json", description="Path to a JSON object representing the spec expected to be output by this connector" | ||
) | ||
configured_catalog_path: str = Field(default="sample_files/configured_catalog.json", description="Path to configured catalog") | ||
|
||
|
||
class BaseConfig(BaseModel): | ||
class Config: | ||
extra = "forbid" | ||
|
||
|
||
class SpecTestConfig(BaseConfig): | ||
spec_path: str = spec_path | ||
|
||
|
||
class ConnectionTestConfig(BaseConfig): | ||
class Status(Enum): | ||
Succeed = "succeed" | ||
Failed = "failed" | ||
Exception = "exception" | ||
|
||
config_path: str = config_path | ||
status: Status = Field(Status.Succeed, description="Indicate if connection check should succeed with provided config") | ||
|
||
|
||
class DiscoveryTestConfig(BaseConfig): | ||
config_path: str = config_path | ||
configured_catalog_path: Optional[str] = configured_catalog_path | ||
|
||
|
||
class BasicReadTestConfig(BaseConfig): | ||
config_path: str = config_path | ||
configured_catalog_path: Optional[str] = configured_catalog_path | ||
validate_output_from_all_streams: bool = Field(False, description="Verify that all streams have records") | ||
|
||
|
||
class FullRefreshConfig(BaseConfig): | ||
config_path: str = config_path | ||
configured_catalog_path: str = configured_catalog_path | ||
|
||
|
||
class IncrementalConfig(BaseConfig): | ||
config_path: str = config_path | ||
configured_catalog_path: str = configured_catalog_path | ||
cursor_paths: Optional[Mapping[str, List[str]]] = Field( | ||
description="For each stream, the path of its cursor field in the output state messages." | ||
) | ||
state_path: Optional[str] = Field(description="Path to state file") | ||
|
||
|
||
class TestConfig(BaseConfig): | ||
spec: Optional[List[SpecTestConfig]] = Field(description="TODO") | ||
connection: Optional[List[ConnectionTestConfig]] = Field(description="TODO") | ||
discovery: Optional[List[DiscoveryTestConfig]] = Field(description="TODO") | ||
basic_read: Optional[List[BasicReadTestConfig]] = Field(description="TODO") | ||
full_refresh: Optional[List[FullRefreshConfig]] = Field(description="TODO") | ||
incremental: Optional[List[IncrementalConfig]] = Field(description="TODO") | ||
|
||
|
||
class Config(BaseConfig): | ||
connector_image: str = Field(description="Docker image to test, for example 'airbyte/source-hubspot:dev'") | ||
base_path: Optional[str] = Field(description="Base path for all relative paths") | ||
tests: TestConfig = Field(description="List of the tests with their configs") |
Oops, something went wrong.