From 592baffda7b5ce33b2830c9da66f1be1ada2683d Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Wed, 16 Jun 2021 17:24:04 +0300 Subject: [PATCH 1/9] added SAT tests, deleted incremental test sfrom pipeline --- .../acceptance-test-config.yml | 18 ++++++++++ .../source-google-adwords-singer/build.gradle | 11 ++---- .../integration_tests/__init__.py | 0 .../integration_tests/acceptance.py | 36 +++++++++++++++++++ .../sample_files/configured_catalog.json | 16 +++------ .../sample_files/invalid_config.json | 9 +++++ 6 files changed, 69 insertions(+), 21 deletions(-) create mode 100644 airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-google-adwords-singer/sample_files/invalid_config.json diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml new file mode 100644 index 000000000000..d77c1ef8e155 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml @@ -0,0 +1,18 @@ +connector_image: airbyte/source-google-adwords-singer:dev +tests: + spec: + - spec_path: "source_google_adwords_singer/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "sample_files/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: yes + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle index 92b5ef9aeff2..3468e9430daf 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle +++ b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle @@ -2,7 +2,7 @@ plugins { id 'java' id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-standard-source-test-file' + id 'airbyte-source-acceptance-test' } airbytePython { @@ -11,7 +11,7 @@ airbytePython { dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } @@ -21,10 +21,3 @@ task('installSingerTap', type: PythonTask) { command = "install tap-adwords==1.12.0" } installReqs.dependsOn installSingerTap - -airbyteStandardSourceTestFile { - // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors - specPath = "source_google_adwords_singer/spec.json" - configPath = "secrets/config.json" - configuredCatalogPath = "sample_files/configured_catalog.json" -} diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/acceptance.py new file mode 100644 index 000000000000..d98ac8aa3a1c --- /dev/null +++ b/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json index fc2a24f9859a..1d899d473555 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json @@ -167,11 +167,9 @@ } } }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "cursor_field": [], "destination_sync_mode": "overwrite" }, { @@ -321,11 +319,9 @@ } } }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "cursor_field": [], "destination_sync_mode": "overwrite" }, { @@ -402,11 +398,9 @@ } } }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "cursor_field": [], "destination_sync_mode": "overwrite" }, { @@ -435,11 +429,9 @@ }, "type": ["null", "object"] }, - "supported_sync_modes": ["full_refresh"], - "source_defined_cursor": false + "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "cursor_field": [], "destination_sync_mode": "overwrite" } ] diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/invalid_config.json new file mode 100644 index 000000000000..5e7dcdaa27f8 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/invalid_config.json @@ -0,0 +1,9 @@ +{ + "developer_token": "hahaha", + "oauth_client_id": "notvalid.apps.googleusercontent.com", + "oauth_client_secret": "notasecret", + "refresh_token": "1//notarefrshtoken", + "customer_ids": ["0000000000"], + "start_date": "2020-06-01T00:00:00Z", + "user_agent": "unknown" +} From a6807763a8f504345a7442c6f5b68203d51cee8e Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Thu, 24 Jun 2021 12:00:24 +0300 Subject: [PATCH 2/9] pr changes and tests uncommented --- .../source-facebook-marketing/unit_tests/test_client.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index a12732a9243e..a75463aa6697 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -33,6 +33,7 @@ FB_API_VERSION = "11.0" + @pytest.fixture(scope="session", name="account_id") def account_id_fixture(): return "unknown_account" From 1f563c8ed13cd64d5ec66f1f5f6bf51f115360af Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Thu, 24 Jun 2021 12:31:37 +0300 Subject: [PATCH 3/9] deleted facebook file from pr --- .../source-facebook-marketing/unit_tests/test_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py index a75463aa6697..a12732a9243e 100644 --- a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py +++ b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py @@ -33,7 +33,6 @@ FB_API_VERSION = "11.0" - @pytest.fixture(scope="session", name="account_id") def account_id_fixture(): return "unknown_account" From 75fe36fcf721c9b49257c0c9195a91beed90afe3 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Thu, 24 Jun 2021 16:35:26 +0300 Subject: [PATCH 4/9] deleted facebook file from pr --- .../acceptance-test-config.yml | 9 ++++++--- .../connectors/source-google-adwords-singer/build.gradle | 7 ------- .../configured_catalog.json | 0 .../invalid_config.json | 0 .../connectors/source-google-adwords-singer/setup.py | 2 +- 5 files changed, 7 insertions(+), 11 deletions(-) rename airbyte-integrations/connectors/source-google-adwords-singer/{sample_files => integration_tests}/configured_catalog.json (100%) rename airbyte-integrations/connectors/source-google-adwords-singer/{sample_files => integration_tests}/invalid_config.json (100%) diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml index d77c1ef8e155..30cee53cd020 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml @@ -5,14 +5,17 @@ tests: connection: - config_path: "secrets/config.json" status: "succeed" - - config_path: "sample_files/invalid_config.json" + - config_path: "integration_tests/invalid_config.json" status: "failed" discovery: - config_path: "secrets/config.json" basic_read: - config_path: "secrets/config.json" - configured_catalog_path: "sample_files/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" validate_output_from_all_streams: yes full_refresh: - config_path: "secrets/config.json" - configured_catalog_path: "sample_files/configured_catalog.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + +# incremental tests are disabled due to unsupported state structure in current tests +# required structure: {stream_name: cursor_value} given {‘bookmarks’: {stream_name: cursor_value}} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle index 3468e9430daf..c62be5100950 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle +++ b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle @@ -14,10 +14,3 @@ dependencies { implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) } - -// used to allow local iteration to work -task('installSingerTap', type: PythonTask) { - module = "pip" - command = "install tap-adwords==1.12.0" -} -installReqs.dependsOn installSingerTap diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/configured_catalog.json similarity index 100% rename from airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json rename to airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/configured_catalog.json diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/invalid_config.json b/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/invalid_config.json similarity index 100% rename from airbyte-integrations/connectors/source-google-adwords-singer/sample_files/invalid_config.json rename to airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/invalid_config.json diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py index aedf77e66bc5..e48bef239c92 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py +++ b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py @@ -31,6 +31,6 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-singer", "base-python", "pytest==6.1.2"], + install_requires=["airbyte-protocol", "base-singer", "base-python", "pytest==6.1.2", "tap-adwords==1.12.0"], package_data={"": ["*.json"]}, ) From a72ec0eaffe3e4f2d840f217e5893612b4b45b13 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Thu, 24 Jun 2021 18:24:14 +0300 Subject: [PATCH 5/9] removed tap adwords from install_requires --- .../connectors/source-google-adwords-singer/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py index e48bef239c92..aedf77e66bc5 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/setup.py +++ b/airbyte-integrations/connectors/source-google-adwords-singer/setup.py @@ -31,6 +31,6 @@ author="Airbyte", author_email="contact@airbyte.io", packages=find_packages(), - install_requires=["airbyte-protocol", "base-singer", "base-python", "pytest==6.1.2", "tap-adwords==1.12.0"], + install_requires=["airbyte-protocol", "base-singer", "base-python", "pytest==6.1.2"], package_data={"": ["*.json"]}, ) From 0fb5047acc029c8711db9ea6e7a44f4ed6b69363 Mon Sep 17 00:00:00 2001 From: Eugene Kulak Date: Fri, 25 Jun 2021 12:39:05 +0300 Subject: [PATCH 6/9] Update airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml --- .../source-google-adwords-singer/acceptance-test-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml index 30cee53cd020..51a76f934f45 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-adwords-singer/acceptance-test-config.yml @@ -18,4 +18,4 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" # incremental tests are disabled due to unsupported state structure in current tests -# required structure: {stream_name: cursor_value} given {‘bookmarks’: {stream_name: cursor_value}} \ No newline at end of file +# required structure: {stream_name: cursor_value} given {‘bookmarks’: {stream_name: cursor_value}} From 907911d42d187bfeae84e26b47d9bb0b6084b1e9 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Fri, 25 Jun 2021 13:02:53 +0300 Subject: [PATCH 7/9] added changelog info --- docs/integrations/sources/google-adwords.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/integrations/sources/google-adwords.md b/docs/integrations/sources/google-adwords.md index e06b49b688fa..bc689e0ede5a 100644 --- a/docs/integrations/sources/google-adwords.md +++ b/docs/integrations/sources/google-adwords.md @@ -60,3 +60,9 @@ When you apply for a token, you need to mention: If for any reason the request gets denied, let us know and we will be able to unblock you. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :----- | :------ | +| 0.1.2 | 2021-06-25 | [4205](https://github.com/airbytehq/airbyte/pull/4205) | Set up CDK SAT tests. Incremental tests are disabled due to unsupported state structure in current tests: required structure: {stream_name: cursor_value} given {‘bookmarks’: {stream_name: cursor_value}} | From 71f21047160fa9a33a256dab686b61345c106d66 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Fri, 25 Jun 2021 14:04:04 +0300 Subject: [PATCH 8/9] version bump --- .../fdc8b827-3257-4b33-83cc-106d234c34d4.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-google-adwords-singer/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json index 9fdd76d2d567..4ab38c4ae487 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "fdc8b827-3257-4b33-83cc-106d234c34d4", "name": "Google Adwords (Deprecated)", "dockerRepository": "airbyte/source-google-adwords-singer", - "dockerImageTag": "0.2.3", + "dockerImageTag": "0.2.4", "documentationUrl": "https://hub.docker.com/r/airbyte/source-google-adwords-singer", "icon": "google-adwords.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1903036841d0..e300dd49ef20 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -28,7 +28,7 @@ - sourceDefinitionId: fdc8b827-3257-4b33-83cc-106d234c34d4 name: Google Adwords (Deprecated) dockerRepository: airbyte/source-google-adwords-singer - dockerImageTag: 0.2.3 + dockerImageTag: 0.2.4 documentationUrl: https://hub.docker.com/r/airbyte/source-google-adwords-singer icon: google-adwords.svg - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile index 1b80990e676d..fa7cdc27a1e6 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile @@ -12,5 +12,5 @@ COPY setup.py ./ RUN pip install tap-adwords==1.12.0 RUN pip install . -LABEL io.airbyte.version=0.2.3 +LABEL io.airbyte.version=0.2.4 LABEL io.airbyte.name=airbyte/source-google-adwords-singer From dfa796cc96ff7bd06faa09a05ea9000b22f54f90 Mon Sep 17 00:00:00 2001 From: po3na4skld Date: Fri, 25 Jun 2021 21:30:56 +0300 Subject: [PATCH 9/9] version bump --- .../fdc8b827-3257-4b33-83cc-106d234c34d4.json | 2 +- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- .../connectors/source-google-adwords-singer/Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json index 4ab38c4ae487..d499cfc99bfa 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "fdc8b827-3257-4b33-83cc-106d234c34d4", "name": "Google Adwords (Deprecated)", "dockerRepository": "airbyte/source-google-adwords-singer", - "dockerImageTag": "0.2.4", + "dockerImageTag": "0.2.5", "documentationUrl": "https://hub.docker.com/r/airbyte/source-google-adwords-singer", "icon": "google-adwords.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e300dd49ef20..0fa8016bc473 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -28,7 +28,7 @@ - sourceDefinitionId: fdc8b827-3257-4b33-83cc-106d234c34d4 name: Google Adwords (Deprecated) dockerRepository: airbyte/source-google-adwords-singer - dockerImageTag: 0.2.4 + dockerImageTag: 0.2.5 documentationUrl: https://hub.docker.com/r/airbyte/source-google-adwords-singer icon: google-adwords.svg - sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile index fa7cdc27a1e6..9ecad9d224ec 100644 --- a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile @@ -12,5 +12,5 @@ COPY setup.py ./ RUN pip install tap-adwords==1.12.0 RUN pip install . -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-google-adwords-singer