diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json
index 383d92f827fb..0d82cc4e196f 100644
--- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json
+++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/fdc8b827-3257-4b33-83cc-106d234c34d4.json
@@ -2,6 +2,6 @@
"sourceDefinitionId": "fdc8b827-3257-4b33-83cc-106d234c34d4",
"name": "Google Adwords",
"dockerRepository": "airbyte/source-google-adwords-singer",
- "dockerImageTag": "0.1.4",
+ "dockerImageTag": "0.1.5",
"documentationUrl": "https://hub.docker.com/r/airbyte/source-google-adwords"
}
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
index ff288bd03778..3d36591b7cff 100644
--- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
+++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -11,7 +11,7 @@
- sourceDefinitionId: fdc8b827-3257-4b33-83cc-106d234c34d4
name: Google Adwords
dockerRepository: airbyte/source-google-adwords-singer
- dockerImageTag: 0.1.4
+ dockerImageTag: 0.1.5
documentationUrl: https://hub.docker.com/r/airbyte/source-google-adwords
- sourceDefinitionId: ef69ef6e-aa7f-4af1-a01d-ef775033524e
name: Github
diff --git a/airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py b/airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py
index 76f6cc565468..c460fc290473 100644
--- a/airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py
+++ b/airbyte-integrations/bases/base-singer/base_singer/singer_helpers.py
@@ -228,6 +228,9 @@ def create_singer_catalog_with_selection(masked_airbyte_catalog: ConfiguredAirby
replication_method = _FULL_TABLE
new_metadata["metadata"]["forced-replication-method"] = replication_method
new_metadata["metadata"]["replication-method"] = replication_method
+ else:
+ if "fieldExclusions" in new_metadata["metadata"]:
+ new_metadata["metadata"]["selected"] = True if not new_metadata["metadata"]["fieldExclusions"] else False
new_metadatas += [new_metadata]
singer_stream["metadata"] = new_metadatas
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile
index 0b86e60b5abd..adb6191d9005 100644
--- a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile
+++ b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile
@@ -12,5 +12,5 @@ COPY setup.py ./
RUN pip install tap-adwords==1.12.0
RUN pip install ".[main]"
-LABEL io.airbyte.version=0.1.4
+LABEL io.airbyte.version=0.1.5
LABEL io.airbyte.name=airbyte/source-google-adwords-singer
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile.test b/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile.test
deleted file mode 100644
index c8e26f13edf1..000000000000
--- a/airbyte-integrations/connectors/source-google-adwords-singer/Dockerfile.test
+++ /dev/null
@@ -1,22 +0,0 @@
-FROM airbyte/base-python-test:dev
-
-RUN apt-get update && rm -rf /var/lib/apt/lists/*
-
-ENV CODE_PATH="integration_tests"
-ENV AIRBYTE_TEST_MODULE="integration_tests"
-ENV AIRBYTE_TEST_PATH="SourceGoogleAdwordsSingerStandardTest"
-ENV AIRBYTE_TEST_CASE=true
-
-LABEL io.airbyte.version=0.1.0
-LABEL io.airbyte.name=airbyte/source-google-adwords-singer-standard-test
-
-WORKDIR /airbyte/integration_code
-COPY $CODE_PATH $CODE_PATH/
-COPY secrets/* $CODE_PATH/
-COPY sample_files/* $CODE_PATH/
-COPY source_google_adwords_singer/*.json $CODE_PATH/
-COPY setup.py ./
-
-RUN pip install ".[tests]"
-
-WORKDIR /airbyte
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle
index 91ff396b7d6e..c1d4f02ddf07 100644
--- a/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle
+++ b/airbyte-integrations/connectors/source-google-adwords-singer/build.gradle
@@ -2,7 +2,7 @@ plugins {
id 'java'
id 'airbyte-python'
id 'airbyte-docker'
- id 'airbyte-source-test'
+ id 'airbyte-standard-source-test-file'
}
airbytePython {
@@ -19,3 +19,10 @@ task('installSingerTap', type: PythonTask) {
command = "install tap-adwords==1.12.0"
}
installReqs.dependsOn installSingerTap
+
+airbyteStandardSourceTestFile {
+ // For more information on standard source tests, see https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/testing-connectors
+ specPath = "source_google_adwords_singer/spec.json"
+ configPath = "secrets/config.json"
+ configuredCatalogPath = "sample_files/configured_catalog.json"
+}
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/__init__.py
deleted file mode 100644
index 9ff2c68fb7b2..000000000000
--- a/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/__init__.py
+++ /dev/null
@@ -1,27 +0,0 @@
-"""
-MIT License
-
-Copyright (c) 2020 Airbyte
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-"""
-
-from .standard_source_test import SourceGoogleAdwordsSingerStandardTest
-
-__all__ = ["SourceGoogleAdwordsSingerStandardTest"]
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/standard_source_test.py b/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/standard_source_test.py
deleted file mode 100644
index 03e5f74f46e6..000000000000
--- a/airbyte-integrations/connectors/source-google-adwords-singer/integration_tests/standard_source_test.py
+++ /dev/null
@@ -1,51 +0,0 @@
-"""
-MIT License
-
-Copyright (c) 2020 Airbyte
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-"""
-
-import json
-import pkgutil
-
-from airbyte_protocol import ConfiguredAirbyteCatalog, ConnectorSpecification
-from base_python_test import StandardSourceTestIface
-
-
-class SourceGoogleAdwordsSingerStandardTest(StandardSourceTestIface):
- def __init__(self):
- pass
-
- def get_spec(self) -> ConnectorSpecification:
- raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json")
- return ConnectorSpecification.parse_obj(json.loads(raw_spec))
-
- def get_config(self) -> object:
- return json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], "config.json"))
-
- def get_catalog(self) -> ConfiguredAirbyteCatalog:
- raw_catalog = pkgutil.get_data(self.__class__.__module__.split(".")[0], "catalog.json")
- return ConfiguredAirbyteCatalog.parse_obj(json.loads(raw_catalog))
-
- def setup(self) -> None:
- pass
-
- def teardown(self) -> None:
- pass
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json
new file mode 100644
index 000000000000..328eb0f1a22c
--- /dev/null
+++ b/airbyte-integrations/connectors/source-google-adwords-singer/sample_files/configured_catalog.json
@@ -0,0 +1,1014 @@
+{
+ "streams": [
+ {
+ "stream": {
+ "name": "campaigns",
+ "json_schema": {
+ "additionalProperties": false,
+ "type": ["null", "object"],
+ "properties": {
+ "campaignTrialType": {
+ "type": ["null", "string"]
+ },
+ "endDate": {
+ "type": ["null", "string"],
+ "format": "date-time"
+ },
+ "settings": {
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "details": {
+ "type": ["null", "array"],
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "criterionTypeGroup": {
+ "type": ["null", "string"]
+ },
+ "targetAll": {
+ "type": ["null", "boolean"]
+ }
+ }
+ }
+ },
+ "languageCode": {
+ "type": ["null", "string"]
+ },
+ "negativeGeoTargetType": {
+ "type": ["null", "string"]
+ },
+ "Setting.Type": {
+ "type": ["null", "string"]
+ },
+ "positiveGeoTargetType": {
+ "type": ["null", "string"]
+ },
+ "domainName": {
+ "type": ["null", "string"]
+ }
+ }
+ },
+ "type": ["null", "array"]
+ },
+ "startDate": {
+ "type": ["null", "string"],
+ "format": "date-time"
+ },
+ "budgetId": {
+ "type": ["null", "integer"]
+ },
+ "name": {
+ "type": ["null", "string"]
+ },
+ "id": {
+ "type": ["null", "integer"]
+ },
+ "adServingOptimizationStatus": {
+ "type": ["null", "string"]
+ },
+ "conversionOptimizerEligibility": {
+ "type": ["null", "object"],
+ "properties": {
+ "eligible": {
+ "type": ["null", "integer"]
+ }
+ }
+ },
+ "advertisingChannelType": {
+ "type": ["null", "string"]
+ },
+ "networkSetting": {
+ "type": ["null", "object"],
+ "properties": {
+ "targetGoogleSearch": {
+ "type": ["null", "integer"]
+ }
+ }
+ },
+ "baseCampaignId": {
+ "type": ["null", "integer"]
+ },
+ "frequencyCap": {
+ "type": ["null", "object"],
+ "properties": {
+ "impressions": {
+ "type": ["null", "integer"]
+ },
+ "timeUnit": {
+ "type": ["null", "string"]
+ }
+ }
+ },
+ "labels": {
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "attribute": {
+ "type": ["null", "object"],
+ "properties": {
+ "LabelAttribute.Type": {
+ "type": ["null", "string"]
+ },
+ "backgroundColor": {
+ "type": ["null", "string"]
+ },
+ "description": {
+ "type": ["null", "string"]
+ }
+ }
+ },
+ "name": {
+ "type": ["null", "string"]
+ },
+ "Label.Type": {
+ "type": ["null", "string"]
+ },
+ "id": {
+ "type": ["null", "integer"]
+ },
+ "status": {
+ "type": ["null", "string"]
+ }
+ }
+ },
+ "type": ["null", "array"]
+ },
+ "status": {
+ "type": ["null", "string"]
+ },
+ "servingStatus": {
+ "type": ["null", "string"]
+ },
+ "urlCustomParameters": {
+ "type": ["null", "object"],
+ "properties": {
+ "parameters": {
+ "type": ["array", "null"],
+ "items": {
+ "type": ["object", "null"],
+ "properties": {
+ "key": {
+ "type": "string"
+ },
+ "value": {
+ "type": ["string", "null"]
+ },
+ "isRemove": {
+ "type": "boolean"
+ }
+ }
+ }
+ },
+ "doReplace": {
+ "type": "boolean"
+ }
+ }
+ }
+ }
+ },
+ "supported_sync_modes": ["full_refresh"],
+ "source_defined_cursor": false
+ },
+ "sync_mode": "full_refresh",
+ "cursor_field": []
+ },
+ {
+ "stream": {
+ "name": "ad_groups",
+ "json_schema": {
+ "additionalProperties": false,
+ "type": ["null", "object"],
+ "properties": {
+ "status": {
+ "type": ["null", "string"]
+ },
+ "campaignName": {
+ "type": ["null", "string"]
+ },
+ "id": {
+ "type": ["null", "integer"]
+ },
+ "campaignId": {
+ "type": ["null", "integer"]
+ },
+ "baseCampaignId": {
+ "type": ["null", "integer"]
+ },
+ "name": {
+ "type": ["null", "string"]
+ },
+ "adGroupType": {
+ "type": ["null", "string"]
+ },
+ "settings": {
+ "type": ["null", "array"],
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "details": {
+ "type": ["null", "array"],
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "targetAll": {
+ "type": ["null", "integer"]
+ },
+ "criterionTypeGroup": {
+ "type": ["null", "string"]
+ }
+ }
+ }
+ },
+ "Setting.Type": {
+ "type": ["null", "string"]
+ },
+ "optIn": {
+ "type": ["null", "integer"]
+ }
+ }
+ }
+ },
+ "biddingStrategyConfiguration": {
+ "type": ["null", "object"],
+ "properties": {
+ "bids": {
+ "type": ["null", "array"],
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "bid": {
+ "type": ["null", "object"],
+ "properties": {
+ "ComparableValue.Type": {
+ "type": ["null", "string"]
+ },
+ "microAmount": {
+ "type": ["null", "integer"]
+ }
+ }
+ },
+ "bidSource": {
+ "type": ["null", "string"]
+ },
+ "Bids.Type": {
+ "type": ["null", "string"]
+ }
+ }
+ }
+ }
+ }
+ },
+ "baseAdGroupId": {
+ "type": ["null", "integer"]
+ },
+ "labels": {
+ "type": ["null", "array"],
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "id": {
+ "type": ["null", "integer"]
+ },
+ "name": {
+ "type": ["null", "string"]
+ },
+ "attribute": {
+ "type": ["null", "object"],
+ "properties": {
+ "backgroundColor": {
+ "type": ["null", "string"]
+ },
+ "LabelAttribute.Type": {
+ "type": ["null", "string"]
+ }
+ }
+ },
+ "Label.Type": {
+ "type": ["null", "string"]
+ },
+ "status": {
+ "type": ["null", "string"]
+ }
+ }
+ }
+ },
+ "urlCustomParameters": {
+ "type": ["null", "object"],
+ "properties": {
+ "parameters": {
+ "type": ["array", "null"],
+ "items": {
+ "type": ["object", "null"],
+ "properties": {
+ "key": {
+ "type": "string"
+ },
+ "value": {
+ "type": ["string", "null"]
+ },
+ "isRemove": {
+ "type": "boolean"
+ }
+ }
+ }
+ },
+ "doReplace": {
+ "type": "boolean"
+ }
+ }
+ }
+ }
+ },
+ "supported_sync_modes": ["full_refresh"],
+ "source_defined_cursor": false
+ },
+ "sync_mode": "full_refresh",
+ "cursor_field": []
+ },
+ {
+ "stream": {
+ "name": "ads",
+ "json_schema": {
+ "type": ["null", "object"],
+ "properties": {
+ "status": {
+ "type": ["null", "string"]
+ },
+ "baseAdGroupId": {
+ "type": ["null", "integer"]
+ },
+ "policySummary": {
+ "type": ["null", "object"],
+ "properties": {
+ "reviewState": {
+ "type": ["null", "string"]
+ },
+ "denormalizedStatus": {
+ "type": ["null", "string"]
+ },
+ "policyTopicEntries": {
+ "type": ["null", "array"],
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "policyTopicName": {
+ "type": ["null", "string"]
+ },
+ "policyTopicEvidences": {
+ "type": ["null", "array"],
+ "items": {
+ "type": ["null", "object"],
+ "properties": {
+ "policyTopicEvidenceType": {
+ "type": ["null", "string"]
+ },
+ "evidenceTextList": {
+ "type": ["null", "array"],
+ "items": {
+ "type": ["null", "string"]
+ }
+ }
+ }
+ }
+ },
+ "policyTopicEntryType": {
+ "type": ["null", "string"]
+ },
+ "policyTopicId": {
+ "type": ["null", "string"]
+ },
+ "policyTopicHelpCenterUrl": {
+ "type": ["null", "string"]
+ }
+ }
+ }
+ },
+ "combinedApprovalStatus": {
+ "type": ["null", "string"]
+ }
+ }
+ },
+ "baseCampaignId": {
+ "type": ["null", "integer"]
+ },
+ "adGroupId": {
+ "type": ["null", "integer"]
+ },
+ "trademarkDisapproved": {
+ "type": ["null", "boolean"]
+ }
+ }
+ },
+ "supported_sync_modes": ["full_refresh"],
+ "source_defined_cursor": false
+ },
+ "sync_mode": "full_refresh",
+ "cursor_field": []
+ },
+ {
+ "stream": {
+ "name": "accounts",
+ "json_schema": {
+ "properties": {
+ "name": {
+ "type": ["null", "string"]
+ },
+ "canManageClients": {
+ "type": ["null", "boolean"]
+ },
+ "customerId": {
+ "type": ["null", "integer"]
+ },
+ "testAccount": {
+ "type": ["null", "boolean"]
+ },
+ "dateTimeZone": {
+ "type": ["null", "string"]
+ },
+ "currencyCode": {
+ "type": ["null", "string"]
+ }
+ },
+ "type": ["null", "object"]
+ },
+ "supported_sync_modes": ["full_refresh"],
+ "source_defined_cursor": false
+ },
+ "sync_mode": "full_refresh",
+ "cursor_field": []
+ },
+ {
+ "stream": {
+ "name": "KEYWORDS_PERFORMANCE_REPORT",
+ "json_schema": {
+ "type": "object",
+ "is_report": "true",
+ "properties": {
+ "imprAbsTop": {
+ "description": "Impr. (Abs. Top) %",
+ "type": ["null", "number"]
+ },
+ "currency": {
+ "description": "Currency",
+ "type": ["null", "string"]
+ },
+ "account": {
+ "description": "Account",
+ "type": ["null", "string"]
+ },
+ "timeZone": {
+ "description": "Time zone",
+ "type": ["null", "string"]
+ },
+ "activeViewAvgCPM": {
+ "description": "Active View avg. CPM",
+ "type": ["null", "integer", "string"]
+ },
+ "activeViewViewableCTR": {
+ "description": "Active View viewable CTR",
+ "type": ["null", "number"]
+ },
+ "activeViewViewableImpressions": {
+ "description": "Active View viewable impressions",
+ "type": ["null", "integer"]
+ },
+ "activeViewMeasurableImprImpr": {
+ "description": "Active View measurable impr. / impr.",
+ "type": ["null", "number"]
+ },
+ "activeViewMeasurableCost": {
+ "description": "Active View measurable cost",
+ "type": ["null", "integer", "string"]
+ },
+ "activeViewMeasurableImpr": {
+ "description": "Active View measurable impr.",
+ "type": ["null", "integer"]
+ },
+ "activeViewViewableImprMeasurableImpr": {
+ "description": "Active View viewable impr. / measurable impr.",
+ "type": ["null", "number"]
+ },
+ "adGroupID": {
+ "description": "Ad group ID",
+ "type": ["null", "integer"]
+ },
+ "adGroup": {
+ "description": "Ad group",
+ "type": ["null", "string"]
+ },
+ "adGroupState": {
+ "description": "Ad group state",
+ "type": ["null", "string"]
+ },
+ "network": {
+ "description": "Network",
+ "type": ["null", "string"]
+ },
+ "networkWithSearchPartners": {
+ "description": "Network (with search partners)",
+ "type": ["null", "string"]
+ },
+ "allConvRate": {
+ "description": "All conv. rate",
+ "type": ["null", "number"]
+ },
+ "allConv": {
+ "description": "All conv.",
+ "type": ["null", "number"]
+ },
+ "allConvValue": {
+ "description": "All conv. value",
+ "type": ["null", "number"]
+ },
+ "approvalStatus": {
+ "description": "Approval Status",
+ "type": ["null", "string"]
+ },
+ "avgCost": {
+ "description": "Avg. Cost",
+ "type": ["null", "integer", "string"]
+ },
+ "avgCPC": {
+ "description": "Avg. CPC",
+ "type": ["null", "integer", "string"]
+ },
+ "avgCPE": {
+ "description": "Avg. CPE",
+ "type": ["null", "number"]
+ },
+ "avgCPM": {
+ "description": "Avg. CPM",
+ "type": ["null", "integer", "string"]
+ },
+ "avgCPV": {
+ "description": "Avg. CPV",
+ "type": ["null", "number"]
+ },
+ "pagesSession": {
+ "description": "Pages / session",
+ "type": ["null", "number"]
+ },
+ "avgPosition": {
+ "description": "Avg. position",
+ "type": ["null", "number"]
+ },
+ "avgSessionDurationSeconds": {
+ "description": "Avg. session duration (seconds)",
+ "type": ["null", "number"]
+ },
+ "baseAdGroupID": {
+ "description": "Base Ad group ID",
+ "type": ["null", "integer"]
+ },
+ "baseCampaignID": {
+ "description": "Base Campaign ID",
+ "type": ["null", "integer"]
+ },
+ "bidStrategyID": {
+ "description": "Bid Strategy ID",
+ "type": ["null", "integer"]
+ },
+ "bidStrategyName": {
+ "description": "Bid Strategy Name",
+ "type": ["null", "string"]
+ },
+ "biddingStrategySource": {
+ "description": "Bidding Strategy Source",
+ "type": ["null", "string"]
+ },
+ "bidStrategyType": {
+ "description": "Bid Strategy Type",
+ "type": ["null", "string"]
+ },
+ "bounceRate": {
+ "description": "Bounce rate",
+ "type": ["null", "number"]
+ },
+ "campaignID": {
+ "description": "Campaign ID",
+ "type": ["null", "integer"]
+ },
+ "campaign": {
+ "description": "Campaign",
+ "type": ["null", "string"]
+ },
+ "campaignState": {
+ "description": "Campaign state",
+ "type": ["null", "string"]
+ },
+ "clickAssistedConv": {
+ "description": "Click Assisted Conv.",
+ "type": ["null", "integer"]
+ },
+ "clickAssistedConvLastClickConv": {
+ "description": "Click Assisted Conv. / Last Click Conv.",
+ "type": ["null", "number"]
+ },
+ "clickAssistedConvValue": {
+ "description": "Click Assisted Conv. Value",
+ "type": ["null", "number"]
+ },
+ "clicks": {
+ "description": "Clicks",
+ "type": ["null", "integer"]
+ },
+ "clickType": {
+ "description": "Click type",
+ "type": ["null", "string"]
+ },
+ "conversionAdjustment": {
+ "description": "Conversion adjustment",
+ "type": ["null", "string"]
+ },
+ "daysToConversionOrAdjustment": {
+ "description": "Days to conversion or adjustment",
+ "type": ["null", "string"]
+ },
+ "conversionCategory": {
+ "description": "Conversion category",
+ "type": ["null", "string"]
+ },
+ "daysToConversion": {
+ "description": "Days to conversion",
+ "type": ["null", "string"]
+ },
+ "convRate": {
+ "description": "Conv. rate",
+ "type": ["null", "number"]
+ },
+ "conversions": {
+ "description": "Conversions",
+ "type": ["null", "number"]
+ },
+ "conversionTrackerId": {
+ "description": "Conversion Tracker Id",
+ "type": ["null", "integer"]
+ },
+ "conversionName": {
+ "description": "Conversion name",
+ "type": ["null", "string"]
+ },
+ "totalConvValue": {
+ "description": "Total conv. value",
+ "type": ["null", "number"]
+ },
+ "cost": {
+ "description": "Cost",
+ "type": ["null", "integer", "string"]
+ },
+ "costAllConv": {
+ "description": "Cost / all conv.",
+ "type": ["null", "integer", "string"]
+ },
+ "costConv": {
+ "description": "Cost / conv.",
+ "type": ["null", "integer", "string"]
+ },
+ "costConvCurrentModel": {
+ "description": "Cost / conv. (current model)",
+ "type": ["null", "number"]
+ },
+ "maxCPC": {
+ "description": "Max. CPC",
+ "type": ["null", "integer", "string"]
+ },
+ "maxCPCSource": {
+ "description": "Max CPC source",
+ "type": ["null", "string"]
+ },
+ "maxCPM": {
+ "description": "Max. CPM",
+ "type": ["null", "integer", "string"]
+ },
+ "adRelevance": {
+ "description": "Ad relevance",
+ "type": ["null", "string"]
+ },
+ "keyword": {
+ "description": "Keyword",
+ "type": ["null", "string"]
+ },
+ "destinationURL": {
+ "description": "Destination URL",
+ "type": ["null", "string"]
+ },
+ "crossDeviceConv": {
+ "description": "Cross-device conv.",
+ "type": ["null", "number"]
+ },
+ "ctr": {
+ "description": "CTR",
+ "type": ["null", "number"]
+ },
+ "conversionsCurrentModel": {
+ "description": "Conversions (current model)",
+ "type": ["null", "number"]
+ },
+ "convValueCurrentModel": {
+ "description": "Conv. value (current model)",
+ "type": ["null", "number"]
+ },
+ "clientName": {
+ "description": "Client name",
+ "type": ["null", "string"]
+ },
+ "day": {
+ "description": "Day",
+ "type": ["null", "string"],
+ "format": "date-time"
+ },
+ "dayOfWeek": {
+ "description": "Day of week",
+ "type": ["null", "string"]
+ },
+ "device": {
+ "description": "Device",
+ "type": ["null", "string"]
+ },
+ "engagementRate": {
+ "description": "Engagement rate",
+ "type": ["null", "number"]
+ },
+ "engagements": {
+ "description": "Engagements",
+ "type": ["null", "integer"]
+ },
+ "enhancedCPCEnabled": {
+ "description": "Enhanced CPC enabled",
+ "type": ["null", "string"]
+ },
+ "estAddClicksWkFirstPositionBid": {
+ "description": "Est. add. clicks/wk (first position bid)",
+ "type": ["null", "integer"]
+ },
+ "estAddCostWkFirstPositionBid": {
+ "description": "Est. add. cost/wk (first position bid)",
+ "type": ["null", "integer", "string"]
+ },
+ "conversionSource": {
+ "description": "Conversion source",
+ "type": ["null", "string"]
+ },
+ "customerID": {
+ "description": "Customer ID",
+ "type": ["null", "integer"]
+ },
+ "appFinalURL": {
+ "description": "App final URL",
+ "type": ["null", "string"]
+ },
+ "mobileFinalURL": {
+ "description": "Mobile final URL",
+ "type": ["null", "string"]
+ },
+ "finalURL": {
+ "description": "Final URL",
+ "type": ["null", "string"]
+ },
+ "finalURLSuffix": {
+ "description": "Final URL suffix",
+ "type": ["null", "string"]
+ },
+ "firstPageCPC": {
+ "description": "First page CPC",
+ "type": ["null", "string"]
+ },
+ "firstPositionCPC": {
+ "description": "First position CPC",
+ "type": ["null", "string"]
+ },
+ "gmailForwards": {
+ "description": "Gmail forwards",
+ "type": ["null", "integer"]
+ },
+ "gmailSaves": {
+ "description": "Gmail saves",
+ "type": ["null", "integer"]
+ },
+ "gmailClicksToWebsite": {
+ "description": "Gmail clicks to website",
+ "type": ["null", "integer"]
+ },
+ "hasQualityScore": {
+ "description": "Has Quality Score",
+ "type": ["null", "string"]
+ },
+ "adRelevanceHist": {
+ "description": "Ad relevance (hist.)",
+ "type": ["null", "string"]
+ },
+ "landingPageExperienceHist": {
+ "description": "Landing page experience (hist.)",
+ "type": ["null", "string"]
+ },
+ "qualScoreHist": {
+ "description": "Qual. score (hist.)",
+ "type": ["null", "integer"]
+ },
+ "expectedClickthroughRateHist": {
+ "description": "Expected clickthrough rate (hist.)",
+ "type": ["null", "string"]
+ },
+ "keywordID": {
+ "description": "Keyword ID",
+ "type": ["null", "integer"]
+ },
+ "imprAssistedConv": {
+ "description": "Impr. Assisted Conv.",
+ "type": ["null", "integer"]
+ },
+ "imprAssistedConvLastClickConv": {
+ "description": "Impr. Assisted Conv. / Last Click Conv.",
+ "type": ["null", "number"]
+ },
+ "imprAssistedConvValue": {
+ "description": "Impr. Assisted Conv. Value",
+ "type": ["null", "number"]
+ },
+ "impressions": {
+ "description": "Impressions",
+ "type": ["null", "integer"]
+ },
+ "interactionRate": {
+ "description": "Interaction Rate",
+ "type": ["null", "number"]
+ },
+ "interactions": {
+ "description": "Interactions",
+ "type": ["null", "integer"]
+ },
+ "interactionTypes": {
+ "description": "Interaction Types",
+ "type": ["null", "string"]
+ },
+ "isNegative": {
+ "description": "Is negative",
+ "type": ["null", "string"]
+ },
+ "matchType": {
+ "description": "Match type",
+ "type": ["null", "string"]
+ },
+ "labelIDs": {
+ "description": "Label IDs",
+ "type": ["null", "string"]
+ },
+ "labels": {
+ "description": "Labels",
+ "type": ["null", "string"]
+ },
+ "month": {
+ "description": "Month",
+ "type": ["null", "string"]
+ },
+ "monthOfYear": {
+ "description": "Month of Year",
+ "type": ["null", "string"]
+ },
+ "newSessions": {
+ "description": "% new sessions",
+ "type": ["null", "number"]
+ },
+ "landingPageExperience": {
+ "description": "Landing page experience",
+ "type": ["null", "string"]
+ },
+ "qualityScore": {
+ "description": "Quality score",
+ "type": ["null", "integer"]
+ },
+ "quarter": {
+ "description": "Quarter",
+ "type": ["null", "string"]
+ },
+ "searchAbsTopIS": {
+ "description": "Search abs. top IS",
+ "type": ["null", "number"]
+ },
+ "searchLostAbsTopISBudget": {
+ "description": "Search lost abs. top IS (budget)",
+ "type": ["null", "number"]
+ },
+ "searchLostTopISBudget": {
+ "description": "Search lost top IS (budget)",
+ "type": ["null", "number"]
+ },
+ "searchExactMatchIS": {
+ "description": "Search Exact match IS",
+ "type": ["null", "number"]
+ },
+ "searchImprShare": {
+ "description": "Search Impr. share",
+ "type": ["null", "number"]
+ },
+ "expectedClickthroughRate": {
+ "description": "Expected clickthrough rate",
+ "type": ["null", "string"]
+ },
+ "searchLostAbsTopISRank": {
+ "description": "Search lost abs. top IS (rank)",
+ "type": ["null", "number"]
+ },
+ "searchLostISRank": {
+ "description": "Search Lost IS (rank)",
+ "type": ["null", "number"]
+ },
+ "searchLostTopISRank": {
+ "description": "Search lost top IS (rank)",
+ "type": ["null", "number"]
+ },
+ "searchTopIS": {
+ "description": "Search top IS",
+ "type": ["null", "number"]
+ },
+ "topVsOther": {
+ "description": "Top vs. Other",
+ "type": ["null", "string"]
+ },
+ "keywordState": {
+ "description": "Keyword state",
+ "type": ["null", "string"]
+ },
+ "criterionServingStatus": {
+ "description": "Criterion serving status",
+ "type": ["null", "string"]
+ },
+ "imprTop": {
+ "description": "Impr. (Top) %",
+ "type": ["null", "number"]
+ },
+ "topOfPageCPC": {
+ "description": "Top of page CPC",
+ "type": ["null", "string"]
+ },
+ "trackingTemplate": {
+ "description": "Tracking template",
+ "type": ["null", "string"]
+ },
+ "customParameter": {
+ "description": "Custom parameter",
+ "type": ["null", "string"]
+ },
+ "valueAllConv": {
+ "description": "Value / all conv.",
+ "type": ["null", "number"]
+ },
+ "valueConv": {
+ "description": "Value / conv.",
+ "type": ["null", "number"]
+ },
+ "valueConvCurrentModel": {
+ "description": "Value / conv. (current model)",
+ "type": ["null", "number"]
+ },
+ "verticalID": {
+ "description": "Vertical ID",
+ "type": ["null", "integer"]
+ },
+ "videoPlayedTo100": {
+ "description": "Video played to 100%",
+ "type": ["null", "number"]
+ },
+ "videoPlayedTo25": {
+ "description": "Video played to 25%",
+ "type": ["null", "number"]
+ },
+ "videoPlayedTo50": {
+ "description": "Video played to 50%",
+ "type": ["null", "number"]
+ },
+ "videoPlayedTo75": {
+ "description": "Video played to 75%",
+ "type": ["null", "number"]
+ },
+ "viewRate": {
+ "description": "View rate",
+ "type": ["null", "number"]
+ },
+ "views": {
+ "description": "Views",
+ "type": ["null", "integer"]
+ },
+ "viewThroughConv": {
+ "description": "View-through conv.",
+ "type": ["null", "integer"]
+ },
+ "week": {
+ "description": "Week",
+ "type": ["null", "string"]
+ },
+ "year": {
+ "description": "Year",
+ "type": ["null", "integer"]
+ }
+ }
+ },
+ "supported_sync_modes": ["incremental"],
+ "source_defined_cursor": false
+ },
+ "sync_mode": "incremental",
+ "cursor_field": []
+ }
+ ]
+}
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/source_google_adwords_singer/source.py b/airbyte-integrations/connectors/source-google-adwords-singer/source_google_adwords_singer/source.py
index 67407c004a3f..fd22b2980cc0 100644
--- a/airbyte-integrations/connectors/source-google-adwords-singer/source_google_adwords_singer/source.py
+++ b/airbyte-integrations/connectors/source-google-adwords-singer/source_google_adwords_singer/source.py
@@ -23,38 +23,139 @@
"""
import json
-import os
+import sys
+from typing import Dict, List
-from airbyte_protocol import AirbyteCatalog, AirbyteConnectionStatus, Status
-from base_python import AirbyteLogger, CatalogHelper
-from base_singer import SingerHelper, SingerSource
+from airbyte_protocol import AirbyteConnectionStatus, Status
+from base_python import AirbyteLogger
+from base_singer import SingerSource, SyncMode, SyncModeInfo
+from googleads import adwords, oauth2
+from tap_adwords import VERSION
class SourceGoogleAdwordsSinger(SingerSource):
+ @staticmethod
+ def _get_accounts(logger: AirbyteLogger, sdk_client: adwords.AdWordsClient, selector: Dict):
+ # obtaining accounts for customer_id
+ managed_customer_page = sdk_client.GetService(service_name="ManagedCustomerService", version=VERSION).get(selector)
+ accounts = managed_customer_page.entries
+ return accounts
+
+ def _check_internal(self, logger: AirbyteLogger, streams: List, config: json):
+ # checking if REPORT syncing will be called for manager account
+ # https://developers.google.com/adwords/api/docs/common-errors#ReportDefinitionError.CUSTOMER_SERVING_TYPE_REPORT_MISMATCH
+ try:
+ customer_ids = config["customer_ids"].split(",")
+ oauth2_client = oauth2.GoogleRefreshTokenClient(
+ config["oauth_client_id"], config["oauth_client_secret"], config["refresh_token"]
+ )
+ for customer_id in customer_ids:
+ sdk_client = adwords.AdWordsClient(
+ config["developer_token"], oauth2_client, user_agent=config["user_agent"], client_customer_id=customer_id
+ )
+ selector = {
+ "fields": ["Name", "CanManageClients", "CustomerId", "TestAccount", "DateTimeZone", "CurrencyCode"],
+ "predicates": [
+ {
+ "field": "CustomerId",
+ "operator": "IN",
+ "values": [
+ customer_id,
+ ],
+ }
+ ],
+ }
+ accounts = self._get_accounts(logger, sdk_client, selector)
+ if accounts:
+ account = accounts[0]
+ is_manager = account.canManageClients
+ for stream in streams:
+ if stream.endswith("REPORT") and is_manager:
+ logger.log_by_prefix(f"Unable to sync {stream} with the manager account {customer_id}", "ERROR")
+ sys.exit(1)
+ else:
+ err = f"No accounts associated with customer id {customer_id}"
+ logger.log_by_prefix(f"Unable to sync with the provided credentials. Error: {err}", "ERROR")
+ sys.exit(1)
+ except Exception as err:
+ logger.log_by_prefix(f"Unable to sync. Error: {err}", "ERROR")
+ sys.exit(1)
+
def check_config(self, logger: AirbyteLogger, config_path: str, config: json) -> AirbyteConnectionStatus:
+ # singer catalog that attempts to pull a stream ("accounts") that should always exists, though it may be empty.
try:
- # singer catalog that attempts to pull a stream ("accounts") that should always exists, though it may be empty.
- singer_check_catalog_path = os.path.abspath(os.path.dirname(__file__)) + "/singer_check_catalog.json"
- read_cmd = self.read_cmd(logger, config_path, singer_check_catalog_path)
- if SingerHelper.read(logger, read_cmd) is not None:
- return AirbyteConnectionStatus(status=Status.SUCCEEDED)
- else:
- return AirbyteConnectionStatus(status=Status.FAILED)
+ customer_ids = config["customer_ids"].split(",")
+ for customer_id in customer_ids:
+ oauth2_client = oauth2.GoogleRefreshTokenClient(
+ config["oauth_client_id"], config["oauth_client_secret"], config["refresh_token"]
+ )
+ sdk_client = adwords.AdWordsClient(
+ config["developer_token"], oauth2_client, user_agent=config["user_agent"], client_customer_id=customer_id
+ )
+ selector = {
+ "fields": ["Name", "CanManageClients", "CustomerId", "TestAccount", "DateTimeZone", "CurrencyCode"],
+ }
+ accounts = self._get_accounts(logger, sdk_client, selector)
+ if not accounts:
+ err = f"No accounts associated with customer id {customer_id}"
+ error_msg = f"Unable to connect with the provided credentials. Error: {err}"
+ return AirbyteConnectionStatus(status=Status.FAILED, message=error_msg)
+ return AirbyteConnectionStatus(status=Status.SUCCEEDED)
except Exception as e:
return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}")
+ def get_sync_mode_overrides(self) -> Dict[str, SyncModeInfo]:
+ incremental_streams = [
+ "ACCOUNT_PERFORMANCE_REPORT",
+ "AD_PERFORMANCE_REPORT",
+ "ADGROUP_PERFORMANCE_REPORT",
+ "AGE_RANGE_PERFORMANCE_REPORT",
+ "AUDIENCE_PERFORMANCE_REPORT",
+ "CALL_METRICS_CALL_DETAILS_REPORT",
+ "CAMPAIGN_PERFORMANCE_REPORT",
+ "CLICK_PERFORMANCE_REPORT",
+ "CRITERIA_PERFORMANCE_REPORT",
+ "DISPLAY_KEYWORD_PERFORMANCE_REPORT",
+ "DISPLAY_TOPICS_PERFORMANCE_REPORT",
+ "FINAL_URL_REPORT",
+ "GENDER_PERFORMANCE_REPORT",
+ "GEO_PERFORMANCE_REPORT",
+ "KEYWORDLESS_QUERY_REPORT",
+ "KEYWORDS_PERFORMANCE_REPORT",
+ "SEARCH_QUERY_PERFORMANCE_REPORT",
+ "VIDEO_PERFORMANCE_REPORT",
+ ]
+
+ full_refresh_streams = [
+ "accounts",
+ "ad_groups",
+ "campaigns",
+ "ads",
+ "PLACEHOLDER_FEED_ITEM_REPORT",
+ "PLACEMENT_PERFORMANCE_REPORT",
+ "SHOPPING_PERFORMANCE_REPORT",
+ "PLACEHOLDER_REPORT",
+ ]
+ overrides = {}
+ for stream_name in incremental_streams:
+ overrides[stream_name] = SyncModeInfo(supported_sync_modes=[SyncMode.incremental])
+ for stream_name in full_refresh_streams:
+ overrides[stream_name] = SyncModeInfo(supported_sync_modes=[SyncMode.full_refresh])
+ return overrides
+
def discover_cmd(self, logger, config_path) -> str:
return f"tap-adwords --config {config_path} --discover"
- def discover(self, logger: AirbyteLogger, config_container) -> AirbyteCatalog:
- catalog = super().discover(logger, config_container)
- return CatalogHelper.coerce_catalog_as_full_refresh(catalog)
-
def read_cmd(self, logger, config_path, catalog_path, state_path=None) -> str:
config_option = f"--config {config_path}"
properties_option = f"--properties {catalog_path}"
- return f"tap-adwords {config_option} {properties_option}"
+ state_option = f"--state {state_path}" if state_path else ""
+ streams = [
+ stream["stream"] for stream in self.read_config(catalog_path).get("streams", []) if stream["schema"].get("selected", False)
+ ]
+ self._check_internal(logger, streams, self.read_config(config_path))
+ return f"tap-adwords {config_option} {properties_option} {state_option}"
def transform_config(self, raw_config):
# required property in the singer tap, but seems like an implementation detail of stitch
diff --git a/airbyte-integrations/connectors/source-google-adwords-singer/source_google_adwords_singer/spec.json b/airbyte-integrations/connectors/source-google-adwords-singer/source_google_adwords_singer/spec.json
index 23167e20a267..e18c7af7f3b8 100644
--- a/airbyte-integrations/connectors/source-google-adwords-singer/source_google_adwords_singer/spec.json
+++ b/airbyte-integrations/connectors/source-google-adwords-singer/source_google_adwords_singer/spec.json
@@ -41,7 +41,8 @@
},
"customer_ids": {
"type": "string",
- "description": "Comma-separated list of a customer ids. More instruction on how to find this value in our docs"
+ "pattern": "^[0-9]{10}$",
+ "description": "Comma-separated list of a customer ids. Each customer id must be specified as a 10-digit number without dashes. More instruction on how to find this value in our docs"
}
}
}