From aa9786df4e40b337aead96a5685e0e7711a4d692 Mon Sep 17 00:00:00 2001 From: vovavovavovavova <39351371+vovavovavovavova@users.noreply.github.com> Date: Fri, 10 Sep 2021 09:17:16 +0300 Subject: [PATCH] :tada: Google Ads improvement: Support user-specified queries (#5302) *Add google ads custom queries stream *Display link to gradle scan on PR comment if test build failed --- .github/workflows/test-command.yml | 1 + .../connectors/source-google-ads/Dockerfile | 2 +- .../acceptance-test-config.yml | 18 +- .../integration_tests/configured_catalog.json | 18 ++ .../configured_catalog_protobuf_msg.json | 16 ++ ...figured_catalog_without_empty_streams.json | 22 ++ .../expected_records_msg.txt | 104 +++++++++ .../connectors/source-google-ads/setup.py | 4 +- .../source_google_ads/custom_query_stream.py | 165 +++++++++++++++ .../source_google_ads/google_ads.py | 40 +++- .../source_google_ads/source.py | 15 +- .../source_google_ads/spec.json | 22 ++ .../unit_tests/test_google_ads.py | 2 +- .../unit_tests/test_source.py | 198 ++++++++++++++++++ docs/integrations/sources/google-ads.md | 1 + tools/bin/ci_integration_test.sh | 18 +- 16 files changed, 628 insertions(+), 18 deletions(-) create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_protobuf_msg.json create mode 100644 airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_msg.txt create mode 100644 airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 96bb437261f6..5c3b241336dc 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -212,6 +212,7 @@ jobs: comment-id: ${{ github.event.inputs.comment-id }} body: | > :x: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + > :bug: ${{env.GRADLE_SCAN_LINK}} # In case of self-hosted EC2 errors, remove this block. stop-test-runner: name: Stop Build EC2 Runner diff --git a/airbyte-integrations/connectors/source-google-ads/Dockerfile b/airbyte-integrations/connectors/source-google-ads/Dockerfile index 4c4431372a61..1b8ee3151fca 100644 --- a/airbyte-integrations/connectors/source-google-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-google-ads/Dockerfile @@ -13,5 +13,5 @@ RUN pip install . ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.8 +LABEL io.airbyte.version=0.1.9 LABEL io.airbyte.name=airbyte/source-google-ads diff --git a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml index 195f07a29000..861e01d01908 100644 --- a/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-google-ads/acceptance-test-config.yml @@ -14,13 +14,17 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog_without_empty_streams.json" -# TODO incremental test is disabled because records output from the report streams can be up to 14 days older than the input state -# incremental: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# future_state_path: "integration_tests/abnormal_state.json" -# cursor_paths: -# ad_group_ad_report: ["segments.date"] + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_protobuf_msg.json" + expect_records: + path: "integration_tests/expected_records_msg.txt" + # TODO incremental test is disabled because records output from the report streams can be up to 14 days older than the input state + # incremental: + # - config_path: "secrets/config.json" + # configured_catalog_path: "integration_tests/configured_catalog.json" + # future_state_path: "integration_tests/abnormal_state.json" + # cursor_paths: + # ad_group_ad_report: ["segments.date"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json index 22fc57b3ac76..940eeffdfe91 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog.json @@ -99,6 +99,24 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "happytable", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "unhappytable", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_protobuf_msg.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_protobuf_msg.json new file mode 100644 index 000000000000..8b7efb166214 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_protobuf_msg.json @@ -0,0 +1,16 @@ +{ + "streams": [ + { + "stream": { + "name": "ad_group_custom", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["segments.date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["segments.date"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_without_empty_streams.json b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_without_empty_streams.json index 3b30a4c060b2..bb3bbd5afe56 100644 --- a/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_without_empty_streams.json +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/configured_catalog_without_empty_streams.json @@ -63,6 +63,28 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "happytable", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["campaign.start_date"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append", + "cursor_field": ["campaign.start_date"] + }, + { + "stream": { + "name": "unhappytable", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["customer.id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_msg.txt b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_msg.txt new file mode 100644 index 000000000000..4da3c9e89bb6 --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/integration_tests/expected_records_msg.txt @@ -0,0 +1,104 @@ +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-05-28"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-05-29"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-05-30"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-05-31"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-01"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-02"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-03"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-04"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-05"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-06"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-07"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-08"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-09"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-10"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-11"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-12"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-13"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-14"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-15"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-16"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-17"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-18"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-19"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-20"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-21"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-22"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-23"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-24"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-25"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-26"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-27"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-28"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-29"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-06-30"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-01"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-02"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-03"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-04"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-05"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-06"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-07"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-08"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-09"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-10"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-11"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-12"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-13"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-14"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-15"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-16"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-17"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-18"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-19"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-20"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-21"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-22"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-23"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-24"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-25"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-26"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-27"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-28"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-29"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-30"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-07-31"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-01"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-02"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-03"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-04"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-05"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-06"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-07"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-08"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-09"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-10"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-11"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-12"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-13"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-14"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-15"}, "emitted_at": 1631175784000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-16"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-17"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-18"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-19"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-20"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-21"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-22"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-23"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-24"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-25"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-26"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-27"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-28"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-29"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-30"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-08-31"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-09-01"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-09-02"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-09-03"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-09-04"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-09-05"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-09-06"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-09-07"}, "emitted_at": 1631175785000} +{"stream": "ad_group_custom", "data": {"ad_group.targeting_setting.target_restrictions": ["targeting_dimension: AUDIENCE\nbid_only: true\n", "targeting_dimension: AGE_RANGE\nbid_only: true\n", "targeting_dimension: GENDER\nbid_only: true\n", "targeting_dimension: PARENTAL_STATUS\nbid_only: true\n", "targeting_dimension: INCOME_RANGE\nbid_only: true\n"], "segments.date": "2021-09-08"}, "emitted_at": 1631175785000} diff --git a/airbyte-integrations/connectors/source-google-ads/setup.py b/airbyte-integrations/connectors/source-google-ads/setup.py index 9fa2efcf1a45..69e3d8ab00a2 100644 --- a/airbyte-integrations/connectors/source-google-ads/setup.py +++ b/airbyte-integrations/connectors/source-google-ads/setup.py @@ -25,9 +25,9 @@ from setuptools import find_packages, setup -MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-ads", "pendulum"] +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-ads==13.0.0", "pendulum"] -TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock", "pendulum"] +TEST_REQUIREMENTS = ["pytest~=6.1", "pytest-mock"] setup( name="source_google_ads", diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py new file mode 100644 index 000000000000..08322d5aa37b --- /dev/null +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/custom_query_stream.py @@ -0,0 +1,165 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +import re +from functools import lru_cache +from typing import Any, Dict, List, Mapping + +from .streams import IncrementalGoogleAdsStream + + +class CustomQuery(IncrementalGoogleAdsStream): + def __init__(self, custom_query_config, **kwargs): + self.custom_query_config = custom_query_config + self.user_defined_query = custom_query_config["query"] + super().__init__(**kwargs) + + @property + def primary_key(self) -> str: + """ + The primary_key option is disabled. Config should not provide the primary key. + It will be ignored if provided. + If you need to enable it, uncomment the next line instead of `return None` and modify your config + """ + # return self.custom_query_config.get("primary_key") or None + return None + + @property + def name(self): + return self.custom_query_config["table_name"] + + def get_query(self, stream_slice: Mapping[str, Any] = None) -> str: + start_date, end_date = self.get_date_params(stream_slice, self.cursor_field) + return self.insert_segments_date_expr(self.user_defined_query, start_date, end_date) + + # IncrementalGoogleAdsStream uses get_json_schema a lot while parsing + # responses, caching plaing crucial role for performance here. + @lru_cache() + def get_json_schema(self) -> Dict[str, Any]: + """ + Compose json schema based on user defined query. + :return Dict object representing jsonschema + """ + + local_json_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {}, + "additionalProperties": True, + } + # full list {'ENUM', 'STRING', 'DATE', 'DOUBLE', 'RESOURCE_NAME', 'INT32', 'INT64', 'BOOLEAN', 'MESSAGE'} + + google_datatype_mapping = { + "INT64": "integer", + "INT32": "integer", + "DOUBLE": "number", + "STRING": "string", + "BOOLEAN": "boolean", + "DATE": "string", + } + fields = CustomQuery.get_query_fields(self.user_defined_query) + fields.append(self.cursor_field) + google_schema = self.google_ads_client.get_fields_metadata(fields) + + for field in fields: + node = google_schema.get(field) + # Data type return in enum format: "GoogleAdsFieldDataType." + google_data_type = str(node.data_type).replace("GoogleAdsFieldDataType.", "") + if google_data_type == "ENUM": + field_value = {"type": "string", "enum": list(node.enum_values)} + elif google_data_type == "MESSAGE": + # Represents protobuf message and could be anything, set custom + # attribute "protobuf_message" to convert it to a string (or + # array of strings) later. + # https://developers.google.com/google-ads/api/reference/rpc/v8/GoogleAdsFieldDataTypeEnum.GoogleAdsFieldDataType?hl=en#message + if node.is_repeated: + output_type = ["array", "null"] + else: + output_type = ["string", "null"] + field_value = {"type": output_type, "protobuf_message": True} + else: + output_type = [google_datatype_mapping.get(google_data_type, "string"), "null"] + field_value = {"type": output_type} + local_json_schema["properties"][field] = field_value + + return local_json_schema + + # Regexp flags for parsing GAQL query + RE_FLAGS = re.DOTALL | re.MULTILINE | re.IGNORECASE + # Regexp for getting query columns + SELECT_EXPR = re.compile("select(.*)from", flags=RE_FLAGS) + WHERE_EXPR = re.compile("where.*", flags=RE_FLAGS) + # list of keywords that can come after WHERE clause, + # according to https://developers.google.com/google-ads/api/docs/query/grammar + KEYWORDS_EXPR = re.compile("(order by|limit|parameters)", flags=RE_FLAGS) + + @staticmethod + def get_query_fields(query: str) -> List[str]: + fields = CustomQuery.SELECT_EXPR.search(query) + if not fields: + return [] + fields = fields.group(1) + return [f.strip() for f in fields.split(",")] + + @staticmethod + def insert_segments_date_expr(query: str, start_date: str, end_date: str) -> str: + """ + Insert segments.date condition to break query into slices for incremental stream. + :param query Origin user defined query + :param start_date start date for metric (inclusive) + :param end_date end date for metric (inclusive) + :return Modified query with date window condition included + """ + # insert segments.date field + columns = CustomQuery.SELECT_EXPR.search(query) + if not columns: + raise Exception("Not valid GAQL expression") + columns = columns.group(1) + new_columns = columns + ", segments.date\n" + result_query = query.replace(columns, new_columns) + + # Modify/insert where condition + where_cond = CustomQuery.WHERE_EXPR.search(result_query) + if not where_cond: + # There is no where condition, insert new one + where_location = len(result_query) + keywords = CustomQuery.KEYWORDS_EXPR.search(result_query) + if keywords: + # where condition is not at the end of expression, insert new condition before keyword begins. + where_location = keywords.start() + result_query = ( + result_query[0:where_location] + + f"\nWHERE segments.date BETWEEN '{start_date}' AND '{end_date}'\n" + + result_query[where_location:] + ) + return result_query + # There is already where condition, add segments.date expression + where_cond = where_cond.group(0) + keywords = CustomQuery.KEYWORDS_EXPR.search(where_cond) + if keywords: + # There is some keywords after WHERE condition + where_cond = where_cond[0 : keywords.start()] + new_where_cond = where_cond + f" AND segments.date BETWEEN '{start_date}' AND '{end_date}'\n" + result_query = result_query.replace(where_cond, new_where_cond) + return result_query diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py index 00146acd3ff4..d32e0fa2c2fd 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/google_ads.py @@ -22,6 +22,7 @@ # SOFTWARE. # + from enum import Enum from typing import Any, List, Mapping @@ -60,10 +61,32 @@ def send_request(self, query: str) -> SearchGoogleAdsResponse: return self.ga_service.search(search_request) + def get_fields_metadata(self, fields: List[str]) -> Mapping[str, Any]: + """ + Issue Google API request to get detailed information on data type for custom query columns. + :params fields list of columns for user defined query. + :return dict of fields type info. + """ + + ga_field_service = self.client.get_service("GoogleAdsFieldService") + request = self.client.get_type("SearchGoogleAdsFieldsRequest") + request.page_size = len(fields) + fields_sql = ",".join([f"'{field}'" for field in fields]) + request.query = f""" + SELECT + name, + data_type, + enum_values, + is_repeated + WHERE name in ({fields_sql}) + """ + response = ga_field_service.search_google_ads_fields(request=request) + return {r.name: r for r in response} + @staticmethod def get_fields_from_schema(schema: Mapping[str, Any]) -> List[str]: properties = schema.get("properties") - return [*properties] + return list(properties.keys()) @staticmethod def convert_schema_into_query( @@ -82,7 +105,7 @@ def convert_schema_into_query( return query_template @staticmethod - def get_field_value(field_value: GoogleAdsRow, field: str) -> str: + def get_field_value(field_value: GoogleAdsRow, field: str, schema_type: Mapping[str, Any]) -> str: field_name = field.split(".") for level_attr in field_name: """ @@ -130,7 +153,6 @@ def get_field_value(field_value: GoogleAdsRow, field: str) -> str: # In GoogleAdsRow there are attributes that add an underscore at the end in their name. # For example, 'ad_group_ad.ad.type' is replaced by 'ad_group_ad.ad.type_'. field_value = getattr(field_value, level_attr + "_", None) - if isinstance(field_value, Enum): field_value = field_value.name elif isinstance(field_value, (Repeated, RepeatedComposite)): @@ -144,13 +166,23 @@ def get_field_value(field_value: GoogleAdsRow, field: str) -> str: # For example: # 1. ad_group_ad.ad.responsive_display_ad.long_headline - type AdTextAsset (https://developers.google.com/google-ads/api/reference/rpc/v6/AdTextAsset?hl=en). # 2. ad_group_ad.ad.legacy_app_install_ad - type LegacyAppInstallAdInfo (https://developers.google.com/google-ads/api/reference/rpc/v7/LegacyAppInstallAdInfo?hl=en). + # if not (isinstance(field_value, (list, int, float, str, bool, dict)) or field_value is None): field_value = str(field_value) + # In case of custom query field has MESSAGE type it represents protobuf + # message and could be anything, convert it to a string or array of + # string if it has "repeated" flag on metadata + if schema_type.get("protobuf_message"): + if "array" in schema_type.get("type"): + field_value = [str(field) for field in field_value] + else: + field_value = str(field_value) return field_value @staticmethod def parse_single_result(schema: Mapping[str, Any], result: GoogleAdsRow): + props = schema.get("properties") fields = GoogleAds.get_fields_from_schema(schema) - single_record = {field: GoogleAds.get_field_value(result, field) for field in fields} + single_record = {field: GoogleAds.get_field_value(result, field, props.get(field)) for field in fields} return single_record diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py index b5737c80362f..49184359ae07 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/source.py @@ -31,6 +31,7 @@ from airbyte_cdk.sources.streams import Stream from google.ads.googleads.errors import GoogleAdsException +from .custom_query_stream import CustomQuery from .google_ads import GoogleAds from .streams import ( AccountPerformanceReport, @@ -61,6 +62,13 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> google_api = GoogleAds(credentials=self.get_credentials(config), customer_id=config["customer_id"]) account_stream = Accounts(api=google_api) list(account_stream.read_records(sync_mode=SyncMode.full_refresh)) + # Check custom query request validity by sending metric request with non-existant time window + for q in config.get("custom_queries", []): + q = q.get("query") + if CustomQuery.cursor_field in q: + raise Exception(f"Custom query should not contain {CustomQuery.cursor_field}") + req_q = CustomQuery.insert_segments_date_expr(q, "1980-01-01", "1980-01-01") + google_api.send_request(req_q) return True, None except GoogleAdsException as error: return False, f"Unable to connect to Google Ads API with the provided credentials - {repr(error.failure)}" @@ -70,6 +78,11 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: incremental_stream_config = dict( api=google_api, conversion_window_days=config["conversion_window_days"], start_date=config["start_date"] ) + + custom_query_streams = [ + CustomQuery(custom_query_config=single_query_config, **incremental_stream_config) + for single_query_config in config.get("custom_queries", []) + ] return [ UserLocationReport(**incremental_stream_config), AccountPerformanceReport(**incremental_stream_config), @@ -81,4 +94,4 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: AdGroups(api=google_api), Accounts(api=google_api), Campaigns(api=google_api), - ] + ] + custom_query_streams diff --git a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json index 994f4206c8cf..8f16a46b34ae 100644 --- a/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json +++ b/airbyte-integrations/connectors/source-google-ads/source_google_ads/spec.json @@ -67,6 +67,28 @@ "maximum": 1095, "default": 14, "examples": [14] + }, + "custom_queries": { + "type": "array", + "title": "Custom GAQL Queries", + "items": { + "type": "object", + "properties": { + "query": { + "type": "string", + "title": "Custom query", + "description": "A custom defined GAQL query for building the report. Should not contain segments.date expression as it used by incremental streams", + "examples": [ + "SELECT segments.ad_destination_type, campaign.advertising_channel_sub_type FROM campaign WHERE campaign.status = 'PAUSED'" + ] + }, + "table_name": { + "type": "string", + "title": "Destination table name", + "description": "The table name in your destination database for choosen query." + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py index d271c266695a..91404bb99350 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_google_ads.py @@ -146,7 +146,7 @@ def test_convert_schema_into_query(): def test_get_field_value(): field = "segment.date" date = "2001-01-01" - response = GoogleAds.get_field_value(MockedDateSegment(date), field) + response = GoogleAds.get_field_value(MockedDateSegment(date), field, {}) assert response == date diff --git a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py index 3cbd14655e40..c5b713bd4320 100644 --- a/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-google-ads/unit_tests/test_source.py @@ -22,6 +22,8 @@ # SOFTWARE. # +import pytest +from source_google_ads.custom_query_stream import CustomQuery from source_google_ads.google_ads import GoogleAds from source_google_ads.streams import AdGroupAdReport, chunk_date_range @@ -49,3 +51,199 @@ def test_get_updated_state(config): latest_record = {"segments.date": "2020-02-01"} new_stream_state = client.get_updated_state(current_state_stream, latest_record) assert new_stream_state == {"segments.date": "2020-02-01"} + + +def get_instance_from_config(config, query): + start_date = "2021-03-04" + conversion_window_days = 14 + google_api = GoogleAds(credentials=config["credentials"], customer_id=config["customer_id"]) + + instance = CustomQuery( + api=google_api, + conversion_window_days=conversion_window_days, + start_date=start_date, + custom_query_config={"query": query, "table_name": "whatever_table"}, + ) + return instance + + +@pytest.mark.parametrize( + "query, fields", + [ + ( + """ + SELecT + campaign.id, + campaign.name, + campaign.status, + metrics.impressions FROM campaign +wheRe campaign.status = 'PAUSED' +AND metrics.impressions > 100 +order by campaign.status + """, + ["campaign.id", "campaign.name", "campaign.status", "metrics.impressions"], + ), + ( + """ + SELECT + campaign.accessible_bidding_strategy, + segments.ad_destination_type, + campaign.start_date, + campaign.end_date + FROM campaign + """, + ["campaign.accessible_bidding_strategy", "segments.ad_destination_type", "campaign.start_date", "campaign.end_date"], + ), + ("""selet aasdasd from aaa""", []), + ], +) +def test_get_query_fields(query, fields): + assert CustomQuery.get_query_fields(query) == fields + + +@pytest.mark.parametrize( + "original_query, expected_query", + [ + ( + """ +SELect + campaign.id, + campaign.name, + campaign.status, + metrics.impressions FROM campaign +wheRe campaign.status = 'PAUSED' +AND metrics.impressions > 100 +order by campaign.status +""", + """ +SELect + campaign.id, + campaign.name, + campaign.status, + metrics.impressions , segments.date +FROM campaign +wheRe campaign.status = 'PAUSED' +AND metrics.impressions > 100 + AND segments.date BETWEEN '1980-01-01' AND '2000-01-01' +order by campaign.status +""", + ), + ( + """ +SELect + campaign.id, + campaign.name, + campaign.status, + metrics.impressions +FROM campaign +order by campaign.status +""", + """ +SELect + campaign.id, + campaign.name, + campaign.status, + metrics.impressions +, segments.date +FROM campaign + +WHERE segments.date BETWEEN '1980-01-01' AND '2000-01-01' +order by campaign.status +""", + ), + ( + """ +SELect + campaign.id, + campaign.name, + campaign.status, + metrics.impressions FROM campaign +wheRe campaign.status = 'PAUSED' +AND metrics.impressions > 100 +""", + """ +SELect + campaign.id, + campaign.name, + campaign.status, + metrics.impressions , segments.date +FROM campaign +wheRe campaign.status = 'PAUSED' +AND metrics.impressions > 100 + AND segments.date BETWEEN '1980-01-01' AND '2000-01-01' +""", + ), + ( + "SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date FROM campaign", + """SELECT campaign.accessible_bidding_strategy, segments.ad_destination_type, campaign.start_date, campaign.end_date , segments.date +FROM campaign +WHERE segments.date BETWEEN '1980-01-01' AND '2000-01-01' +""", + ), + ], +) +def test_insert_date(original_query, expected_query): + assert CustomQuery.insert_segments_date_expr(original_query, "1980-01-01", "2000-01-01") == expected_query + + +def test_get_json_schema_parse_query(config): + query = """ + SELECT + campaign.accessible_bidding_strategy, + segments.ad_destination_type, + campaign.start_date, + campaign.end_date + FROM campaign + """ + final_fields = [ + "campaign.accessible_bidding_strategy", + "segments.ad_destination_type", + "campaign.start_date", + "campaign.end_date", + "segments.date", + ] + + instance = get_instance_from_config(config=config, query=query) + final_schema = instance.get_json_schema() + schema_keys = final_schema["properties"] + assert set(schema_keys) == set(final_fields) # test 1 + + +def test_google_type_conversion(config): + """ + query may be invalid (fields incompatibility did not checked). + But we are just testing types, without submitting the query and further steps. + Doing that with all possible types. + """ + desired_mapping = { + "accessible_bidding_strategy.target_impression_share.location": "string", # "ENUM" + "campaign.name": ["string", "null"], # STRING + "campaign.end_date": ["string", "null"], # DATE + "campaign.optimization_score": ["number", "null"], # DOUBLE + "campaign.resource_name": ["string", "null"], # RESOURCE_NAME + "campaign.shopping_setting.campaign_priority": ["integer", "null"], # INT32 + "campaign.shopping_setting.merchant_id": ["integer", "null"], # INT64 + "campaign_budget.explicitly_shared": ["boolean", "null"], # BOOLEAN + "bidding_strategy.enhanced_cpc": ["string", "null"], # MESSAGE + "segments.date": ["string", "null"], # autoadded, should be DATE + } + + # query is select field of each type + query = """ + SELECT + accessible_bidding_strategy.target_impression_share.location, + campaign.name, + campaign.end_date, + campaign.optimization_score, + campaign.resource_name, + campaign.shopping_setting.campaign_priority, + campaign.shopping_setting.merchant_id, + campaign_budget.explicitly_shared, + bidding_strategy.enhanced_cpc + FROM campaign + """ + instance = get_instance_from_config(config=config, query=query) + final_schema = instance.get_json_schema() + schema_properties = final_schema.get("properties") + for prop, value in schema_properties.items(): + assert desired_mapping[prop] == value.get("type"), f"{prop} should be {value}" diff --git a/docs/integrations/sources/google-ads.md b/docs/integrations/sources/google-ads.md index 5a38dde6970c..9fec2abc6cd4 100644 --- a/docs/integrations/sources/google-ads.md +++ b/docs/integrations/sources/google-ads.md @@ -87,6 +87,7 @@ The Google Ads Query Language can query the Google Ads API. Check out [Google Ad | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| `0.1.9` | 2021-09-07 | [#5302](https://github.com/airbytehq/airbyte/pull/5302) | Add custom query stream support | | `0.1.8` | 2021-08-03 | [#5509](https://github.com/airbytehq/airbyte/pull/5509) | additionalProperties in spec.json | | `0.1.7` | 2021-08-03 | [#5422](https://github.com/airbytehq/airbyte/pull/5422) | Correct query to not skip dates | | `0.1.6` | 2021-08-03 | [#5423](https://github.com/airbytehq/airbyte/pull/5423) | Added new stream UserLocationReport | diff --git a/tools/bin/ci_integration_test.sh b/tools/bin/ci_integration_test.sh index 861f69fb33ec..854e37325830 100755 --- a/tools/bin/ci_integration_test.sh +++ b/tools/bin/ci_integration_test.sh @@ -8,7 +8,7 @@ set -e connector="$1" all_integration_tests=$(./gradlew integrationTest --dry-run | grep 'integrationTest SKIPPED' | cut -d: -f 4) - +run() { if [[ "$connector" == "all" ]] ; then echo "Running: ./gradlew --no-daemon --scan integrationTest" ./gradlew --no-daemon --scan integrationTest @@ -36,6 +36,20 @@ else ./gradlew --no-daemon --scan "$integrationTestCommand" else echo "Connector '$connector' not found..." - exit 1 + return 1 fi fi +} + +# Copy command output to extract gradle scan link. +run | tee build.out +# return status of "run" command, not "tee" +# https://tldp.org/LDP/abs/html/internalvariables.html#PIPESTATUSREF +run_status=${PIPESTATUS[0]} +test $run_status == "0" || { + # Save gradle scan link to github GRADLE_SCAN_LINK variable for next job. + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#setting-an-environment-variable + LINK=$(cat build.out | grep -A1 "Publishing build scan..." | tail -n1 | tr -d "\n") + echo "GRADLE_SCAN_LINK=$LINK" >> $GITHUB_ENV +} +exit $run_status