diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json index 69023fb04c98..ffc80a647046 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1", "name": "Google Analytics", "dockerRepository": "airbyte/source-googleanalytics-singer", - "dockerImageTag": "0.2.4", + "dockerImageTag": "0.2.5", "documentationUrl": "https://hub.docker.com/r/airbyte/source-googleanalytics-singer", "icon": "google-analytics.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 1472012dd8ae..c280f647bbbe 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -100,7 +100,7 @@ - sourceDefinitionId: 39f092a6-8c87-4f6f-a8d9-5cef45b7dbe1 name: Google Analytics dockerRepository: airbyte/source-googleanalytics-singer - dockerImageTag: 0.2.4 + dockerImageTag: 0.2.5 documentationUrl: https://hub.docker.com/r/airbyte/source-googleanalytics-singer icon: google-analytics.svg - sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile b/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile index 48ba0055582c..a4371c49c760 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/Dockerfile @@ -8,7 +8,7 @@ ENV CODE_PATH="source_googleanalytics_singer" ENV AIRBYTE_IMPL_MODULE="source_googleanalytics_singer" ENV AIRBYTE_IMPL_PATH="GoogleAnalyticsSingerSource" -LABEL io.airbyte.version=0.2.4 +LABEL io.airbyte.version=0.2.5 LABEL io.airbyte.name=airbyte/source-googleanalytics-singer WORKDIR /airbyte/integration_code diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml b/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml new file mode 100644 index 000000000000..0f077cd3b443 --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/acceptance-test-config.yml @@ -0,0 +1,27 @@ +connector_image: airbyte/source-googleanalytics-singer:dev +tests: + spec: + - spec_path: "source_googleanalytics_singer/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "sample_files/configured_catalog.json" + validate_output_from_all_streams: yes +# Commented out due to the state structure is not supported by the SAT +# incremental: +# - config_path: "secrets/config.json" +# configured_catalog_path: "sample_files/configured_catalog.json" +# future_state_path: "sample_files/abnormal_state.json" +# cursor_paths: +# website_overview: ["report_start_date"] +# traffic_sources: ["report_start_date"] +# pages: ["report_start_date"] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle index 18cabfffe42d..a7f77917049c 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/build.gradle @@ -1,7 +1,7 @@ plugins { id 'airbyte-python' id 'airbyte-docker' - id 'airbyte-standard-source-test-file' + id 'airbyte-source-acceptance-test' } airbytePython { @@ -10,10 +10,11 @@ airbytePython { dependencies { - implementation files(project(':airbyte-integrations:bases:base-standard-source-test-file').airbyteDocker.outputs) implementation files(project(':airbyte-integrations:bases:base-singer').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) } + // TODO(sherifnada) re-enable tests. Currently they're disabled because they hog up so much bandwidth that they prevent local iteration at times. // airbyteStandardSourceTestFile { // specPath = "source_googleanalytics_singer/spec.json" diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/__init__.py b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/acceptance.py new file mode 100644 index 000000000000..d98ac8aa3a1c --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/acceptance.py @@ -0,0 +1,36 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..9381437e5c2b --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/configured_catalog.json @@ -0,0 +1,523 @@ +{ + "streams": [ + { + "stream": { + "name": "website_overview", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "traffic_sources", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_source": { + "type": ["string"] + }, + "ga_medium": { + "type": ["string"] + }, + "ga_socialNetwork": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "pages", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_hostname": { + "type": ["string"] + }, + "ga_pagePath": { + "type": ["string"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_uniquePageviews": { + "type": ["null", "integer"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_entrances": { + "type": ["null", "integer"] + }, + "ga_entranceRate": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exits": { + "type": ["null", "integer"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "locations", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_continent": { + "type": ["string"] + }, + "ga_subContinent": { + "type": ["string"] + }, + "ga_country": { + "type": ["string"] + }, + "ga_region": { + "type": ["string"] + }, + "ga_metro": { + "type": ["string"] + }, + "ga_city": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "monthly_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_30dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "four_weekly_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_28dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "two_weekly_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_14dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "weekly_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_7dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "daily_active_users", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_1dayUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "devices", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_deviceCategory": { + "type": ["string"] + }, + "ga_operatingSystem": { + "type": ["string"] + }, + "ga_browser": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "ga_pageviews": { + "type": ["null", "integer"] + }, + "ga_pageviewsPerSession": { + "type": ["null", "number"] + }, + "ga_avgTimeOnPage": { + "type": ["null", "number"] + }, + "ga_bounceRate": { + "type": ["null", "number"] + }, + "ga_exitRate": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users_per_day", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_users": { + "type": ["null", "integer"] + }, + "ga_newUsers": { + "type": ["null", "integer"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "sessions_per_country_day", + "json_schema": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "ga_date": { + "type": ["string"] + }, + "ga_country": { + "type": ["string"] + }, + "ga_sessions": { + "type": ["null", "integer"] + }, + "ga_sessionsPerUser": { + "type": ["null", "number"] + }, + "ga_avgSessionDuration": { + "type": ["null", "number"] + }, + "report_start_date": { + "type": ["string"] + }, + "report_end_date": { + "type": ["string"] + } + } + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true + }, + "sync_mode": "incremental", + "cursor_field": ["report_start_date"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json new file mode 100644 index 000000000000..c9d1a02acd5a --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "credentials_json": "{\n \"type\": \"service_account\",\n \"project_id\": \"dataline-integration-testing\",\n \"private_key_id\": \"8e24553ba1c810d663a74dc5e0b4463911ae3dbb\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAooBAQDp7M8lWSO2fn/3\\nriGVX5GnTNqB4rGHs5IhKtoOAQ+/Vt+4hS9+VxyMyx8+k9/I/3KJRd5gOTA3bD45\\nXyqvLecOoyAVKyHtdtePmm8zjQgtAS9Hx0zuxDZA605qh5IR76j564wgdZtZfF+o\\nvnK6HqM0pCJQuYNnZmetGuOh+1vE7asF2SUkLo3qTVsnEreQdjV6M/aeUehtsJK0\\nnSZ2VAoXAgS4i+7vmrxCxZ9iRnzKSBaBXDthAFA4kap1DWMpnWI+GH00vSF3Ofb9\\nYyvArv9Fqj66uDqD0fMZJvji4+USQTziXGyZ8aO+7bJ5c3aR+qytkQmd52fS37U9\\nJlPeUBOzAgMBAAECggEAAJ4UaefTVU7+LXwzMH7BFqz4QrGDgHUz5IyFaNYN4ZZs\\nHsOTPYUjWSfMHFEtnQ03ky8xCBX+j5bkjyg2J6I6YuqBUiz7+PGAXiutMXik1eGT\\n61+WT2cLMZ6tZVCYTR3R118MzqUg1NqkMgDWsrEUAzaPtKDYvPQjP+y7Pxk3w8xP\\nhS/EIEN5EWczWencfouYfx11ZC6I9IwkywQR3kKxWhaaxOVUr3LYTkW9r5g7dTgE\\n0u+NCJRTJ7uZmQ0pSWcAkWV/42jEY8j/PLAdzpEFO0wf/9pPBAJrtSvEAH8qWx6H\\ne5sjAcqW0S3d0CMVstAUjlJcGGiQNfYB6oWql3+tgQKBgQD/6Hp97W1F6kAAiRCH\\nt5ES0JynbwCGvtoExPnip5RNHQuxIs+FI7xNSsPUJii7nIOIpjBog5coFOhRF7fp\\n5oraALS6d2AQRDfTn8R1XPsfX6WWElpp7eSuU7xLB0yHZed/iP7SRsArRxeTVNHj\\nRIh6zPM0avAZvOBGVk4CjunocwKBgQDqAk9klJLq9UveJdUwua/NQ42ubBjylA5I\\nJpeZJTmnGpTfbCeTjmnlLTWlvnT7s7inFFPSrAxS96F1/vwv61alPsUBBmHNytom\\nypmKdzH0JR7bALJbNpNr2W7c0ncHbOPya6+QEq5ZvF/AbsVVTcX++GrnTKCP5lJ7\\nb5n/rbSXwQKBgQDaKxhCw2elc0+dJT0ydwz3PWJQXBHWzl6QMl3XHYcRNvIA0eyZ\\nVR3zxwmsk9umFokMtqIOeCElyq1EdTQV9LXrpS5uydbcB0yQ3mReqiZtWN2SU5NB\\nO875z+l3DYHw7K+vyttz6V1rh/BRar6FQqgYCSJOTdzkFGIflPRsueowgQKBgQDn\\nP7ktcBVv01BSC5kOObGdavCMOY69ycoDSv/s+3hpxj7wO3UwwD+tlmu5iukYA5aq\\npc/gFN1o8AXDEWQuBqbtTYZpaOEl+Rxz0SrCRuA6oKRJT/rxYJF2trxzxBiUYesx\\nGXG5MnzRePI2vECN9/l96gfa40KYccd46+SHYiVhAQKBgQCK66RqjCFxJSoeD++B\\nGnIG3hFM2wJg0BsYlJPwg82OF/efhPmPMzbFtmZw4BbVZ8KGSPAJ3oa7HskKKsmD\\nlnsrNtbRQ8EGXpBRr6d3cQhoShIiicqh0b2jvZgPHKOlgexMi6A3G+rk6v3PlyaL\\nXlnklGqG4MilnFgyKs5nEah1ow==\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"google-analytics-access@dataline-integration-testing.iam.gserviceaccount.com\",\n \"client_id\": \"103293192021796062643\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/google-analytics-access%40dataline-integration-testing.iam.gserviceaccount.com\"\n}\n", + "view_id": "222222222", + "start_date": "2020-02-13T00:00:00Z", + "custom_reports": "[{\"name\": \"users_per_day\", \"dimensions\": [\"ga:date\"], \"metrics\": [\"ga:users\", \"ga:newUsers\"]}, {\"name\": \"sessions_per_country_day\", \"dimensions\": [\"ga:date\", \"ga:country\"], \"metrics\": [\"ga:sessions\", \"ga:sessionsPerUser\", \"ga:avgSessionDuration\"]}]" +} diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json new file mode 100644 index 000000000000..b94062f8e2a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "website_overview": { "report_start_date": "2050-05-01T00:00:00Z" }, + "traffic_sources": { "report_start_date": "2050-05-01T00:00:00Z" }, + "pages": { "report_start_date": "2050-05-01T00:00:00Z" } +} diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/configured_catalog.json index 9381437e5c2b..6cfc5158b0e4 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/sample_files/configured_catalog.json @@ -118,406 +118,6 @@ "sync_mode": "incremental", "cursor_field": ["report_start_date"], "destination_sync_mode": "append" - }, - { - "stream": { - "name": "pages", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_hostname": { - "type": ["string"] - }, - "ga_pagePath": { - "type": ["string"] - }, - "ga_pageviews": { - "type": ["null", "integer"] - }, - "ga_uniquePageviews": { - "type": ["null", "integer"] - }, - "ga_avgTimeOnPage": { - "type": ["null", "number"] - }, - "ga_entrances": { - "type": ["null", "integer"] - }, - "ga_entranceRate": { - "type": ["null", "number"] - }, - "ga_bounceRate": { - "type": ["null", "number"] - }, - "ga_exits": { - "type": ["null", "integer"] - }, - "ga_exitRate": { - "type": ["null", "number"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "locations", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_continent": { - "type": ["string"] - }, - "ga_subContinent": { - "type": ["string"] - }, - "ga_country": { - "type": ["string"] - }, - "ga_region": { - "type": ["string"] - }, - "ga_metro": { - "type": ["string"] - }, - "ga_city": { - "type": ["string"] - }, - "ga_users": { - "type": ["null", "integer"] - }, - "ga_newUsers": { - "type": ["null", "integer"] - }, - "ga_sessions": { - "type": ["null", "integer"] - }, - "ga_sessionsPerUser": { - "type": ["null", "number"] - }, - "ga_avgSessionDuration": { - "type": ["null", "number"] - }, - "ga_pageviews": { - "type": ["null", "integer"] - }, - "ga_pageviewsPerSession": { - "type": ["null", "number"] - }, - "ga_avgTimeOnPage": { - "type": ["null", "number"] - }, - "ga_bounceRate": { - "type": ["null", "number"] - }, - "ga_exitRate": { - "type": ["null", "number"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "monthly_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_30dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "four_weekly_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_28dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "two_weekly_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_14dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "weekly_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_7dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "daily_active_users", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_1dayUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "devices", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_deviceCategory": { - "type": ["string"] - }, - "ga_operatingSystem": { - "type": ["string"] - }, - "ga_browser": { - "type": ["string"] - }, - "ga_users": { - "type": ["null", "integer"] - }, - "ga_newUsers": { - "type": ["null", "integer"] - }, - "ga_sessions": { - "type": ["null", "integer"] - }, - "ga_sessionsPerUser": { - "type": ["null", "number"] - }, - "ga_avgSessionDuration": { - "type": ["null", "number"] - }, - "ga_pageviews": { - "type": ["null", "integer"] - }, - "ga_pageviewsPerSession": { - "type": ["null", "number"] - }, - "ga_avgTimeOnPage": { - "type": ["null", "number"] - }, - "ga_bounceRate": { - "type": ["null", "number"] - }, - "ga_exitRate": { - "type": ["null", "number"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "users_per_day", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_users": { - "type": ["null", "integer"] - }, - "ga_newUsers": { - "type": ["null", "integer"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" - }, - { - "stream": { - "name": "sessions_per_country_day", - "json_schema": { - "type": ["null", "object"], - "additionalProperties": false, - "properties": { - "ga_date": { - "type": ["string"] - }, - "ga_country": { - "type": ["string"] - }, - "ga_sessions": { - "type": ["null", "integer"] - }, - "ga_sessionsPerUser": { - "type": ["null", "number"] - }, - "ga_avgSessionDuration": { - "type": ["null", "number"] - }, - "report_start_date": { - "type": ["string"] - }, - "report_end_date": { - "type": ["string"] - } - } - }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true - }, - "sync_mode": "incremental", - "cursor_field": ["report_start_date"], - "destination_sync_mode": "append" } ] } diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py index 0780c9d6a0e6..771dceead871 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/source.py @@ -29,6 +29,7 @@ from pathlib import Path from typing import List +from airbyte_protocol import ConfiguredAirbyteCatalog from base_singer import AirbyteLogger, BaseSingerSource from jsonschema.validators import Draft4Validator from tap_google_analytics import GAClient @@ -42,6 +43,7 @@ class GoogleAnalyticsSingerSource(BaseSingerSource): tap_cmd = "tap-google-analytics" tap_name = "Google Analytics API" api_error = Exception + reports_to_read = None # can be overridden to change an input config def configure(self, raw_config: json, temp_dir: str) -> json: @@ -59,19 +61,28 @@ def _validate_custom_reports(self, custom_reports_data: List[dict]): error_messages.append(error.message) raise Exception("An error occurred during custom_reports data validation: " + "; ".join(error_messages)) + def read_catalog(self, catalog_path: str) -> ConfiguredAirbyteCatalog: + catalog = ConfiguredAirbyteCatalog.parse_obj(self.read_config(catalog_path)) + if not self.reports_to_read: + self.reports_to_read = [i.stream.name for i in catalog.streams] + return catalog_path + def _get_reports_file_path(self, temp_dir: str, custom_reports_data: List[dict]) -> str: report_definition = ( json.loads(pkgutil.get_data("tap_google_analytics", "defaults/default_report_definition.json")) + custom_reports_data ) + if self.reports_to_read: + report_definition = [i for i in report_definition if i["name"] in self.reports_to_read] + custom_reports = os.path.join(temp_dir, "custom_reports.json") with open(custom_reports, "w") as file: file.write(json.dumps(report_definition)) + return custom_reports def _check_custom_reports(self, config: dict = None, config_path: str = None): if config_path: config = self.read_config(config_path) - custom_reports = config.pop("custom_reports") if custom_reports.strip() and json.loads(custom_reports): custom_reports_data = json.loads(custom_reports) diff --git a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/spec.json b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/spec.json index 40c7467a1e06..164651b7645b 100644 --- a/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/spec.json +++ b/airbyte-integrations/connectors/source-googleanalytics-singer/source_googleanalytics_singer/spec.json @@ -27,7 +27,6 @@ "custom_reports": { "title": "Custom Reports", "type": "string", - "title": "Custom Reports", "description": "A JSON array describing the custom reports you want to sync from GA. Check out the docs to get more information about this field." } } diff --git a/docs/troubleshooting/running-sync.md b/docs/troubleshooting/running-sync.md index 1576d1c9dcb9..1dc4a40fcff1 100644 --- a/docs/troubleshooting/running-sync.md +++ b/docs/troubleshooting/running-sync.md @@ -29,4 +29,4 @@ Several things to check: * **Is Airbyte updated to your latest version?** You can see the latest version [here](https://github.com/airbytehq/airbyte/tags). If not, please upgrade to the latest one, [upgrading instructions are here](../operator-guides/upgrading-airbyte.md) * **Is the connector that is failing updated to the latest version?** You can check the latest version available for the connectors [in the yamls here](https://github.com/airbytehq/airbyte/tree/master/airbyte-config/init/src/main/resources/seed). If you don't have the latest connector version, make sure you first update to the latest Airbyte version, and then go to the Admin section in the web app and put the right version in the cell for the connector. Then try again. -If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). +If the above workaround does not fix your problem, please report it [here](https://github.com/airbytehq/airbyte/issues/1462) or in our [Slack](https://slack.airbyte.io). \ No newline at end of file