From daff4c1dd97971b0411cac3ff6157fbcb2dd2b47 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 26 Aug 2021 00:03:22 +0300 Subject: [PATCH 1/3] Fix validation of empty custom reports --- .../eff3616a-f9c3-11eb-9a03-0242ac130003.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../source-google-analytics-v4/Dockerfile | 2 +- .../source-google-analytics-v4/setup.py | 1 + .../source_google_analytics_v4/source.py | 13 ++++++---- .../unit_tests/unit_test.py | 24 ++++++++++++++++++- .../sources/google-analytics-v4.md | 1 + 7 files changed, 37 insertions(+), 8 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json index f730aef5bc97..950d3300794e 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/eff3616a-f9c3-11eb-9a03-0242ac130003.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "eff3616a-f9c3-11eb-9a03-0242ac130003", "name": "Google Analytics v4", "dockerRepository": "airbyte/source-google-analytics-v4", - "dockerImageTag": "0.1.0", + "dockerImageTag": "0.1.1", "documentationUrl": "https://docs.airbyte.io/integrations/sources/source-google-analytics-v4", "icon": "google-analytics.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c3f0ffdab74a..34fb2fccbab6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -131,7 +131,7 @@ - sourceDefinitionId: eff3616a-f9c3-11eb-9a03-0242ac130003 name: Google Analytics v4 dockerRepository: airbyte/source-google-analytics-v4 - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 documentationUrl: https://docs.airbyte.io/integrations/sources/source-google-analytics-v4 icon: google-analytics.svg - sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile index 93e1b12babb5..00d2bcc8e1a6 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile +++ b/airbyte-integrations/connectors/source-google-analytics-v4/Dockerfile @@ -12,5 +12,5 @@ RUN pip install . ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.version=0.1.1 LABEL io.airbyte.name=airbyte/source-google-analytics-v4 diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py index 503e4f82818c..8d53d72dbab9 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/setup.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/setup.py @@ -30,6 +30,7 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "requests-mock", + "pytest-mock", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index 0e7bb8d0f0ee..f7d5dbdeefc7 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -474,7 +474,7 @@ class SourceGoogleAnalyticsV4(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: try: url = f"{GoogleAnalyticsV4TypesList.url_base}" - + print("ll") authenticator = GoogleAnalyticsOauth2Authenticator(config) session = requests.get(url, headers=authenticator.get_auth_header()) @@ -495,9 +495,14 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: authenticator = GoogleAnalyticsOauth2Authenticator(config) config["authenticator"] = authenticator - config["ga_streams"] = json.loads(pkgutil.get_data("source_google_analytics_v4", "defaults/default_reports.json")) + json.loads( - config["custom_reports"] - ) + + reports = json.loads(pkgutil.get_data("source_google_analytics_v4", "defaults/default_reports.json")) + + if config.get("custom_reports"): + custom_reports = json.loads(config["custom_reports"]) + reports += custom_reports + + config["ga_streams"] = reports for stream in config["ga_streams"]: config["metrics"] = stream["metrics"] diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py index 885ad5ee374b..a338df5d25c5 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py @@ -24,10 +24,11 @@ import json from pathlib import Path +from unittest.mock import MagicMock, patch import pytest from airbyte_cdk.sources.streams.http.auth import NoAuth -from source_google_analytics_v4.source import GoogleAnalyticsV4Stream, GoogleAnalyticsV4TypesList +from source_google_analytics_v4.source import GoogleAnalyticsV4Stream, GoogleAnalyticsV4TypesList, SourceGoogleAnalyticsV4 def read_file(file_name): @@ -76,3 +77,24 @@ def test_lookup_metrics_dimensions_data_type(metrics_dimensions_mapping, mock_me test = g.lookup_data_type(field_type, attribute) assert test == expected + + +class GoogleAnalyticsOauth2AuthenticatorMock: + def refresh_access_token(self): + return MagicMock(), 0 + + +@patch( + "source_google_analytics_v4.source.GoogleAnalyticsOauth2Authenticator.refresh_access_token", + new=GoogleAnalyticsOauth2AuthenticatorMock.refresh_access_token, +) +def test_check_connection(mocker, mock_metrics_dimensions_type_list_link): + test_config = json.loads(read_file("../integration_tests/sample_config.json")) + test_config["credentials_json"] = '{"client_email": "", "private_key": "", "private_key_id": ""}' + + del test_config["custom_reports"] + + source = SourceGoogleAnalyticsV4() + logger_mock, config_mock = MagicMock(), test_config + + assert source.check_connection(logger_mock, config_mock) == (True, None) diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index 4a1e88eadc90..90a149f585c6 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -128,4 +128,5 @@ Incremental sync supports only if you add `ga:date` dimension to your custom rep | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | +| 0.1.1 | 2021-08-25 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Corrected validation of empty custom report| | 0.1.0 | 2021-08-10 | [5290](https://github.com/airbytehq/airbyte/pull/5290) | Initial Release| From 4980794f39ec48287ebf8b223f1bb4ffe47a6f5d Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 26 Aug 2021 00:09:35 +0300 Subject: [PATCH 2/3] Fix typo --- .../source_google_analytics_v4/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py index f7d5dbdeefc7..aa4f19e11286 100644 --- a/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py +++ b/airbyte-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py @@ -474,7 +474,7 @@ class SourceGoogleAnalyticsV4(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: try: url = f"{GoogleAnalyticsV4TypesList.url_base}" - print("ll") + authenticator = GoogleAnalyticsOauth2Authenticator(config) session = requests.get(url, headers=authenticator.get_auth_header()) From 900390cb18fcab105489d5911f83691dbd7871bb Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 26 Aug 2021 00:11:35 +0300 Subject: [PATCH 3/3] Updated PR number in docs --- docs/integrations/sources/google-analytics-v4.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/google-analytics-v4.md b/docs/integrations/sources/google-analytics-v4.md index 90a149f585c6..afbe6e4600ae 100644 --- a/docs/integrations/sources/google-analytics-v4.md +++ b/docs/integrations/sources/google-analytics-v4.md @@ -128,5 +128,5 @@ Incremental sync supports only if you add `ga:date` dimension to your custom rep | Version | Date | Pull Request | Subject | | :------ | :-------- | :----- | :------ | -| 0.1.1 | 2021-08-25 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Corrected validation of empty custom report| +| 0.1.1 | 2021-08-25 | [5655](https://github.com/airbytehq/airbyte/pull/5655) | Corrected validation of empty custom report| | 0.1.0 | 2021-08-10 | [5290](https://github.com/airbytehq/airbyte/pull/5290) | Initial Release|