Skip to content

Commit

Permalink
#12013 expose isDataGolden field, assume missing field equals False
Browse files Browse the repository at this point in the history
  • Loading branch information
@davydov-d
davydov-d committed May 1, 2022
1 parent 62596be commit 37fc7f1
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 16 deletions.
12 changes: 6 additions & 6 deletions ...e-integrations/connectors/source-google-analytics-v4/source_google_analytics_v4/source.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -231,10 +231,10 @@ def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs: Any) -
if stream_state:
prev_end_date = pendulum.parse(stream_state.get(self.cursor_field)).date()
start_date = prev_end_date.add(days=1) # do not include previous `end_date`
# always resync 2 previous days to be sure data is golden
# https://support.google.com/analytics/answer/1070983?hl=en#DataProcessingLatency&zippy=%2Cin-this-article
# https://github.com/airbytehq/airbyte/issues/12013#issuecomment-1111255503
start_date = start_date.subtract(days=2)
# always resync 2 previous days to be sure data is golden
# https://support.google.com/analytics/answer/1070983?hl=en#DataProcessingLatency&zippy=%2Cin-this-article
# https://github.com/airbytehq/airbyte/issues/12013#issuecomment-1111255503
start_date = start_date.subtract(days=2)

date_slices = []
slice_start_date = start_date
Expand Down Expand Up @@ -404,11 +404,11 @@ def parse_response(self, response: requests.Response, **kwargs: Any) -> Iterable
record[metric_name.replace("ga:", "ga_")] = value

record["view_id"] = self.view_id
record["isDataGolden"] = report.get("data", {}).get("isDataGolden", True)
record["isDataGolden"] = report.get("data", {}).get("isDataGolden", False)
yield record

def check_for_sampled_result(self, data: Mapping) -> None:
if not data.get("isDataGolden", True):
if not data.get("isDataGolden", False):
self.logger.warning(DATA_IS_NOT_GOLDEN_MSG)
if data.get("samplesReadCounts", False):
self.logger.warning(RESULT_IS_SAMPLED_MSG)
Expand Down
1 change: 1 addition & 0 deletions ...-v4/unit_tests/response_with_records.json → ...s-v4/unit_tests/response_golden_data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
]
}
],
"isDataGolden": true,
"totals": [
{
"values": ["158"]
Expand Down
1 change: 0 additions & 1 deletion ..._tests/response_is_data_golden_false.json → .../unit_tests/response_non_golden_data.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
]
}
],
"isDataGolden": false,
"totals": [
{
"values": ["158"]
Expand Down
27 changes: 18 additions & 9 deletions airbyte-integrations/connectors/source-google-analytics-v4/unit_tests/unit_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

import pendulum
import pytest
from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode
from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode, Type
from airbyte_cdk.sources.streams.http.auth import NoAuth
from freezegun import freeze_time
from source_google_analytics_v4.source import (
Expand Down Expand Up @@ -81,10 +81,10 @@ def mock_api_returns_no_records(requests_mock):
@pytest.fixture
def mock_api_returns_valid_records(requests_mock):
"""API returns valid data for given date based slice"""
yield requests_mock.post(
"https://analyticsreporting.googleapis.com/v4/reports:batchGet",
json=json.loads(read_file("response_with_records.json")),
)
response = json.loads(read_file("response_golden_data.json"))
for report in response["reports"]:
assert report["data"]["isDataGolden"] is True
yield requests_mock.post("https://analyticsreporting.googleapis.com/v4/reports:batchGet", json=response)


@pytest.fixture
Expand All @@ -99,10 +99,10 @@ def mock_api_returns_sampled_results(requests_mock):
@pytest.fixture
def mock_api_returns_is_data_golden_false(requests_mock):
"""API returns valid data for given date based slice"""
yield requests_mock.post(
"https://analyticsreporting.googleapis.com/v4/reports:batchGet",
json=json.loads(read_file("response_is_data_golden_false.json")),
)
response = json.loads(read_file("response_non_golden_data.json"))
for report in response["reports"]:
assert "isDataGolden" not in report["data"]
yield requests_mock.post("https://analyticsreporting.googleapis.com/v4/reports:batchGet", json=response)


@pytest.fixture
Expand Down Expand Up @@ -372,3 +372,12 @@ def test_connection_fail_due_to_http_status(
assert "Please check the permissions for the requested view_id" in error
assert test_config["view_id"] in error
assert json_resp["error"] in error


def test_is_data_golden_flag_missing_equals_false(
mock_api_returns_is_data_golden_false, test_config, configured_catalog, mock_metrics_dimensions_type_list_link, mock_auth_call
):
source = SourceGoogleAnalyticsV4()
for message in source.read(logging.getLogger(), test_config, configured_catalog):
if message.type == Type.RECORD:
assert message.record.data["isDataGolden"] is False

1 comment on commit 37fc7f1

@github-actions
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

SonarQube Report

SonarQube report for Airbyte Connectors Source Google Analytics V4(#12426)

Measures

Name Value Name Value Name Value
Lines of Code 340 Code Smells 16 Coverage 100.0
Duplicated Blocks 0 Vulnerabilities 0 Security Rating A
Quality Gate Status OK Duplicated Lines (%) 0.0 Bugs 0
Reliability Rating A Lines to Cover 2 Blocker Issues 0
Critical Issues 1 Major Issues 5 Minor Issues 10

Detected Issues

Rule File Description Message
python:mypy_attr_defined (MINOR) source_google_analytics_v4/source.py:232 Check that attribute exists Module has no attribute "parse" . Code line: prev_end_date = pendulum.parse(stream_state.get(self.curso...
python:mypy_list_item (MINOR) source_google_analytics_v4/source.py:248 Check list items in a list expression [item, ...] List item 0 has incompatible type "None"; expected "Dict[str, str]" . Code line: return date_slices or [None]
python:mypy_no_any_return (MINOR) source_google_analytics_v4/source.py:252 Reject returning value with "Any" type if return type is not "Any" Returning Any from function declared to return "List[MutableMapping[Any, Any]]" . Code line: return report_body.get("data", {}).get("rows", [])
python:mypy_no_any_return (MINOR) source_google_analytics_v4/source.py:432 Reject returning value with "Any" type if return type is not "Any" Returning Any from function declared to return "Iterable[Mapping[str, Any]]" . Code line: return super().read_records(sync_mode, cursor_field, stream_sl...
python:mypy_return (MINOR) source_google_analytics_v4/source.py:290 Check that function always returns a value Missing return statement . Code line: def lookup_data_format(attribute: str) -> Union[str, None]:
python:S1192 (CRITICAL) source_google_analytics_v4/source.py:291 String literals should not be duplicated Define a constant instead of duplicating this literal "ga:date" 4 times.
python:mypy_attr_defined (MINOR) source_google_analytics_v4/source.py:513 Check that attribute exists Module has no attribute "parse" . Code line: start_date = pendulum.parse(self.start_date).date()
python:mypy_arg_type (MINOR) source_google_analytics_v4/source.py:579 Check argument types in calls Argument 1 to "loads" has incompatible type "Optional[bytes]"; expected "Union[str, bytes]" . Code line: reports = json.loads(pkgutil.get_data("source_google_analytics...
python:mypy_import (MINOR) source_google_analytics_v4/source.py:16 Require that imported module can be found or has stubs Library stubs not installed for "requests" (or incompatible with Python 3.9) . Code line: import requests
python:S112 (MAJOR) source_google_analytics_v4/source.py:81 "Exception" and "BaseException" should not be raised Replace this generic exception class with a more specific one.
python:mypy_return (MINOR) source_google_analytics_v4/source.py:132 Check that function always returns a value Missing return statement . Code line: def next_page_token(self, response: requests.Response) -> Optional...
python:mypy_attr_defined (MINOR) source_google_analytics_v4/source.py:230 Check that attribute exists Module has no attribute "parse" . Code line: start_date = pendulum.parse(self.start_date).date()
python:S1871 (MAJOR) source_google_analytics_v4/source.py:275 Two branches in a conditional structure should not have exactly the same implementation Either merge this branch with the identical one on line "272" or change one of the implementations.
python:S1871 (MAJOR) source_google_analytics_v4/source.py:277 Two branches in a conditional structure should not have exactly the same implementation Either merge this branch with the identical one on line "272" or change one of the implementations.
python:S112 (MAJOR) source_google_analytics_v4/source.py:466 "Exception" and "BaseException" should not be raised Replace this generic exception class with a more specific one.
python:S112 (MAJOR) source_google_analytics_v4/source.py:471 "Exception" and "BaseException" should not be raised Replace this generic exception class with a more specific one.

Coverage (100.0%)

File Coverage File Coverage
source_google_analytics_v4/init.py 100.0 source_google_analytics_v4/source.py 91.7

Please sign in to comment.