From ccdbbfca6fb6895dde5da41909561f5f0f278c39 Mon Sep 17 00:00:00 2001 From: midavadim Date: Fri, 21 Oct 2022 22:25:53 +0300 Subject: [PATCH] :tada: Source Linkedin Ads - for adDirectSponsoredContents stream skip accounts which are not part of organization (#18111) * for adDirectSponsoredContents stream skip accounts which are part of organization * updated PR number * updated log message, added unit tests * additional formatting * updated connector version in source_definitions.yaml * auto-bump connector version Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-linkedin-ads/Dockerfile | 2 +- .../source_linkedin_ads/source.py | 18 ++++++++++++++++++ .../unit_tests/source_tests/test_source.py | 13 +++++++++++++ docs/integrations/sources/linkedin-ads.md | 1 + 6 files changed, 35 insertions(+), 3 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b8f9c1193a40..b8883ed27b28 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -594,7 +594,7 @@ - name: LinkedIn Ads sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451 dockerRepository: airbyte/source-linkedin-ads - dockerImageTag: 0.1.11 + dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads icon: linkedin.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index f872c4bad656..2d842b06fcab 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5864,7 +5864,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-linkedin-ads:0.1.11" +- dockerImage: "airbyte/source-linkedin-ads:0.1.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/linkedin-ads" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile index 068b60e44cce..d0bfa957d1b1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile @@ -33,5 +33,5 @@ COPY source_linkedin_ads ./source_linkedin_ads ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 0274264e6337..a4a3bcb5a692 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -254,6 +254,24 @@ def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[ params["q"] = self.search_param return params + def read_records( + self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + parent_stream = self.parent_stream(config=self.config) + for record in parent_stream.read_records(**kwargs): + + if record.get("reference", "").startswith("urn:li:person"): + self.logger.warn( + f'Skip {record.get("name")} account, ORGANIZATION permissions required, but referenced to PERSON {record.get("reference")}' + ) + continue + + child_stream_slice = super(LinkedInAdsStreamSlicing, self).read_records( + stream_slice=get_parent_stream_values(record, self.parent_values_map), **kwargs + ) + yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=child_stream_slice) + class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream): """ diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py index 2f666c1605c3..e01aa865e85e 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py @@ -177,6 +177,19 @@ def test_request_headers(self): assert result == expected +class TestAccountUsers: + stream: AccountUsers = AccountUsers(TEST_CONFIG) + + def test_state_checkpoint_interval(self): + assert self.stream.state_checkpoint_interval == 500 + + def test_get_updated_state(self): + state = self.stream.get_updated_state( + current_stream_state={"lastModified": "2021-01-01"}, latest_record={"lastModified": "2021-08-01"} + ) + assert state == {"lastModified": "2021-08-01"} + + class TestLinkedInAdsStreamSlicing: @pytest.mark.parametrize( "stream_cls, slice, expected", diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index b044c4138fb5..30ce727ef6c9 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -182,6 +182,7 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------| +| 0.1.12 | 2022-10-18 | [18111](https://github.com/airbytehq/airbyte/pull/18111) | for adDirectSponsoredContents stream skip accounts which are part of organization | | 0.1.11 | 2022-10-07 | [17724](https://github.com/airbytehq/airbyte/pull/17724) | Retry 429/5xx errors when refreshing access token | | 0.1.10 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | | 0.1.9 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas |