From fed7f0309831146b9a9e5946d2eb15845b81286d Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 18 Oct 2022 18:26:59 +0300 Subject: [PATCH 1/6] for adDirectSponsoredContents stream skip accounts which are part of organization --- .../connectors/source-linkedin-ads/Dockerfile | 2 +- .../source_linkedin_ads/source.py | 14 ++++++++++++++ docs/integrations/sources/linkedin-ads.md | 1 + 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile index 068b60e44cce..d0bfa957d1b1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile @@ -33,5 +33,5 @@ COPY source_linkedin_ads ./source_linkedin_ads ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.11 +LABEL io.airbyte.version=0.1.12 LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 0274264e6337..3ff6bc9092f1 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -254,6 +254,20 @@ def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[ params["q"] = self.search_param return params + def read_records( + self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs + ) -> Iterable[Mapping[str, Any]]: + stream_state = stream_state or {} + parent_stream = self.parent_stream(config=self.config) + for record in parent_stream.read_records(**kwargs): + + if record.get('reference', '').startswith('urn:li:person'): + self.logger.warn(f'Skip {record.get("name")} account because it has reference to person {record.get("reference")} instead of organization') + continue + + child_stream_slice = super(LinkedInAdsStreamSlicing, self).read_records(stream_slice=get_parent_stream_values(record, self.parent_values_map), **kwargs) + yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=child_stream_slice) + class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream): """ diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index 440e04fdafb6..33cbd7725d16 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -182,6 +182,7 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------| +| 0.1.12 | 2022-10-18 | [17724](https://github.com/airbytehq/airbyte/pull/17724) | for adDirectSponsoredContents stream skip accounts which are part of organization | | 0.1.11 | 2022-10-07 | [17724](https://github.com/airbytehq/airbyte/pull/17724) | Retry 429/5xx errors when refreshing access token | | 0.1.10 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | | 0.1.9 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas | From bfe64e458df31e593f01ac0f5f34d19fbb705500 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Tue, 18 Oct 2022 18:29:55 +0300 Subject: [PATCH 2/6] updated PR number --- docs/integrations/sources/linkedin-ads.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index 33cbd7725d16..d8c72072fc65 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -182,7 +182,7 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s | Version | Date | Pull Request | Subject | |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------| -| 0.1.12 | 2022-10-18 | [17724](https://github.com/airbytehq/airbyte/pull/17724) | for adDirectSponsoredContents stream skip accounts which are part of organization | +| 0.1.12 | 2022-10-18 | [18111](https://github.com/airbytehq/airbyte/pull/18111) | for adDirectSponsoredContents stream skip accounts which are part of organization | | 0.1.11 | 2022-10-07 | [17724](https://github.com/airbytehq/airbyte/pull/17724) | Retry 429/5xx errors when refreshing access token | | 0.1.10 | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states. | | 0.1.9 | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas | From a6aaae06a15eb93e5943947d0256d1598b2b5762 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 19 Oct 2022 14:51:37 +0300 Subject: [PATCH 3/6] updated log message, added unit tests --- .../source_linkedin_ads/source.py | 2 +- .../unit_tests/source_tests/test_source.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 3ff6bc9092f1..8fe0d29aec71 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -262,7 +262,7 @@ def read_records( for record in parent_stream.read_records(**kwargs): if record.get('reference', '').startswith('urn:li:person'): - self.logger.warn(f'Skip {record.get("name")} account because it has reference to person {record.get("reference")} instead of organization') + self.logger.warn(f'Skip {record.get("name")} account, ORGANIZATION permissions required, but referenced to PERSON {record.get("reference")}') continue child_stream_slice = super(LinkedInAdsStreamSlicing, self).read_records(stream_slice=get_parent_stream_values(record, self.parent_values_map), **kwargs) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py index 2f666c1605c3..894a5b414841 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py @@ -176,6 +176,18 @@ def test_request_headers(self): result = self.stream.request_headers(stream_state={}) assert result == expected +class TestAccountUsers: + stream: AccountUsers = AccountUsers(TEST_CONFIG) + def test_state_checkpoint_interval(self): + assert self.stream.state_checkpoint_interval == 500 + + def test_get_updated_state(self): + state = self.stream.get_updated_state( + current_stream_state={'lastModified': '2021-01-01'}, + latest_record={'lastModified': "2021-08-01"} + ) + assert state == {'lastModified': '2021-08-01'} + class TestLinkedInAdsStreamSlicing: @pytest.mark.parametrize( From 623a36ce0082a3d6c684f107d2ee1e37dd599fe6 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 19 Oct 2022 14:54:00 +0300 Subject: [PATCH 4/6] additional formatting --- .../source-linkedin-ads/source_linkedin_ads/source.py | 10 +++++++--- .../unit_tests/source_tests/test_source.py | 7 ++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 8fe0d29aec71..a4a3bcb5a692 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -261,11 +261,15 @@ def read_records( parent_stream = self.parent_stream(config=self.config) for record in parent_stream.read_records(**kwargs): - if record.get('reference', '').startswith('urn:li:person'): - self.logger.warn(f'Skip {record.get("name")} account, ORGANIZATION permissions required, but referenced to PERSON {record.get("reference")}') + if record.get("reference", "").startswith("urn:li:person"): + self.logger.warn( + f'Skip {record.get("name")} account, ORGANIZATION permissions required, but referenced to PERSON {record.get("reference")}' + ) continue - child_stream_slice = super(LinkedInAdsStreamSlicing, self).read_records(stream_slice=get_parent_stream_values(record, self.parent_values_map), **kwargs) + child_stream_slice = super(LinkedInAdsStreamSlicing, self).read_records( + stream_slice=get_parent_stream_values(record, self.parent_values_map), **kwargs + ) yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=child_stream_slice) diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py index 894a5b414841..e01aa865e85e 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py @@ -176,17 +176,18 @@ def test_request_headers(self): result = self.stream.request_headers(stream_state={}) assert result == expected + class TestAccountUsers: stream: AccountUsers = AccountUsers(TEST_CONFIG) + def test_state_checkpoint_interval(self): assert self.stream.state_checkpoint_interval == 500 def test_get_updated_state(self): state = self.stream.get_updated_state( - current_stream_state={'lastModified': '2021-01-01'}, - latest_record={'lastModified': "2021-08-01"} + current_stream_state={"lastModified": "2021-01-01"}, latest_record={"lastModified": "2021-08-01"} ) - assert state == {'lastModified': '2021-08-01'} + assert state == {"lastModified": "2021-08-01"} class TestLinkedInAdsStreamSlicing: From 24c3dd53a0e6839f981fbf5d67c062b39d59dd99 Mon Sep 17 00:00:00 2001 From: Vadym Ratniuk Date: Wed, 19 Oct 2022 14:59:25 +0300 Subject: [PATCH 5/6] updated connector version in source_definitions.yaml --- .../init/src/main/resources/seed/source_definitions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0cefad4524aa..64e5f775aeb0 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -566,7 +566,7 @@ - name: LinkedIn Ads sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451 dockerRepository: airbyte/source-linkedin-ads - dockerImageTag: 0.1.11 + dockerImageTag: 0.1.12 documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads icon: linkedin.svg sourceType: api From dc858e594cf1e3f9a361c9d31a37b83cfd9db5c1 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Fri, 21 Oct 2022 14:35:42 +0000 Subject: [PATCH 6/6] auto-bump connector version --- airbyte-config/init/src/main/resources/seed/source_specs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 666bbc8c2f78..175ea299c1dd 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5673,7 +5673,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-linkedin-ads:0.1.11" +- dockerImage: "airbyte/source-linkedin-ads:0.1.12" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/linkedin-ads" connectionSpecification: