🎉 Source Linkedin Ads - for adDirectSponsoredContents stream skip acc…

…ounts which are not part of organization (#18111) * for adDirectSponsoredContents stream skip accounts which are part of organization * updated PR number * updated log message, added unit tests * additional formatting * updated connector version in source_definitions.yaml * auto-bump connector version Co-authored-by: Octavia Squidington III <octavia-squidington-iii@users.noreply.github.com>
airbytehq · Oct 21, 2022 · ccdbbfc · ccdbbfc
1 parent e1c220a
commit ccdbbfc
Show file tree

Hide file tree

Showing 6 changed files with 35 additions and 3 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -594,7 +594,7 @@
 - name: LinkedIn Ads
   sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451
   dockerRepository: airbyte/source-linkedin-ads
-  dockerImageTag: 0.1.11
+  dockerImageTag: 0.1.12
   documentationUrl: https://docs.airbyte.com/integrations/sources/linkedin-ads
   icon: linkedin.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -5864,7 +5864,7 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-linkedin-ads:0.1.11"
+- dockerImage: "airbyte/source-linkedin-ads:0.1.12"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/linkedin-ads"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile
@@ -33,5 +33,5 @@ COPY source_linkedin_ads ./source_linkedin_ads
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.11
+LABEL io.airbyte.version=0.1.12
 LABEL io.airbyte.name=airbyte/source-linkedin-ads
diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py
@@ -254,6 +254,24 @@ def request_params(self, stream_state: Mapping[str, Any], stream_slice: Mapping[
         params["q"] = self.search_param
         return params
 
+    def read_records(
+        self, stream_state: Mapping[str, Any] = None, stream_slice: Optional[Mapping[str, Any]] = None, **kwargs
+    ) -> Iterable[Mapping[str, Any]]:
+        stream_state = stream_state or {}
+        parent_stream = self.parent_stream(config=self.config)
+        for record in parent_stream.read_records(**kwargs):
+
+            if record.get("reference", "").startswith("urn:li:person"):
+                self.logger.warn(
+                    f'Skip {record.get("name")} account, ORGANIZATION permissions required, but referenced to PERSON {record.get("reference")}'
+                )
+                continue
+
+            child_stream_slice = super(LinkedInAdsStreamSlicing, self).read_records(
+                stream_slice=get_parent_stream_values(record, self.parent_values_map), **kwargs
+            )
+            yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=child_stream_slice)
+
 
 class LinkedInAdsAnalyticsStream(IncrementalLinkedinAdsStream):
     """

diff --git a/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py b/airbyte-integrations/connectors/source-linkedin-ads/unit_tests/source_tests/test_source.py
@@ -177,6 +177,19 @@ def test_request_headers(self):
         assert result == expected
 
 
+class TestAccountUsers:
+    stream: AccountUsers = AccountUsers(TEST_CONFIG)
+
+    def test_state_checkpoint_interval(self):
+        assert self.stream.state_checkpoint_interval == 500
+
+    def test_get_updated_state(self):
+        state = self.stream.get_updated_state(
+            current_stream_state={"lastModified": "2021-01-01"}, latest_record={"lastModified": "2021-08-01"}
+        )
+        assert state == {"lastModified": "2021-08-01"}
+
+
 class TestLinkedInAdsStreamSlicing:
     @pytest.mark.parametrize(
         "stream_cls, slice, expected",

diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md
@@ -182,6 +182,7 @@ After 5 unsuccessful attempts - the connector will stop the sync operation. In s
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                           |
 |:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------------------------------------------------------------------------|
+| 0.1.12  | 2022-10-18 | [18111](https://github.com/airbytehq/airbyte/pull/18111) | for adDirectSponsoredContents stream skip accounts which are part of organization                                 |
 | 0.1.11  | 2022-10-07 | [17724](https://github.com/airbytehq/airbyte/pull/17724) | Retry 429/5xx errors when refreshing access token                                                                 |
 | 0.1.10  | 2022-09-28 | [17326](https://github.com/airbytehq/airbyte/pull/17326) | Migrate to per-stream states.                                                                                     |
 | 0.1.9   | 2022-07-21 | [14924](https://github.com/airbytehq/airbyte/pull/14924) | Remove `additionalProperties` field from schemas                                                                  |