🐛 Source Hubspot: Fix non exists updated at field error (#11321)

* Fix updated at field non exists issue * Updated PR number * Change state default value to self._state * Bumped seed version
airbytehq · Mar 24, 2022 · 6528926 · 6528926
1 parent 0191af9
commit 6528926
Show file tree

Hide file tree

Showing 7 changed files with 43 additions and 6 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -335,7 +335,7 @@
 - name: HubSpot
   sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.50
+  dockerImageTag: 0.1.51
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3359,7 +3359,7 @@
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
-- dockerImage: "airbyte/source-hubspot:0.1.50"
+- dockerImage: "airbyte/source-hubspot:0.1.51"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
     connectionSpecification:

diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.50
+LABEL io.airbyte.version=0.1.51
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py
@@ -667,7 +667,7 @@ def state(self) -> Optional[Mapping[str, Any]]:
 
     @state.setter
     def state(self, value):
-        state_value = value[self.updated_at_field]
+        state_value = value.get(self.updated_at_field, self._state)
         self._state = (
             pendulum.parse(str(pendulum.from_timestamp(state_value / 1000)))
             if isinstance(state_value, int)

diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py
@@ -2,6 +2,7 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
+import pendulum
 import pytest
 from source_hubspot.streams import (
     Campaigns,
@@ -32,6 +33,41 @@
 from .utils import read_full_refresh, read_incremental
 
 
+def test_updated_at_field_non_exist_handler(requests_mock, common_params, fake_properties_list):
+    stream = ContactLists(**common_params)
+
+    responses = [
+        {
+            "json": {
+                stream.data_field: [
+                    {
+                        "id": "test_id",
+                        "createdAt": "2022-03-25T16:43:11Z",
+                    },
+                ],
+            }
+        }
+    ]
+    properties_response = [
+        {
+            "json": [
+                {"name": property_name, "type": "string", "updatedAt": 1571085954360, "createdAt": 1565059306048}
+                for property_name in fake_properties_list
+            ],
+            "status_code": 200,
+        }
+    ]
+
+    requests_mock.register_uri("GET", stream.url, responses)
+    requests_mock.register_uri("GET", "/properties/v2/contact/properties", properties_response)
+
+    _, stream_state = read_incremental(stream, {})
+
+    expected = int(pendulum.parse(common_params["start_date"]).timestamp() * 1000)
+
+    assert stream_state[stream.updated_at_field] == expected
+
+
 @pytest.mark.parametrize(
     "stream, endpoint",
     [
@@ -94,7 +130,7 @@ def test_streams_read(stream, endpoint, requests_mock, common_params, fake_prope
     requests_mock.register_uri("GET", "/email/public/v1/campaigns/test_id", responses)
     requests_mock.register_uri("GET", f"/properties/v2/{endpoint}/properties", properties_response)
 
-    records = read_incremental(stream, {})
+    records, _ = read_incremental(stream, {})
 
     assert records
 

diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/utils.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/utils.py
@@ -16,7 +16,7 @@ def read_incremental(stream_instance: Stream, stream_state: MutableMapping[str,
         for record in records:
             stream_state = stream_instance.get_updated_state(stream_state, record)
             res.append(record)
-    return res
+    return res, stream_state
 
 
 def read_full_refresh(stream_instance: Stream):

diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md
@@ -143,6 +143,7 @@ If you are using OAuth, most of the streams require the appropriate [scopes](htt
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.1.51  | 2022-03-24 | [11321](https://github.com/airbytehq/airbyte/pull/11321) | Fix updated at field non exists issue                                                                                                     |
 | 0.1.50  | 2022-03-22 | [11266](https://github.com/airbytehq/airbyte/pull/11266) | Fix Engagements Stream Pagination                                                                                                       |
 | 0.1.49  | 2022-03-17 | [11218](https://github.com/airbytehq/airbyte/pull/11218) | Anchor hyperlink in input configuration                                                                                                        |
 | 0.1.48  | 2022-03-16 | [11105](https://github.com/airbytehq/airbyte/pull/11105) | Fix float numbers, upd docs                                                                                                                    |