#268: fix URI too long (HubSpot)

airbytehq · Jun 10, 2022 · f3922df · f3922df
1 parent ab4028d
commit f3922df
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 3 deletions.
diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.68
+LABEL io.airbyte.version=0.1.69
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py
@@ -64,13 +64,13 @@ def split_properties(properties_list: List[str]) -> Iterator[Tuple[str]]:
     summary_length = 0
     local_properties = []
     for property_ in properties_list:
-        if len(property_) + summary_length + len(urllib.parse.quote(",")) >= PROPERTIES_PARAM_MAX_LENGTH:
+        if len(urllib.parse.quote(f"property={property_}&")) + summary_length >= PROPERTIES_PARAM_MAX_LENGTH:
             yield local_properties
             local_properties = []
             summary_length = 0
 
         local_properties.append(property_)
-        summary_length += len(property_) + len(urllib.parse.quote(","))
+        summary_length += len(urllib.parse.quote(f"property={property_}&"))
 
     if local_properties:
         yield local_properties

diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_split_properties.py
@@ -0,0 +1,23 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+import pytest
+from source_hubspot.streams import split_properties
+
+lorem_ipsum = """Lorem ipsum dolor sit amet, consectetur adipiscing elit"""
+lorem_ipsum = lorem_ipsum.lower().replace(",", "")
+
+many_properties = lorem_ipsum.split(" ") * 100
+few_properties = ["firstname", "lastname", "age", "dob", "id"]
+
+
+@pytest.mark.parametrize(("properties", "chunks_expected"), ((few_properties, 1), (many_properties, 2)))
+def test_split_properties(properties, chunks_expected):
+    chunked_properties = set()
+    index = 0
+    for index, chunk in enumerate(split_properties(properties)):
+        chunked_properties |= set(chunk)
+    chunks = index + 1
+    assert chunked_properties == set(properties)
+    assert chunks == chunks_expected
diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md
@@ -133,6 +133,7 @@ HubSpot's API will [rate limit](https://developers.hubspot.com/docs/api/usage-de
 
 | Version | Date       | Pull Request                                             | Subject                                                                                                                                        |
 |:--------|:-----------|:---------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------|
+| 0.1.69  | 2022-06-10 | [00000](https://github.com/airbytehq/airbyte/pull/00000) | Fix the `URI Too Long` issue                                                                                                                   |
 | 0.1.68  | 2022-06-08 | [13596](https://github.com/airbytehq/airbyte/pull/13596) | Fix for the `property_history` which did not emit records                                                                                      |
 | 0.1.67  | 2022-06-07 | [13566](https://github.com/airbytehq/airbyte/pull/13566) | Report which scopes are missing to the user                                                                                                    |
 | 0.1.66  | 2022-06-05 | [13475](https://github.com/airbytehq/airbyte/pull/13475) | Scope `crm.objects.feedback_submissions.read` added for `feedback_submissions` stream                                                          |