🐛 Source HubSpot: Fix empty string inside number / float datatype (…

…#5334) #5293 - Source Hubspot fails in normalization step Co-authored-by: Oleksandr Bazarnov <oleksandr.bazarnov@globallogic.com>
airbytehq · Aug 12, 2021 · 6f56833 · 6f56833
1 parent acab9dd
commit 6f56833
Show file tree

Hide file tree

Showing 24 changed files with 298 additions and 232 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/36c891d9-4bd9-43ac-bad2-10e12756272c.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "36c891d9-4bd9-43ac-bad2-10e12756272c",
   "name": "Hubspot",
   "dockerRepository": "airbyte/source-hubspot",
-  "dockerImageTag": "0.1.8",
+  "dockerImageTag": "0.1.9",
   "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot",
   "icon": "hubspot.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -137,7 +137,7 @@
 - sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
   name: Hubspot
   dockerRepository: airbyte/source-hubspot
-  dockerImageTag: 0.1.8
+  dockerImageTag: 0.1.9
   documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
   icon: hubspot.svg
 - sourceDefinitionId: 95e8cffd-b8c4-4039-968e-d32fb4a69bde

diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile
@@ -14,5 +14,5 @@ RUN pip install .
 
 ENV AIRBYTE_ENTRYPOINT "/airbyte/base.sh"
 
-LABEL io.airbyte.version=0.1.8
+LABEL io.airbyte.version=0.1.9
 LABEL io.airbyte.name=airbyte/source-hubspot
diff --git a/airbyte-integrations/connectors/source-hubspot/README.md b/airbyte-integrations/connectors/source-hubspot/README.md
@@ -53,12 +53,51 @@ python main_dev.py discover --config secrets/config.json
 python main_dev.py read --config secrets/config.json --catalog sample_files/configured_catalog.json
 ```
 
+## Testing
+Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+First install test dependencies into your virtual environment:
+```
+pip install .[tests]
+```
+
 ### Unit Tests
 To run unit tests locally, from the connector directory run:
 ```
 python -m pytest unit_tests
 ```
 
+### Integration Tests
+There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector).
+
+#### Custom Integration tests
+Place custom tests inside `integration_tests/` folder, then, from the connector root, run
+```
+python -m pytest integration_tests
+```
+
+#### Acceptance Tests
+Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information.
+If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py.
+
+To run your integration tests with acceptance tests, from the connector root, run
+```
+python -m pytest integration_tests -p integration_tests.acceptance
+```
+
+To run your integration tests with docker
+
+### Using gradle to run tests
+All commands should be run from airbyte project root.
+To run unit tests:
+```
+./gradlew :airbyte-integrations:connectors:source-hubspot:unitTest
+```
+
+To run acceptance and custom integration tests:
+```
+./gradlew :airbyte-integrations:connectors:source-hubspot:integrationTest
+```
+
 ### Locally running the connector docker image
 
 #### Build
@@ -85,16 +124,16 @@ docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/sample_files:/sample_files
 
 ### Integration Tests
 1. From the airbyte project root, run `./gradlew :airbyte-integrations:connectors:source-hubspot:integrationTest` to run the standard integration test suite.
-1. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`.
-   Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
+2. To run additional integration tests, place your integration tests in a new directory `integration_tests` and run them with `python -m pytest -s integration_tests`.
+Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named.
 
 ## Dependency Management
 All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development.
 
 ### Publishing a new version of the connector
 You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what?
 1. Make sure your changes are passing unit and integration tests
-1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer).
-1. Create a Pull Request
-1. Pat yourself on the back for being an awesome contributor
-1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master
+2. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use SemVer).
+3. Create a Pull Request
+4. Pat yourself on the back for being an awesome contributor
+5. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master
diff --git a/airbyte-integrations/connectors/source-hubspot/setup.py b/airbyte-integrations/connectors/source-hubspot/setup.py
@@ -28,9 +28,9 @@
 MAIN_REQUIREMENTS = [
     "airbyte-protocol",
     "base-python",
-    "backoff==1.10.0",
-    "pendulum==1.2.0",
-    "requests==2.25.1",
+    "backoff==1.11.1",
+    "pendulum==2.1.2",
+    "requests==2.26.0",
 ]
 
 TEST_REQUIREMENTS = [

diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py
@@ -273,12 +273,17 @@ def _cast_value(declared_field_types: List, field_name: str, field_value):
         target_type = CUSTOM_FIELD_VALUE_TYPE_CAST_REVERSED.get(target_type_name)
 
         if target_type_name == "number":
-            if field_name.endswith("_id"):
-                # do not cast numeric IDs into float, use integer instead
-                target_type = int
+            # do not cast numeric IDs into float, use integer instead
+            target_type = int if field_name.endswith("_id") else target_type
+
+        if target_type_name != "string" and field_value == '':
+            # do not cast empty strings, return None instead to be properly casted.
+            field_value = None
+            return field_value
 
         try:
             casted_value = target_type(field_value)
+            print(casted_value)
         except ValueError:
             logger.exception(f"Could not cast `{field_value}` to `{target_type}`")
             return field_value
@@ -451,7 +456,7 @@ def read(self, getter: Callable, params: Mapping[str, Any] = None) -> Iterator:
                 self._start_date = self._state
 
     def read_chunked(
-        self, getter: Callable, params: Mapping[str, Any] = None, chunk_size: pendulum.Interval = pendulum.interval(days=1)
+        self, getter: Callable, params: Mapping[str, Any] = None, chunk_size: pendulum.duration = pendulum.duration(days=1)
     ) -> Iterator:
         params = {**params} if params else {}
         now_ts = int(pendulum.now().timestamp() * 1000)

diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/campaigns.json
@@ -86,7 +86,7 @@
       "type": ["null", "string"]
     },
     "lastUpdatedTime": {
-      "type": "integer"
+      "type": ["null", "integer"]
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/companies.json
@@ -6,7 +6,7 @@
       "type": ["null", "integer"]
     },
     "companyId": {
-      "type": "integer"
+      "type": ["null", "integer"]
     },
     "isDeleted": {
       "type": ["null", "boolean"]
@@ -24,10 +24,10 @@
       "type": ["null", "array"]
     },
     "createdAt": {
-      "type": "string"
+      "type": ["null", "string"]
     },
     "updatedAt": {
-      "type": "string"
+      "type": ["null", "string"]
     }
   }
 }
diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/contact_lists.json
@@ -6,7 +6,7 @@
       "type": ["null", "integer"]
     },
     "metaData": {
-      "type": "object",
+      "type": ["null", "object"],
       "properties": {
         "processing": {
           "type": ["null", "string"]
@@ -21,7 +21,7 @@
           "type": ["null", "integer"]
         },
         "lastSizeChangeAt": {
-          "type": "integer"
+          "type": ["null", "integer"]
         }
       }
     },
@@ -32,11 +32,11 @@
       "type": ["null", "string"]
     },
     "filters": {
-      "type": "array",
+      "type": ["null", "array"],
       "items": {
-        "type": "array",
+        "type": ["null", "array"],
         "items": {
-          "type": "object",
+          "type": ["null", "object"],
           "properties": {
             "filterFamily": {
               "type": ["null", "string"]
@@ -67,13 +67,13 @@
       "type": ["null", "integer"]
     },
     "createdAt": {
-      "type": "integer"
+      "type": ["null", "integer"]
     },
     "listId": {
       "type": ["null", "integer"]
     },
     "updatedAt": {
-      "type": "integer"
+      "type": ["null", "integer"]
     },
     "internalListId": {
       "type": ["null", "integer"]