Hubspot Source

.github/workflows/gradle.yml

@@ -47,6 +47,7 @@ jobs:
           STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }}
           GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }}
           SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }}
+          HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }}
           GSHEETS_INTEGRATION_TESTS_CREDS: ${{ secrets.GSHEETS_INTEGRATION_TESTS_CREDS }}
 
       - name: Build

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/57eb1576-8f52-463d-beb6-2e107cdf571d.json

@@ -0,0 +1,7 @@
+{
+  "sourceId": "57eb1576-8f52-463d-beb6-2e107cdf571d",
+  "name": "Hubspot",
+  "dockerRepository": "airbyte/source-hubspot-singer",
+  "dockerImageTag": "0.1.0",
+  "documentationUrl": "https://https://docs.airbyte.io/integrations/sources/hubspot"
+}

airbyte-integrations/connectors/source-hubspot-singer/.dockerignore

@@ -0,0 +1 @@
+build

airbyte-integrations/connectors/source-hubspot-singer/Dockerfile

@@ -0,0 +1,21 @@
+FROM airbyte/integration-base-singer:dev
+
+RUN apt-get update && apt-get install -y \
+    bash \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CODE_PATH="source_hubspot_singer"
+ENV AIRBYTE_IMPL_MODULE="source_hubspot_singer"
+ENV AIRBYTE_IMPL_PATH="SourceHubspotSinger"
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-hubspot-singer
+
+WORKDIR /airbyte/integration_code
+COPY $CODE_PATH ./$CODE_PATH
+COPY resources ./$CODE_PATH
+COPY setup.py ./
+RUN pip install ".[main]"
+
+WORKDIR /airbyte
+

airbyte-integrations/connectors/source-hubspot-singer/Dockerfile.test

@@ -0,0 +1,24 @@
+
+FROM airbyte/base-python-test:dev
+
+RUN apt-get update && apt-get install -y \
+    bash \
+    && rm -rf /var/lib/apt/lists/*
+
+ENV CODE_PATH="standardtest"
+ENV AIRBYTE_IMPL_MODULE="standardtest"
+ENV AIRBYTE_IMPL_PATH="HubspotStandardSourceTest"
+
+LABEL io.airbyte.version=0.1.0
+LABEL io.airbyte.name=airbyte/source-hubspot-singer-standard-test
+
+WORKDIR /airbyte/integration_code
+COPY secrets ./$CODE_PATH
+COPY resources ./$CODE_PATH
+COPY resourcesstandardtest ./$CODE_PATH
+COPY $CODE_PATH ./$CODE_PATH
+COPY setup.py ./
+RUN pip install ".[standardtest]"
+
+WORKDIR /airbyte
+

airbyte-integrations/connectors/source-hubspot-singer/README.md

@@ -0,0 +1,61 @@
+# Hubspot Test Configuration
+
+This integration wraps the existing singer [tap-hubspot](https://github.com/singer-io/tap-hubspot). In order to test the Hubspot source, you will need api credentials, see the [docs](https://docs.airbyte.io/integrations/sources/hubspot) for details.
+
+## Community Contributor
+
+1. Create a file at `secrets/config.json` with one of the following two format:
+    1. If using an api key:
+        ```
+        {
+          "start_date": "2017-01-01T00:00:00Z",
+          "credentials": {
+            "api_key": "your hubspot api key"
+          }
+        }
+        ```
+   1. If using oauth:
+       ```
+        {
+          "start_date": "2017-01-01T00:00:00Z",
+          "credentials": {
+            "redirect_uri": "<redirect uri>",
+            "client_id": "<hubspot client id>",
+            "client_secret": "<hubspot client secret>",
+            "refresh_token": "<hubspot refresh token>",
+          }
+        }
+       ```
+
+## Airbyte Employee
+
+1. Access the api key credentials in the `hubspot-integration-test-api-key` secret on Rippling under the `Engineering` folder
+    1. If we ever need a new api key it can be found in settings -> integrations (under the account banner) -> api key
+1. Access the oauth config in the `hubspot-integration-test-oauth-config` secret on Rippling under the `Engineering` folder
+    1. If we ever need to regenerate the refresh token for auth follow these hubspot [instructions](https://developers.hubspot.com/docs/api/oauth-quickstart-guide). Going to lay out the process because it wasn't 100% clear in their docs.
+    1. Make sure you create or have developer account. I had an account but it wasn't a developer account, which means I couldn't access the oauth info.
+    1. To get to an app (which can be tricky to find) go here https://app.hubspot.com/developer/8665273/applications.
+    1. Then either use the existing app or create a new one.
+    1. Then basic info -> auth will get you the client_id and client_secret
+    1. Then you need to get a refresh token.
+    1. Put this url in a browser https://app.hubspot.com/oauth/authorize?scope=contacts%20social&redirect_uri=https://www.example.com/auth-callback&client_id=<client-id>.
+    1. It will direct you to a page in hubspot. Before you hit authorized on the next page open the developer console network tab. Okay, now hit authorize.
+    1. In the developer console search for example.com and in the url params for the http request example.com there will be a code `https://www.example.com/auth-callback?code=<code>`. Save that code somewhere (I will refer to it as code below).
+    1. Use this javascript snipped to get the refresh token. In the response body will be a refresh token.
+       ```
+       var request = require("request");
+
+       const formData = {
+         grant_type: 'authorization_code',
+         client_id: '<client-id>',
+         client_secret: '<client-secret>',
+         redirect_uri: 'https://www.example.com/auth-callback',
+         code: '<code>'
+       };
+
+       request.post('https://api.hubapi.com/oauth/v1/token', { form: formData }, (err, data) => {
+         console.log(data)
+       })
+       ```
+   1. Finally! You have all the pieces you need to do oauth with hubspot (client_id, client_secret, and refresh_token).
+

airbyte-integrations/connectors/source-hubspot-singer/build.gradle

@@ -0,0 +1,20 @@
+project.ext.pyModule = 'source_hubspot_singer'
+apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle')
+apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle')
+
+standardSourceTestPython {
+    ext {
+        imageName = "${extractImageName(project.file('Dockerfile'))}:dev"
+        pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev"
+    }
+}
+
+standardSourceTestPython.dependsOn(buildTestImage)
+build.dependsOn ':airbyte-integrations:bases:base-singer:build'
+buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage'
+build.dependsOn ':airbyte-integrations:bases:base-python-test:build'
+buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage'
+integrationTest.dependsOn(buildImage)

airbyte-integrations/connectors/source-hubspot-singer/main_dev.py

@@ -0,0 +1,32 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+import sys
+
+from base_python.entrypoint import launch
+from source_hubspot_singer import SourceHubspotSinger
+
+if __name__ == "__main__":
+    source = SourceHubspotSinger()
+    launch(source, sys.argv[1:])

airbyte-integrations/connectors/source-hubspot-singer/requirements.standard-test.txt

@@ -0,0 +1,2 @@
+-e ../../bases/base-python-test
+-e .

airbyte-integrations/connectors/source-hubspot-singer/requirements.txt

@@ -0,0 +1,4 @@
+-e ../../bases/airbyte-protocol
+-e ../../bases/base-singer
+-e ../../bases/base-python-test
+-e .

airbyte-integrations/connectors/source-hubspot-singer/resources/spec.json

@@ -0,0 +1,64 @@
+{
+  "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot",
+  "connectionSpecification": {
+    "$schema": "http://json-schema.org/draft-07/schema#",
+    "title": "Hubspot Source Spec",
+    "type": "object",
+    "required": ["start_date", "credentials"],
+    "additionalProperties": false,
+    "properties": {
+      "start_date": {
+        "type": "string",
+        "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$",
+        "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.",
+        "examples": ["2017-01-25T00:00:00Z"]
+      },
+      "credentials": {
+        "type": "object",
+        "oneOf": [
+          {
+            "title": "api key",
+            "required": ["api_key"],
+            "properties": {
+              "api_key": {
+                "description": "Hubspot API Key. See our <a href=\"https://docs.airbyte.io/integrations/sources/hubspot\">docs</a> if you need help finding this key.",
+                "type": "string"
+              }
+            }
+          },
+          {
+            "title": "oauth",
+            "required": [
+              "redirect_uri",
+              "client_id",
+              "client_secret",
+              "refresh_token"
+            ],
+            "properties": {
+              "redirect_uri": {
+                "description": "Hubspot API Key. See our <a href=\"https://docs.airbyte.io/integrations/sources/hubspot\">docs</a> if you need help finding this key.",
+                "type": "string",
+                "examples": ["https://api.hubspot.com/"]
+              },
+              "client_id": {
+                "description": "Hubspot client_id. See our <a href=\"https://docs.airbyte.io/integrations/sources/hubspot\">docs</a> if you need help finding this id.",
+                "type": "string",
+                "examples": ["123456789000,"]
+              },
+              "client_secret": {
+                "description": "Hubspot client_secret. See our <a href=\"https://docs.airbyte.io/integrations/sources/hubspot\">docs</a> if you need help finding this secret.",
+                "type": "string",
+                "examples": ["secret"]
+              },
+              "refresh_token": {
+                "description": "Hubspot refresh_token. See our <a href=\"https://docs.airbyte.io/integrations/sources/hubspot\">docs</a> if you need help generating the token.",
+                "type": "string",
+                "examples": ["refresh_token"]
+              }
+            }
+          }
+        ]
+      }
+    }
+  }
+}

airbyte-integrations/connectors/source-hubspot-singer/resourcesstandardtest/catalog.json

@@ -0,0 +1,79 @@
+{
+  "streams": [
+    {
+      "name": "owners",
+      "json_schema": {
+        "type": "object",
+        "properties": {
+          "portalId": {
+            "type": ["null", "integer"]
+          },
+          "ownerId": {
+            "type": ["null", "integer"]
+          },
+          "type": {
+            "type": ["null", "string"]
+          },
+          "firstName": {
+            "type": ["null", "string"]
+          },
+          "lastName": {
+            "type": ["null", "string"]
+          },
+          "email": {
+            "type": ["null", "string"]
+          },
+          "createdAt": {
+            "type": ["null", "string"],
+            "format": "date-time"
+          },
+          "signature": {
+            "type": ["null", "string"]
+          },
+          "updatedAt": {
+            "type": ["null", "string"],
+            "format": "date-time"
+          },
+          "hasContactsAccess": {
+            "type": ["null", "boolean"]
+          },
+          "isActive": {
+            "type": ["null", "boolean"]
+          },
+          "activeUserId": {
+            "type": ["null", "integer"]
+          },
+          "userIdIncludingInactive": {
+            "type": ["null", "integer"]
+          },
+          "remoteList": {
+            "type": "array",
+            "items": {
+              "type": "object",
+              "properties": {
+                "id": {
+                  "type": ["null", "integer"]
+                },
+                "portalId": {
+                  "type": ["null", "integer"]
+                },
+                "ownerId": {
+                  "type": ["null", "integer"]
+                },
+                "remoteId": {
+                  "type": ["null", "string"]
+                },
+                "remoteType": {
+                  "type": ["null", "string"]
+                },
+                "active": {
+                  "type": ["null", "boolean"]
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  ]
+}

airbyte-integrations/connectors/source-hubspot-singer/setup.py

@@ -0,0 +1,38 @@
+"""
+MIT License
+
+Copyright (c) 2020 Airbyte
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+"""
+
+from setuptools import find_packages, setup
+
+setup(
+    name="source_github_singer",
+    description="Source implementation for Github.",
+    author="Airbyte",
+    author_email="contact@airbyte.io",
+    packages=find_packages(),
+    package_data={"": ["*.json"]},
+    extras_require={
+        "main": ["tap-hubspot==2.8.1", "requests", "airbyte-protocol", "base-singer"],
+        "standardtest": ["airbyte_python_test"],
+    },
+)