diff --git a/.github/workflows/gradle.yml b/.github/workflows/gradle.yml index 8f3a10d7ea1c..5eec5e69b6fc 100644 --- a/.github/workflows/gradle.yml +++ b/.github/workflows/gradle.yml @@ -47,6 +47,7 @@ jobs: STRIPE_INTEGRATION_TEST_CREDS: ${{ secrets.STRIPE_INTEGRATION_TEST_CREDS }} GH_INTEGRATION_TEST_CREDS: ${{ secrets.GH_INTEGRATION_TEST_CREDS }} SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} + HUBSPOT_INTEGRATION_TESTS_CREDS: ${{ secrets.HUBSPOT_INTEGRATION_TESTS_CREDS }} GSHEETS_INTEGRATION_TESTS_CREDS: ${{ secrets.GSHEETS_INTEGRATION_TESTS_CREDS }} - name: Build diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/57eb1576-8f52-463d-beb6-2e107cdf571d.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/57eb1576-8f52-463d-beb6-2e107cdf571d.json new file mode 100644 index 000000000000..4928db1c16b1 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE/57eb1576-8f52-463d-beb6-2e107cdf571d.json @@ -0,0 +1,7 @@ +{ + "sourceId": "57eb1576-8f52-463d-beb6-2e107cdf571d", + "name": "Hubspot", + "dockerRepository": "airbyte/source-hubspot-singer", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://https://docs.airbyte.io/integrations/sources/hubspot" +} diff --git a/airbyte-integrations/connectors/source-hubspot-singer/.dockerignore b/airbyte-integrations/connectors/source-hubspot-singer/.dockerignore new file mode 100644 index 000000000000..378eac25d311 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/.dockerignore @@ -0,0 +1 @@ +build diff --git a/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile new file mode 100644 index 000000000000..90c20f3c2ea7 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile @@ -0,0 +1,21 @@ +FROM airbyte/integration-base-singer:dev + +RUN apt-get update && apt-get install -y \ + bash \ + && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="source_hubspot_singer" +ENV AIRBYTE_IMPL_MODULE="source_hubspot_singer" +ENV AIRBYTE_IMPL_PATH="SourceHubspotSinger" + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-hubspot-singer + +WORKDIR /airbyte/integration_code +COPY $CODE_PATH ./$CODE_PATH +COPY resources ./$CODE_PATH +COPY setup.py ./ +RUN pip install ".[main]" + +WORKDIR /airbyte + diff --git a/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile.test b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile.test new file mode 100644 index 000000000000..7afe987e83ed --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/Dockerfile.test @@ -0,0 +1,24 @@ + +FROM airbyte/base-python-test:dev + +RUN apt-get update && apt-get install -y \ + bash \ + && rm -rf /var/lib/apt/lists/* + +ENV CODE_PATH="standardtest" +ENV AIRBYTE_IMPL_MODULE="standardtest" +ENV AIRBYTE_IMPL_PATH="HubspotStandardSourceTest" + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-hubspot-singer-standard-test + +WORKDIR /airbyte/integration_code +COPY secrets ./$CODE_PATH +COPY resources ./$CODE_PATH +COPY resourcesstandardtest ./$CODE_PATH +COPY $CODE_PATH ./$CODE_PATH +COPY setup.py ./ +RUN pip install ".[standardtest]" + +WORKDIR /airbyte + diff --git a/airbyte-integrations/connectors/source-hubspot-singer/README.md b/airbyte-integrations/connectors/source-hubspot-singer/README.md new file mode 100644 index 000000000000..de1fae919a4c --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/README.md @@ -0,0 +1,61 @@ +# Hubspot Test Configuration + +This integration wraps the existing singer [tap-hubspot](https://github.com/singer-io/tap-hubspot). In order to test the Hubspot source, you will need api credentials, see the [docs](https://docs.airbyte.io/integrations/sources/hubspot) for details. + +## Community Contributor + +1. Create a file at `secrets/config.json` with one of the following two format: + 1. If using an api key: + ``` + { + "start_date": "2017-01-01T00:00:00Z", + "credentials": { + "api_key": "your hubspot api key" + } + } + ``` + 1. If using oauth: + ``` + { + "start_date": "2017-01-01T00:00:00Z", + "credentials": { + "redirect_uri": "", + "client_id": "", + "client_secret": "", + "refresh_token": "", + } + } + ``` + +## Airbyte Employee + +1. Access the api key credentials in the `hubspot-integration-test-api-key` secret on Rippling under the `Engineering` folder + 1. If we ever need a new api key it can be found in settings -> integrations (under the account banner) -> api key +1. Access the oauth config in the `hubspot-integration-test-oauth-config` secret on Rippling under the `Engineering` folder + 1. If we ever need to regenerate the refresh token for auth follow these hubspot [instructions](https://developers.hubspot.com/docs/api/oauth-quickstart-guide). Going to lay out the process because it wasn't 100% clear in their docs. + 1. Make sure you create or have developer account. I had an account but it wasn't a developer account, which means I couldn't access the oauth info. + 1. To get to an app (which can be tricky to find) go here https://app.hubspot.com/developer/8665273/applications. + 1. Then either use the existing app or create a new one. + 1. Then basic info -> auth will get you the client_id and client_secret + 1. Then you need to get a refresh token. + 1. Put this url in a browser https://app.hubspot.com/oauth/authorize?scope=contacts%20social&redirect_uri=https://www.example.com/auth-callback&client_id=. + 1. It will direct you to a page in hubspot. Before you hit authorized on the next page open the developer console network tab. Okay, now hit authorize. + 1. In the developer console search for example.com and in the url params for the http request example.com there will be a code `https://www.example.com/auth-callback?code=`. Save that code somewhere (I will refer to it as code below). + 1. Use this javascript snipped to get the refresh token. In the response body will be a refresh token. + ``` + var request = require("request"); + + const formData = { + grant_type: 'authorization_code', + client_id: '', + client_secret: '', + redirect_uri: 'https://www.example.com/auth-callback', + code: '' + }; + + request.post('https://api.hubapi.com/oauth/v1/token', { form: formData }, (err, data) => { + console.log(data) + }) + ``` + 1. Finally! You have all the pieces you need to do oauth with hubspot (client_id, client_secret, and refresh_token). + diff --git a/airbyte-integrations/connectors/source-hubspot-singer/build.gradle b/airbyte-integrations/connectors/source-hubspot-singer/build.gradle new file mode 100644 index 000000000000..d88fa050983b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/build.gradle @@ -0,0 +1,20 @@ +project.ext.pyModule = 'source_hubspot_singer' +apply from: rootProject.file('tools/gradle/commons/integrations/python.gradle') +apply from: rootProject.file('tools/gradle/commons/integrations/image.gradle') +apply from: rootProject.file('tools/gradle/commons/integrations/test-image.gradle') +apply from: rootProject.file('tools/gradle/commons/integrations/integration-test.gradle') +apply from: rootProject.file('tools/gradle/commons/integrations/standard-source-test-python.gradle') + +standardSourceTestPython { + ext { + imageName = "${extractImageName(project.file('Dockerfile'))}:dev" + pythonContainerName = "${extractImageName(project.file('Dockerfile.test'))}:dev" + } +} + +standardSourceTestPython.dependsOn(buildTestImage) +build.dependsOn ':airbyte-integrations:bases:base-singer:build' +buildImage.dependsOn ':airbyte-integrations:bases:base-singer:buildImage' +build.dependsOn ':airbyte-integrations:bases:base-python-test:build' +buildTestImage.dependsOn ':airbyte-integrations:bases:base-python-test:buildImage' +integrationTest.dependsOn(buildImage) diff --git a/airbyte-integrations/connectors/source-hubspot-singer/main_dev.py b/airbyte-integrations/connectors/source-hubspot-singer/main_dev.py new file mode 100644 index 000000000000..bf981e765e22 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/main_dev.py @@ -0,0 +1,32 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import sys + +from base_python.entrypoint import launch +from source_hubspot_singer import SourceHubspotSinger + +if __name__ == "__main__": + source = SourceHubspotSinger() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-hubspot-singer/requirements.standard-test.txt b/airbyte-integrations/connectors/source-hubspot-singer/requirements.standard-test.txt new file mode 100644 index 000000000000..bf756121ea4e --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/requirements.standard-test.txt @@ -0,0 +1,2 @@ +-e ../../bases/base-python-test +-e . diff --git a/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt b/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt new file mode 100644 index 000000000000..a2c4c5f66b9b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/requirements.txt @@ -0,0 +1,4 @@ +-e ../../bases/airbyte-protocol +-e ../../bases/base-singer +-e ../../bases/base-python-test +-e . diff --git a/airbyte-integrations/connectors/source-hubspot-singer/resources/spec.json b/airbyte-integrations/connectors/source-hubspot-singer/resources/spec.json new file mode 100644 index 000000000000..06481e95f44b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/resources/spec.json @@ -0,0 +1,64 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/hubspot", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Hubspot Source Spec", + "type": "object", + "required": ["start_date", "credentials"], + "additionalProperties": false, + "properties": { + "start_date": { + "type": "string", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", + "examples": ["2017-01-25T00:00:00Z"] + }, + "credentials": { + "type": "object", + "oneOf": [ + { + "title": "api key", + "required": ["api_key"], + "properties": { + "api_key": { + "description": "Hubspot API Key. See our docs if you need help finding this key.", + "type": "string" + } + } + }, + { + "title": "oauth", + "required": [ + "redirect_uri", + "client_id", + "client_secret", + "refresh_token" + ], + "properties": { + "redirect_uri": { + "description": "Hubspot API Key. See our docs if you need help finding this key.", + "type": "string", + "examples": ["https://api.hubspot.com/"] + }, + "client_id": { + "description": "Hubspot client_id. See our docs if you need help finding this id.", + "type": "string", + "examples": ["123456789000,"] + }, + "client_secret": { + "description": "Hubspot client_secret. See our docs if you need help finding this secret.", + "type": "string", + "examples": ["secret"] + }, + "refresh_token": { + "description": "Hubspot refresh_token. See our docs if you need help generating the token.", + "type": "string", + "examples": ["refresh_token"] + } + } + } + ] + } + } + } +} diff --git a/airbyte-integrations/connectors/source-hubspot-singer/resourcesstandardtest/catalog.json b/airbyte-integrations/connectors/source-hubspot-singer/resourcesstandardtest/catalog.json new file mode 100644 index 000000000000..f1e79f8e4bde --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/resourcesstandardtest/catalog.json @@ -0,0 +1,79 @@ +{ + "streams": [ + { + "name": "owners", + "json_schema": { + "type": "object", + "properties": { + "portalId": { + "type": ["null", "integer"] + }, + "ownerId": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "signature": { + "type": ["null", "string"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "hasContactsAccess": { + "type": ["null", "boolean"] + }, + "isActive": { + "type": ["null", "boolean"] + }, + "activeUserId": { + "type": ["null", "integer"] + }, + "userIdIncludingInactive": { + "type": ["null", "integer"] + }, + "remoteList": { + "type": "array", + "items": { + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "portalId": { + "type": ["null", "integer"] + }, + "ownerId": { + "type": ["null", "integer"] + }, + "remoteId": { + "type": ["null", "string"] + }, + "remoteType": { + "type": ["null", "string"] + }, + "active": { + "type": ["null", "boolean"] + } + } + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot-singer/setup.py b/airbyte-integrations/connectors/source-hubspot-singer/setup.py new file mode 100644 index 000000000000..b0d9afcc3a9d --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/setup.py @@ -0,0 +1,38 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from setuptools import find_packages, setup + +setup( + name="source_github_singer", + description="Source implementation for Github.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + package_data={"": ["*.json"]}, + extras_require={ + "main": ["tap-hubspot==2.8.1", "requests", "airbyte-protocol", "base-singer"], + "standardtest": ["airbyte_python_test"], + }, +) diff --git a/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/__init__.py b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/__init__.py new file mode 100644 index 000000000000..72edd0735fd9 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourceHubspotSinger + +__all__ = ["SourceHubspotSinger"] diff --git a/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/source.py b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/source.py new file mode 100644 index 000000000000..750e0ce0632a --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/source_hubspot_singer/source.py @@ -0,0 +1,104 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import requests +from airbyte_protocol import AirbyteConnectionStatus, Status +from base_singer import SingerSource + + +class SourceHubspotSinger(SingerSource): + def __init__(self): + pass + + def check(self, logger, config_container) -> AirbyteConnectionStatus: + try: + json_config = config_container.rendered_config + api_key = json_config.get("hapikey", None) + if api_key: + logger.info("checking with api key") + r = requests.get(f"https://api.hubapi.com/contacts/v1/lists/all/contacts/all?hapikey={api_key}") + else: + logger.info("checking with oauth") + # borrowing from tap-hubspot + # https://github.com/singer-io/tap-hubspot/blob/master/tap_hubspot/__init__.py#L208-L229 + payload = { + "grant_type": "refresh_token", + "redirect_uri": json_config["redirect_uri"], + "refresh_token": json_config["refresh_token"], + "client_id": json_config["client_id"], + "client_secret": json_config["client_secret"], + } + resp = requests.post("https://api.hubapi.com/oauth/v1/token", data=payload) + + if resp.status_code == 403: + return AirbyteConnectionStatus(status=Status.FAILED, message=resp.content) + + try: + resp.raise_for_status() + except Exception as e: + return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}") + + auth = resp.json() + headers = {"Authorization": "Bearer {}".format(auth["access_token"])} + + r = requests.get("https://api.hubapi.com/contacts/v1/lists/all/contacts/all", headers=headers) + + if r.status_code == 200: + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + else: + return AirbyteConnectionStatus(status=Status.FAILED, message=r.text) + except Exception as e: + return AirbyteConnectionStatus(status=Status.FAILED, message=f"{str(e)}") + + def discover_cmd(self, logger, config_path) -> str: + return f"tap-hubspot --config {config_path} --discover" + + def read_cmd(self, logger, config_path, catalog_path, state_path=None) -> str: + config_option = f"--config {config_path}" + properties_option = f"--properties {catalog_path}" + state_option = f"--state {state_path}" if state_path else "" + return f"tap-hubspot {config_option} {properties_option} {state_option}" + + def transform_config(self, raw_config): + rendered_config = dict(raw_config) + singer_config = dict() + + # the singer integration requires if the hapikey field is passed that all + # of the oauth fields get passed as well. it just checks existence. we + # do not bother our users to provide the extra oauth creds if they are + # using an api key. so we fill them in with dummy values for them. + if "api_key" in rendered_config["credentials"]: + singer_config["hapikey"] = rendered_config["credentials"]["api_key"] + singer_config["redirect_uri"] = "placeholder" + singer_config["client_id"] = "placeholder" + singer_config["client_secret"] = "placeholder" + singer_config["refresh_token"] = "placeholder" + else: + singer_config = rendered_config["credentials"] + + singer_config["start_date"] = rendered_config["start_date"] + # always turn off singer tracking. + singer_config["disable_collection"] = True + + return singer_config diff --git a/airbyte-integrations/connectors/source-hubspot-singer/standardtest/__init__.py b/airbyte-integrations/connectors/source-hubspot-singer/standardtest/__init__.py new file mode 100644 index 000000000000..758a9ddc400b --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/standardtest/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .standard_source_test import HubspotStandardSourceTest + +__all__ = ["HubspotStandardSourceTest"] diff --git a/airbyte-integrations/connectors/source-hubspot-singer/standardtest/standard_source_test.py b/airbyte-integrations/connectors/source-hubspot-singer/standardtest/standard_source_test.py new file mode 100644 index 000000000000..f062cf9e30b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot-singer/standardtest/standard_source_test.py @@ -0,0 +1,48 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import json +import pkgutil + +from airbyte_protocol import AirbyteCatalog, ConnectorSpecification +from base_python_test import StandardSourceTestIface + + +class HubspotStandardSourceTest(StandardSourceTestIface): + def get_spec(self) -> ConnectorSpecification: + raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "spec.json") + return ConnectorSpecification.parse_obj(json.loads(raw_spec)) + + def get_config(self) -> object: + return json.loads(pkgutil.get_data(self.__class__.__module__.split(".")[0], "config.json")) + + def get_catalog(self) -> AirbyteCatalog: + raw_spec = pkgutil.get_data(self.__class__.__module__.split(".")[0], "catalog.json") + return AirbyteCatalog.parse_obj(json.loads(raw_spec)) + + def setup(self) -> None: + pass + + def teardown(self) -> None: + pass diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md new file mode 100644 index 000000000000..9c3813fc6612 --- /dev/null +++ b/docs/integrations/sources/hubspot.md @@ -0,0 +1,37 @@ +# Hubspot + +## Overview + +The Hubspot source supports Full Refresh syncs. That is, every time a sync is run, Airbyte will copy all rows in the tables and columns you set up for replication into the destination in a new table. + +This Hubspot source wraps the [Singer Hubspot Tap](https://github.com/singer-io/tap-hubspot). + +### Output schema + +Several output streams are available from this source \(campaigns, contacts, deals, etc.\) For a comprehensive output schema [look at the Singer tap schema files](https://github.com/singer-io/tap-hubspot/tree/master/tap_hubspot/schemas). + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | No | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | + +### Performance considerations + +The connector is restricted by normal Hubspot [rate limitations](https://legacydocs.hubspot.com/apps/api_guidelines). + +## Getting started + +### Requirements + +* Hubspot Account +* Api credentials + +### Setup guide +*There are two ways of performing auth with hubspot (api key and oauth): + * For api key auth, in Hubspot, for the account to go settings -> integrations (under the account banner) -> api key. If you already have an api key you can use that. Otherwise generated a new one. + * Note: The Hubspot [docs](https://legacydocs.hubspot.com/docs/methods/auth/oauth-overview) recommends that api key auth is only used for testing purposes. + * For oauth follow the [oauth instruction](https://developers.hubspot.com/docs/api/oauth-quickstart-guide) in Hubspot to get client_id, client_secret, redirect_uri, and refresh_token. diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index f9371018e866..fff433b12688 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -15,5 +15,8 @@ jq --arg v "$GH_INTEGRATION_TEST_CREDS" '.access_token = $v' airbyte-integration mkdir airbyte-integrations/connectors/source-salesforce-singer/secrets/ echo "$SALESFORCE_INTEGRATION_TESTS_CREDS" > airbyte-integrations/connectors/source-salesforce-singer/secrets/config.json +mkdir airbyte-integrations/connectors/source-hubspot-singer/secrets/ +echo "$HUBSPOT_INTEGRATION_TESTS_CREDS" > airbyte-integrations/connectors/source-hubspot-singer/secrets/config.json + mkdir airbyte-integrations/connectors/source-google-sheets/secrets echo "$GSHEETS_INTEGRATION_TESTS_CREDS" > airbyte-integrations/connectors/source-google-sheets/secrets/creds.json