From 8bb311e8517fe01d22a35539b03cac0b0c15f9ae Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Fri, 9 Jul 2021 22:49:28 +0300 Subject: [PATCH 01/23] Added Intercom implementation --- .../connectors/source-intercom/.dockerignore | 7 + .../connectors/source-intercom/Dockerfile | 16 + .../connectors/source-intercom/README.md | 131 ++ .../acceptance-test-config.yml | 26 + .../source-intercom/acceptance-test-docker.sh | 7 + .../connectors/source-intercom/build.gradle | 14 + .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 21 + .../integration_tests/acceptance.py | 34 + .../integration_tests/catalog.json | 181 +++ .../integration_tests/configured_catalog.json | 1341 +++++++++++++++++ .../integration_tests/invalid_config.json | 4 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 20 + .../connectors/source-intercom/main.py | 31 + .../source-intercom/requirements.txt | 2 + .../connectors/source-intercom/setup.py | 46 + .../source_intercom/__init__.py | 27 + .../source_intercom/schemas/admins.json | 65 + .../source_intercom/schemas/companies.json | 116 ++ .../schemas/company_attributes.json | 63 + .../schemas/company_segments.json | 29 + .../schemas/contact_attributes.json | 104 ++ .../source_intercom/schemas/contacts.json | 534 +++++++ .../schemas/conversation_parts.json | 208 +++ .../schemas/conversations.json | 827 ++++++++++ .../source_intercom/schemas/segments.json | 29 + .../source_intercom/schemas/tags.json | 15 + .../source_intercom/schemas/teams.json | 28 + .../source-intercom/source_intercom/source.py | 318 ++++ .../source-intercom/source_intercom/spec.json | 23 + .../source-intercom/unit_tests/unit_test.py | 25 + docs/integrations/sources/intercom.md | 2 +- tools/bin/ci_credentials.sh | 1 + 34 files changed, 4297 insertions(+), 1 deletion(-) create mode 100644 airbyte-integrations/connectors/source-intercom/.dockerignore create mode 100644 airbyte-integrations/connectors/source-intercom/Dockerfile create mode 100644 airbyte-integrations/connectors/source-intercom/README.md create mode 100644 airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml create mode 100755 airbyte-integrations/connectors/source-intercom/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-intercom/build.gradle create mode 100644 airbyte-integrations/connectors/source-intercom/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-intercom/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-intercom/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-intercom/main.py create mode 100644 airbyte-integrations/connectors/source-intercom/requirements.txt create mode 100644 airbyte-integrations/connectors/source-intercom/setup.py create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/__init__.py create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/admins.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/tags.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/schemas/teams.json create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/source.py create mode 100644 airbyte-integrations/connectors/source-intercom/source_intercom/spec.json create mode 100644 airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py diff --git a/airbyte-integrations/connectors/source-intercom/.dockerignore b/airbyte-integrations/connectors/source-intercom/.dockerignore new file mode 100644 index 000000000000..e06cadbdaf9f --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_intercom +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-intercom/Dockerfile b/airbyte-integrations/connectors/source-intercom/Dockerfile new file mode 100644 index 000000000000..fd3069a3bb2f --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_intercom ./source_intercom +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-intercom diff --git a/airbyte-integrations/connectors/source-intercom/README.md b/airbyte-integrations/connectors/source-intercom/README.md new file mode 100644 index 000000000000..356e26dd6c39 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/README.md @@ -0,0 +1,131 @@ +# Intercom Source + +This is the repository for the Intercom source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/intercom). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-intercom:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/intercom) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_intercom/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source intercom test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-intercom:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-intercom:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-intercom:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-intercom:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-intercom:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-intercom:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-intercom:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-intercom:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml new file mode 100644 index 000000000000..324d34c68901 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml @@ -0,0 +1,26 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/contributing-to-airbyte/building-new-connector/source-acceptance-tests.md) +# for more information about how to configure these tests +connector_image: airbyte/source-intercom:dev +tests: + spec: + - spec_path: "source_intercom/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" +# basic_read: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + cursor_paths: + charges: [ "updated_at" ] +# full_refresh: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-intercom/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-intercom/acceptance-test-docker.sh new file mode 100755 index 000000000000..1425ff74f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-intercom/build.gradle b/airbyte-integrations/connectors/source-intercom/build.gradle new file mode 100644 index 000000000000..78d760d044a1 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_intercom' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/__init__.py b/airbyte-integrations/connectors/source-intercom/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..7ebe7b3d2bd9 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -0,0 +1,21 @@ +{ + "companies": { + "updated_at": 1657363380 + }, + "company_segments": { + "updated_at": 1657363380 + }, + "conversations": { + "updated_at": 1657363380 + }, + "conversation_parts": { + "updated_at": 1657363380 + }, + "contacts": { + "updated_at": 1657363380 + }, + "segments": { + "updated_at": 1657363380 + } +} + diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py new file mode 100644 index 000000000000..df2783d1750f --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py @@ -0,0 +1,34 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/catalog.json new file mode 100644 index 000000000000..38ca3d8e410f --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/catalog.json @@ -0,0 +1,181 @@ +{ + "streams": [ + { + "name": "admins", + "json_schema": { + "properties": { + "admin_ids": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "avatar": { + "properties": { + "image_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "away_mode_enabled": { + "type": ["null", "boolean"] + }, + "away_mode_reassign": { + "type": ["null", "boolean"] + }, + "email": { + "type": ["null", "string"] + }, + "has_inbox_seat": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "job_title": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "team_ids": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": "object", + "additionalProperties": false + } + }, + { + "name": "companies", + "json_schema": { + "properties": { + "company_id": { + "type": ["null", "string"] + }, + "created_at": { + "format": "date-time", + "type": ["null", "string"] + }, + "custom_attributes": { + "type": ["null", "object"], + "additionalProperties": true + }, + "id": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "monthly_spend": { + "multipleOf": 1e-8, + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "plan": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "remote_created_at": { + "format": "date-time", + "type": ["null", "string"] + }, + "segments": { + "anyOf": [ + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + }, + { + "type": "null" + } + ] + }, + "session_count": { + "type": ["null", "integer"] + }, + "size": { + "type": ["null", "integer"] + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + }, + { + "type": "null" + } + ] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "format": "date-time", + "type": ["null", "string"] + }, + "user_count": { + "type": ["null", "integer"] + }, + "website": { + "type": ["null", "string"] + } + }, + "type": "object", + "additionalProperties": false + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..937ff492be66 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json @@ -0,0 +1,1341 @@ +{ + "streams": [ + { + "stream": { + "name": "admins", + "json_schema": { + "properties": { + "admin_ids": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "avatar": { + "properties": { + "image_url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "away_mode_enabled": { + "type": ["null", "boolean"] + }, + "away_mode_reassign": { + "type": ["null", "boolean"] + }, + "email": { + "type": ["null", "string"] + }, + "has_inbox_seat": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "job_title": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "team_ids": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "companies", + "json_schema": { + "properties": { + "company_id": { + "type": ["null", "string"] + }, + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "custom_attributes": { + "type": ["null", "object"], + "additionalProperties": true + }, + "id": { + "type": ["null", "string"] + }, + "industry": { + "type": ["null", "string"] + }, + "monthly_spend": { + "multipleOf": 1e-8, + "type": ["null", "number"] + }, + "name": { + "type": ["null", "string"] + }, + "plan": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "remote_created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "segments": { + "anyOf": [ + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + }, + { + "type": "null" + } + ] + }, + "session_count": { + "type": ["null", "integer"] + }, + "size": { + "type": ["null", "integer"] + }, + "tags": { + "anyOf": [ + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + }, + { + "type": "null" + } + ] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "integer"] + }, + "user_count": { + "type": ["null", "integer"] + }, + "website": { + "type": ["null", "string"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "company_attributes", + "json_schema": { + "properties": { + "admin_id": { + "type": ["null", "string"] + }, + "api_writable": { + "type": ["null", "boolean"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "custom": { + "type": ["null", "boolean"] + }, + "data_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "full_name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "model": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "options": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "type": { + "type": ["null", "string"] + }, + "ui_writable": { + "type": ["null", "boolean"] + }, + "updated_at": { + "format": "date-time", + "type": ["null", "integer"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "company_segments", + "json_schema": { + "properties": { + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "count": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "format": "date-time", + "type": ["null", "integer"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "conversations", + "json_schema": { + "properties": { + "assignee": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "source": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "delivered_as": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "body": { + "type": ["null", "string"] + }, + "author": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "attachments": { + "items": { + "properties": {}, + "type": ["null", "object"], + "additionalProperties": true + }, + "type": ["null", "array"] + }, + "url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "contacts": { + "items": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "type": ["null", "array"] + }, + "teammates": { + "properties": { + "admins": { + "items": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "type": ["null", "array"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "first_contact_reply": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "priority": { + "type": ["null", "string"] + }, + "conversation_message": { + "properties": { + "attachments": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "content_type": { + "type": ["null", "string"] + }, + "filesize": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + } + } + } + }, + { + "type": "null" + } + ] + }, + "author": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "body": { + "type": ["null", "string"] + }, + "delivered_as": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "subject": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "conversation_rating": { + "properties": { + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "customer": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "rating": { + "type": ["null", "integer"] + }, + "remark": { + "type": ["null", "string"] + }, + "teammate": { + "properties": { + "id": { + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "customer_first_reply": { + "properties": { + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "type": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "customers": { + "anyOf": [ + { + "type": "array", + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } + }, + { + "type": "null" + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "open": { + "type": ["null", "boolean"] + }, + "read": { + "type": ["null", "boolean"] + }, + "sent_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "snoozed_until": { + "format": "date-time", + "type": ["null", "integer"] + }, + "sla_applied": { + "properties": { + "sla_name": { + "type": ["null", "string"] + }, + "sla_status": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "state": { + "type": ["null", "string"] + }, + "statistics": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "time_to_assignment": { + "type": ["null", "integer"] + }, + "time_to_admin_reply": { + "type": ["null", "integer"] + }, + "time_to_first_close": { + "type": ["null", "integer"] + }, + "time_to_last_close": { + "type": ["null", "integer"] + }, + "median_time_to_reply": { + "type": ["null", "integer"] + }, + "first_contact_reply_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "first_assignment_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "first_admin_reply_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "first_close_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_assignment_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_assignment_admin_reply_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_contact_reply_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_admin_reply_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_close_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_closed_by_id": { + "type": ["null", "integer"] + }, + "count_reopens": { + "type": ["null", "integer"] + }, + "count_assignments": { + "type": ["null", "integer"] + }, + "count_conversation_parts": { + "type": ["null", "integer"] + } + }, + "type": ["null", "object"] + }, + "tags": { + "items": { + "properties": { + "applied_at": { + "format": "date-time", + "type": ["null", "string"] + }, + "applied_by": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "type": ["null", "array"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "user": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "waiting_since": { + "format": "date-time", + "type": ["null", "integer"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "conversation_parts", + "json_schema": { + "properties": { + "assigned_to": { + "type": ["null", "string"] + }, + "attachments": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "content_type": { + "type": ["null", "string"] + }, + "filesize": { + "type": ["null", "integer"] + }, + "height": { + "type": ["null", "integer"] + }, + "width": { + "type": ["null", "integer"] + } + } + } + }, + { + "type": "null" + } + ] + }, + "author": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "body": { + "type": ["null", "string"] + }, + "conversation_id": { + "type": ["null", "string"] + }, + "conversation_created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "conversation_updated_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "conversation_total_parts": { + "type": ["null", "integer"] + }, + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "external_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "notified_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "part_type": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "format": "date-time", + "type": ["null", "integer"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["full_refresh","incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "contact_attributes", + "json_schema": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "model": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "full_name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "data_type": { + "type": ["null", "string"] + }, + "options": { + "items": { + "type": ["null", "string"] + }, + "type": ["null", "array"] + }, + "api_writable": { + "type": ["null", "boolean"] + }, + "ui_writable": { + "type": ["null", "boolean"] + }, + "custom": { + "type": ["null", "boolean"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "admin_id": { + "type": ["null", "string"] + }, + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "updated_at": { + "format": "date-time", + "type": ["null", "integer"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "contacts", + "json_schema": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "workspace_id": { + "type": ["null", "string"] + }, + "external_id": { + "type": ["null", "string"] + }, + "role": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "avatar": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "social_profiles": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "data": { + "items": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "type": ["null", "array"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "has_hard_bounced": { + "type": ["null", "boolean"] + }, + "marked_email_as_spam": { + "type": ["null", "boolean"] + }, + "unsubscribed_from_emails": { + "type": ["null", "boolean"] + }, + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "updated_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "signed_up_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_seen_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_replied_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_contacted_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_email_opened_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "last_email_clicked_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "language_override": { + "type": ["null", "string"] + }, + "browser": { + "type": ["null", "string"] + }, + "browser_version": { + "type": ["null", "string"] + }, + "browser_language": { + "type": ["null", "string"] + }, + "os": { + "type": ["null", "string"] + }, + "location": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "android_app_name": { + "type": ["null", "string"] + }, + "android_app_version": { + "type": ["null", "string"] + }, + "android_device": { + "type": ["null", "string"] + }, + "android_os_version": { + "type": ["null", "string"] + }, + "android_sdk_version": { + "type": ["null", "string"] + }, + "android_last_seen_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "ios_app_name": { + "type": ["null", "string"] + }, + "ios_app_version": { + "type": ["null", "string"] + }, + "ios_device": { + "type": ["null", "string"] + }, + "ios_os_version": { + "type": ["null", "string"] + }, + "ios_sdk_version": { + "type": ["null", "string"] + }, + "ios_last_seen_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "custom_attributes": { + "properties": {}, + "type": ["null", "object"], + "additionalProperties": true + }, + "tags": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "data": { + "items": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "type": ["null", "array"] + }, + "url": { + "type": ["null", "string"] + }, + "total_count": { + "type": ["null", "integer"] + }, + "has_more": { + "type": ["null", "boolean"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "notes": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "data": { + "items": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "type": ["null", "array"] + }, + "url": { + "type": ["null", "string"] + }, + "total_count": { + "type": ["null", "integer"] + }, + "has_more": { + "type": ["null", "boolean"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "companies": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "data": { + "items": { + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "type": ["null", "array"] + }, + "url": { + "type": ["null", "string"] + }, + "total_count": { + "type": ["null", "integer"] + }, + "has_more": { + "type": ["null", "boolean"] + } + }, + "type": ["null", "object"], + "additionalProperties": false + } + }, + "type": ["null", "object"], + "additionalProperties": false + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "segments", + "json_schema": { + "properties": { + "created_at": { + "format": "date-time", + "type": ["null", "integer"] + }, + "count": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "updated_at": { + "format": "date-time", + "type": ["null", "integer"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_at"] + }, + "sync_mode": "incremental", + "cursor_field": ["updated_at"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "tags", + "json_schema": { + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "teams", + "json_schema": { + "properties": { + "admin_ids": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + }, + "type": "object", + "additionalProperties": false + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-intercom/integration_tests/invalid_config.json new file mode 100644 index 000000000000..b289e80e0206 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "start_date": "2021-11-22T20:32:05Z", + "access_token": "invalid_access_token" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json new file mode 100644 index 000000000000..8cc62593aa08 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json new file mode 100644 index 000000000000..755fe1883bbf --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json @@ -0,0 +1,20 @@ +{ + "companies": { + "updated_at": 1637613125 + }, + "company_segments": { + "updated_at": 1637613125 + }, + "conversations": { + "updated_at": 1637613125 + }, + "conversation_parts": { + "updated_at": 1637613125 + }, + "contacts": { + "updated_at": 1637613125 + }, + "segments": { + "updated_at": 1637613125 + } +} diff --git a/airbyte-integrations/connectors/source-intercom/main.py b/airbyte-integrations/connectors/source-intercom/main.py new file mode 100644 index 000000000000..8289653cf681 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/main.py @@ -0,0 +1,31 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_intercom import SourceIntercom + +if __name__ == "__main__": + source = SourceIntercom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-intercom/requirements.txt b/airbyte-integrations/connectors/source-intercom/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-intercom/setup.py b/airbyte-integrations/connectors/source-intercom/setup.py new file mode 100644 index 000000000000..b1549eba1dbc --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/setup.py @@ -0,0 +1,46 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_intercom", + description="Source implementation for Intercom.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/__init__.py b/airbyte-integrations/connectors/source-intercom/source_intercom/__init__.py new file mode 100644 index 000000000000..9a3ebdd08f65 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourceIntercom + +__all__ = ["SourceIntercom"] diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/admins.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/admins.json new file mode 100644 index 000000000000..8ca4c3791821 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/admins.json @@ -0,0 +1,65 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "admin_ids": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "avatar": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "image_url": { + "type": ["null", "string"] + } + } + }, + "away_mode_enabled": { + "type": ["null", "boolean"] + }, + "away_mode_reassign": { + "type": ["null", "boolean"] + }, + "email": { + "type": ["null", "string"] + }, + "has_inbox_seat": { + "type": ["null", "boolean"] + }, + "id": { + "type": ["null", "string"] + }, + "job_title": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "team_ids": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json new file mode 100644 index 000000000000..4eb46d750ec8 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json @@ -0,0 +1,116 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": ["null", "string"] + }, + "company_id": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "string"] + }, + "app_id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "integer"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "integer"], + "format": "date-time" + }, + "monthly_spend": { + "type": ["null", "number"], + "multipleOf": 1e-8 + }, + "session_count": { + "type": ["null", "integer"] + }, + "user_count": { + "type": ["null", "integer"] + }, + "size": { + "type": ["null", "integer"] + }, + "tags": { + "anyOf": [ + { + "type": [ + "null", + "object" + ], + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + }, + { + "type": "null" + } + ] + }, + + "segments": { + "anyOf": [ + { + "type": [ + "null", + "object" + ], + "items": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + } + } + } + }, + { + "type": "null" + } + ] + }, + "plan": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + }, + "custom_attributes": { + "type": ["null", "object"], + "additionalProperties": true + }, + "industry": { + "type": ["null", "string"] + }, + "remote_created_at": { + "type": ["null", "integer"], + "format": "date-time" + }, + "website": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json new file mode 100644 index 000000000000..bc2330b485f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json @@ -0,0 +1,63 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "admin_id": { + "type": ["null", "string"] + }, + "api_writable": { + "type": ["null", "boolean"] + }, + "archived": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "integer"], + "format": "date-time" + }, + "custom": { + "type": ["null", "boolean"] + }, + "data_type": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "full_name": { + "type": ["null", "string"] + }, + "label": { + "type": ["null", "string"] + }, + "model": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "options": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "string" + } + }, + { + "type": "null" + } + ] + }, + "type": { + "type": ["null", "string"] + }, + "ui_writable": { + "type": ["null", "boolean"] + }, + "updated_at": { + "type": ["null", "integer"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json new file mode 100644 index 000000000000..f25f34364280 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json @@ -0,0 +1,29 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "created_at": { + "type": ["null", "integer"], + "format": "date-time" + }, + "count": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "person_type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "integer"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json new file mode 100644 index 000000000000..1ae5ae2e994f --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json @@ -0,0 +1,104 @@ +{ + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "model": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "full_name": { + "type": [ + "null", + "string" + ] + }, + "label": { + "type": [ + "null", + "string" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "data_type": { + "type": [ + "null", + "string" + ] + }, + "options": { + "type": [ + "null", + "array" + ], + "items": { + "type": ["null", "string"] + } + }, + "api_writable": { + "type": [ + "null", + "boolean" + ] + }, + "ui_writable": { + "type": [ + "null", + "boolean" + ] + }, + "custom": { + "type": [ + "null", + "boolean" + ] + }, + "archived": { + "type": [ + "null", + "boolean" + ] + }, + "admin_id": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "updated_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json new file mode 100644 index 000000000000..0e34dc8f9aa7 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json @@ -0,0 +1,534 @@ +{ + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "workspace_id": { + "type": [ + "null", + "string" + ] + }, + "external_id": { + "type": [ + "null", + "string" + ] + }, + "role": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + }, + "phone": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "avatar": { + "type": [ + "null", + "string" + ] + }, + "owner_id": { + "type": [ + "null", + "integer" + ] + }, + "social_profiles": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "data": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + } + } + }, + "has_hard_bounced": { + "type": [ + "null", + "boolean" + ] + }, + "marked_email_as_spam": { + "type": [ + "null", + "boolean" + ] + }, + "unsubscribed_from_emails": { + "type": [ + "null", + "boolean" + ] + }, + "created_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "updated_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "signed_up_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_seen_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_replied_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_contacted_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_email_opened_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_email_clicked_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "language_override": { + "type": [ + "null", + "string" + ] + }, + "browser": { + "type": [ + "null", + "string" + ] + }, + "browser_version": { + "type": [ + "null", + "string" + ] + }, + "browser_language": { + "type": [ + "null", + "string" + ] + }, + "os": { + "type": [ + "null", + "string" + ] + }, + "location": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "country": { + "type": [ + "null", + "string" + ] + }, + "region": { + "type": [ + "null", + "string" + ] + }, + "city": { + "type": [ + "null", + "string" + ] + } + } + }, + "android_app_name": { + "type": [ + "null", + "string" + ] + }, + "android_app_version": { + "type": [ + "null", + "string" + ] + }, + "android_device": { + "type": [ + "null", + "string" + ] + }, + "android_os_version": { + "type": [ + "null", + "string" + ] + }, + "android_sdk_version": { + "type": [ + "null", + "string" + ] + }, + "android_last_seen_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "ios_app_name": { + "type": [ + "null", + "string" + ] + }, + "ios_app_version": { + "type": [ + "null", + "string" + ] + }, + "ios_device": { + "type": [ + "null", + "string" + ] + }, + "ios_os_version": { + "type": [ + "null", + "string" + ] + }, + "ios_sdk_version": { + "type": [ + "null", + "string" + ] + }, + "ios_last_seen_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "custom_attributes": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": {} + }, + "tags": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "data": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "url": { + "type": [ + "null", + "string" + ] + }, + "total_count": { + "type": [ + "null", + "integer" + ] + }, + "has_more": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "notes": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "data": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "url": { + "type": [ + "null", + "string" + ] + }, + "total_count": { + "type": [ + "null", + "integer" + ] + }, + "has_more": { + "type": [ + "null", + "boolean" + ] + } + } + }, + "companies": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "data": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "url": { + "type": [ + "null", + "string" + ] + }, + "total_count": { + "type": [ + "null", + "integer" + ] + }, + "has_more": { + "type": [ + "null", + "boolean" + ] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json new file mode 100644 index 000000000000..a5132ac41dc3 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json @@ -0,0 +1,208 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "assigned_to": { + "anyOf": [ + { + "type": ["null", "object"], + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + } + }, + "id": { + "type": [ + "null", + "string" + ] + } + } + }, + { + "type": "null" + } + ] + }, + "attachments": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + }, + "content_type": { + "type": [ + "null", + "string" + ] + }, + "filesize": { + "type": [ + "null", + "integer" + ] + }, + "height": { + "type": [ + "null", + "integer" + ] + }, + "width": { + "type": [ + "null", + "integer" + ] + } + } + } + }, + { + "type": "null" + } + ] + }, + "author": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + } + } + }, + "body": { + "type": [ + "null", + "string" + ] + }, + "conversation_id": { + "type": [ + "null", + "string" + ] + }, + "conversation_created_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "conversation_updated_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "conversation_total_parts": { + "type": [ + "null", + "integer" + ] + }, + "created_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "external_id": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "notified_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "part_type": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "redacted": { + "type": [ + "null", + "boolean" + ] + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json new file mode 100644 index 000000000000..539585966b0c --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json @@ -0,0 +1,827 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "assignee": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + } + } + }, + "source": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "redacted": { + "type": [ + "null", + "boolean" + ] + }, + "delivered_as": { + "type": [ + "null", + "string" + ] + }, + "subject": { + "type": [ + "null", + "string" + ] + }, + "body": { + "type": [ + "null", + "string" + ] + }, + "author": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + } + } + }, + "attachments": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": {} + } + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + }, + "contacts": { + "type": [ + "null", + "object" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "teammates": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "admins": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + }, + "first_contact_reply": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + } + } + }, + "priority": { + "type": [ + "null", + "string" + ] + }, + "conversation_message": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "attachments": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + }, + "content_type": { + "type": [ + "null", + "string" + ] + }, + "filesize": { + "type": [ + "null", + "integer" + ] + }, + "height": { + "type": [ + "null", + "integer" + ] + }, + "width": { + "type": [ + "null", + "integer" + ] + } + } + } + }, + { + "type": "null" + } + ] + }, + "author": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "email": { + "type": [ + "null", + "string" + ] + } + } + }, + "body": { + "type": [ + "null", + "string" + ] + }, + "delivered_as": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "subject": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + }, + "conversation_rating": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "created_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "customer": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + }, + "rating": { + "type": [ + "null", + "integer" + ] + }, + "remark": { + "type": [ + "null", + "string" + ] + }, + "teammate": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "integer" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + } + }, + "created_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "customer_first_reply": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "created_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "url": { + "type": [ + "null", + "string" + ] + } + } + }, + "customers": { + "anyOf": [ + { + "type": "array", + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + }, + { + "type": "null" + } + ] + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "open": { + "type": [ + "null", + "boolean" + ] + }, + "read": { + "type": [ + "null", + "boolean" + ] + }, + "sent_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "snoozed_until": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "sla_applied": { + "type": [ + "null", + "object" + ], + "properties": { + "sla_name": { + "type": [ + "null", + "string" + ] + }, + "sla_status": { + "type": [ + "null", + "string" + ] + } + } + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "statistics": { + "type": [ + "null", + "object" + ], + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "time_to_assignment": { + "type": [ + "null", + "integer" + ] + }, + "time_to_admin_reply": { + "type": [ + "null", + "integer" + ] + }, + "time_to_first_close": { + "type": [ + "null", + "integer" + ] + }, + "time_to_last_close": { + "type": [ + "null", + "integer" + ] + }, + "median_time_to_reply": { + "type": [ + "null", + "integer" + ] + }, + "first_contact_reply_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "first_assignment_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "first_admin_reply_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "first_close_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_assignment_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_assignment_admin_reply_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_contact_reply_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_admin_reply_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_close_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "last_closed_by_id": { + "type": [ + "null", + "integer" + ] + }, + "count_reopens": { + "type": [ + "null", + "integer" + ] + }, + "count_assignments": { + "type": [ + "null", + "integer" + ] + }, + "count_conversation_parts": { + "type": [ + "null", + "integer" + ] + } + } + }, + "tags": { + "type": [ + "null", + "object" + ], + "items": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "applied_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "applied_by": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + }, + "id": { + "type": [ + "null", + "string" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + } + }, + "type": { + "type": [ + "null", + "string" + ] + }, + "updated_at": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "user": { + "type": [ + "null", + "object" + ], + "additionalProperties": false, + "properties": { + "id": { + "type": [ + "null", + "string" + ] + }, + "type": { + "type": [ + "null", + "string" + ] + } + } + }, + "waiting_since": { + "type": [ + "null", + "integer" + ], + "format": "date-time" + }, + "admin_assignee_id": { + "type": [ + "null", + "integer" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "team_assignee_id": { + "type": [ + "null", + "integer" + ] + }, + "redacted": { + "type": [ + "null", + "boolean" + ] + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json new file mode 100644 index 000000000000..f25f34364280 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json @@ -0,0 +1,29 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "created_at": { + "type": ["null", "integer"], + "format": "date-time" + }, + "count": { + "type": ["null", "integer"] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "person_type": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "integer"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/tags.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/tags.json new file mode 100644 index 000000000000..ab2658ecee5e --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/tags.json @@ -0,0 +1,15 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/teams.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/teams.json new file mode 100644 index 000000000000..9783dcb3da9c --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/teams.json @@ -0,0 +1,28 @@ +{ + "type": "object", + "additionalProperties": false, + "properties": { + "admin_ids": { + "anyOf": [ + { + "type": "array", + "items": { + "type": "integer" + } + }, + { + "type": "null" + } + ] + }, + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py new file mode 100644 index 000000000000..ce0023144099 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -0,0 +1,318 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import time +from abc import ABC +from datetime import date, timedelta, datetime +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union + +import requests +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import HttpAuthenticator, TokenAuthenticator + + +class IntercomStream(HttpStream, ABC): + url_base = "https://api.intercom.io/" + + # https://developers.intercom.com/intercom-api-reference/reference#rate-limiting + rate_limit = 1000 # 1000 queries per hour == 1 req in 3,6 secs + + def __init__( + self, + authenticator: HttpAuthenticator, + start_date: Union[date, str] = None, + **kwargs, + ): + self.start_date = start_date + + super().__init__(authenticator=authenticator) + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + return {"Accept": "application/json"} + + def _send_request(self, request: requests.PreparedRequest) -> requests.Response: + # wait for 3,6 seconds before + time.sleep(3600 / self.rate_limit) + try: + return super()._send_request(request) + except requests.exceptions.HTTPError as e: + error_message = e.response.text + if error_message: + self.logger.error(f"Stream {self.name}: {e.response.status_code} {e.response.reason} - {error_message}") + exit(1) + else: + raise e + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + data = response.json() + for data_field in self.data_fields: + if data_field is not None: + data = data.get(data_field, []) + if isinstance(data, list): + data = data + elif isinstance(data, dict): + data = [data] + + for record in data: + yield record + + +class IncrementalIntercomStream(IntercomStream, ABC): + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: + # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state + # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + current_stream_state = current_stream_state or {} + current_stream_state_date = current_stream_state.get("updated_at", self.start_date.timestamp()) + latest_record_date = latest_record.get(self.cursor_field, self.start_date.timestamp()) + return {"updated_at": max(current_stream_state_date, latest_record_date)} + + +class StreamMixin: + stream = None + + def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + for item in self.stream(authenticator=self.authenticator).read_records(sync_mode=sync_mode): + yield {"id": item["id"]} + + yield from [] + + +class Admins(IntercomStream): + """Return list of all admins. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-admins + Endpoint: https://api.intercom.io/admins + """ + + primary_key = "id" + data_fields = ["admins"] + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "admins" + + +class Companies(IncrementalIntercomStream): + """Return list of all companies. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-companies + Endpoint: https://api.intercom.io/companies + """ + + primary_key = "id" + data_fields = ["data"] + cursor_field = "updated_at" + + def path(self, **kwargs) -> str: + return "companies" + + +class CompanySegments(StreamMixin, IncrementalIntercomStream): + """Return list of all company segments. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-attached-segments-1 + Endpoint: https://api.intercom.io/companies//segments + """ + + primary_key = "id" + data_fields = ["data"] + cursor_field = "updated_at" + stream = Companies + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"/companies/{stream_slice['id']}/segments" + + +class Conversations(IncrementalIntercomStream): + """Return list of all conversations. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-conversations + Endpoint: https://api.intercom.io/conversations + """ + + primary_key = "id" + data_fields = ["conversations"] + cursor_field = "updated_at" + + def path(self, **kwargs) -> str: + return "conversations" + + +class ConversationParts(StreamMixin, IncrementalIntercomStream): + """Return list of all conversation parts. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#retrieve-a-conversation + Endpoint: https://api.intercom.io/conversations/ + """ + + primary_key = "id" + data_fields = ["conversation_parts", "conversation_parts"] + cursor_field = "updated_at" + stream = Conversations + + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + return f"/conversations/{stream_slice['id']}" + + +class Segments(IncrementalIntercomStream): + """Return list of all conversations. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-segments + Endpoint: https://api.intercom.io/segments + """ + + primary_key = "id" + data_fields = ["segments"] + cursor_field = "updated_at" + + def path(self, **kwargs) -> str: + return "segments" + + +class Contacts(IncrementalIntercomStream): + """Return list of all contacts. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-contacts + Endpoint: https://api.intercom.io/contacts + """ + + primary_key = "id" + data_fields = ["data"] + cursor_field = "updated_at" + + def path(self, **kwargs) -> str: + return "contacts" + + +class DataAttributes(IntercomStream): + primary_key = "name" + data_fields = ["data"] + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "data_attributes" + + +class CompanyAttributes(DataAttributes): + """Return list of all data attributes belonging to a workspace for companies. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-data-attributes + Endpoint: https://api.intercom.io/data_attributes?model=company + """ + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {"model": "company"} + + +class ContactAttributes(DataAttributes): + """Return list of all data attributes belonging to a workspace for contacts. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-data-attributes + Endpoint: https://api.intercom.io/data_attributes?model=contact + """ + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return {"model": "contact"} + + +class Tags(IntercomStream): + """Return list of all tags. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-tags-for-an-app + Endpoint: https://api.intercom.io/tags + """ + + primary_key = "name" + data_fields = ["data"] + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "tags" + + +class Teams(IntercomStream): + """Return list of all teams. + API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-teams + Endpoint: https://api.intercom.io/teams + """ + + primary_key = "name" + data_fields = ["teams"] + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return "teams" + + +class SourceIntercom(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + authenticator = TokenAuthenticator(token=config["access_token"]) + try: + url = f"{IntercomStream.url_base}/tags" + auth_headers = {"Accept": "application/json", + **authenticator.get_auth_header()} + session = requests.get(url, headers=auth_headers) + session.raise_for_status() + return True, None + except requests.exceptions.RequestException as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + + now = date.today() + + start_date = config.get("start_date") + if start_date and isinstance(start_date, str): + start_date = datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S%z") + config["start_date"] = start_date or now - timedelta(days=365) # set to 1 year ago by default + + AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") + + auth = TokenAuthenticator(token=config["access_token"]) + return [Admins(authenticator=auth, **config), + Conversations(authenticator=auth, **config), + ConversationParts(authenticator=auth, **config), + CompanySegments(authenticator=auth, **config), + CompanyAttributes(authenticator=auth, **config), + ContactAttributes(authenticator=auth, **config), + Segments(authenticator=auth, **config), + Contacts(authenticator=auth, **config), + Companies(authenticator=auth, **config), + Tags(authenticator=auth, **config), + Teams(authenticator=auth, **config)] diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json b/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json new file mode 100644 index 000000000000..69d52942ef60 --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json @@ -0,0 +1,23 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/intercom", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Source Intercom Spec", + "type": "object", + "required": ["access_token", "start_date"], + "additionalProperties": false, + "properties": { + "access_token": { + "type": "string", + "description": "Intercom Access Token. See the docs for more information on how to obtain this key.", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "description": "The date from which you'd like to replicate data for Intercom API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "examples": ["2020-11-16T00:00:00Z"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py new file mode 100644 index 000000000000..f03f99f7c46e --- /dev/null +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py @@ -0,0 +1,25 @@ +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + + +def test_example_method(): + assert True diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index 653aec89d0a2..7780d1a29854 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -4,7 +4,7 @@ The Intercom source supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. -This Intercom source wraps the [Singer Intercom Tap](https://github.com/singer-io/tap-intercom). +This Source Connector is based on a [Airbyte CDK](https://docs.airbyte.io/contributing-to-airbyte/python). ### Output schema diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 4d76b90bbde7..4cf4316dc253 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -61,6 +61,7 @@ write_standard_creds source-greenhouse "$GREENHOUSE_TEST_CREDS" write_standard_creds source-harvest "$HARVEST_INTEGRATION_TESTS_CREDS" write_standard_creds source-hubspot "$HUBSPOT_INTEGRATION_TESTS_CREDS" write_standard_creds source-instagram "$INSTAGRAM_INTEGRATION_TESTS_CREDS" +write_standard_creds source-intercom "$INTERCOM_INTEGRATION_TEST_CREDS" write_standard_creds source-intercom-singer "$INTERCOM_INTEGRATION_TEST_CREDS" write_standard_creds source-iterable "$ITERABLE_INTEGRATION_TEST_CREDS" write_standard_creds source-jira "$JIRA_INTEGRATION_TEST_CREDS" From f87738055642e6321c26d1e4d526345648915a89 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 12 Jul 2021 12:28:21 +0300 Subject: [PATCH 02/23] Updated segments docs --- .../connectors/source-intercom/source_intercom/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index ce0023144099..2cf5f21ddb91 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -178,7 +178,7 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Segments(IncrementalIntercomStream): - """Return list of all conversations. + """Return list of all segments. API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-segments Endpoint: https://api.intercom.io/segments """ From 129e05784635e98d795a173725ed9bb74c2283d7 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 12 Jul 2021 13:01:40 +0300 Subject: [PATCH 03/23] Updated _send_request method to new airbyte-cdk version --- .../source-intercom/source_intercom/source.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 2cf5f21ddb91..a00b442d913b 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -57,18 +57,14 @@ def request_headers( ) -> Mapping[str, Any]: return {"Accept": "application/json"} - def _send_request(self, request: requests.PreparedRequest) -> requests.Response: - # wait for 3,6 seconds before - time.sleep(3600 / self.rate_limit) + def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: try: - return super()._send_request(request) + return super()._send_request(request, request_kwargs) except requests.exceptions.HTTPError as e: error_message = e.response.text if error_message: self.logger.error(f"Stream {self.name}: {e.response.status_code} {e.response.reason} - {error_message}") - exit(1) - else: - raise e + raise e def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: data = response.json() @@ -83,6 +79,9 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp for record in data: yield record + # wait for 3,6 seconds according to API limit + time.sleep(3600 / self.rate_limit) + class IncrementalIntercomStream(IntercomStream, ABC): def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: From 61c48033cc069b829eae60909c7681ca9d304278 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 12 Jul 2021 14:56:18 +0300 Subject: [PATCH 04/23] Updated cursor field to datetime string --- .../source-intercom/acceptance-test-config.yml | 14 +++++++------- .../integration_tests/abnormal_state.json | 12 ++++++------ .../integration_tests/sample_state.json | 12 ++++++------ .../source_intercom/schemas/companies.json | 2 +- .../schemas/company_attributes.json | 2 +- .../source_intercom/schemas/company_segments.json | 2 +- .../source_intercom/schemas/contacts.json | 2 +- .../schemas/conversation_parts.json | 2 +- .../source_intercom/schemas/conversations.json | 2 +- .../source_intercom/schemas/segments.json | 2 +- .../source-intercom/source_intercom/source.py | 11 +++++++++-- 11 files changed, 35 insertions(+), 28 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml index 324d34c68901..e1185300f21a 100644 --- a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml @@ -11,16 +11,16 @@ tests: status: "failed" discovery: - config_path: "secrets/config.json" -# basic_read: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" -# validate_output_from_all_streams: yes + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" cursor_paths: charges: [ "updated_at" ] -# full_refresh: -# - config_path: "secrets/config.json" -# configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json index 7ebe7b3d2bd9..af68083cd95f 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -1,21 +1,21 @@ { "companies": { - "updated_at": 1657363380 + "updated_at": "2022-07-12T10:44:09+00:00" }, "company_segments": { - "updated_at": 1657363380 + "updated_at": "2022-07-12T10:44:09+00:00" }, "conversations": { - "updated_at": 1657363380 + "updated_at": "2022-07-12T10:44:09+00:00" }, "conversation_parts": { - "updated_at": 1657363380 + "updated_at": "2022-07-12T10:44:09+00:00" }, "contacts": { - "updated_at": 1657363380 + "updated_at": "2022-07-12T10:44:09+00:00" }, "segments": { - "updated_at": 1657363380 + "updated_at": "2022-07-12T10:44:09+00:00" } } diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json index 755fe1883bbf..c047b88df194 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json @@ -1,20 +1,20 @@ { "companies": { - "updated_at": 1637613125 + "updated_at": "2021-07-12T11:22:46+00:00" }, "company_segments": { - "updated_at": 1637613125 + "updated_at": "2021-07-12T11:22:46+00:00" }, "conversations": { - "updated_at": 1637613125 + "updated_at": "2021-07-12T11:22:46+00:00" }, "conversation_parts": { - "updated_at": 1637613125 + "updated_at": "2021-07-12T11:22:46+00:00" }, "contacts": { - "updated_at": 1637613125 + "updated_at": "2021-07-12T11:22:46+00:00" }, "segments": { - "updated_at": 1637613125 + "updated_at": "2021-07-12T11:22:46+00:00" } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json index 4eb46d750ec8..2cf3687adcab 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json @@ -22,7 +22,7 @@ "format": "date-time" }, "updated_at": { - "type": ["null", "integer"], + "type": ["null", "string"], "format": "date-time" }, "monthly_spend": { diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json index bc2330b485f4..b69b6fbd1471 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_attributes.json @@ -56,7 +56,7 @@ "type": ["null", "boolean"] }, "updated_at": { - "type": ["null", "integer"], + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json index f25f34364280..c20c129a5297 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/company_segments.json @@ -22,7 +22,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "integer"], + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json index 0e34dc8f9aa7..3156aa797cd8 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json @@ -141,7 +141,7 @@ "updated_at": { "type": [ "null", - "integer" + "string" ], "format": "date-time" }, diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json index a5132ac41dc3..b9fb142c6376 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json @@ -194,7 +194,7 @@ "updated_at": { "type": [ "null", - "integer" + "string" ], "format": "date-time" }, diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json index 539585966b0c..c147b47bdc22 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json @@ -767,7 +767,7 @@ "updated_at": { "type": [ "null", - "integer" + "string" ], "format": "date-time" }, diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json index f25f34364280..c20c129a5297 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/segments.json @@ -22,7 +22,7 @@ "type": ["null", "string"] }, "updated_at": { - "type": ["null", "integer"], + "type": ["null", "string"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index a00b442d913b..7b6b9b3f76e3 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -77,6 +77,9 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp data = [data] for record in data: + updated_at = record.get("updated_at", 0) + if updated_at: + record["updated_at"] = datetime.fromtimestamp(record["updated_at"]).isoformat() # convert timestamp to datetime string yield record # wait for 3,6 seconds according to API limit @@ -88,8 +91,8 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. current_stream_state = current_stream_state or {} - current_stream_state_date = current_stream_state.get("updated_at", self.start_date.timestamp()) - latest_record_date = latest_record.get(self.cursor_field, self.start_date.timestamp()) + current_stream_state_date = current_stream_state.get("updated_at", str(self.start_date)) + latest_record_date = latest_record.get(self.cursor_field, str(self.start_date)) return {"updated_at": max(current_stream_state_date, latest_record_date)} @@ -269,6 +272,10 @@ def path( class SourceIntercom(AbstractSource): + """ + Source Intercom fetch data from messaging platform + """ + def check_connection(self, logger, config) -> Tuple[bool, any]: """ See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 From 3375acffd75c7a1769481c6829629b633649f466 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 12 Jul 2021 20:41:46 +0300 Subject: [PATCH 05/23] Updated to review comments --- .../integration_tests/configured_catalog.json | 7 +- .../source-intercom/source_intercom/source.py | 78 +++++++++++-------- 2 files changed, 46 insertions(+), 39 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json index 937ff492be66..25c240ed48a2 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json @@ -254,12 +254,9 @@ "type": "object", "additionalProperties": false }, - "supported_sync_modes": ["incremental"], - "source_defined_cursor": true, - "default_cursor_field": ["updated_at"] + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "cursor_field": ["updated_at"], + "sync_mode": "full_refresh", "destination_sync_mode": "append" }, { diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 7b6b9b3f76e3..8521ae6a034b 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -22,6 +22,7 @@ import time from abc import ABC +from urllib.parse import parse_qsl, urlparse from datetime import date, timedelta, datetime from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union @@ -39,6 +40,9 @@ class IntercomStream(HttpStream, ABC): # https://developers.intercom.com/intercom-api-reference/reference#rate-limiting rate_limit = 1000 # 1000 queries per hour == 1 req in 3,6 secs + primary_key = "id" + data_fields = ["data"] + def __init__( self, authenticator: HttpAuthenticator, @@ -50,7 +54,16 @@ def __init__( super().__init__(authenticator=authenticator) def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - return None + """ + Abstract method of HttpStream - should be overwritten. + Returning None means there are no more pages to read in response. + """ + + next_page = response.links.get("next", None) + if next_page: + return dict(parse_qsl(urlparse(next_page.get("url")).query)) + else: + return None def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -66,7 +79,7 @@ def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mappi self.logger.error(f"Stream {self.name}: {e.response.status_code} {e.response.reason} - {error_message}") raise e - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + def get_data(self, response: requests.Response, **kwargs) -> List: data = response.json() for data_field in self.data_fields: if data_field is not None: @@ -76,10 +89,12 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp elif isinstance(data, dict): data = [data] + return data + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + data = self.get_data(response, **kwargs) + for record in data: - updated_at = record.get("updated_at", 0) - if updated_at: - record["updated_at"] = datetime.fromtimestamp(record["updated_at"]).isoformat() # convert timestamp to datetime string yield record # wait for 3,6 seconds according to API limit @@ -87,20 +102,35 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalIntercomStream(IntercomStream, ABC): + cursor_field = "updated_at" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + data = self.get_data(response, **kwargs) + + for record in data: + updated_at = record.get(self.cursor_field, None) + if updated_at: + record[self.cursor_field] = datetime.fromtimestamp(record[self.cursor_field]).isoformat() # convert timestamp to datetime string + + yield record + + # wait for 3,6 seconds according to API limit + time.sleep(3600 / self.rate_limit) + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. current_stream_state = current_stream_state or {} - current_stream_state_date = current_stream_state.get("updated_at", str(self.start_date)) - latest_record_date = latest_record.get(self.cursor_field, str(self.start_date)) - return {"updated_at": max(current_stream_state_date, latest_record_date)} + current_stream_state_date = current_stream_state.get(self.cursor_field, self.start_date) + latest_record_date = latest_record.get(self.cursor_field, self.start_date) + return {self.cursor_field: max(current_stream_state_date, latest_record_date)} class StreamMixin: stream = None def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.stream(authenticator=self.authenticator).read_records(sync_mode=sync_mode): + for item in self.stream(authenticator=self.authenticator, start_date=self.start_date).read_records(sync_mode=sync_mode): yield {"id": item["id"]} yield from [] @@ -112,7 +142,6 @@ class Admins(IntercomStream): Endpoint: https://api.intercom.io/admins """ - primary_key = "id" data_fields = ["admins"] def path( @@ -127,10 +156,6 @@ class Companies(IncrementalIntercomStream): Endpoint: https://api.intercom.io/companies """ - primary_key = "id" - data_fields = ["data"] - cursor_field = "updated_at" - def path(self, **kwargs) -> str: return "companies" @@ -141,9 +166,6 @@ class CompanySegments(StreamMixin, IncrementalIntercomStream): Endpoint: https://api.intercom.io/companies//segments """ - primary_key = "id" - data_fields = ["data"] - cursor_field = "updated_at" stream = Companies def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -156,9 +178,7 @@ class Conversations(IncrementalIntercomStream): Endpoint: https://api.intercom.io/conversations """ - primary_key = "id" data_fields = ["conversations"] - cursor_field = "updated_at" def path(self, **kwargs) -> str: return "conversations" @@ -170,9 +190,7 @@ class ConversationParts(StreamMixin, IncrementalIntercomStream): Endpoint: https://api.intercom.io/conversations/ """ - primary_key = "id" data_fields = ["conversation_parts", "conversation_parts"] - cursor_field = "updated_at" stream = Conversations def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: @@ -185,9 +203,7 @@ class Segments(IncrementalIntercomStream): Endpoint: https://api.intercom.io/segments """ - primary_key = "id" data_fields = ["segments"] - cursor_field = "updated_at" def path(self, **kwargs) -> str: return "segments" @@ -199,17 +215,12 @@ class Contacts(IncrementalIntercomStream): Endpoint: https://api.intercom.io/contacts """ - primary_key = "id" - data_fields = ["data"] - cursor_field = "updated_at" - def path(self, **kwargs) -> str: return "contacts" class DataAttributes(IntercomStream): primary_key = "name" - data_fields = ["data"] def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -248,7 +259,6 @@ class Tags(IntercomStream): """ primary_key = "name" - data_fields = ["data"] def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -273,7 +283,7 @@ def path( class SourceIntercom(AbstractSource): """ - Source Intercom fetch data from messaging platform + Source Intercom fetch data from messaging platform. """ def check_connection(self, logger, config) -> Tuple[bool, any]: @@ -312,13 +322,13 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: auth = TokenAuthenticator(token=config["access_token"]) return [Admins(authenticator=auth, **config), + Companies(authenticator=auth, **config), + CompanySegments(authenticator=auth, **config), Conversations(authenticator=auth, **config), ConversationParts(authenticator=auth, **config), - CompanySegments(authenticator=auth, **config), - CompanyAttributes(authenticator=auth, **config), - ContactAttributes(authenticator=auth, **config), Segments(authenticator=auth, **config), Contacts(authenticator=auth, **config), - Companies(authenticator=auth, **config), + CompanyAttributes(authenticator=auth, **config), + ContactAttributes(authenticator=auth, **config), Tags(authenticator=auth, **config), Teams(authenticator=auth, **config)] From ec44b41247c387d66a5597722f3a29718c660d8a Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Tue, 13 Jul 2021 20:45:05 +0300 Subject: [PATCH 06/23] Added filtering by state for incremental sync --- .../integration_tests/sample_state.json | 12 +- .../source-intercom/source_intercom/source.py | 189 ++++++++++++++---- 2 files changed, 156 insertions(+), 45 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json index c047b88df194..e3e8395eaea0 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_state.json @@ -1,20 +1,20 @@ { "companies": { - "updated_at": "2021-07-12T11:22:46+00:00" + "updated_at": "2019-07-12T11:22:46+00:00" }, "company_segments": { - "updated_at": "2021-07-12T11:22:46+00:00" + "updated_at": "2019-07-12T11:22:46+00:00" }, "conversations": { - "updated_at": "2021-07-12T11:22:46+00:00" + "updated_at": "2019-07-12T11:22:46+00:00" }, "conversation_parts": { - "updated_at": "2021-07-12T11:22:46+00:00" + "updated_at": "2019-07-12T11:22:46+00:00" }, "contacts": { - "updated_at": "2021-07-12T11:22:46+00:00" + "updated_at": "2019-07-12T11:22:46+00:00" }, "segments": { - "updated_at": "2021-07-12T11:22:46+00:00" + "updated_at": "2019-07-12T11:22:46+00:00" } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 8521ae6a034b..f2e42a736fe7 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -27,6 +27,7 @@ from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union import requests +from airbyte_cdk.models import SyncMode from airbyte_cdk.logger import AirbyteLogger from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream @@ -60,26 +61,35 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, """ next_page = response.links.get("next", None) + if next_page: return dict(parse_qsl(urlparse(next_page.get("url")).query)) else: return None def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None ) -> Mapping[str, Any]: return {"Accept": "application/json"} - def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: + def _send_request( + self, + request: requests.PreparedRequest, + request_kwargs: Mapping[str, Any] + ) -> requests.Response: try: return super()._send_request(request, request_kwargs) except requests.exceptions.HTTPError as e: error_message = e.response.text if error_message: - self.logger.error(f"Stream {self.name}: {e.response.status_code} {e.response.reason} - {error_message}") + self.logger.error(f"Stream {self.name}: {e.response.status_code} " + f"{e.response.reason} - {error_message}") raise e - def get_data(self, response: requests.Response, **kwargs) -> List: + def get_data(self, response: requests.Response) -> List: data = response.json() for data_field in self.data_fields: if data_field is not None: @@ -91,8 +101,15 @@ def get_data(self, response: requests.Response, **kwargs) -> List: return data - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - data = self.get_data(response, **kwargs) + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + + data = self.get_data(response) for record in data: yield record @@ -104,33 +121,91 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalIntercomStream(IntercomStream, ABC): cursor_field = "updated_at" - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - data = self.get_data(response, **kwargs) + def filter_by_state( + self, + stream_state: Mapping[str, Any] = None, + records: Mapping[str, Any] = None + ) -> Iterable: + + if stream_state: + for record in records: + if record[self.cursor_field] >= stream_state.get(self.cursor_field): + yield record + else: + yield from records + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + + data = self.get_data(response) for record in data: updated_at = record.get(self.cursor_field, None) + if updated_at: - record[self.cursor_field] = datetime.fromtimestamp(record[self.cursor_field]).isoformat() # convert timestamp to datetime string + record[self.cursor_field] = datetime.fromtimestamp( + record[self.cursor_field] + ).isoformat() # convert timestamp to datetime string yield record # wait for 3,6 seconds according to API limit time.sleep(3600 / self.rate_limit) - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: - # This method is called once for each record returned from the API to compare the cursor field value in that record with the current state - # we then return an updated state object. If this is the first time we run a sync or no state was passed, current_stream_state will be None. + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any] + ) -> Mapping[str, any]: + """ + This method is called once for each record returned from the API to + compare the cursor field value in that record with the current state + we then return an updated state object. If this is the first time we + run a sync or no state was passed, current_stream_state will be None. + """ + current_stream_state = current_stream_state or {} - current_stream_state_date = current_stream_state.get(self.cursor_field, self.start_date) - latest_record_date = latest_record.get(self.cursor_field, self.start_date) + + current_stream_state_date = current_stream_state.get( + self.cursor_field, str(self.start_date) + ) + latest_record_date = latest_record.get( + self.cursor_field, str(self.start_date) + ) + return {self.cursor_field: max(current_stream_state_date, latest_record_date)} + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + """ + Endpoint does not provide query filtering params, but they provide us + updated_at field in most cases, so we used that as incremental filtering + during the slicing. + """ + + records = super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + + yield from self.filter_by_state(stream_state=stream_state, records=records) + class StreamMixin: stream = None def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.stream(authenticator=self.authenticator, start_date=self.start_date).read_records(sync_mode=sync_mode): + for item in self.stream( + authenticator=self.authenticator, + start_date=self.start_date + ).read_records(sync_mode=sync_mode): yield {"id": item["id"]} yield from [] @@ -145,7 +220,10 @@ class Admins(IntercomStream): data_fields = ["admins"] def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None ) -> str: return "admins" @@ -156,7 +234,12 @@ class Companies(IncrementalIntercomStream): Endpoint: https://api.intercom.io/companies """ - def path(self, **kwargs) -> str: + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None + ) -> str: return "companies" @@ -168,7 +251,12 @@ class CompanySegments(StreamMixin, IncrementalIntercomStream): stream = Companies - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None + ) -> str: return f"/companies/{stream_slice['id']}/segments" @@ -180,7 +268,12 @@ class Conversations(IncrementalIntercomStream): data_fields = ["conversations"] - def path(self, **kwargs) -> str: + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None + ) -> str: return "conversations" @@ -193,7 +286,12 @@ class ConversationParts(StreamMixin, IncrementalIntercomStream): data_fields = ["conversation_parts", "conversation_parts"] stream = Conversations - def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None + ) -> str: return f"/conversations/{stream_slice['id']}" @@ -205,7 +303,12 @@ class Segments(IncrementalIntercomStream): data_fields = ["segments"] - def path(self, **kwargs) -> str: + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None + ) -> str: return "segments" @@ -215,7 +318,12 @@ class Contacts(IncrementalIntercomStream): Endpoint: https://api.intercom.io/contacts """ - def path(self, **kwargs) -> str: + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None + ) -> str: return "contacts" @@ -223,7 +331,10 @@ class DataAttributes(IntercomStream): primary_key = "name" def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None ) -> str: return "data_attributes" @@ -235,7 +346,10 @@ class CompanyAttributes(DataAttributes): """ def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, any] = None, + next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: return {"model": "company"} @@ -247,7 +361,10 @@ class ContactAttributes(DataAttributes): """ def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, any] = None, + next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: return {"model": "contact"} @@ -261,7 +378,10 @@ class Tags(IntercomStream): primary_key = "name" def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None ) -> str: return "tags" @@ -276,7 +396,10 @@ class Teams(IntercomStream): data_fields = ["teams"] def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None ) -> str: return "teams" @@ -287,14 +410,6 @@ class SourceIntercom(AbstractSource): """ def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 - for an example. - - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ authenticator = TokenAuthenticator(token=config["access_token"]) try: url = f"{IntercomStream.url_base}/tags" @@ -307,10 +422,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - now = date.today() start_date = config.get("start_date") From 1fb7bf5f01a207af20c05c0694c00bd4a2a04e19 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Tue, 13 Jul 2021 23:12:01 +0300 Subject: [PATCH 07/23] Code formated --- .../integration_tests/abnormal_state.json | 1 - .../integration_tests/acceptance.py | 2 + .../integration_tests/configured_catalog.json | 2 +- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../connectors/source-intercom/main.py | 2 + .../connectors/source-intercom/setup.py | 2 + .../source_intercom/schemas/companies.json | 10 +- .../schemas/contact_attributes.json | 80 +-- .../source_intercom/schemas/contacts.json | 410 +++--------- .../schemas/conversation_parts.json | 137 +--- .../schemas/conversations.json | 600 ++++-------------- .../source-intercom/source_intercom/source.py | 172 ++--- .../source-intercom/source_intercom/spec.json | 2 +- .../source-intercom/unit_tests/unit_test.py | 2 + 15 files changed, 318 insertions(+), 1108 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json index af68083cd95f..1bb4ec8dd2e0 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/abnormal_state.json @@ -18,4 +18,3 @@ "updated_at": "2022-07-12T10:44:09+00:00" } } - diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py index df2783d1750f..eeb4a2d3e02e 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py @@ -1,3 +1,4 @@ +# # MIT License # # Copyright (c) 2020 Airbyte @@ -19,6 +20,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# import pytest diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json index 25c240ed48a2..9224d780226c 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json @@ -867,7 +867,7 @@ "type": "object", "additionalProperties": false }, - "supported_sync_modes": ["full_refresh","incremental"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"] }, diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-intercom/integration_tests/invalid_config.json index b289e80e0206..ab7c557e58f9 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { "start_date": "2021-11-22T20:32:05Z", "access_token": "invalid_access_token" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json index 8cc62593aa08..ecc4913b84c7 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { "fix-me": "TODO" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-intercom/main.py b/airbyte-integrations/connectors/source-intercom/main.py index 8289653cf681..1c18fbae560d 100644 --- a/airbyte-integrations/connectors/source-intercom/main.py +++ b/airbyte-integrations/connectors/source-intercom/main.py @@ -1,3 +1,4 @@ +# # MIT License # # Copyright (c) 2020 Airbyte @@ -19,6 +20,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# import sys diff --git a/airbyte-integrations/connectors/source-intercom/setup.py b/airbyte-integrations/connectors/source-intercom/setup.py index b1549eba1dbc..a5107673bff9 100644 --- a/airbyte-integrations/connectors/source-intercom/setup.py +++ b/airbyte-integrations/connectors/source-intercom/setup.py @@ -1,3 +1,4 @@ +# # MIT License # # Copyright (c) 2020 Airbyte @@ -19,6 +20,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# from setuptools import find_packages, setup diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json index 2cf3687adcab..bb23f31f35b9 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/companies.json @@ -41,10 +41,7 @@ "tags": { "anyOf": [ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "items": { "type": ["null", "object"], "additionalProperties": false, @@ -64,10 +61,7 @@ "segments": { "anyOf": [ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "items": { "type": ["null", "object"], "additionalProperties": false, diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json index 1ae5ae2e994f..4babd19e5a3e 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contact_attributes.json @@ -1,103 +1,55 @@ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "model": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "full_name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "label": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "description": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "data_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "options": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { "type": ["null", "string"] } }, "api_writable": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "ui_writable": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "custom": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "archived": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "admin_id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "created_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "updated_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json index 3156aa797cd8..9c30ab910370 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/contacts.json @@ -1,112 +1,58 @@ { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "workspace_id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "external_id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "role": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "email": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "phone": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "avatar": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "owner_id": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "social_profiles": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "data": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } @@ -114,419 +60,227 @@ } }, "has_hard_bounced": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "marked_email_as_spam": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "unsubscribed_from_emails": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "created_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "updated_at": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "signed_up_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_seen_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_replied_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_contacted_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_email_opened_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_email_clicked_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "language_override": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "browser": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "browser_version": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "browser_language": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "os": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "location": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "country": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "region": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "city": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "android_app_name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "android_app_version": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "android_device": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "android_os_version": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "android_sdk_version": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "android_last_seen_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "ios_app_name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ios_app_version": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ios_device": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ios_os_version": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ios_sdk_version": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "ios_last_seen_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "custom_attributes": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": true, "properties": {} }, "tags": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "data": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "total_count": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "has_more": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } } }, "notes": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "data": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "total_count": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "has_more": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } } }, "companies": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "data": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "total_count": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "has_more": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json index b9fb142c6376..76d51823b164 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversation_parts.json @@ -11,18 +11,12 @@ "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } }, "id": { - "type": [ - "null", - "string" - ] - } + "type": ["null", "string"] + } } }, { @@ -39,46 +33,25 @@ "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "content_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "filesize": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "height": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "width": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } } } @@ -89,120 +62,66 @@ ] }, "author": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "email": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "body": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "conversation_id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "conversation_created_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "conversation_updated_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "conversation_total_parts": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "created_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "external_id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "notified_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "part_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "updated_at": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "redacted": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json index c147b47bdc22..b48e7a2d40e2 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/schemas/conversations.json @@ -3,244 +3,136 @@ "additionalProperties": false, "properties": { "assignee": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "email": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "source": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "redacted": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "delivered_as": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "subject": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "body": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "author": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "email": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "attachments": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": true, "properties": {} } }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "contacts": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "teammates": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "admins": { - "type": [ - "null", - "array" - ], + "type": ["null", "array"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "first_contact_reply": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "created_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" } } }, "priority": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "conversation_message": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "attachments": { @@ -252,46 +144,25 @@ "additionalProperties": false, "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "content_type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "filesize": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "height": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "width": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } } } @@ -302,178 +173,100 @@ ] }, "author": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "email": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "body": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "delivered_as": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "subject": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "conversation_rating": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "created_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "customer": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "rating": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "remark": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "teammate": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } } }, "created_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "customer_first_reply": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "created_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "url": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, @@ -482,23 +275,14 @@ { "type": "array", "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } @@ -509,319 +293,175 @@ ] }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "open": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "read": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] }, "sent_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "snoozed_until": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "sla_applied": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "sla_name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "sla_status": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "state": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "statistics": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "properties": { "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "time_to_assignment": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "time_to_admin_reply": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "time_to_first_close": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "time_to_last_close": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "median_time_to_reply": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "first_contact_reply_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "first_assignment_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "first_admin_reply_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "first_close_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_assignment_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_assignment_admin_reply_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_contact_reply_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_admin_reply_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_close_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "last_closed_by_id": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "count_reopens": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "count_assignments": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "count_conversation_parts": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] } } }, "tags": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "items": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "applied_at": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "applied_by": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "name": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } } }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "updated_at": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time" }, "user": { - "type": [ - "null", - "object" - ], + "type": ["null", "object"], "additionalProperties": false, "properties": { "id": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "type": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] } } }, "waiting_since": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "format": "date-time" }, "admin_assignee_id": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "title": { - "type": [ - "null", - "string" - ] + "type": ["null", "string"] }, "team_assignee_id": { - "type": [ - "null", - "integer" - ] + "type": ["null", "integer"] }, "redacted": { - "type": [ - "null", - "boolean" - ] + "type": ["null", "boolean"] } } } diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index f2e42a736fe7..c767135d4e7a 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -1,3 +1,4 @@ +# # MIT License # # Copyright (c) 2020 Airbyte @@ -19,16 +20,17 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# import time from abc import ABC -from urllib.parse import parse_qsl, urlparse -from datetime import date, timedelta, datetime +from datetime import date, datetime, timedelta from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union +from urllib.parse import parse_qsl, urlparse import requests -from airbyte_cdk.models import SyncMode from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream @@ -45,10 +47,10 @@ class IntercomStream(HttpStream, ABC): data_fields = ["data"] def __init__( - self, - authenticator: HttpAuthenticator, - start_date: Union[date, str] = None, - **kwargs, + self, + authenticator: HttpAuthenticator, + start_date: Union[date, str] = None, + **kwargs, ): self.start_date = start_date @@ -68,25 +70,17 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, return None def request_headers( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> Mapping[str, Any]: return {"Accept": "application/json"} - def _send_request( - self, - request: requests.PreparedRequest, - request_kwargs: Mapping[str, Any] - ) -> requests.Response: + def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: try: return super()._send_request(request, request_kwargs) except requests.exceptions.HTTPError as e: error_message = e.response.text if error_message: - self.logger.error(f"Stream {self.name}: {e.response.status_code} " - f"{e.response.reason} - {error_message}") + self.logger.error(f"Stream {self.name}: {e.response.status_code} " f"{e.response.reason} - {error_message}") raise e def get_data(self, response: requests.Response) -> List: @@ -102,11 +96,11 @@ def get_data(self, response: requests.Response) -> List: return data def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> Iterable[Mapping]: data = self.get_data(response) @@ -121,11 +115,7 @@ def parse_response( class IncrementalIntercomStream(IntercomStream, ABC): cursor_field = "updated_at" - def filter_by_state( - self, - stream_state: Mapping[str, Any] = None, - records: Mapping[str, Any] = None - ) -> Iterable: + def filter_by_state(self, stream_state: Mapping[str, Any] = None, records: Mapping[str, Any] = None) -> Iterable: if stream_state: for record in records: @@ -135,11 +125,11 @@ def filter_by_state( yield from records def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, ) -> Iterable[Mapping]: data = self.get_data(response) @@ -157,11 +147,7 @@ def parse_response( # wait for 3,6 seconds according to API limit time.sleep(3600 / self.rate_limit) - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any] - ) -> Mapping[str, any]: + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: """ This method is called once for each record returned from the API to compare the cursor field value in that record with the current state @@ -171,21 +157,17 @@ def get_updated_state( current_stream_state = current_stream_state or {} - current_stream_state_date = current_stream_state.get( - self.cursor_field, str(self.start_date) - ) - latest_record_date = latest_record.get( - self.cursor_field, str(self.start_date) - ) + current_stream_state_date = current_stream_state.get(self.cursor_field, str(self.start_date)) + latest_record_date = latest_record.get(self.cursor_field, str(self.start_date)) return {self.cursor_field: max(current_stream_state_date, latest_record_date)} def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, ) -> Iterable[Mapping[str, Any]]: """ Endpoint does not provide query filtering params, but they provide us @@ -202,10 +184,7 @@ class StreamMixin: stream = None def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.stream( - authenticator=self.authenticator, - start_date=self.start_date - ).read_records(sync_mode=sync_mode): + for item in self.stream(authenticator=self.authenticator, start_date=self.start_date).read_records(sync_mode=sync_mode): yield {"id": item["id"]} yield from [] @@ -220,10 +199,7 @@ class Admins(IntercomStream): data_fields = ["admins"] def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "admins" @@ -235,10 +211,7 @@ class Companies(IncrementalIntercomStream): """ def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "companies" @@ -252,10 +225,7 @@ class CompanySegments(StreamMixin, IncrementalIntercomStream): stream = Companies def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return f"/companies/{stream_slice['id']}/segments" @@ -269,10 +239,7 @@ class Conversations(IncrementalIntercomStream): data_fields = ["conversations"] def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "conversations" @@ -287,10 +254,7 @@ class ConversationParts(StreamMixin, IncrementalIntercomStream): stream = Conversations def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return f"/conversations/{stream_slice['id']}" @@ -304,10 +268,7 @@ class Segments(IncrementalIntercomStream): data_fields = ["segments"] def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "segments" @@ -319,10 +280,7 @@ class Contacts(IncrementalIntercomStream): """ def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "contacts" @@ -331,10 +289,7 @@ class DataAttributes(IntercomStream): primary_key = "name" def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "data_attributes" @@ -346,10 +301,7 @@ class CompanyAttributes(DataAttributes): """ def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: return {"model": "company"} @@ -361,10 +313,7 @@ class ContactAttributes(DataAttributes): """ def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: return {"model": "contact"} @@ -378,10 +327,7 @@ class Tags(IntercomStream): primary_key = "name" def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "tags" @@ -396,10 +342,7 @@ class Teams(IntercomStream): data_fields = ["teams"] def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> str: return "teams" @@ -413,8 +356,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: authenticator = TokenAuthenticator(token=config["access_token"]) try: url = f"{IntercomStream.url_base}/tags" - auth_headers = {"Accept": "application/json", - **authenticator.get_auth_header()} + auth_headers = {"Accept": "application/json", **authenticator.get_auth_header()} session = requests.get(url, headers=auth_headers) session.raise_for_status() return True, None @@ -432,14 +374,16 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") auth = TokenAuthenticator(token=config["access_token"]) - return [Admins(authenticator=auth, **config), - Companies(authenticator=auth, **config), - CompanySegments(authenticator=auth, **config), - Conversations(authenticator=auth, **config), - ConversationParts(authenticator=auth, **config), - Segments(authenticator=auth, **config), - Contacts(authenticator=auth, **config), - CompanyAttributes(authenticator=auth, **config), - ContactAttributes(authenticator=auth, **config), - Tags(authenticator=auth, **config), - Teams(authenticator=auth, **config)] + return [ + Admins(authenticator=auth, **config), + Companies(authenticator=auth, **config), + CompanySegments(authenticator=auth, **config), + Conversations(authenticator=auth, **config), + ConversationParts(authenticator=auth, **config), + Segments(authenticator=auth, **config), + Contacts(authenticator=auth, **config), + CompanyAttributes(authenticator=auth, **config), + ContactAttributes(authenticator=auth, **config), + Tags(authenticator=auth, **config), + Teams(authenticator=auth, **config), + ] diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json b/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json index 69d52942ef60..a233d03a9084 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/spec.json @@ -20,4 +20,4 @@ } } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py index f03f99f7c46e..b8a8150b507f 100644 --- a/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py +++ b/airbyte-integrations/connectors/source-intercom/unit_tests/unit_test.py @@ -1,3 +1,4 @@ +# # MIT License # # Copyright (c) 2020 Airbyte @@ -19,6 +20,7 @@ # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +# def test_example_method(): From 68ea9e4107b480fe1e19f467f20c3da0659056e9 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Wed, 14 Jul 2021 00:15:19 +0300 Subject: [PATCH 08/23] Updated to review --- .../source-intercom/source_intercom/source.py | 58 ++++++------------- 1 file changed, 17 insertions(+), 41 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index c767135d4e7a..7409233fca35 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -86,8 +86,7 @@ def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mappi def get_data(self, response: requests.Response) -> List: data = response.json() for data_field in self.data_fields: - if data_field is not None: - data = data.get(data_field, []) + data = data.get(data_field, []) if isinstance(data, list): data = data elif isinstance(data, dict): @@ -115,14 +114,15 @@ def parse_response( class IncrementalIntercomStream(IntercomStream, ABC): cursor_field = "updated_at" - def filter_by_state(self, stream_state: Mapping[str, Any] = None, records: Mapping[str, Any] = None) -> Iterable: + def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> Iterable: + """ + Endpoint does not provide query filtering params, but they provide us + updated_at field in most cases, so we used that as incremental filtering + during the slicing. + """ - if stream_state: - for record in records: - if record[self.cursor_field] >= stream_state.get(self.cursor_field): - yield record - else: - yield from records + if not stream_state or record[self.cursor_field] >= stream_state.get(self.cursor_field): + yield record def parse_response( self, @@ -135,14 +135,14 @@ def parse_response( data = self.get_data(response) for record in data: - updated_at = record.get(self.cursor_field, None) + updated_at = record.get(self.cursor_field) if updated_at: record[self.cursor_field] = datetime.fromtimestamp( record[self.cursor_field] ).isoformat() # convert timestamp to datetime string - yield record + yield from self.filter_by_state(stream_state=stream_state, record=record) # wait for 3,6 seconds according to API limit time.sleep(3600 / self.rate_limit) @@ -157,34 +157,17 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late current_stream_state = current_stream_state or {} - current_stream_state_date = current_stream_state.get(self.cursor_field, str(self.start_date)) - latest_record_date = latest_record.get(self.cursor_field, str(self.start_date)) + current_stream_state_date = current_stream_state.get(self.cursor_field, self.start_date) + latest_record_date = latest_record.get(self.cursor_field, self.start_date) return {self.cursor_field: max(current_stream_state_date, latest_record_date)} - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: - """ - Endpoint does not provide query filtering params, but they provide us - updated_at field in most cases, so we used that as incremental filtering - during the slicing. - """ - - records = super().read_records(sync_mode, cursor_field, stream_slice, stream_state) - - yield from self.filter_by_state(stream_state=stream_state, records=records) - class StreamMixin: - stream = None + slicing_stream: Optional[IntercomStream] = None def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.stream(authenticator=self.authenticator, start_date=self.start_date).read_records(sync_mode=sync_mode): + for item in self.slicing_stream(authenticator=self.authenticator, start_date=self.start_date).read_records(sync_mode=sync_mode): yield {"id": item["id"]} yield from [] @@ -222,7 +205,7 @@ class CompanySegments(StreamMixin, IncrementalIntercomStream): Endpoint: https://api.intercom.io/companies//segments """ - stream = Companies + slicing_stream = Companies def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -251,7 +234,7 @@ class ConversationParts(StreamMixin, IncrementalIntercomStream): """ data_fields = ["conversation_parts", "conversation_parts"] - stream = Conversations + slicing_stream = Conversations def path( self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -364,13 +347,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: - now = date.today() - - start_date = config.get("start_date") - if start_date and isinstance(start_date, str): - start_date = datetime.strptime(config["start_date"], "%Y-%m-%dT%H:%M:%S%z") - config["start_date"] = start_date or now - timedelta(days=365) # set to 1 year ago by default - AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") auth = TokenAuthenticator(token=config["access_token"]) From 7428cac535d82a1815d42f8c0f3137d479b4841a Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Wed, 14 Jul 2021 00:20:11 +0300 Subject: [PATCH 09/23] Code formated --- .../connectors/source-intercom/source_intercom/source.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 7409233fca35..7218729ec304 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -24,13 +24,12 @@ import time from abc import ABC -from datetime import date, datetime, timedelta +from datetime import date, datetime from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union from urllib.parse import parse_qsl, urlparse import requests from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream From 79313724eaa977c7f4be4c760ec7caaaae413324 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Wed, 14 Jul 2021 16:37:10 +0300 Subject: [PATCH 10/23] Updated cursor paths for test incremental sync --- .../connectors/source-intercom/acceptance-test-config.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml index e1185300f21a..0843f6349f8e 100644 --- a/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-intercom/acceptance-test-config.yml @@ -20,7 +20,12 @@ tests: configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" cursor_paths: - charges: [ "updated_at" ] + companies: ["updated_at"] + company_segments: ["updated_at"] + conversations: ["updated_at"] + conversation_parts: ["updated_at"] + contacts: ["updated_at"] + segments: ["updated_at"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" \ No newline at end of file From 0b8ca5eef1020b69647fc8df97ba17e11f19ed70 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Wed, 14 Jul 2021 16:38:08 +0300 Subject: [PATCH 11/23] Added dict type validation to get_data method --- .../connectors/source-intercom/source_intercom/source.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 7218729ec304..eb6510122210 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -84,8 +84,11 @@ def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mappi def get_data(self, response: requests.Response) -> List: data = response.json() + for data_field in self.data_fields: - data = data.get(data_field, []) + if data and isinstance(data, dict): + data = data.get(data_field, []) + if isinstance(data, list): data = data elif isinstance(data, dict): From 2d38c0258df7eb21a1af526846de07505008824c Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Wed, 14 Jul 2021 17:12:04 +0300 Subject: [PATCH 12/23] Updated catalog --- .../integration_tests/configured_catalog.json | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json index 9224d780226c..e6dc584fdbf9 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/configured_catalog.json @@ -181,11 +181,12 @@ "type": "object", "additionalProperties": false }, - "supported_sync_modes": ["incremental"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"] }, "sync_mode": "incremental", + "cursor_field": ["updated_at"], "destination_sync_mode": "append" }, { @@ -288,7 +289,7 @@ "type": "object", "additionalProperties": false }, - "supported_sync_modes": ["incremental"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"] }, @@ -751,7 +752,7 @@ "type": "object", "additionalProperties": false }, - "supported_sync_modes": ["incremental"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"] }, @@ -1230,7 +1231,7 @@ "type": ["null", "object"], "additionalProperties": false }, - "supported_sync_modes": ["incremental"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"] }, @@ -1267,7 +1268,7 @@ "type": "object", "additionalProperties": false }, - "supported_sync_modes": ["incremental"], + "supported_sync_modes": ["full_refresh", "incremental"], "source_defined_cursor": true, "default_cursor_field": ["updated_at"] }, From 2c82fff7c52873dc25e6d18b457b77191b05df8c Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Wed, 14 Jul 2021 18:04:15 +0300 Subject: [PATCH 13/23] Updated typing for start_date --- .../connectors/source-intercom/source_intercom/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index eb6510122210..797debb3f26f 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -48,7 +48,7 @@ class IntercomStream(HttpStream, ABC): def __init__( self, authenticator: HttpAuthenticator, - start_date: Union[date, str] = None, + start_date: str = None, **kwargs, ): self.start_date = start_date From 4590932dc7b2f52909b4e69de1b160c6da2a5d26 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Wed, 14 Jul 2021 18:05:10 +0300 Subject: [PATCH 14/23] Updated singer seed to cdk seed --- .../d8313939-3782-41b0-be29-b3ca20d8dd3a.json | 6 +++--- .../init/src/main/resources/seed/source_definitions.yaml | 6 +++--- airbyte-integrations/builds.md | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json index 500de9798fde..feeedd810bac 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8313939-3782-41b0-be29-b3ca20d8dd3a.json @@ -1,8 +1,8 @@ { "sourceDefinitionId": "d8313939-3782-41b0-be29-b3ca20d8dd3a", "name": "Intercom", - "dockerRepository": "airbyte/source-intercom-singer", - "dockerImageTag": "0.2.3", - "documentationUrl": "https://hub.docker.com/r/airbyte/source-intercom-singer", + "dockerRepository": "airbyte/source-intercom", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-intercom", "icon": "intercom.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index be08178f40ac..20eb19d2a397 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -184,9 +184,9 @@ icon: zendesk.svg - sourceDefinitionId: d8313939-3782-41b0-be29-b3ca20d8dd3a name: Intercom - dockerRepository: airbyte/source-intercom-singer - dockerImageTag: 0.2.3 - documentationUrl: https://hub.docker.com/r/airbyte/source-intercom-singer + dockerRepository: airbyte/source-intercom + dockerImageTag: 0.1.0 + documentationUrl: https://hub.docker.com/r/airbyte/source-intercom icon: intercom.svg - sourceDefinitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 name: Jira diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 423ecb7b1ae9..4d7ae5ee6c8f 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -55,7 +55,7 @@ Instagram [![source-instagram](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-instagram%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-instagram) - Intercom [![source-intercom-singer](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-intercom-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-intercom-singer) + Intercom [![source-intercom](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-intercom-singer%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-intercom) Iterable [![source-iterable](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-iterable%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-iterable) From 29f47a0ba1a72e9b98636118530d483c52aaeee3 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 15 Jul 2021 16:29:15 +0300 Subject: [PATCH 15/23] Updated connector docs --- docs/integrations/sources/intercom.md | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/docs/integrations/sources/intercom.md b/docs/integrations/sources/intercom.md index 7780d1a29854..59e46043cab2 100644 --- a/docs/integrations/sources/intercom.md +++ b/docs/integrations/sources/intercom.md @@ -10,19 +10,18 @@ This Source Connector is based on a [Airbyte CDK](https://docs.airbyte.io/contri Several output streams are available from this source: -* [Admins](https://developers.intercom.com/intercom-api-reference/reference#list-admins) -* [Companies](https://developers.intercom.com/intercom-api-reference/reference#list-companies) -* [Conversations](https://developers.intercom.com/intercom-api-reference/reference#list-conversations) - * [Conversation Parts](https://developers.intercom.com/intercom-api-reference/reference#get-a-single-conversation) -* [Data Attributes](https://developers.intercom.com/intercom-api-reference/reference#data-attributes) - * [Customer Attributes](https://developers.intercom.com/intercom-api-reference/reference#list-customer-data-attributes) - * [Company Attributes](https://developers.intercom.com/intercom-api-reference/reference#list-company-data-attributes) -* [Leads](https://developers.intercom.com/intercom-api-reference/reference#list-leads) -* [Segments](https://developers.intercom.com/intercom-api-reference/reference#list-segments) - * [Company Segments](https://developers.intercom.com/intercom-api-reference/reference#list-segments) -* [Tags](https://developers.intercom.com/intercom-api-reference/reference#list-tags-for-an-app) -* [Teams](https://developers.intercom.com/intercom-api-reference/reference#list-teams) -* [Users](https://developers.intercom.com/intercom-api-reference/reference#list-users) +* [Admins](https://developers.intercom.com/intercom-api-reference/reference#list-admins) \(Full table\) +* [Companies](https://developers.intercom.com/intercom-api-reference/reference#list-companies) \(Incremental\) + * [Company Segments](https://developers.intercom.com/intercom-api-reference/reference#list-attached-segments-1) \(Incremental\) +* [Conversations](https://developers.intercom.com/intercom-api-reference/reference#list-conversations) \(Incremental\) + * [Conversation Parts](https://developers.intercom.com/intercom-api-reference/reference#get-a-single-conversation) \(Incremental\) +* [Data Attributes](https://developers.intercom.com/intercom-api-reference/reference#data-attributes) \(Full table\) + * [Customer Attributes](https://developers.intercom.com/intercom-api-reference/reference#list-customer-data-attributes) \(Full table\) + * [Company Attributes](https://developers.intercom.com/intercom-api-reference/reference#list-company-data-attributes) \(Full table\) +* [Contacts](https://developers.intercom.com/intercom-api-reference/reference#list-contacts) \(Incremental\) +* [Segments](https://developers.intercom.com/intercom-api-reference/reference#list-segments) \(Incremental\) +* [Tags](https://developers.intercom.com/intercom-api-reference/reference#list-tags-for-an-app) \(Full table\) +* [Teams](https://developers.intercom.com/intercom-api-reference/reference#list-teams) \(Full table\) If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) From 96d889c7225e114e230e9d16b6838e5d1c7e98e1 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 15 Jul 2021 19:57:08 +0300 Subject: [PATCH 16/23] Updated sample config file --- .../source-intercom/integration_tests/sample_config.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json index ecc4913b84c7..80049770c24b 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/sample_config.json @@ -1,3 +1,4 @@ { - "fix-me": "TODO" + "start_date": "2020-11-22T20:32:05Z", + "access_token": "access_token" } From 98ccd38f7a9533ab91c84c9adf3507d0bd9efdc8 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 15 Jul 2021 20:00:24 +0300 Subject: [PATCH 17/23] Sorted streams alphabetically --- .../connectors/source-intercom/source_intercom/source.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 797debb3f26f..277b1823cd85 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -358,10 +358,10 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: CompanySegments(authenticator=auth, **config), Conversations(authenticator=auth, **config), ConversationParts(authenticator=auth, **config), - Segments(authenticator=auth, **config), Contacts(authenticator=auth, **config), CompanyAttributes(authenticator=auth, **config), ContactAttributes(authenticator=auth, **config), + Segments(authenticator=auth, **config), Tags(authenticator=auth, **config), Teams(authenticator=auth, **config), ] From 9a803e2fa530f3413faebefc9608ebb815dbbca5 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 15 Jul 2021 20:05:25 +0300 Subject: [PATCH 18/23] Removed a placeholder comments --- .../connectors/source-intercom/integration_tests/acceptance.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py index eeb4a2d3e02e..496a799cf8ed 100644 --- a/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-intercom/integration_tests/acceptance.py @@ -30,7 +30,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies From 091f7cb30845602887003e5849f80f3ea93f2f08 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 15 Jul 2021 20:09:01 +0300 Subject: [PATCH 19/23] Renamed rate_limit to queries_per_hour --- .../connectors/source-intercom/source_intercom/source.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 277b1823cd85..9c68f7f40c03 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -40,7 +40,7 @@ class IntercomStream(HttpStream, ABC): url_base = "https://api.intercom.io/" # https://developers.intercom.com/intercom-api-reference/reference#rate-limiting - rate_limit = 1000 # 1000 queries per hour == 1 req in 3,6 secs + queries_per_hour = 1000 # 1000 queries per hour == 1 req in 3,6 secs primary_key = "id" data_fields = ["data"] @@ -110,7 +110,7 @@ def parse_response( yield record # wait for 3,6 seconds according to API limit - time.sleep(3600 / self.rate_limit) + time.sleep(3600 / self.queries_per_hour) class IncrementalIntercomStream(IntercomStream, ABC): @@ -147,7 +147,7 @@ def parse_response( yield from self.filter_by_state(stream_state=stream_state, record=record) # wait for 3,6 seconds according to API limit - time.sleep(3600 / self.rate_limit) + time.sleep(3600 / self.queries_per_hour) def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: """ From d48477ea9061b08708c09340b990ce9a483e557c Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Thu, 15 Jul 2021 20:22:14 +0300 Subject: [PATCH 20/23] Updated common sleep time to backoff_time method --- .../source-intercom/source_intercom/source.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 9c68f7f40c03..7ca623ad135c 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -22,9 +22,8 @@ # SOFTWARE. # -import time from abc import ABC -from datetime import date, datetime +from datetime import datetime from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union from urllib.parse import parse_qsl, urlparse @@ -109,8 +108,9 @@ def parse_response( for record in data: yield record - # wait for 3,6 seconds according to API limit - time.sleep(3600 / self.queries_per_hour) + def backoff_time(self, response: requests.Response) -> Optional[float]: + wait_time = 3600 / self.queries_per_hour # wait for 3,6 seconds according to API limit + return wait_time class IncrementalIntercomStream(IntercomStream, ABC): @@ -146,9 +146,6 @@ def parse_response( yield from self.filter_by_state(stream_state=stream_state, record=record) - # wait for 3,6 seconds according to API limit - time.sleep(3600 / self.queries_per_hour) - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, any]: """ This method is called once for each record returned from the API to From fd996e190632aab315e6411592c1475105647d1d Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 19 Jul 2021 09:48:21 +0300 Subject: [PATCH 21/23] Updated to review --- .../source-intercom/source_intercom/source.py | 76 ++++++------------- 1 file changed, 25 insertions(+), 51 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 7ca623ad135c..8bc926333aae 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -22,6 +22,7 @@ # SOFTWARE. # +import time from abc import ABC from datetime import datetime from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union @@ -67,9 +68,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, else: return None - def request_headers( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: + def request_headers(self, **kwargs) -> Mapping[str, Any]: return {"Accept": "application/json"} def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: @@ -108,9 +107,8 @@ def parse_response( for record in data: yield record - def backoff_time(self, response: requests.Response) -> Optional[float]: - wait_time = 3600 / self.queries_per_hour # wait for 3,6 seconds according to API limit - return wait_time + # wait for 3,6 seconds according to API limit + time.sleep(3600 / self.queries_per_hour) class IncrementalIntercomStream(IntercomStream, ABC): @@ -134,9 +132,9 @@ def parse_response( next_page_token: Mapping[str, Any] = None, ) -> Iterable[Mapping]: - data = self.get_data(response) + record = super().parse_response(response, stream_state, stream_slice, next_page_token) - for record in data: + for record in record: updated_at = record.get(self.cursor_field) if updated_at: @@ -162,11 +160,11 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late return {self.cursor_field: max(current_stream_state_date, latest_record_date)} -class StreamMixin: - slicing_stream: Optional[IntercomStream] = None +class ChildStreamMixin: + parent_stream_class: Optional[IntercomStream] = None def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.slicing_stream(authenticator=self.authenticator, start_date=self.start_date).read_records(sync_mode=sync_mode): + for item in self.parent_stream_class(authenticator=self.authenticator, start_date=self.start_date).read_records(sync_mode=sync_mode): yield {"id": item["id"]} yield from [] @@ -180,9 +178,7 @@ class Admins(IntercomStream): data_fields = ["admins"] - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "admins" @@ -192,23 +188,19 @@ class Companies(IncrementalIntercomStream): Endpoint: https://api.intercom.io/companies """ - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "companies" -class CompanySegments(StreamMixin, IncrementalIntercomStream): +class CompanySegments(ChildStreamMixin, IncrementalIntercomStream): """Return list of all company segments. API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-attached-segments-1 Endpoint: https://api.intercom.io/companies//segments """ - slicing_stream = Companies + parent_stream_class = Companies - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"/companies/{stream_slice['id']}/segments" @@ -220,24 +212,20 @@ class Conversations(IncrementalIntercomStream): data_fields = ["conversations"] - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "conversations" -class ConversationParts(StreamMixin, IncrementalIntercomStream): +class ConversationParts(ChildStreamMixin, IncrementalIntercomStream): """Return list of all conversation parts. API Docs: https://developers.intercom.com/intercom-api-reference/reference#retrieve-a-conversation Endpoint: https://api.intercom.io/conversations/ """ data_fields = ["conversation_parts", "conversation_parts"] - slicing_stream = Conversations + parent_stream_class = Conversations - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"/conversations/{stream_slice['id']}" @@ -249,9 +237,7 @@ class Segments(IncrementalIntercomStream): data_fields = ["segments"] - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "segments" @@ -261,18 +247,14 @@ class Contacts(IncrementalIntercomStream): Endpoint: https://api.intercom.io/contacts """ - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "contacts" class DataAttributes(IntercomStream): primary_key = "name" - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "data_attributes" @@ -282,9 +264,7 @@ class CompanyAttributes(DataAttributes): Endpoint: https://api.intercom.io/data_attributes?model=company """ - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: + def request_params(self, **kwargs) -> MutableMapping[str, Any]: return {"model": "company"} @@ -294,9 +274,7 @@ class ContactAttributes(DataAttributes): Endpoint: https://api.intercom.io/data_attributes?model=contact """ - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: + def request_params(self, **kwargs) -> MutableMapping[str, Any]: return {"model": "contact"} @@ -308,9 +286,7 @@ class Tags(IntercomStream): primary_key = "name" - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "tags" @@ -323,9 +299,7 @@ class Teams(IntercomStream): primary_key = "name" data_fields = ["teams"] - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: + def path(self, **kwargs) -> str: return "teams" From 92c50f15c016f679feb96930aeeac82e58829670 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 19 Jul 2021 12:00:43 +0300 Subject: [PATCH 22/23] Updated to review --- .../source-intercom/source_intercom/source.py | 34 ++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 8bc926333aae..7f55cfb146dc 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -61,23 +61,30 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Returning None means there are no more pages to read in response. """ - next_page = response.links.get("next", None) + next_page = response.json().get("pages", {}).get('next') if next_page: - return dict(parse_qsl(urlparse(next_page.get("url")).query)) + return {"starting_after": next_page["starting_after"]} else: return None + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = {} + if next_page_token: + params.update(**next_page_token) + return params + def request_headers(self, **kwargs) -> Mapping[str, Any]: return {"Accept": "application/json"} - def _send_request(self, request: requests.PreparedRequest, request_kwargs: Mapping[str, Any]) -> requests.Response: + def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: try: - return super()._send_request(request, request_kwargs) + yield from super().read_records(*args, **kwargs) except requests.exceptions.HTTPError as e: error_message = e.response.text if error_message: - self.logger.error(f"Stream {self.name}: {e.response.status_code} " f"{e.response.reason} - {error_message}") + self.logger.error( + f"Stream {self.name}: {e.response.status_code} " f"{e.response.reason} - {error_message}") raise e def get_data(self, response: requests.Response) -> List: @@ -184,12 +191,23 @@ def path(self, **kwargs) -> str: class Companies(IncrementalIntercomStream): """Return list of all companies. - API Docs: https://developers.intercom.com/intercom-api-reference/reference#list-companies - Endpoint: https://api.intercom.io/companies + API Docs: https://developers.intercom.com/intercom-api-reference/reference#iterating-over-all-companies + Endpoint: https://api.intercom.io/companies/scroll """ + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """For reset scroll needs to iterate pages untill the last. + Another way need wait 1 min for the scroll to expire to get a new list for companies segments.""" + + data = response.json().get("data") + + if data: + return {"scroll_param": response.json()["scroll_param"]} + else: + return None + def path(self, **kwargs) -> str: - return "companies" + return "companies/scroll" class CompanySegments(ChildStreamMixin, IncrementalIntercomStream): From a91862ee92776d9477ff2c450b590cb44639faa4 Mon Sep 17 00:00:00 2001 From: lazebnyi Date: Mon, 19 Jul 2021 12:14:26 +0300 Subject: [PATCH 23/23] Updated to review --- .../source-intercom/source_intercom/source.py | 32 ++++++------------- 1 file changed, 9 insertions(+), 23 deletions(-) diff --git a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py index 7f55cfb146dc..f3d65f4be67e 100644 --- a/airbyte-integrations/connectors/source-intercom/source_intercom/source.py +++ b/airbyte-integrations/connectors/source-intercom/source_intercom/source.py @@ -25,8 +25,7 @@ import time from abc import ABC from datetime import datetime -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple, Union -from urllib.parse import parse_qsl, urlparse +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple import requests from airbyte_cdk.logger import AirbyteLogger @@ -61,7 +60,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Returning None means there are no more pages to read in response. """ - next_page = response.json().get("pages", {}).get('next') + next_page = response.json().get("pages", {}).get("next") if next_page: return {"starting_after": next_page["starting_after"]} @@ -83,8 +82,7 @@ def read_records(self, *args, **kwargs) -> Iterable[Mapping[str, Any]]: except requests.exceptions.HTTPError as e: error_message = e.response.text if error_message: - self.logger.error( - f"Stream {self.name}: {e.response.status_code} " f"{e.response.reason} - {error_message}") + self.logger.error(f"Stream {self.name}: {e.response.status_code} " f"{e.response.reason} - {error_message}") raise e def get_data(self, response: requests.Response) -> List: @@ -101,14 +99,7 @@ def get_data(self, response: requests.Response) -> List: return data - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: data = self.get_data(response) for record in data: @@ -131,15 +122,8 @@ def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mappin if not stream_state or record[self.cursor_field] >= stream_state.get(self.cursor_field): yield record - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - - record = super().parse_response(response, stream_state, stream_slice, next_page_token) + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + record = super().parse_response(response, stream_state, **kwargs) for record in record: updated_at = record.get(self.cursor_field) @@ -171,7 +155,9 @@ class ChildStreamMixin: parent_stream_class: Optional[IntercomStream] = None def stream_slices(self, sync_mode, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - for item in self.parent_stream_class(authenticator=self.authenticator, start_date=self.start_date).read_records(sync_mode=sync_mode): + for item in self.parent_stream_class(authenticator=self.authenticator, start_date=self.start_date).read_records( + sync_mode=sync_mode + ): yield {"id": item["id"]} yield from []