diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index e7f8de2da4b7..8a437ff54ab9 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -110,6 +110,7 @@ jobs: MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }} PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} + PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }} SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} diff --git a/.github/workflows/test-command.yml b/.github/workflows/test-command.yml index 8c3984f01c45..4b5a768e76d7 100644 --- a/.github/workflows/test-command.yml +++ b/.github/workflows/test-command.yml @@ -109,6 +109,7 @@ jobs: MSSQL_RDS_TEST_CREDS: ${{ secrets.MSSQL_RDS_TEST_CREDS }} PAYPAL_TRANSACTION_CREDS: ${{ secrets.SOURCE_PAYPAL_TRANSACTION_CREDS }} POSTHOG_TEST_CREDS: ${{ secrets.POSTHOG_TEST_CREDS }} + PIPEDRIVE_INTEGRATION_TESTS_CREDS: ${{ secrets.PIPEDRIVE_INTEGRATION_TESTS_CREDS }} RECHARGE_INTEGRATION_TEST_CREDS: ${{ secrets.RECHARGE_INTEGRATION_TEST_CREDS }} QUICKBOOKS_TEST_CREDS: ${{ secrets.QUICKBOOKS_TEST_CREDS }} SALESFORCE_INTEGRATION_TESTS_CREDS: ${{ secrets.SALESFORCE_INTEGRATION_TESTS_CREDS }} diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8286229-c680-4063-8c59-23b9b391c700.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8286229-c680-4063-8c59-23b9b391c700.json new file mode 100644 index 000000000000..7eddb5f3c104 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/d8286229-c680-4063-8c59-23b9b391c700.json @@ -0,0 +1,7 @@ +{ + "sourceDefinitionId": "d8286229-c680-4063-8c59-23b9b391c700", + "name": "Pipedrive", + "dockerRepository": "airbyte/source-pipedrive", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-pipedrive" +} diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 0ab6f14de2e0..335fab970b64 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -43,6 +43,11 @@ dockerImageTag: 0.3.2 documentationUrl: https://hub.docker.com/r/airbyte/source-mssql icon: mssql.svg +- sourceDefinitionId: d8286229-c680-4063-8c59-23b9b391c700 + name: Pipedrive + dockerRepository: airbyte/source-pipedrive + dockerImageTag: 0.1.0 + documentationUrl: https://hub.docker.com/r/airbyte/source-pipedrive - sourceDefinitionId: decd338e-5647-4c0b-adf4-da0e75f5a750 name: Postgres dockerRepository: airbyte/source-postgres diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 7366aed40e40..397ad5b4fdfd 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -79,6 +79,8 @@ Paypal Transaction [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-paypal-transaction) + Pipedrive [![source-pipedrive](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-pipedrive) + Plaid [![source-plaid](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-plaid%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-plaid) Postgres [![source-postgres](https://img.shields.io/endpoint?url=https%3A%2F%2Fstatus-api.airbyte.io%2Ftests%2Fsummary%2Fsource-postgres%2Fbadge.json)](https://status-api.airbyte.io/tests/summary/source-postgres) diff --git a/airbyte-integrations/connectors/source-pipedrive/.dockerignore b/airbyte-integrations/connectors/source-pipedrive/.dockerignore new file mode 100644 index 000000000000..576890429ae3 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_pipedrive +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-pipedrive/Dockerfile b/airbyte-integrations/connectors/source-pipedrive/Dockerfile new file mode 100644 index 000000000000..69ca6bdd68d5 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.7-slim + +# Bash is installed for more convenient debugging. +RUN apt-get update && apt-get install -y bash && rm -rf /var/lib/apt/lists/* + +WORKDIR /airbyte/integration_code +COPY source_pipedrive ./source_pipedrive +COPY main.py ./ +COPY setup.py ./ +RUN pip install . + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.name=airbyte/source-pipedrive diff --git a/airbyte-integrations/connectors/source-pipedrive/README.md b/airbyte-integrations/connectors/source-pipedrive/README.md new file mode 100644 index 000000000000..e20fe30561e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/README.md @@ -0,0 +1,131 @@ +# Pipedrive Source + +This is the repository for the Pipedrive source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/pipedrive). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install . +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-pipedrive:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/pipedrive) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_pipedrive/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source pipedrive test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-pipedrive:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-pipedrive:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-pipedrive:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pipedrive:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-pipedrive:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-pipedrive:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](source-acceptance-tests.md) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-pipedrive:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-pipedrive:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml new file mode 100644 index 000000000000..10426feb7e77 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-config.yml @@ -0,0 +1,22 @@ +connector_image: airbyte/source-pipedrive:dev +tests: + spec: + - spec_path: "source_pipedrive/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + validate_output_from_all_streams: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh new file mode 100644 index 000000000000..1425ff74f151 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/acceptance-test-docker.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env sh +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input diff --git a/airbyte-integrations/connectors/source-pipedrive/build.gradle b/airbyte-integrations/connectors/source-pipedrive/build.gradle new file mode 100644 index 000000000000..e694e7144b04 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_pipedrive' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..f77fa7857faf --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/abnormal_state.json @@ -0,0 +1,20 @@ +{ + "deals": { + "update_time": "2217-06-26T21:20:07Z" + }, + "activities": { + "update_time": "2217-06-26T21:20:07Z" + }, + "persons": { + "update_time": "2217-06-26T21:20:07Z" + }, + "pipelines": { + "update_time": "2217-06-26T21:20:07Z" + }, + "stages": { + "update_time": "2217-06-26T21:20:07Z" + }, + "users": { + "modified": "2217-06-26T21:20:07Z" + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py new file mode 100644 index 000000000000..d6cbdc97c495 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/acceptance.py @@ -0,0 +1,34 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..9522f196eaa7 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/configured_catalog.json @@ -0,0 +1,126 @@ +{ + "streams": [ + { + "stream": { + "name": "deals", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] + }, + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "leads", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "activity_fields", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "activities", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] + }, + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "persons", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] + }, + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "pipelines", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] + }, + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "stages", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["update_time"] + }, + "sync_mode": "incremental", + "cursor_field": ["update_time"], + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": {} + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["modified"] + }, + "sync_mode": "incremental", + "cursor_field": ["modified"], + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json new file mode 100644 index 000000000000..6fe81a91ed38 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_token": "wrong-api-token", + "replication_start_date": "2021-06-01T10:10:10Z" +} diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json new file mode 100644 index 000000000000..5d3a28f22a86 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_token": "", + "replication_start_date": "2021-06-01T10:10:10Z" +} diff --git a/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json new file mode 100644 index 000000000000..98370c15bdec --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/integration_tests/sample_state.json @@ -0,0 +1,20 @@ +{ + "deals": { + "update_time": "2021-06-01T10:10:10Z" + }, + "activities": { + "update_time": "2021-06-01T10:10:10Z" + }, + "persons": { + "update_time": "2021-06-01T10:10:10Z" + }, + "pipelines": { + "update_time": "2021-06-01T10:10:10Z" + }, + "stages": { + "update_time": "2021-06-01T10:10:10Z" + }, + "users": { + "modified": "2021-06-01T10:10:10Z" + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/main.py b/airbyte-integrations/connectors/source-pipedrive/main.py new file mode 100644 index 000000000000..ed6472dce13b --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/main.py @@ -0,0 +1,33 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_pipedrive import SourcePipedrive + +if __name__ == "__main__": + source = SourcePipedrive() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-pipedrive/setup.py b/airbyte-integrations/connectors/source-pipedrive/setup.py new file mode 100644 index 000000000000..8dc15a1add40 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/setup.py @@ -0,0 +1,49 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "pendulum~=2.1", + "requests~=2.25", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", +] + +setup( + name="source_pipedrive", + description="Source implementation for Pipedrive.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py new file mode 100644 index 000000000000..7e21a5c1b775 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/__init__.py @@ -0,0 +1,27 @@ +""" +MIT License + +Copyright (c) 2020 Airbyte + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from .source import SourcePipedrive + +__all__ = ["SourcePipedrive"] diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activities.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activities.json new file mode 100644 index 000000000000..454675e50d3c --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activities.json @@ -0,0 +1,237 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "company_id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "done": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "reference_type": { + "type": ["null", "string"] + }, + "reference_id": { + "type": ["null", "integer"] + }, + "conference_meeting_client": { + "type": ["null", "string"] + }, + "conference_meeting_url": { + "type": ["null", "string"] + }, + "conference_meeting_id": { + "type": ["null", "string"] + }, + "due_date": { + "type": ["null", "string"] + }, + "due_time": { + "type": ["null", "string"] + }, + "duration": { + "type": ["null", "string"] + }, + "busy_flag": { + "type": ["null", "boolean"] + }, + "add_time": { + "type": ["null", "string"] + }, + "marked_as_done_time": { + "type": ["null", "string"] + }, + "last_notification_time": { + "type": ["null", "string"] + }, + "last_notification_user_id": { + "type": ["null", "integer"] + }, + "notification_language_id": { + "type": ["null", "integer"] + }, + "subject": { + "type": ["null", "string"] + }, + "public_description": { + "type": ["null", "string"] + }, + "calendar_sync_include_context": { + "type": ["null", "boolean"] + }, + "location": { + "type": ["null", "string"] + }, + "org_id": { + "type": ["null", "integer"] + }, + "person_id": { + "type": ["null", "integer"] + }, + "deal_id": { + "type": ["null", "integer"] + }, + "lead_id": { + "type": ["null", "integer"] + }, + "active_flag": { + "type": ["null", "boolean"] + }, + "update_time": { + "type": ["null", "string"] + }, + "update_user_id": { + "type": ["null", "integer"] + }, + "gcal_event_id": { + "type": ["null", "string"] + }, + "google_calendar_id": { + "type": ["null", "string"] + }, + "google_calendar_etag": { + "type": ["null", "string"] + }, + "source_timezone": { + "type": ["null", "string"] + }, + "rec_rule": { + "type": ["null", "string"] + }, + "rec_rule_extension": { + "type": ["null", "string"] + }, + "rec_master_activity_id": { + "type": ["null", "integer"] + }, + "series": { + "type": ["null", "string"] + }, + "note": { + "type": ["null", "string"] + }, + "created_by_user_id": { + "type": ["null", "integer"] + }, + "location_subpremise": { + "type": ["null", "string"] + }, + "location_street_number": { + "type": ["null", "string"] + }, + "location_route": { + "type": ["null", "string"] + }, + "location_sublocality": { + "type": ["null", "string"] + }, + "location_locality": { + "type": ["null", "string"] + }, + "location_lat": { + "type": ["null", "number"] + }, + "location_long": { + "type": ["null", "number"] + }, + "location_admin_area_level_1": { + "type": ["null", "string"] + }, + "location_admin_area_level_2": { + "type": ["null", "string"] + }, + "location_country": { + "type": ["null", "string"] + }, + "location_postal_code": { + "type": ["null", "string"] + }, + "location_formatted_address": { + "type": ["null", "string"] + }, + "attendees": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "email_address": { + "type": ["null", "string"] + }, + "is_organizer": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "person_id": { + "type": ["null", "integer"] + }, + "status": { + "type": ["null", "string"] + }, + "user_id": { + "type": ["null", "integer"] + } + } + } + }, + "participants": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "person_id": { + "type": ["null", "integer"] + }, + "primary_flag": { + "type": ["null", "boolean"] + } + } + } + }, + "org_name": { + "type": ["null", "string"] + }, + "person_name": { + "type": ["null", "string"] + }, + "deal_title": { + "type": ["null", "string"] + }, + "owner_name": { + "type": ["null", "string"] + }, + "person_dropbox_bcc": { + "type": ["null", "string"] + }, + "deal_dropbox_bcc": { + "type": ["null", "string"] + }, + "assigned_to_user_id": { + "type": ["null", "integer"] + }, + "file": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "string"] + }, + "clean_name": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json new file mode 100644 index 000000000000..10406b6f63ba --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/activity_fields.json @@ -0,0 +1,79 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "key": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "order_nr": { + "type": ["null", "integer"] + }, + "field_type": { + "type": ["null", "string"] + }, + "add_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_updated_by_user_id": { + "type": ["null", "integer"] + }, + "active_flag": { + "type": ["null", "boolean"] + }, + "edit_flag": { + "type": ["null", "boolean"] + }, + "index_visible_flag": { + "type": ["null", "boolean"] + }, + "details_visible_flag": { + "type": ["null", "boolean"] + }, + "add_visible_flag": { + "type": ["null", "boolean"] + }, + "important_flag": { + "type": ["null", "boolean"] + }, + "bulk_edit_allowed": { + "type": ["null", "boolean"] + }, + "searchable_flag": { + "type": ["null", "boolean"] + }, + "filtering_allowed": { + "type": ["null", "boolean"] + }, + "sortable_flag": { + "type": ["null", "boolean"] + }, + "options": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer", "string", "boolean"] + }, + "label": { + "type": ["null", "string"] + } + } + } + }, + "mandatory_flag": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deals.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deals.json new file mode 100644 index 000000000000..0cdcdc43b00a --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/deals.json @@ -0,0 +1,192 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "creator_user_id": { + "type": ["null", "integer"] + }, + "user_id": { + "type": ["null", "integer"] + }, + "person_id": { + "type": ["null", "integer"] + }, + "org_id": { + "type": ["null", "integer"] + }, + "stage_id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + }, + "add_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "stage_change_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "active": { + "type": ["null", "boolean"] + }, + "deleted": { + "type": ["null", "boolean"] + }, + "status": { + "type": ["null", "string"] + }, + "probability": { + "type": ["null", "number"] + }, + "next_activity_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "next_activity_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "next_activity_id": { + "type": ["null", "integer"] + }, + "last_activity_id": { + "type": ["null", "integer"] + }, + "last_activity_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "lost_reason": { + "type": ["null", "string"] + }, + "visible_to": { + "type": ["null", "string"] + }, + "close_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "pipeline_id": { + "type": ["null", "integer"] + }, + "won_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "first_won_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "lost_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "products_count": { + "type": ["null", "integer"] + }, + "files_count": { + "type": ["null", "integer"] + }, + "notes_count": { + "type": ["null", "integer"] + }, + "followers_count": { + "type": ["null", "integer"] + }, + "email_messages_count": { + "type": ["null", "integer"] + }, + "activities_count": { + "type": ["null", "integer"] + }, + "done_activities_count": { + "type": ["null", "integer"] + }, + "undone_activities_count": { + "type": ["null", "integer"] + }, + "participants_count": { + "type": ["null", "integer"] + }, + "expected_close_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_incoming_mail_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_outgoing_mail_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "label": { + "type": ["null", "string"] + }, + "stage_order_nr": { + "type": ["null", "integer"] + }, + "person_name": { + "type": ["null", "string"] + }, + "org_name": { + "type": ["null", "string"] + }, + "next_activity_subject": { + "type": ["null", "string"] + }, + "next_activity_type": { + "type": ["null", "string"] + }, + "next_activity_duration": { + "type": ["null", "string"], + "format": "date-time" + }, + "next_activity_note": { + "type": ["null", "string"] + }, + "formatted_value": { + "type": ["null", "string"] + }, + "weighted_value": { + "type": ["null", "integer"] + }, + "formatted_weighted_value": { + "type": ["null", "string"] + }, + "weighted_value_currency": { + "type": ["null", "string"] + }, + "rotten_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "owner_name": { + "type": ["null", "string"] + }, + "cc_email": { + "type": ["null", "string"] + }, + "org_hidden": { + "type": ["null", "boolean"] + }, + "person_hidden": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/leads.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/leads.json new file mode 100644 index 000000000000..10bde9a3bee1 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/leads.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "creator_id": { + "type": ["null", "integer"] + }, + "label_ids": { + "type": ["null", "array"] + }, + "person_id": { + "type": ["null", "integer"] + }, + "organization_id": { + "type": ["null", "integer"] + }, + "source_name": { + "type": ["null", "string"] + }, + "is_archived": { + "type": ["null", "boolean"] + }, + "was_seen": { + "type": ["null", "boolean"] + }, + "value": { + "type": ["null", "object"], + "properties": { + "amount": { + "type": ["null", "integer"] + }, + "currency": { + "type": ["null", "string"] + } + } + }, + "expected_close_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "next_activity_id": { + "type": ["null", "integer"] + }, + "add_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "update_time": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json new file mode 100644 index 000000000000..a0ec22180f92 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/persons.json @@ -0,0 +1,177 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "company_id": { + "type": ["null", "integer"] + }, + "owner_id": { + "type": ["null", "integer"] + }, + "org_id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "open_deals_count": { + "type": ["null", "integer"] + }, + "related_open_deals_count": { + "type": ["null", "integer"] + }, + "closed_deals_count": { + "type": ["null", "integer"] + }, + "related_closed_deals_count": { + "type": ["null", "integer"] + }, + "participant_open_deals_count": { + "type": ["null", "integer"] + }, + "participant_closed_deals_count": { + "type": ["null", "integer"] + }, + "email_messages_count": { + "type": ["null", "integer"] + }, + "activities_count": { + "type": ["null", "integer"] + }, + "done_activities_count": { + "type": ["null", "integer"] + }, + "undone_activities_count": { + "type": ["null", "integer"] + }, + "files_count": { + "type": ["null", "integer"] + }, + "notes_count": { + "type": ["null", "integer"] + }, + "followers_count": { + "type": ["null", "integer"] + }, + "won_deals_count": { + "type": ["null", "integer"] + }, + "related_won_deals_count": { + "type": ["null", "integer"] + }, + "lost_deals_count": { + "type": ["null", "integer"] + }, + "related_lost_deals_count": { + "type": ["null", "integer"] + }, + "active_flag": { + "type": ["null", "boolean"] + }, + "phone": { + "type": ["null", "array"] + }, + "email": { + "type": ["null", "array"] + }, + "first_char": { + "type": ["null", "string"] + }, + "update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "add_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "visible_to": { + "type": ["null", "string"] + }, + "picture_id": { + "type": ["null", "object"], + "properties": { + "item_type": { + "type": ["null", "string"] + }, + "item_id": { + "type": ["null", "integer"] + }, + "active_flag": { + "type": ["null", "boolean"] + }, + "add_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "update_time": { + "type": ["null", "string"] + }, + "added_by_user_id": { + "type": ["null", "integer"] + }, + "pictures": { + "type": ["null", "object"], + "properties": { + "128": { + "type": ["null", "string"] + }, + "512": { + "type": ["null", "string"] + } + } + }, + "value": { + "type": ["null", "integer"] + } + } + }, + "next_activity_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "next_activity_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "next_activity_id": { + "type": ["null", "integer"] + }, + "last_activity_id": { + "type": ["null", "integer"] + }, + "last_activity_date": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_incoming_mail_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "last_outgoing_mail_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "label": { + "type": ["null", "integer"] + }, + "org_name": { + "type": ["null", "string"] + }, + "owner_name": { + "type": ["null", "string"] + }, + "cc_email": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/pipelines.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/pipelines.json new file mode 100644 index 000000000000..01a4cfde2a92 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/pipelines.json @@ -0,0 +1,35 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "url_title": { + "type": ["null", "string"] + }, + "order_nr": { + "type": ["null", "integer"] + }, + "active": { + "type": ["null", "boolean"] + }, + "deal_probability": { + "type": ["null", "boolean"] + }, + "add_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "selected": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/stages.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/stages.json new file mode 100644 index 000000000000..95e4058dab65 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/stages.json @@ -0,0 +1,44 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "order_nr": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "active_flag": { + "type": ["null", "boolean"] + }, + "deal_probability": { + "type": ["null", "integer"] + }, + "pipeline_id": { + "type": ["null", "integer"] + }, + "rotten_flag": { + "type": ["null", "boolean"] + }, + "rotten_days": { + "type": ["null", "integer"] + }, + "add_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "update_time": { + "type": ["null", "string"], + "format": "date-time" + }, + "pipeline_name": { + "type": ["null", "string"] + }, + "pipeline_deal_probability": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json new file mode 100644 index 000000000000..d28d6bb7d446 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/schemas/users.json @@ -0,0 +1,69 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "default_currency": { + "type": ["null", "string"] + }, + "locale": { + "type": ["null", "string"] + }, + "lang": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "activated": { + "type": ["null", "boolean"] + }, + "last_login": { + "type": ["null", "string"], + "format": "date-time" + }, + "created": { + "type": ["null", "string"], + "format": "date-time" + }, + "modified": { + "type": ["null", "string"], + "format": "date-time" + }, + "signup_flow_variation": { + "type": ["null", "string"] + }, + "has_created_company": { + "type": ["null", "boolean"] + }, + "is_admin": { + "type": ["null", "integer"] + }, + "active_flag": { + "type": ["null", "boolean"] + }, + "timezone_name": { + "type": ["null", "string"] + }, + "timezone_offset": { + "type": ["null", "string"] + }, + "role_id": { + "type": ["null", "integer"] + }, + "icon_url": { + "type": ["null", "string"] + }, + "is_you": { + "type": ["null", "boolean"] + } + } +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py new file mode 100644 index 000000000000..5df0f1e1fa00 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/source.py @@ -0,0 +1,62 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +from typing import Any, List, Mapping, Tuple + +import pendulum +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from source_pipedrive.streams import Activities, ActivityFields, Deals, Leads, Persons, Pipelines, Stages, Users + + +class SourcePipedrive(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + try: + deals = Deals(api_token=config["api_token"], replication_start_date=pendulum.parse(config["replication_start_date"])) + deals_gen = deals.read_records(sync_mode=SyncMode.full_refresh) + next(deals_gen) + return True, None + except Exception as error: + return False, f"Unable to connect to Pipedrive API with the provided credentials - {repr(error)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + stream_kwargs = {"api_token": config["api_token"]} + incremental_stream_kwargs = {**stream_kwargs, "replication_start_date": pendulum.parse(config["replication_start_date"])} + streams = [ + Activities(**incremental_stream_kwargs), + ActivityFields(**stream_kwargs), + Deals(**incremental_stream_kwargs), + Leads(**stream_kwargs), + Persons(**incremental_stream_kwargs), + Pipelines(**incremental_stream_kwargs), + Stages(**incremental_stream_kwargs), + Users(**incremental_stream_kwargs), + ] + return streams diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json new file mode 100644 index 000000000000..2d3d47f4b302 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/spec.json @@ -0,0 +1,27 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/pipedrive", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Pipedrive Spec", + "type": "object", + "required": ["api_token", "replication_start_date"], + "additionalProperties": false, + "properties": { + "api_token": { + "title": "API Token", + "description": "Pipedrive API Token", + "airbyte_secret": true, + "type": "string" + }, + "replication_start_date": { + "title": "Replication Start Date", + "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated. When specified and not None, then stream will behave as incremental", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": ["2017-01-25T00:00:00Z"], + "type": "string" + } + } + }, + "supportsIncremental": true, + "supported_destination_sync_modes": ["append"] +} diff --git a/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py new file mode 100644 index 000000000000..5078ed9d2182 --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/source_pipedrive/streams.py @@ -0,0 +1,173 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union + +import pendulum +import requests +from airbyte_cdk.sources.streams.http import HttpStream + +PIPEDRIVE_URL_BASE = "https://api.pipedrive.com/v1/" + + +class PipedriveStream(HttpStream, ABC): + url_base = PIPEDRIVE_URL_BASE + primary_key = "id" + data_field = "data" + page_size = 50 + + def __init__(self, api_token: str, replication_start_date: pendulum.datetime = None, **kwargs): + super().__init__(**kwargs) + self._api_token = api_token + self._replication_start_date = replication_start_date + + @property + def cursor_field(self) -> Union[str, List[str]]: + if self._replication_start_date: + return "update_time" + return [] + + def path(self, **kwargs) -> str: + if self._replication_start_date: + return "recents" + + class_name = self.__class__.__name__ + return f"{class_name[0].lower()}{class_name[1:]}" + + @property + def path_param(self): + return self.name[:-1] + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query + the next page in the response. + If there are no more pages in the result, return None. + """ + pagination_data = response.json().get("additional_data", {}).get("pagination", {}) + if pagination_data.get("more_items_in_collection") and pagination_data.get("start") is not None: + start = pagination_data.get("start") + self.page_size + return {"start": start} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + next_page_token = next_page_token or {} + params = {"api_token": self._api_token, "limit": self.page_size, **next_page_token} + + replication_start_date = self._replication_start_date + if replication_start_date: + if stream_state.get(self.cursor_field): + replication_start_date = max(pendulum.parse(stream_state[self.cursor_field]), replication_start_date) + + params.update( + { + "items": self.path_param, + "since_timestamp": replication_start_date.strftime("%Y-%m-%d %H:%M:%S"), + } + ) + + return params + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + :return an iterable containing each record in the response + """ + records = response.json().get(self.data_field) or [] + for record in records: + if record.get(self.data_field): + yield record.get(self.data_field) + else: + yield record + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + latest_benchmark = latest_record[self.cursor_field] + if current_stream_state.get(self.cursor_field): + return {self.cursor_field: max(latest_benchmark, current_stream_state[self.cursor_field])} + return {self.cursor_field: latest_benchmark} + + +class Deals(PipedriveStream): + """ + API docs: https://developers.pipedrive.com/docs/api/v1/Deals#getDeals, + retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents + """ + + +class Leads(PipedriveStream): + """https://developers.pipedrive.com/docs/api/v1/Leads#getLeads""" + + +class Activities(PipedriveStream): + """ + API docs: https://developers.pipedrive.com/docs/api/v1/Activities#getActivities, + retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents + """ + + path_param = "activity" + + +class ActivityFields(PipedriveStream): + """https://developers.pipedrive.com/docs/api/v1/ActivityFields#getActivityFields""" + + +class Persons(PipedriveStream): + """ + API docs: https://developers.pipedrive.com/docs/api/v1/Persons#getPersons, + retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents + """ + + +class Pipelines(PipedriveStream): + """ + API docs: https://developers.pipedrive.com/docs/api/v1/Pipelines#getPipelines, + retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents + """ + + +class Stages(PipedriveStream): + """ + API docs: https://developers.pipedrive.com/docs/api/v1/Stages#getStages, + retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents + """ + + +class Users(PipedriveStream): + """ + API docs: https://developers.pipedrive.com/docs/api/v1/Users#getUsers, + retrieved by https://developers.pipedrive.com/docs/api/v1/Recents#getRecents + """ + + cursor_field = "modified" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + record_gen = super().parse_response(response=response, **kwargs) + for records in record_gen: + yield from records diff --git a/airbyte-integrations/connectors/source-pipedrive/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-pipedrive/unit_tests/unit_test.py new file mode 100644 index 000000000000..b8a8150b507f --- /dev/null +++ b/airbyte-integrations/connectors/source-pipedrive/unit_tests/unit_test.py @@ -0,0 +1,27 @@ +# +# MIT License +# +# Copyright (c) 2020 Airbyte +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + + +def test_example_method(): + assert True diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 76c7974c6c78..622aa707dcd2 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -76,6 +76,7 @@ * [Oracle DB](integrations/sources/oracle.md) * [Paypal Transaction](integrations/sources/paypal-transaction.md) * [Plaid](integrations/sources/plaid.md) + * [Pipedrive](integrations/sources/pipedrive.md) * [PokéAPI](integrations/sources/pokeapi.md) * [Postgres](integrations/sources/postgres.md) * [PostHog](integrations/sources/posthog.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 24f92ed1131c..be22e023b567 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -54,6 +54,7 @@ Airbyte uses a grading system for connectors to help users understand what to ex |[Okta](./sources/okta.md)| Beta | |[Oracle DB](./sources/oracle.md)| Certified | |[PayPal Transaction](./sources/paypal-transaction.md)| Beta | +|[Pipedrive](./sources/pipedrive.md)| Alpha | |[Plaid](./sources/plaid.md)| Alpha | |[PokéAPI](./sources/pokeapi.md)| Beta | |[Postgres](./sources/postgres.md)| Certified | diff --git a/docs/integrations/sources/pipedrive.md b/docs/integrations/sources/pipedrive.md new file mode 100644 index 000000000000..5494b39b470e --- /dev/null +++ b/docs/integrations/sources/pipedrive.md @@ -0,0 +1,80 @@ +# Pipedrive + +## Overview + +The Pipedrive connector can be used to sync your Pipedrive data. It supports full refresh sync for Deals, Leads, Activities, ActivityFields, +Persons, Pipelines, Stages, Users streams and incremental sync for Activities, Deals, Persons, Pipelines, Stages, Users streams. + +There was a priority to include at least a single stream of each stream type which is present on Pipedrive, so the list of the supported +streams is meant to be easily extendable. By the way, we can only support incremental stream support for the streams listed +[there](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents). + +### Output schema + +Several output streams are available from this source: + +* [Activities](https://developers.pipedrive.com/docs/api/v1/Activities#getActivities), + retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) (incremental) +* [ActivityFields](https://developers.pipedrive.com/docs/api/v1/ActivityFields#getActivityFields) +* [Deals](https://developers.pipedrive.com/docs/api/v1/Deals#getDeals), + retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) (incremental) +* [Leads](https://developers.pipedrive.com/docs/api/v1/Leads#getLeads) +* [Persons](https://developers.pipedrive.com/docs/api/v1/Persons#getPersons), + retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) (incremental) +* [Pipelines](https://developers.pipedrive.com/docs/api/v1/Pipelines#getPipelines), + retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) (incremental) +* [Stages](https://developers.pipedrive.com/docs/api/v1/Stages#getStages), + retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) (incremental) +* [Users](https://developers.pipedrive.com/docs/api/v1/Users#getUsers), + retrieved by [getRecents](https://developers.pipedrive.com/docs/api/v1/Recents#getRecents) (incremental) + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| Replicate Incremental Deletes | No | +| SSL connection | Yes | +| Namespaces | No | + +### Performance considerations + +The Pipedrive connector will gracefully handle rate limits. For more information, see [the Pipedrive docs for rate limitations](https://pipedrive.readme.io/docs/core-api-concepts-rate-limiting). + +## Getting started + +### Requirements + +* Pipedrive Account with wright to generate API Token + +### Setup guide + +This connector supports only authentication with API Token. To obtain API Token follow the instructions below: + +#### Enable API: +1. Click Manage users from the left-side menu. +1. Click on the Permission sets tab. +1. Choose the set where the user (who needs the API enabled) belongs to. +1. Lastly, click on "use API" on the right-hand side section (you need to scroll down a bit). + Now all users who belong in the set that has the API enabled can find their API token under + Settings > Personal Preferences > API in their Pipedrive web app. + +See [Enabling API for company users](https://pipedrive.readme.io/docs/enabling-api-for-company-users) for more info. + +#### How to find the API token: +1. Account name (on the top right) +1. Company settings +1. Personal preferences +1. API +1. Copy API Token + +See [How to find the API token](https://pipedrive.readme.io/docs/how-to-find-the-api-token) for more info. + + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :----- | :------ | +| 0.1.1 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Update spec.json | +| 0.1.0 | 2021-07-19 | [4686](https://github.com/airbytehq/airbyte/pull/4686) | Release Pipedrive connector! | \ No newline at end of file diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 4d76b90bbde7..b546a1eac7bb 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -75,6 +75,7 @@ write_standard_creds source-okta "$SOURCE_OKTA_TEST_CREDS" write_standard_creds source-plaid "$PLAID_INTEGRATION_TEST_CREDS" write_standard_creds source-paypal-transaction "$PAYPAL_TRANSACTION_CREDS" write_standard_creds source-posthog "$POSTHOG_TEST_CREDS" +write_standard_creds source-pipedrive "$PIPEDRIVE_INTEGRATION_TESTS_CREDS" write_standard_creds source-quickbooks-singer "$QUICKBOOKS_TEST_CREDS" write_standard_creds source-recharge "$RECHARGE_INTEGRATION_TEST_CREDS" write_standard_creds source-recurly "$SOURCE_RECURLY_INTEGRATION_TEST_CREDS"