diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 4080996ee356..b720c03db4ed 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1097,6 +1097,14 @@ icon: woocommerce.svg sourceType: api releaseStage: alpha +- name: Wrike + sourceDefinitionId: 9c13f986-a13b-4988-b808-4705badf71c2 + dockerRepository: airbyte/source-wrike + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/wrike + icon: wrike.svg + sourceType: api + releaseStage: alpha - name: Zendesk Chat sourceDefinitionId: 40d24d0f-b8f9-4fe0-9e6c-b06c0f3f45e4 dockerRepository: airbyte/source-zendesk-chat diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index fc970108ccb1..10f9a74a2544 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -11136,6 +11136,43 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-wrike:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Wrike Spec" + type: "object" + required: + - "access_token" + - "wrike_instance" + properties: + access_token: + type: "string" + title: "Permanent Access Token" + description: "Permanent access token. You can find documentation on how\ + \ to acquire a permanent access token here" + airbyte_secret: true + order: 0 + wrike_instance: + type: "string" + title: "Wrike Instance (hostname)" + description: "Wrike's instance such as `app-us2.wrike.com`" + default: "app-us2.wrike.com" + order: 1 + start_date: + type: "string" + title: "Start date for comments" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + description: "UTC date and time in the format 2017-01-25T00:00:00Z. Only\ + \ comments after this date will be replicated." + examples: + - "2017-01-25T00:00:00Z" + order: 2 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-zendesk-chat:0.1.9" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/zendesk-chat" diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 7bf5b9737db7..dc0027e825b2 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -112,6 +112,7 @@ | Twilio | [![source-twilio](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-twilio%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-twilio) | | Typeform | [![source-typeform](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-typeform%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-typeform) | | US Census | [![source-us-census](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-us-census%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-us-census) | +| Wrike | [![source-wrike](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-wrike%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-wrike) | | YouTube Analytics | [![source-youtube-analytics](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-youtube-analytics%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-youtube-analytics) | | Zendesk Chat | [![source-zendesk-chat](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-zendesk-chat%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-zendesk-chat) | | Zendesk Support | [![source-zendesk-support](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-zendesk-support%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-zendesk-support) | diff --git a/airbyte-integrations/connectors/source-wrike/.dockerignore b/airbyte-integrations/connectors/source-wrike/.dockerignore new file mode 100644 index 000000000000..642cbde41617 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_wrike +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-wrike/Dockerfile b/airbyte-integrations/connectors/source-wrike/Dockerfile new file mode 100644 index 000000000000..eae358751af0 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_wrike ./source_wrike + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-wrike diff --git a/airbyte-integrations/connectors/source-wrike/README.md b/airbyte-integrations/connectors/source-wrike/README.md new file mode 100644 index 000000000000..85464f5f1b6a --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/README.md @@ -0,0 +1,147 @@ +# Wrike Source + +This is the repository for the Wrike source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/wrike). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-wrike:build +``` + +### Create credentials + +#### Generating access token + +First get a Wrike account, you can get a trial here: Register for trial: https://www.wrike.com/free-trial/ + +To generate a token: + +1. Navigate to the ‘API apps’ section. +2. Select the required application from the list. +3. Click ‘Configure’. +4. Click ‘Obtain token’ in the “Permanent access token” section. +5. You will see a pop-up with a warning about using the permanent token, and after confirming, you will be able to copy and paste it into a secure storage (Wrike will not display it again). If your permanent token gets lost, you should generate a new one. + + +### Configuration files + +Then create a file `secrets/config.json` conforming to the `source_wrike/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source wrike test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-wrike:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-wrike:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-wrike:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wrike:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-wrike:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-wrike:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-wrike:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-wrike:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-wrike/acceptance-test-config.yml b/airbyte-integrations/connectors/source-wrike/acceptance-test-config.yml new file mode 100644 index 000000000000..2e89cfb7056c --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-wrike:dev +tests: + spec: + - spec_path: "source_wrike/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-wrike/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-wrike/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-wrike/bootstrap.md b/airbyte-integrations/connectors/source-wrike/bootstrap.md new file mode 100644 index 000000000000..1ba4e4fdcee2 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/bootstrap.md @@ -0,0 +1,28 @@ +# Wrike + +The connector uses the v4 API documented here: https://developers.wrike.com/overview/ . It is +straightforward HTTP REST API with Bearer token authentication. + +## Generating access token + +First get a Wrike account, you can get a trial here: Register for trial: https://www.wrike.com/free-trial/ + +To generate a token: + +1. Navigate to the ‘API apps’ section. +2. Select the required application from the list. +3. Click ‘Configure’. +4. Click ‘Obtain token’ in the “Permanent access token” section. +5. You will see a pop-up with a warning about using the permanent token, and after confirming, you will be able to copy and paste it into a secure storage (Wrike will not display it again). If your permanent token gets lost, you should generate a new one. + +Auth is done by TokenAuthenticator. + +## Implementation details + +I wrote a longer blog post on the implementation details: https://medium.com/starschema-blog/extending-airbyte-creating-a-source-connector-for-wrike-8e6c1337365a + +In a nutshell: + * We use only GET methods, all endpoints are straightforward. We emit what we receive as HTTP response. + * The `comments` endpoint is the only trick one: it allows only 7 days data to be retrieved once. Thus, the codes creates 7 days slices, starting from start replication date. + * It uses cursor based pagination. If you provide the next_page_token then no other parameters needed, the API will assume all parameters from the first page. + diff --git a/airbyte-integrations/connectors/source-wrike/build.gradle b/airbyte-integrations/connectors/source-wrike/build.gradle new file mode 100644 index 000000000000..5deab3c5278d --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_wrike' +} diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/__init__.py b/airbyte-integrations/connectors/source-wrike/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/catalog.json b/airbyte-integrations/connectors/source-wrike/integration_tests/catalog.json new file mode 100644 index 000000000000..0bbacc5360a5 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/catalog.json @@ -0,0 +1,320 @@ +{ + "streams": [ + [ + { + "name": "tasks", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "accountId": { + "type": "string" + }, + "title": { + "type": "string" + }, + "status": { + "type": "string" + }, + "importance": { + "type": ["null", "string"] + }, + "scope": { + "type": ["null", "string"] + }, + "briefDescription": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "customStatusId": { + "type": ["null", "string"] + }, + "permalink": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "responsibleIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "parentIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "superTaskIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "authorIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "customFields": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "id": { + "type": ["string"] + }, + "value": { + "type": ["string", "null"] + } + } + } + }, + "dates": { + "type": ["object", "null"], + "properties": { + "type": { + "type": ["string", "null"] + } + } + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + { + "name": "customfields", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "accountId": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "spaceId": { + "type": ["null", "string"] + }, + "sharedIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "settings": { + "type": ["object", "null"], + "properties": { + "inheritanceType": { + "type": ["string", "null"] + }, + "values": { + "type": ["array", "null"], + "items": { + "type": ["string", "integer", "number", "boolean", "null"] + } + }, + "decimalPlaces": { + "type": ["integer", "null"] + }, + "useThousandsSeparator": { + "type": ["boolean", "null"] + }, + "readOnly": { + "type": ["boolean", "null"] + }, + "allowTime": { + "type": ["boolean", "null"] + }, + "allowOtherValues": { + "type": ["boolean", "null"] + }, + "aggregation": { + "type": ["string", "null"] + }, + "currency": { + "type": ["string", "null"] + } + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + { + "name": "contacts", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "avatarUrl": { + "type": ["null", "string"] + }, + "timezone": { + "type": ["null", "string"] + }, + "locale": { + "type": ["null", "string"] + }, + "deleted": { + "type": ["null", "boolean"] + }, + "type": { + "type": ["null", "string"] + }, + "profiles": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "accountId": { + "type": ["string"] + }, + "role": { + "type": ["string"] + }, + "external": { + "type": ["boolean", "null"] + }, + "admin": { + "type": ["boolean", "null"] + }, + "owner": { + "type": ["boolean", "null"] + }, + "email": { + "type": ["string", "null"] + } + } + } + }, + "title": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + { + "name": "folders", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "scope": { + "type": ["null", "string"] + }, + "customStatusId": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "childIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "title": { + "type": ["null", "string"] + }, + "project": { + "type": ["object", "null"], + "properties": { + "authorId": { + "type": ["string", "null"] + }, + "customStatusId": { + "type": ["string", "null"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "ownerIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + } + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + { + "name": "comments", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "authorId": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "string"] + }, + "taskId": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + } + ] + ] +} diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-wrike/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..350d75ef56f1 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/configured_catalog.json @@ -0,0 +1,133 @@ +{ + "streams": [ + { + "stream": { + "name": "tasks", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "accountId": { + "type": "string" + }, + "title": { + "type": "string" + }, + "status": { + "type": "string" + }, + "importance": { + "type": ["null", "string"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "customfields", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "folders", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "accountId": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "contacts", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "comments", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "text": { + "type": ["null", "string"] + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-wrike/integration_tests/invalid_config.json new file mode 100644 index 000000000000..41219931b49a --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "access_token": "your access token here", + "wrike_instance": "app-us2.wrike.com", + "start_date": "2022-05-01T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-wrike/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-wrike/integration_tests/sample_config.json new file mode 100644 index 000000000000..1e6995012488 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/integration_tests/sample_config.json @@ -0,0 +1,5 @@ +{ + "access_token": "your_wrike_access_token", + "wrike_instance": "app-us2.wrike.com", + "start_date": "2021-01-01T00:00:00Z" +} diff --git a/airbyte-integrations/connectors/source-wrike/main.py b/airbyte-integrations/connectors/source-wrike/main.py new file mode 100644 index 000000000000..fe60920c8900 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_wrike import SourceWrike + +if __name__ == "__main__": + source = SourceWrike() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-wrike/requirements.txt b/airbyte-integrations/connectors/source-wrike/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-wrike/setup.py b/airbyte-integrations/connectors/source-wrike/setup.py new file mode 100644 index 000000000000..39c707d08a5a --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1.56", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_wrike", + description="Source implementation for Wrike.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/__init__.py b/airbyte-integrations/connectors/source-wrike/source_wrike/__init__.py new file mode 100644 index 000000000000..9cabc286cdb1 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceWrike + +__all__ = ["SourceWrike"] diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/comments.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/comments.json new file mode 100644 index 000000000000..1a5d17b5d3ae --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/comments.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "authorId": { + "type": "string" + }, + "text": { + "type": "string" + }, + "taskId": { + "type": "string" + }, + "createdDate": { + "type": "string", + "format": "date-time" + }, + "updatedDate": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/contacts.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/contacts.json new file mode 100644 index 000000000000..08709d869df3 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/contacts.json @@ -0,0 +1,59 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "firstName": { + "type": "string" + }, + "lastName": { + "type": "string" + }, + "avatarUrl": { + "type": "string" + }, + "timezone": { + "type": "string" + }, + "locale": { + "type": "string" + }, + "deleted": { + "type": "boolean" + }, + "type": { + "type": "string" + }, + "profiles": { + "type": "array", + "items": { + "type": ["object", "null"], + "properties": { + "accountId": { + "type": "string" + }, + "role": { + "type": "string" + }, + "external": { + "type": ["boolean", "null"] + }, + "admin": { + "type": ["boolean", "null"] + }, + "owner": { + "type": ["boolean", "null"] + }, + "email": { + "type": ["string", "null"] + } + } + } + }, + "title": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/customfields.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/customfields.json new file mode 100644 index 000000000000..e31b380b4d1e --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/customfields.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "accountId": { + "type": "string" + }, + "type": { + "type": "string" + }, + "title": { + "type": "string" + }, + "spaceId": { + "type": ["null", "string"] + }, + "sharedIds": { + "type": "array", + "items": { + "type": "string" + } + }, + "settings": { + "type": ["object", "null"], + "properties": { + "inheritanceType": { + "type": ["string", "null"] + }, + "values": { + "type": ["array", "null"], + "items": { + "type": ["string", "integer", "number", "boolean", "null"] + } + }, + "decimalPlaces": { + "type": ["integer", "null"] + }, + "useThousandsSeparator": { + "type": ["boolean", "null"] + }, + "readOnly": { + "type": ["boolean", "null"] + }, + "allowTime": { + "type": ["boolean", "null"] + }, + "allowOtherValues": { + "type": ["boolean", "null"] + }, + "aggregation": { + "type": ["string", "null"] + }, + "currency": { + "type": ["string", "null"] + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/folders.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/folders.json new file mode 100644 index 000000000000..77a093303914 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/folders.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "childIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "space": { + "type": "boolean" + }, + "title": { + "type": "string" + }, + "project": { + "type": ["object", "null"], + "properties": { + "authorId": { + "type": ["string", "null"] + }, + "customStatusId": { + "type": ["string", "null"] + }, + "createdDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "ownerIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/tasks.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/tasks.json new file mode 100644 index 000000000000..aaf086c8b645 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/tasks.json @@ -0,0 +1,93 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "accountId": { + "type": "string" + }, + "title": { + "type": "string" + }, + "status": { + "type": "string" + }, + "importance": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "briefDescription": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "string"] + }, + "customStatusId": { + "type": "string" + }, + "permalink": { + "type": "string" + }, + "priority": { + "type": "string" + }, + "createdDate": { + "type": "string", + "format": "date-time" + }, + "responsibleIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "parentIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "superTaskIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "authorIds": { + "type": ["array", "null"], + "items": { + "type": ["string", "null"] + } + }, + "customFields": { + "type": ["array", "null"], + "items": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "string" + }, + "value": { + "type": ["string", "null"] + } + } + } + }, + "dates": { + "type": "object", + "properties": { + "type": { + "type": "string" + } + } + }, + "updatedDate": { + "type": "string", + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/workflows.json b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/workflows.json new file mode 100644 index 000000000000..264a755be4ab --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/schemas/workflows.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "standard": { + "type": "string" + }, + "hidden": { + "type": "string" + }, + "customStatuses": { + "type": "array", + "items": { + "type": ["object", "null"], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "standardName": { + "type": "boolean" + }, + "color": { + "type": ["string", "null"] + }, + "standard": { + "type": "string" + }, + "group": { + "type": "string" + }, + "hidden": { + "type": "string" + } + } + } + }, + "title": { + "type": "string" + } + } +} diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/source.py b/airbyte-integrations/connectors/source-wrike/source_wrike/source.py new file mode 100644 index 000000000000..34325398aef6 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/source.py @@ -0,0 +1,151 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import pendulum +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from pendulum import DateTime + + +# Basic full refresh stream +class WrikeStream(HttpStream, ABC): + """ + Wrike API Reference: https://developers.wrike.com/overview/ + """ + + primary_key = "id" + url_base = "" + + def __init__(self, wrike_instance: str, **kwargs): + super().__init__(**kwargs) + self.url_base = f"https://{wrike_instance}/api/v4/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + nextPageToken = response.json().get("nextPageToken") + + if nextPageToken: + return {"nextPageToken": nextPageToken} + else: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + return next_page_token + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + + for record in response.json()["data"]: + yield record + + def path(self, **kwargs) -> str: + """ + This one is tricky, the API path is the class name by default. Airbyte will load `url_base`/`classname` by + default, like https://app-us2.wrike.com/api/v4/tasks if the class name is Tasks + """ + return self.__class__.__name__.lower() + + +class Tasks(WrikeStream): + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + return next_page_token or {"fields": "[customFields,parentIds,authorIds,responsibleIds,description,briefDescription,superTaskIds]"} + + +class Customfields(WrikeStream): + pass + + +class Contacts(WrikeStream): + pass + + +class Workflows(WrikeStream): + pass + + +def to_utc_z(date: DateTime): + return date.strftime("%Y-%m-%dT%H:%M:%SZ") + + +class Comments(WrikeStream): + def __init__(self, start_date: DateTime, **kwargs): + self._start_date = start_date + super().__init__(**kwargs) + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + Yields a list of the beginning timestamps of each 7 days period between the start date and now, + as the comments endpoint limits the requests for 7 days intervals. + """ + start_date = self._start_date + now = pendulum.now() + + while start_date <= now: + end_date = start_date + pendulum.duration(days=7) + yield {"start": to_utc_z(start_date)} + start_date = end_date + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + + slice_params = {"updatedDate": '{"start":"' + stream_slice["start"] + '"}'} + return next_page_token or slice_params + + +class Folders(WrikeStream): + pass + + +# Source + + +class SourceWrike(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + :param config: the user-input config object conforming to the connector's spec.yaml + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + + try: + headers = { + "Accept": "application/json", + } | TokenAuthenticator(token=config["access_token"]).get_auth_header() + + resp = requests.get(f"https://{config['wrike_instance']}/api/v4/version", headers=headers) + resp.raise_for_status() + return True, None + + except requests.exceptions.RequestException as e: + error = e.response.json() + message = error.get("errorDescription") or error.get("error") + return False, message + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + start_date = pendulum.parse(config.get("start_date")) if config.get("start_date") else pendulum.now().subtract(days=7) + + args = {"authenticator": TokenAuthenticator(token=config["access_token"]), "wrike_instance": config["wrike_instance"]} + return [ + Tasks(**args), + Customfields(**args), + Contacts(**args), + Workflows(**args), + Folders(**args), + Comments(start_date=start_date, **args), + ] diff --git a/airbyte-integrations/connectors/source-wrike/source_wrike/spec.yaml b/airbyte-integrations/connectors/source-wrike/source_wrike/spec.yaml new file mode 100644 index 000000000000..0a64cd00fba9 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/source_wrike/spec.yaml @@ -0,0 +1,33 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Wrike Spec + type: object + required: + - access_token + - wrike_instance + properties: + access_token: + type: string + title: Permanent Access Token + description: >- + Permanent access token. You can find documentation on how to acquire a permanent access token + here + airbyte_secret: true + order: 0 + wrike_instance: + type: string + title: Wrike Instance (hostname) + description: Wrike's instance such as `app-us2.wrike.com` + default: app-us2.wrike.com + order: 1 + start_date: + type: string + title: Start date for comments + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + description: >- + UTC date and time in the format 2017-01-25T00:00:00Z. Only comments + after this date will be replicated. + examples: + - "2017-01-25T00:00:00Z" + order: 2 diff --git a/airbyte-integrations/connectors/source-wrike/unit_tests/__init__.py b/airbyte-integrations/connectors/source-wrike/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-wrike/unit_tests/test_source.py b/airbyte-integrations/connectors/source-wrike/unit_tests/test_source.py new file mode 100644 index 000000000000..053e95945451 --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/unit_tests/test_source.py @@ -0,0 +1,39 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pendulum +from pytest import fixture +from source_wrike.source import SourceWrike + + +@fixture() +def config(request): + args = {"access_token": "foo", "wrike_instance": "app-us2.wrike.com"} + return args + + +def test_check_connection(mocker, config): + source = SourceWrike() + logger_mock = MagicMock() + (connection_status, _) = source.check_connection(logger_mock, config) + expected_status = False + assert connection_status == expected_status + + +def test_streams_without_date(mocker, config): + source = SourceWrike() + streams = source.streams(config) + expected_streams_number = 6 + assert len(streams) == expected_streams_number + assert streams[-1]._start_date is not None + + +def test_streams_with_date(mocker, config): + source = SourceWrike() + streams = source.streams(config | {"start_date": "2022-05-01T00:00:00Z"}) + expected_streams_number = 6 + assert len(streams) == expected_streams_number + assert streams[-1]._start_date == pendulum.parse("2022-05-01T00:00:00Z") diff --git a/airbyte-integrations/connectors/source-wrike/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-wrike/unit_tests/test_streams.py new file mode 100644 index 000000000000..1b67363ed86d --- /dev/null +++ b/airbyte-integrations/connectors/source-wrike/unit_tests/test_streams.py @@ -0,0 +1,122 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pendulum +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from pytest import fixture, mark +from source_wrike.source import Comments, Tasks, WrikeStream, to_utc_z + + +@fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(WrikeStream, "__abstractmethods__", set()) + + +@fixture() +def args(request): + args = {"wrike_instance": "app-us2.wrike.com", "authenticator": TokenAuthenticator(token="tokkk")} + return args + + +def test_request_params(args): + stream = WrikeStream(**args) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = None + assert stream.request_params(**inputs) == expected_params + + +def test_tasks_request_params(args): + stream = Tasks(**args) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + assert stream.request_params(**inputs).get("fields") + + +def test_comments_slices(args): + stream = Comments(start_date=pendulum.parse("2022-05-01"), **args) + inputs = {"stream_state": None} + slices = list(stream.stream_slices(**inputs)) + assert slices[0].get("start") == "2022-05-01T00:00:00Z" + assert len(slices) > 1 + + +def test_next_page_token(args): + stream = WrikeStream(**args) + + response = MagicMock() + # first page + response.json.return_value = { + "kind": "tasks", + "nextPageToken": "ADE7SXYAAAAAUAAAAAAQAAAAMAAAAAABVB5K4QPE7SXKM", + "responseSize": 96, + "data": [{"id": "IEAFHZ6ZKQ233LQK"}], + } + inputs = {"response": response} + expected_token = {"nextPageToken": "ADE7SXYAAAAAUAAAAAAQAAAAMAAAAAABVB5K4QPE7SXKM"} + assert stream.next_page_token(**inputs) == expected_token + # next page + response.json.return_value = { + "kind": "tasks", + "responseSize": 96, + "data": [{"id": "IEAFHZ6ZKQ233LQK"}], + } + inputs = {"response": response} + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(args): + stream = WrikeStream(**args) + response = MagicMock() + response.json.return_value = { + "kind": "tasks", + "responseSize": 96, + "data": [{"id": "IEAFHZ6ZKQ233LQK"}], + } + inputs = {"response": response} + expected_parsed_object = {"id": "IEAFHZ6ZKQ233LQK"} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(args): + stream = WrikeStream(**args) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(args): + stream = WrikeStream(**args) + expected_method = "GET" + assert stream.http_method == expected_method + + +@mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(args, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = WrikeStream(**args) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(args): + response_mock = MagicMock() + stream = WrikeStream(**args) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time + + +def test_to_utc_z(): + assert to_utc_z(pendulum.parse("2022-05-01")) == "2022-05-01T00:00:00Z" diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 03bb2d5ee62d..1444b0a1e274 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -166,6 +166,7 @@ For more information about the grading system, see [Product Release Stages](http | [Webflow](sources/webflow.md ) | Alpha | Yes | | [WooCommerce](sources/woocommerce.md) | Alpha | No | | [Wordpress](sources/wordpress.md) | Alpha | No | +| [Wrike](sources/wrike.md) | Alpha | No | | [YouTube Analytics](sources/youtube-analytics.md) | Beta | Yes | | [Zencart](sources/zencart.md) | Alpha | No | | [Zendesk Chat](sources/zendesk-chat.md) | Beta | Yes | diff --git a/docs/integrations/sources/wrike.md b/docs/integrations/sources/wrike.md new file mode 100644 index 000000000000..5674fbf38be2 --- /dev/null +++ b/docs/integrations/sources/wrike.md @@ -0,0 +1,50 @@ +# Wrike + +This page guides you through the process of setting up the Wrike source connector. + +## Prerequisites + +* Your [Wrike `Permanent Access Token`](https://help.wrike.com/hc/en-us/community/posts/211849065-Get-Started-with-Wrike-s-API) + +## Set up the Wrike source connector + +1. Log into your [Airbyte Cloud](https://cloud.airbyte.io/workspaces) or Airbyte OSS account. +2. Click **Sources** and then click **+ New source**. +3. On the Set up the source page, select **Wrike** from the Source type dropdown. +4. Enter a name for your source. +5. For **Permanent Access Token**, enter your [Wrike `Permanent Access Token`](https://help.wrike.com/hc/en-us/community/posts/211849065-Get-Started-with-Wrike-s-API). + + Permissions granted to the permanent token are equal to the permissions of the user who generates the token. + +6. For **Wrike Instance (hostname)**, add the hostname of the Wrike instance you are currently using. This could be `www.wrike.com`, `app-us2.wrike.com`, or anything similar. +7. For **Start date for comments**, enter the date in YYYY-MM-DDTHH:mm:ssZ format. The comments added on and after this date will be replicated. If this field is blank, Airbyte will replicate comments from the last seven days. +8. Click **Set up source**. + +## Supported sync modes + +The Wrike source connector supports on full sync refresh. + +## Supported Streams + +The Wrike source connector supports the following streams: + +* [Tasks](https://developers.wrike.com/api/v4/tasks/)\(Full Refresh\) +* [Customfields](https://developers.wrike.com/api/v4/customfields/)\(Full Refresh\) +* [Comments](https://developers.wrike.com/api/v4/comments/)\(Full Refresh\) +* [Contacts](https://developers.wrike.com/api/v4/contacts/)\(Full Refresh\) +* [Folders](https://developers.wrike.com/api/v4/folders/)\(Full Refresh\) + +### Data type mapping + +Currencies are number and the date is a string. + +### Performance considerations + +The Wrike connector should not run into Wrike API limitations under normal usage. [Create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.0 | 2022-08-16 | [15638](https://github.com/airbytehq/airbyte/pull/15638) | Initial version/release of the connector. +