From 9cb315e79169c86a6a60178badc1c9e5970e168c Mon Sep 17 00:00:00 2001 From: lgomezm Date: Mon, 25 Oct 2021 14:45:29 -0500 Subject: [PATCH 01/10] Generated connector source code --- .../connectors/source-outreach/.dockerignore | 7 + .../connectors/source-outreach/Dockerfile | 38 ++++ .../connectors/source-outreach/README.md | 132 +++++++++++ .../acceptance-test-config.yml | 30 +++ .../source-outreach/acceptance-test-docker.sh | 16 ++ .../connectors/source-outreach/build.gradle | 14 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/catalog.json | 39 ++++ .../integration_tests/configured_catalog.json | 22 ++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-outreach/main.py | 13 ++ .../source-outreach/requirements.txt | 2 + .../connectors/source-outreach/setup.py | 29 +++ .../source_outreach/__init__.py | 8 + .../source_outreach/schemas/TODO.md | 25 +++ .../source_outreach/schemas/customers.json | 16 ++ .../source_outreach/schemas/employees.json | 19 ++ .../source-outreach/source_outreach/source.py | 206 ++++++++++++++++++ .../source-outreach/source_outreach/spec.json | 16 ++ .../source-outreach/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 +++++ .../source-outreach/unit_tests/test_source.py | 22 ++ .../unit_tests/test_streams.py | 83 +++++++ 27 files changed, 834 insertions(+) create mode 100644 airbyte-integrations/connectors/source-outreach/.dockerignore create mode 100644 airbyte-integrations/connectors/source-outreach/Dockerfile create mode 100644 airbyte-integrations/connectors/source-outreach/README.md create mode 100644 airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-outreach/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-outreach/build.gradle create mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-outreach/main.py create mode 100644 airbyte-integrations/connectors/source-outreach/requirements.txt create mode 100644 airbyte-integrations/connectors/source-outreach/setup.py create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/__init__.py create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/source.py create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/spec.json create mode 100644 airbyte-integrations/connectors/source-outreach/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-outreach/.dockerignore b/airbyte-integrations/connectors/source-outreach/.dockerignore new file mode 100644 index 000000000000..6f589badb0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_outreach +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-outreach/Dockerfile b/airbyte-integrations/connectors/source-outreach/Dockerfile new file mode 100644 index 000000000000..b167af246ed3 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_outreach ./source_outreach + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-outreach diff --git a/airbyte-integrations/connectors/source-outreach/README.md b/airbyte-integrations/connectors/source-outreach/README.md new file mode 100644 index 000000000000..6ff5ace7e644 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/README.md @@ -0,0 +1,132 @@ +# Outreach Source + +This is the repository for the Outreach source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/outreach). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-outreach:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/outreach) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_outreach/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source outreach test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-outreach:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-outreach:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-outreach:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-outreach:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-outreach:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-outreach:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-outreach:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-outreach:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml b/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml new file mode 100644 index 000000000000..a6bf61f7b040 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-outreach:dev +tests: + spec: + - spec_path: "source_outreach/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-outreach/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-outreach/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-outreach/build.gradle b/airbyte-integrations/connectors/source-outreach/build.gradle new file mode 100644 index 000000000000..a8307dd19b20 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_outreach' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/__init__.py b/airbyte-integrations/connectors/source-outreach/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py new file mode 100644 index 000000000000..58c194c5d137 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/catalog.json b/airbyte-integrations/connectors/source-outreach/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..36f0468db0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-outreach/main.py b/airbyte-integrations/connectors/source-outreach/main.py new file mode 100644 index 000000000000..7f2e3d552302 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_outreach import SourceOutreach + +if __name__ == "__main__": + source = SourceOutreach() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-outreach/requirements.txt b/airbyte-integrations/connectors/source-outreach/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-outreach/setup.py b/airbyte-integrations/connectors/source-outreach/setup.py new file mode 100644 index 000000000000..0f163f7eace3 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_outreach", + description="Source implementation for Outreach.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/__init__.py b/airbyte-integrations/connectors/source-outreach/source_outreach/__init__.py new file mode 100644 index 000000000000..87e486950f0e --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceOutreach + +__all__ = ["SourceOutreach"] diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/TODO.md b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/customers.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/customers.json new file mode 100644 index 000000000000..9a4b13485836 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/customers.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/employees.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/employees.json new file mode 100644 index 000000000000..2fa01a0fa1ff --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/employees.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py new file mode 100644 index 000000000000..c94d10417e47 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -0,0 +1,206 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.json file. +""" + + +# Basic full refresh stream +class OutreachStream(HttpStream, ABC): + """ + TODO remove this comment + + This class represents a stream output by the connector. + This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, + parsing responses etc.. + + Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. + + Typically for REST APIs each stream corresponds to a resource in the API. For example if the API + contains the endpoints + - GET v1/customers + - GET v1/employees + + then you should have three classes: + `class OutreachStream(HttpStream, ABC)` which is the current class + `class Customers(OutreachStream)` contains behavior to pull data for customers using v1/customers + `class Employees(OutreachStream)` contains behavior to pull data for employees using v1/employees + + If some streams implement incremental sync, it is typical to create another class + `class IncrementalOutreachStream((OutreachStream), ABC)` then have concrete stream implementations extend it. An example + is provided below. + + See the reference docs for the full list of configurable options. + """ + + # TODO: Fill in the url base. Required. + url_base = "https://example-api.com/v1/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + yield {} + + +class Customers(OutreachStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "customer_id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this + should return "customers". Required. + """ + return "customers" + + +# Basic incremental stream +class IncrementalOutreachStream(OutreachStream, ABC): + """ + TODO fill in details of this class to implement functionality related to incremental syncs for your connector. + if you do not need to implement incremental sync for any streams, remove this class. + """ + + # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + + :return str: The name of the cursor field. + """ + return [] + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} + + +class Employees(IncrementalOutreachStream): + """ + TODO: Change class name to match the table/data source this stream corresponds to. + """ + + # TODO: Fill in the cursor_field. Required. + cursor_field = "start_date" + + # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. + primary_key = "employee_id" + + def path(self, **kwargs) -> str: + """ + TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should + return "single". Required. + """ + return "employees" + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + """ + TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. + + Slices control when state is saved. Specifically, state is saved after a slice has been fully read. + This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" + section of the docs for more information. + + The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the + necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. + This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + + An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help + craft that specific request. + + For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement + this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date + till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into + the date query param. + """ + raise NotImplementedError("Implement stream slices or delete this method!") + + +# Source +class SourceOutreach(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + """ + TODO: Replace the streams below with your own streams. + + :param config: A Mapping of the user input configuration as defined in the connector spec. + """ + # TODO remove the authenticator if not required. + auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support + return [Customers(authenticator=auth), Employees(authenticator=auth)] diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json b/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json new file mode 100644 index 000000000000..10561bb1fda4 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json @@ -0,0 +1,16 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Outreach Spec", + "type": "object", + "required": ["TODO"], + "additionalProperties": false, + "properties": { + "TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.": { + "type": "string", + "description": "describe me" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/__init__.py b/airbyte-integrations/connectors/source-outreach/unit_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..e07e10478d24 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_outreach.source import IncrementalOutreachStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalOutreachStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalOutreachStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalOutreachStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalOutreachStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalOutreachStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalOutreachStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalOutreachStream, "cursor_field", "dummy_field") + stream = IncrementalOutreachStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalOutreachStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalOutreachStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py new file mode 100644 index 000000000000..a5cf801463dc --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_outreach.source import SourceOutreach + + +def test_check_connection(mocker): + source = SourceOutreach() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceOutreach() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py new file mode 100644 index 000000000000..4fbb6bc0001b --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_outreach.source import OutreachStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(OutreachStream, "path", "v0/example_endpoint") + mocker.patch.object(OutreachStream, "primary_key", "test_primary_key") + mocker.patch.object(OutreachStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = OutreachStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = OutreachStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = OutreachStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = OutreachStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = OutreachStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = OutreachStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = OutreachStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time From 96dd33c9a6bc9fc5d864921d0d76fe52df52f0a2 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Mon, 25 Oct 2021 17:05:34 -0500 Subject: [PATCH 02/10] Added streams: prospect, sequence, sequenceState --- .../source-outreach/source_outreach/source.py | 236 ++++++------------ 1 file changed, 81 insertions(+), 155 deletions(-) diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index c94d10417e47..644e34237819 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -5,202 +5,128 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from urllib import parse import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator - -""" -TODO: Most comments in this class are instructive and should be deleted after the source is implemented. - -This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an -incremental syncs from an HTTP API. - -The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental -stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. - -The approach here is not authoritative, and devs are free to use their own judgement. - -There are additional required TODOs in the files within the integration_tests folder and the spec.json file. -""" +from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator +from airbyte_cdk.sources.streams.http.auth.oauth import Oauth2Authenticator # Basic full refresh stream class OutreachStream(HttpStream, ABC): - """ - TODO remove this comment - - This class represents a stream output by the connector. - This is an abstract base class meant to contain all the common functionality at the API level e.g: the API base URL, pagination strategy, - parsing responses etc.. - - Each stream should extend this class (or another abstract subclass of it) to specify behavior unique to that stream. - - Typically for REST APIs each stream corresponds to a resource in the API. For example if the API - contains the endpoints - - GET v1/customers - - GET v1/employees - - then you should have three classes: - `class OutreachStream(HttpStream, ABC)` which is the current class - `class Customers(OutreachStream)` contains behavior to pull data for customers using v1/customers - `class Employees(OutreachStream)` contains behavior to pull data for employees using v1/employees - If some streams implement incremental sync, it is typical to create another class - `class IncrementalOutreachStream((OutreachStream), ABC)` then have concrete stream implementations extend it. An example - is provided below. - - See the reference docs for the full list of configurable options. - """ - - # TODO: Fill in the url base. Required. - url_base = "https://example-api.com/v1/" + url_base = "https://api.outreach.io/api/v2" def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. - - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ - return None + try: + params = parse.parse_qs(parse.urlparse(response.authentication_url).query) + if not params or 'page[after]' not in params: + return {} + return {"after": params['page[after]'][0]} + except Exception as e: + raise KeyError(f"error parsing next_page token: {e}") def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {} + params = {"page[size]": 1, "count": False} + if next_page_token and "after" in next_page_token: + params["page[after]"] = next_page_token["after"] + return params def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - """ - TODO: Override this method to define how a response is parsed. - :return an iterable containing each record in the response - """ - yield {} - - -class Customers(OutreachStream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ - - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "customer_id" - - def path( - self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this - should return "customers". Required. - """ - return "customers" + data = response.json().get("data") + if not data: + return + for element in data: + yield element # Basic incremental stream class IncrementalOutreachStream(OutreachStream, ABC): - """ - TODO fill in details of this class to implement functionality related to incremental syncs for your connector. - if you do not need to implement incremental sync for any streams, remove this class. - """ - - # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. - state_checkpoint_interval = None - @property def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] + return "updatedAt" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} + current_stream_state = current_stream_state or {} + current_stream_state_date = current_stream_state.get(self.cursor_field, self.start_date) + latest_record_date = latest_record.get('attributes', {}).get(self.cursor_field, self.start_date) -class Employees(IncrementalOutreachStream): - """ - TODO: Change class name to match the table/data source this stream corresponds to. - """ + return {self.cursor_field: max(current_stream_state_date, latest_record_date)} - # TODO: Fill in the cursor_field. Required. - cursor_field = "start_date" + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) + if self.cursor_field in stream_state: + params["filter[updatedAt]"] = stream_state[self.cursor_field] + '..inf' + return params - # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. - primary_key = "employee_id" + +class Prospects(IncrementalOutreachStream): + primary_key = "id" + cursor_field = "updatedAt" def path(self, **kwargs) -> str: - """ - TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should - return "single". Required. - """ - return "employees" + return "prospects" - def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: - """ - TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. - Slices control when state is saved. Specifically, state is saved after a slice has been fully read. - This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" - section of the docs for more information. +class Sequences(IncrementalOutreachStream): + primary_key = "id" + cursor_field = "updatedAt" - The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the - necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. - This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. + def path(self, **kwargs) -> str: + return "sequences" - An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help - craft that specific request. - For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement - this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date - till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into - the date query param. - """ - raise NotImplementedError("Implement stream slices or delete this method!") +class SequenceStates(IncrementalOutreachStream): + primary_key = "id" + cursor_field = "updatedAt" + + def path(self, **kwargs) -> str: + return "sequenceStates" + + +class OutreachAuthenticator(Oauth2Authenticator): + def __init__(self, redirect_uri: str, *args, **kwargs): + super(Oauth2Authenticator, self).__init__(*args, **kwargs) + self.redirect_uri = redirect_uri + + def get_refresh_request_body(self) -> Mapping[str, Any]: + payload = super().get_refresh_request_body() + payload['redirect_uri'] = self.redirect_uri + return payload # Source class SourceOutreach(AbstractSource): - def check_connection(self, logger, config) -> Tuple[bool, any]: - """ - TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + def _create_authenticator(self, config): + return OutreachAuthenticator( + redirect_uri=config["redirect_uri"], + token_refresh_endpoint="https://api.outreach.io/oauth/token", + client_id=config["client_id"], + client_secret=config["client_secret"], + refresh_token=config["refresh_token"], + ) - See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 - for an example. - - :param config: the user-input config object conforming to the connector's spec.json - :param logger: logger object - :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. - """ - return True, None + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + access_token, _ = self._create_authenticator(config).refresh_access_token() + response = requests.get("https://api.outreach.io/api/v2", headers={"Authorization": f"Bearer {access_token}"}) + response.raise_for_status() + return True, None + except Exception as e: + return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: - """ - TODO: Replace the streams below with your own streams. - - :param config: A Mapping of the user input configuration as defined in the connector spec. - """ - # TODO remove the authenticator if not required. - auth = TokenAuthenticator(token="api_key") # Oauth2Authenticator is also available if you need oauth support - return [Customers(authenticator=auth), Employees(authenticator=auth)] + auth = self._create_authenticator(config) + return [ + Prospects(authenticator=auth, **config), + Sequences(authenticator=auth, **config), + SequenceStates(authenticator=auth, **config), + ] From 7084c7c22d8a9fa3e0239dc91e60ed848404f474 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Tue, 26 Oct 2021 14:29:58 -0500 Subject: [PATCH 03/10] Updated spec and schemas --- .../source_outreach/schemas/TODO.md | 25 - .../source_outreach/schemas/customers.json | 16 - .../source_outreach/schemas/employees.json | 19 - .../source_outreach/schemas/prospects.json | 1445 +++++++++++++++++ .../schemas/sequence_states.json | 154 ++ .../source_outreach/schemas/sequences.json | 280 ++++ .../source-outreach/source_outreach/source.py | 24 +- .../source-outreach/source_outreach/spec.json | 31 +- 8 files changed, 1927 insertions(+), 67 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/TODO.md delete mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/customers.json delete mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequence_states.json create mode 100644 airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/TODO.md b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c..000000000000 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/customers.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/customers.json deleted file mode 100644 index 9a4b13485836..000000000000 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/customers.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "signup_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/employees.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/employees.json deleted file mode 100644 index 2fa01a0fa1ff..000000000000 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/employees.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "years_of_service": { - "type": ["null", "integer"] - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json new file mode 100644 index 000000000000..00009c921fa6 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json @@ -0,0 +1,1445 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "attributes": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "addedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "addressCity": { + "type": [ + "null", + "string" + ] + }, + "addressCountry": { + "type": [ + "null", + "string" + ] + }, + "addressState": { + "type": [ + "null", + "string" + ] + }, + "addressStreet": { + "type": [ + "null", + "string" + ] + }, + "addressStreet2": { + "type": [ + "null", + "string" + ] + }, + "addressZip": { + "type": [ + "null", + "string" + ] + }, + "angelListUrl": { + "type": [ + "null", + "string" + ] + }, + "availableAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "callOptedOut": { + "type": [ + "null", + "boolean" + ] + }, + "callsOptStatus": { + "type": [ + "null", + "string" + ] + }, + "callsOptedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "campaignName": { + "type": [ + "null", + "string" + ] + }, + "clickCount": { + "type": [ + "null", + "integer" + ] + }, + "company": { + "type": [ + "null", + "string" + ] + }, + "contactHistogram": { + "type": [ + "null", + "array" + ], + "items": { + "type": "array", + "items": { + "type": "integer" + } + } + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "custom1": { + "type": [ + "null", + "string" + ] + }, + "custom10": { + "type": [ + "null", + "string" + ] + }, + "custom100": { + "type": [ + "null", + "string" + ] + }, + "custom101": { + "type": [ + "null", + "string" + ] + }, + "custom102": { + "type": [ + "null", + "string" + ] + }, + "custom103": { + "type": [ + "null", + "string" + ] + }, + "custom104": { + "type": [ + "null", + "string" + ] + }, + "custom105": { + "type": [ + "null", + "string" + ] + }, + "custom106": { + "type": [ + "null", + "string" + ] + }, + "custom107": { + "type": [ + "null", + "string" + ] + }, + "custom108": { + "type": [ + "null", + "string" + ] + }, + "custom109": { + "type": [ + "null", + "string" + ] + }, + "custom11": { + "type": [ + "null", + "string" + ] + }, + "custom110": { + "type": [ + "null", + "string" + ] + }, + "custom111": { + "type": [ + "null", + "string" + ] + }, + "custom112": { + "type": [ + "null", + "string" + ] + }, + "custom113": { + "type": [ + "null", + "string" + ] + }, + "custom114": { + "type": [ + "null", + "string" + ] + }, + "custom115": { + "type": [ + "null", + "string" + ] + }, + "custom116": { + "type": [ + "null", + "string" + ] + }, + "custom117": { + "type": [ + "null", + "string" + ] + }, + "custom118": { + "type": [ + "null", + "string" + ] + }, + "custom119": { + "type": [ + "null", + "string" + ] + }, + "custom12": { + "type": [ + "null", + "string" + ] + }, + "custom120": { + "type": [ + "null", + "string" + ] + }, + "custom121": { + "type": [ + "null", + "string" + ] + }, + "custom122": { + "type": [ + "null", + "string" + ] + }, + "custom123": { + "type": [ + "null", + "string" + ] + }, + "custom124": { + "type": [ + "null", + "string" + ] + }, + "custom125": { + "type": [ + "null", + "string" + ] + }, + "custom126": { + "type": [ + "null", + "string" + ] + }, + "custom127": { + "type": [ + "null", + "string" + ] + }, + "custom128": { + "type": [ + "null", + "string" + ] + }, + "custom129": { + "type": [ + "null", + "string" + ] + }, + "custom13": { + "type": [ + "null", + "string" + ] + }, + "custom130": { + "type": [ + "null", + "string" + ] + }, + "custom131": { + "type": [ + "null", + "string" + ] + }, + "custom132": { + "type": [ + "null", + "string" + ] + }, + "custom133": { + "type": [ + "null", + "string" + ] + }, + "custom134": { + "type": [ + "null", + "string" + ] + }, + "custom135": { + "type": [ + "null", + "string" + ] + }, + "custom136": { + "type": [ + "null", + "string" + ] + }, + "custom137": { + "type": [ + "null", + "string" + ] + }, + "custom138": { + "type": [ + "null", + "string" + ] + }, + "custom139": { + "type": [ + "null", + "string" + ] + }, + "custom14": { + "type": [ + "null", + "string" + ] + }, + "custom140": { + "type": [ + "null", + "string" + ] + }, + "custom141": { + "type": [ + "null", + "string" + ] + }, + "custom142": { + "type": [ + "null", + "string" + ] + }, + "custom143": { + "type": [ + "null", + "string" + ] + }, + "custom144": { + "type": [ + "null", + "string" + ] + }, + "custom145": { + "type": [ + "null", + "string" + ] + }, + "custom146": { + "type": [ + "null", + "string" + ] + }, + "custom147": { + "type": [ + "null", + "string" + ] + }, + "custom148": { + "type": [ + "null", + "string" + ] + }, + "custom149": { + "type": [ + "null", + "string" + ] + }, + "custom15": { + "type": [ + "null", + "string" + ] + }, + "custom150": { + "type": [ + "null", + "string" + ] + }, + "custom16": { + "type": [ + "null", + "string" + ] + }, + "custom17": { + "type": [ + "null", + "string" + ] + }, + "custom18": { + "type": [ + "null", + "string" + ] + }, + "custom19": { + "type": [ + "null", + "string" + ] + }, + "custom2": { + "type": [ + "null", + "string" + ] + }, + "custom20": { + "type": [ + "null", + "string" + ] + }, + "custom21": { + "type": [ + "null", + "string" + ] + }, + "custom22": { + "type": [ + "null", + "string" + ] + }, + "custom23": { + "type": [ + "null", + "string" + ] + }, + "custom24": { + "type": [ + "null", + "string" + ] + }, + "custom25": { + "type": [ + "null", + "string" + ] + }, + "custom26": { + "type": [ + "null", + "string" + ] + }, + "custom27": { + "type": [ + "null", + "string" + ] + }, + "custom28": { + "type": [ + "null", + "string" + ] + }, + "custom29": { + "type": [ + "null", + "string" + ] + }, + "custom3": { + "type": [ + "null", + "string" + ] + }, + "custom30": { + "type": [ + "null", + "string" + ] + }, + "custom31": { + "type": [ + "null", + "string" + ] + }, + "custom32": { + "type": [ + "null", + "string" + ] + }, + "custom33": { + "type": [ + "null", + "string" + ] + }, + "custom34": { + "type": [ + "null", + "string" + ] + }, + "custom35": { + "type": [ + "null", + "string" + ] + }, + "custom36": { + "type": [ + "null", + "string" + ] + }, + "custom37": { + "type": [ + "null", + "string" + ] + }, + "custom38": { + "type": [ + "null", + "string" + ] + }, + "custom39": { + "type": [ + "null", + "string" + ] + }, + "custom4": { + "type": [ + "null", + "string" + ] + }, + "custom40": { + "type": [ + "null", + "string" + ] + }, + "custom41": { + "type": [ + "null", + "string" + ] + }, + "custom42": { + "type": [ + "null", + "string" + ] + }, + "custom43": { + "type": [ + "null", + "string" + ] + }, + "custom44": { + "type": [ + "null", + "string" + ] + }, + "custom45": { + "type": [ + "null", + "string" + ] + }, + "custom46": { + "type": [ + "null", + "string" + ] + }, + "custom47": { + "type": [ + "null", + "string" + ] + }, + "custom48": { + "type": [ + "null", + "string" + ] + }, + "custom49": { + "type": [ + "null", + "string" + ] + }, + "custom50": { + "type": [ + "null", + "string" + ] + }, + "custom51": { + "type": [ + "null", + "string" + ] + }, + "custom52": { + "type": [ + "null", + "string" + ] + }, + "custom53": { + "type": [ + "null", + "string" + ] + }, + "custom54": { + "type": [ + "null", + "string" + ] + }, + "custom55": { + "type": [ + "null", + "string" + ] + }, + "custom56": { + "type": [ + "null", + "string" + ] + }, + "custom57": { + "type": [ + "null", + "string" + ] + }, + "custom58": { + "type": [ + "null", + "string" + ] + }, + "custom59": { + "type": [ + "null", + "string" + ] + }, + "custom6": { + "type": [ + "null", + "string" + ] + }, + "custom60": { + "type": [ + "null", + "string" + ] + }, + "custom61": { + "type": [ + "null", + "string" + ] + }, + "custom62": { + "type": [ + "null", + "string" + ] + }, + "custom63": { + "type": [ + "null", + "string" + ] + }, + "custom64": { + "type": [ + "null", + "string" + ] + }, + "custom65": { + "type": [ + "null", + "string" + ] + }, + "custom66": { + "type": [ + "null", + "string" + ] + }, + "custom67": { + "type": [ + "null", + "string" + ] + }, + "custom68": { + "type": [ + "null", + "string" + ] + }, + "custom69": { + "type": [ + "null", + "string" + ] + }, + "custom7": { + "type": [ + "null", + "string" + ] + }, + "custom70": { + "type": [ + "null", + "string" + ] + }, + "custom71": { + "type": [ + "null", + "string" + ] + }, + "custom72": { + "type": [ + "null", + "string" + ] + }, + "custom73": { + "type": [ + "null", + "string" + ] + }, + "custom74": { + "type": [ + "null", + "string" + ] + }, + "custom75": { + "type": [ + "null", + "string" + ] + }, + "custom76": { + "type": [ + "null", + "string" + ] + }, + "custom77": { + "type": [ + "null", + "string" + ] + }, + "custom78": { + "type": [ + "null", + "string" + ] + }, + "custom79": { + "type": [ + "null", + "string" + ] + }, + "custom8": { + "type": [ + "null", + "string" + ] + }, + "custom80": { + "type": [ + "null", + "string" + ] + }, + "custom81": { + "type": [ + "null", + "string" + ] + }, + "custom82": { + "type": [ + "null", + "string" + ] + }, + "custom83": { + "type": [ + "null", + "string" + ] + }, + "custom84": { + "type": [ + "null", + "string" + ] + }, + "custom85": { + "type": [ + "null", + "string" + ] + }, + "custom86": { + "type": [ + "null", + "string" + ] + }, + "custom87": { + "type": [ + "null", + "string" + ] + }, + "custom88": { + "type": [ + "null", + "string" + ] + }, + "custom89": { + "type": [ + "null", + "string" + ] + }, + "custom9": { + "type": [ + "null", + "string" + ] + }, + "custom90": { + "type": [ + "null", + "string" + ] + }, + "custom91": { + "type": [ + "null", + "string" + ] + }, + "custom92": { + "type": [ + "null", + "string" + ] + }, + "custom93": { + "type": [ + "null", + "string" + ] + }, + "custom94": { + "type": [ + "null", + "string" + ] + }, + "custom95": { + "type": [ + "null", + "string" + ] + }, + "custom96": { + "type": [ + "null", + "string" + ] + }, + "custom97": { + "type": [ + "null", + "string" + ] + }, + "custom98": { + "type": [ + "null", + "string" + ] + }, + "custom99": { + "type": [ + "null", + "string" + ] + }, + "dateOfBirth": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "degree": { + "type": [ + "null", + "string" + ] + }, + "emailOptedOut": { + "type": [ + "null", + "boolean" + ] + }, + "emails": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "emailsOptStatus": { + "type": [ + "null", + "string" + ] + }, + "emailsOptedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "engagedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "engagedScore": { + "type": [ + "null", + "string" + ] + }, + "eventName": { + "type": [ + "null", + "string" + ] + }, + "externalId": { + "type": [ + "null", + "string" + ] + }, + "externalOwner": { + "type": [ + "null", + "string" + ] + }, + "externalSource": { + "type": [ + "null", + "string" + ] + }, + "facebookUrl": { + "type": [ + "null", + "string" + ] + }, + "firstName": { + "type": [ + "null", + "string" + ] + }, + "gender": { + "type": [ + "null", + "string" + ] + }, + "githubUrl": { + "type": [ + "null", + "string" + ] + }, + "githubUsername": { + "type": [ + "null", + "string" + ] + }, + "googlePlusUrl": { + "type": [ + "null", + "string" + ] + }, + "graduationDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "homePhones": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "jobStartDate": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "lastName": { + "type": [ + "null", + "string" + ] + }, + "linkedInConnections": { + "type": [ + "null", + "integer" + ] + }, + "linkedInId": { + "type": [ + "null", + "string" + ] + }, + "linkedInSlug": { + "type": [ + "null", + "string" + ] + }, + "linkedInUrl": { + "type": [ + "null", + "string" + ] + }, + "middleName": { + "type": [ + "null", + "string" + ] + }, + "mobilePhones": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "nickname": { + "type": [ + "null", + "string" + ] + }, + "occupation": { + "type": [ + "null", + "string" + ] + }, + "openCount": { + "type": [ + "null", + "integer" + ] + }, + "optedOut": { + "type": [ + "null", + "boolean" + ] + }, + "optedOutAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "otherPhones": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "personalNote1": { + "type": [ + "null", + "string" + ] + }, + "personalNote2": { + "type": [ + "null", + "string" + ] + }, + "preferredContact": { + "type": [ + "null", + "string" + ] + }, + "quoraUrl": { + "type": [ + "null", + "string" + ] + }, + "region": { + "type": [ + "null", + "string" + ] + }, + "replyCount": { + "type": [ + "null", + "integer" + ] + }, + "school": { + "type": [ + "null", + "string" + ] + }, + "score": { + "type": [ + "null", + "number" + ] + }, + "sharingTeamId": { + "type": [ + "null", + "string" + ] + }, + "source": { + "type": [ + "null", + "string" + ] + }, + "specialties": { + "type": [ + "null", + "string" + ] + }, + "stackOverflowId": { + "type": [ + "null", + "string" + ] + }, + "stackOverflowUrl": { + "type": [ + "null", + "string" + ] + }, + "tags": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "timeZone": { + "type": [ + "null", + "string" + ] + }, + "timeZoneIana": { + "type": [ + "null", + "string" + ] + }, + "timeZoneInferred": { + "type": [ + "null", + "string" + ] + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "touchedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "trashedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "twitterUrl": { + "type": [ + "null", + "string" + ] + }, + "twitterUsername": { + "type": [ + "null", + "string" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "voipPhones": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + }, + "websiteUrl1": { + "type": [ + "null", + "string" + ] + }, + "websiteUrl2": { + "type": [ + "null", + "string" + ] + }, + "websiteUrl3": { + "type": [ + "null", + "string" + ] + }, + "workPhones": { + "type": [ + "null", + "array" + ], + "items": { + "type": "string" + } + } + } + } + }, + "additionalProperties": true + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequence_states.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequence_states.json new file mode 100644 index 000000000000..e4e89eb7c82c --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequence_states.json @@ -0,0 +1,154 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "attributes": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "activeAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "bounceCount": { + "type": [ + "null", + "integer" + ] + }, + "callCompletedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "clickCount": { + "type": [ + "null", + "integer" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "deliverCount": { + "type": [ + "null", + "integer" + ] + }, + "errorReason": { + "type": [ + "null", + "string" + ] + }, + "failureCount": { + "type": [ + "null", + "integer" + ] + }, + "negativeReplyCount": { + "type": [ + "null", + "integer" + ] + }, + "neutralReplyCount": { + "type": [ + "null", + "integer" + ] + }, + "openCount": { + "type": [ + "null", + "integer" + ] + }, + "optOutCount": { + "type": [ + "null", + "integer" + ] + }, + "pauseReason": { + "type": [ + "null", + "string" + ] + }, + "positiveReplyCount": { + "type": [ + "null", + "integer" + ] + }, + "repliedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "replyCount": { + "type": [ + "null", + "integer" + ] + }, + "scheduleCount": { + "type": [ + "null", + "integer" + ] + }, + "state": { + "type": [ + "null", + "string" + ] + }, + "stateChangedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + } + }, + "additionalProperties": true + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json new file mode 100644 index 000000000000..c02f01c9fed3 --- /dev/null +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json @@ -0,0 +1,280 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": [ + "null", + "string" + ] + }, + "id": { + "type": [ + "null", + "integer" + ] + }, + "attributes": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "automationPercentage": { + "type": [ + "null", + "number" + ] + }, + "bounceCount": { + "type": [ + "null", + "integer" + ] + }, + "clickCount": { + "type": [ + "null", + "integer" + ] + }, + "createdAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "deliverCount": { + "type": [ + "null", + "integer" + ] + }, + "description": { + "type": [ + "null", + "string" + ] + }, + "durationInDays": { + "type": [ + "null", + "integer" + ] + }, + "enabled": { + "type": [ + "null", + "boolean" + ] + }, + "enabledAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "failureCount": { + "type": [ + "null", + "integer" + ] + }, + "finishOnReply": { + "type": [ + "null", + "boolean" + ] + }, + "lastUsedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "locked": { + "type": [ + "null", + "boolean" + ] + }, + "lockedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "maxActivations": { + "type": [ + "null", + "integer" + ] + }, + "name": { + "type": [ + "null", + "string" + ] + }, + "negativeReplyCount": { + "type": [ + "null", + "integer" + ] + }, + "neutralReplyCount": { + "type": [ + "null", + "integer" + ] + }, + "numContactedProspects": { + "type": [ + "null", + "integer" + ] + }, + "numRepliedProspects": { + "type": [ + "null", + "integer" + ] + }, + "openCount": { + "type": [ + "null", + "integer" + ] + }, + "optOutCount": { + "type": [ + "null", + "integer" + ] + }, + "positiveReplyCount": { + "type": [ + "null", + "integer" + ] + }, + "primaryReplyAction": { + "type": [ + "null", + "string" + ] + }, + "primaryReplyPauseDuration": { + "type": [ + "null", + "integer" + ] + }, + "replyCount": { + "type": [ + "null", + "integer" + ] + }, + "scheduleCount": { + "type": [ + "null", + "integer" + ] + }, + "scheduleIntervalType": { + "type": [ + "null", + "string" + ] + }, + "secondaryReplyAction": { + "type": [ + "null", + "string" + ] + }, + "secondaryReplyPauseDuration": { + "type": [ + "null", + "integer" + ] + }, + "sequenceStepCount": { + "type": [ + "null", + "integer" + ] + }, + "sequenceType": { + "type": [ + "null", + "string" + ] + }, + "shareType": { + "type": [ + "null", + "string" + ] + }, + "tags": { + "type": [ + "null", + "array" + ], + "items": { + "type": [ + "null", + "string" + ] + } + }, + "throttleCapacity": { + "type": [ + "null", + "integer" + ] + }, + "throttleMaxAddsPerDay": { + "type": [ + "null", + "integer" + ] + }, + "throttlePaused": { + "type": [ + "null", + "boolean" + ] + }, + "throttlePausedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + }, + "transactional": { + "type": [ + "null", + "boolean" + ] + }, + "updatedAt": { + "type": [ + "null", + "string" + ], + "format": "date-time" + } + } + } + }, + "additionalProperties": true + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index 644e34237819..5d5c8bca4c3d 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -19,6 +19,15 @@ class OutreachStream(HttpStream, ABC): url_base = "https://api.outreach.io/api/v2" + + def __init__( + self, + authenticator: HttpAuthenticator, + start_date: str = None, + **kwargs, + ): + self.start_date = start_date + super().__init__(authenticator=authenticator) def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: try: @@ -93,8 +102,17 @@ def path(self, **kwargs) -> str: class OutreachAuthenticator(Oauth2Authenticator): - def __init__(self, redirect_uri: str, *args, **kwargs): - super(Oauth2Authenticator, self).__init__(*args, **kwargs) + def __init__(self, redirect_uri: str, + token_refresh_endpoint: str, + client_id: str, + client_secret: str, + refresh_token: str): + super().__init__( + token_refresh_endpoint=token_refresh_endpoint, + client_id=client_id, + client_secret=client_secret, + refresh_token=refresh_token + ) self.redirect_uri = redirect_uri def get_refresh_request_body(self) -> Mapping[str, Any]: @@ -118,9 +136,11 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: try: access_token, _ = self._create_authenticator(config).refresh_access_token() response = requests.get("https://api.outreach.io/api/v2", headers={"Authorization": f"Bearer {access_token}"}) + print(f'New access token: {access_token}') response.raise_for_status() return True, None except Exception as e: + logger.error(f'Failed to check connection. Error: {e}') return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json b/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json index 10561bb1fda4..702ded909148 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json @@ -1,15 +1,36 @@ { - "documentationUrl": "https://docsurl.com", + "documentationUrl": "https://docs.airbyte.io/integrations/sources/outreach", "connectionSpecification": { "$schema": "http://json-schema.org/draft-07/schema#", - "title": "Outreach Spec", + "title": "Source Outreach Spec", "type": "object", - "required": ["TODO"], + "required": ["client_id", "client_secret", "refresh_token", "redirect_uri", "start_date"], "additionalProperties": false, "properties": { - "TODO: This schema defines the configuration required for the source. This usually involves metadata such as database and/or authentication information.": { + "client_id": { "type": "string", - "description": "describe me" + "description": "Outreach client id." + }, + "client_secret": { + "type": "string", + "description": "Outreach client secret.", + "airbyte_secret": true + }, + "refresh_token": { + "type": "string", + "description": "Outreach refresh token.", + "airbyte_secret": true + }, + "redirect_uri": { + "type": "string", + "description": "Outreach redirect URI.", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "description": "The date from which you'd like to replicate data for Outreach API, in the format YYYY-MM-DDT00:00:00Z. All data generated after this date will be replicated.", + "examples": ["2020-11-16T00:00:00Z"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" } } } From 41f437bf0a57c1c4c522b85c90494b3d318e53b4 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Tue, 26 Oct 2021 15:44:10 -0500 Subject: [PATCH 04/10] Removed print statement --- .../connectors/source-outreach/source_outreach/source.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index 5d5c8bca4c3d..29f91ac348c3 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -18,7 +18,7 @@ # Basic full refresh stream class OutreachStream(HttpStream, ABC): - url_base = "https://api.outreach.io/api/v2" + url_base = "https://api.outreach.io/api/v2/" def __init__( self, @@ -31,7 +31,8 @@ def __init__( def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: try: - params = parse.parse_qs(parse.urlparse(response.authentication_url).query) + next_page_url = response.json().get('links').get('next') + params = parse.parse_qs(parse.urlparse(next_page_url).query) if not params or 'page[after]' not in params: return {} return {"after": params['page[after]'][0]} @@ -41,7 +42,7 @@ def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - params = {"page[size]": 1, "count": False} + params = {"page[size]": 100, "count": "false"} if next_page_token and "after" in next_page_token: params["page[after]"] = next_page_token["after"] return params @@ -136,7 +137,6 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: try: access_token, _ = self._create_authenticator(config).refresh_access_token() response = requests.get("https://api.outreach.io/api/v2", headers={"Authorization": f"Bearer {access_token}"}) - print(f'New access token: {access_token}') response.raise_for_status() return True, None except Exception as e: From 866b90d9918608affde4389385f96d59ecc8b166 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Wed, 27 Oct 2021 13:14:49 -0500 Subject: [PATCH 05/10] Added unit tests --- .../unit_tests/test_incremental_streams.py | 25 ++++------ .../source-outreach/unit_tests/test_source.py | 9 +--- .../unit_tests/test_streams.py | 47 ++++++++++--------- 3 files changed, 37 insertions(+), 44 deletions(-) diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py index e07e10478d24..0df0f950a6a6 100644 --- a/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py @@ -3,6 +3,7 @@ # +from unittest.mock import MagicMock from airbyte_cdk.models import SyncMode from pytest import fixture from source_outreach.source import IncrementalOutreachStream @@ -17,43 +18,37 @@ def patch_incremental_base_class(mocker): def test_cursor_field(patch_incremental_base_class): - stream = IncrementalOutreachStream() - # TODO: replace this with your expected cursor field - expected_cursor_field = [] + stream = IncrementalOutreachStream(authenticator=MagicMock()) + expected_cursor_field = 'updatedAt' assert stream.cursor_field == expected_cursor_field def test_get_updated_state(patch_incremental_base_class): - stream = IncrementalOutreachStream() - # TODO: replace this with your input parameters - inputs = {"current_stream_state": None, "latest_record": None} - # TODO: replace this with your expected updated stream state - expected_state = {} + stream = IncrementalOutreachStream(authenticator=MagicMock(), start_date="2021-10-27T00:00:00.000Z") + inputs = {"current_stream_state": {}, "latest_record": {}} + expected_state = {"updatedAt": "2021-10-27T00:00:00.000Z"} assert stream.get_updated_state(**inputs) == expected_state def test_stream_slices(patch_incremental_base_class): - stream = IncrementalOutreachStream() - # TODO: replace this with your input parameters + stream = IncrementalOutreachStream(authenticator=MagicMock()) inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - # TODO: replace this with your expected stream slices list expected_stream_slice = [None] assert stream.stream_slices(**inputs) == expected_stream_slice def test_supports_incremental(patch_incremental_base_class, mocker): mocker.patch.object(IncrementalOutreachStream, "cursor_field", "dummy_field") - stream = IncrementalOutreachStream() + stream = IncrementalOutreachStream(authenticator=MagicMock()) assert stream.supports_incremental def test_source_defined_cursor(patch_incremental_base_class): - stream = IncrementalOutreachStream() + stream = IncrementalOutreachStream(authenticator=MagicMock()) assert stream.source_defined_cursor def test_stream_checkpoint_interval(patch_incremental_base_class): - stream = IncrementalOutreachStream() - # TODO: replace this with your expected checkpoint interval + stream = IncrementalOutreachStream(authenticator=MagicMock()) expected_checkpoint_interval = None assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py index a5cf801463dc..1d143758d96a 100644 --- a/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_source.py @@ -7,16 +7,9 @@ from source_outreach.source import SourceOutreach -def test_check_connection(mocker): - source = SourceOutreach() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) - - def test_streams(mocker): source = SourceOutreach() config_mock = MagicMock() streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 + expected_streams_number = 3 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py index 4fbb6bc0001b..0a96ed9f0988 100644 --- a/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py @@ -18,44 +18,49 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): - stream = OutreachStream() - # TODO: replace this with your input parameters + stream = OutreachStream(authenticator=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters - expected_params = {} + expected_params = {'count': 'false', 'page[size]': 100} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): - stream = OutreachStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token - expected_token = None + stream = OutreachStream(authenticator=MagicMock()) + response = MagicMock() + response.json.return_value = {"links": {"next": "http://api.outreach.io/api/v2/prospects?page[after]=100"}} + inputs = {"response": response} + expected_token = {'after': '100'} assert stream.next_page_token(**inputs) == expected_token def test_parse_response(patch_base_class): - stream = OutreachStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} + stream = OutreachStream(authenticator=MagicMock()) + response = MagicMock() + response.json.return_value = { + "data": [{ + "id": 123, + "attributes": {"name": "John Doe"}, + "relationships": {"account": {"data": {"type": "account", "id": 4}}} + }] + } + inputs = {"response": response} + expected_parsed_object = { + "id": 123, + "attributes": {"name": "John Doe"}, + "relationships": {"account": {"data": {"type": "account", "id": 4}}} + } assert next(stream.parse_response(**inputs)) == expected_parsed_object def test_request_headers(patch_base_class): - stream = OutreachStream() - # TODO: replace this with your input parameters + stream = OutreachStream(authenticator=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers expected_headers = {} assert stream.request_headers(**inputs) == expected_headers def test_http_method(patch_base_class): - stream = OutreachStream() - # TODO: replace this with your expected http request method + stream = OutreachStream(authenticator=MagicMock()) expected_method = "GET" assert stream.http_method == expected_method @@ -72,12 +77,12 @@ def test_http_method(patch_base_class): def test_should_retry(patch_base_class, http_status, should_retry): response_mock = MagicMock() response_mock.status_code = http_status - stream = OutreachStream() + stream = OutreachStream(authenticator=MagicMock()) assert stream.should_retry(response_mock) == should_retry def test_backoff_time(patch_base_class): response_mock = MagicMock() - stream = OutreachStream() + stream = OutreachStream(authenticator=MagicMock()) expected_backoff_time = None assert stream.backoff_time(response_mock) == expected_backoff_time From 02e7c241b8f64944bd92e7ad3b70ec365bd01700 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Wed, 27 Oct 2021 15:52:57 -0500 Subject: [PATCH 06/10] Updated acceptance test files --- .../acceptance-test-config.yml | 8 +--- .../integration_tests/abnormal_state.json | 6 +-- .../integration_tests/acceptance.py | 2 - .../integration_tests/catalog.json | 39 ------------------- .../integration_tests/configured_catalog.json | 35 +++++++++++++---- .../integration_tests/invalid_config.json | 8 +++- .../integration_tests/sample_config.json | 6 ++- .../integration_tests/sample_state.json | 6 +-- .../source_outreach/schemas/prospects.json | 2 +- .../source-outreach/source_outreach/source.py | 18 ++++----- 10 files changed, 56 insertions(+), 74 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-outreach/integration_tests/catalog.json diff --git a/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml b/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml index a6bf61f7b040..e5a2567c35e2 100644 --- a/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-outreach/acceptance-test-config.yml @@ -15,13 +15,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json index 52b0f2c2118f..357973ceb1f4 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/abnormal_state.json @@ -1,5 +1,5 @@ { - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" - } + "prospects": { "updatedAt": "2040-11-16T00:00:00Z" }, + "sequences": { "updatedAt": "2040-11-16T00:00:00Z" }, + "sequence_states": { "updatedAt": "2040-11-16T00:00:00Z" } } diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py index 58c194c5d137..108075487440 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/acceptance.py @@ -11,6 +11,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """ This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/catalog.json b/airbyte-integrations/connectors/source-outreach/integration_tests/catalog.json deleted file mode 100644 index 6799946a6851..000000000000 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/catalog.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "streams": [ - { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json index 36f0468db0d8..fbc4d4b7fa1f 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/configured_catalog.json @@ -2,21 +2,42 @@ "streams": [ { "stream": { - "name": "customers", + "name": "prospects", "json_schema": {}, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["attributes/updatedAt"], + "source_defined_primary_key": [["id"]] }, - "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["attributes", "updatedAt"] + }, + { + "stream": { + "name": "sequences", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["attributes/updatedAt"], + "source_defined_primary_key": [["id"]] + }, + "sync_mode": "incremental", + "destination_sync_mode": "overwrite", + "cursor_field": ["attributes", "updatedAt"] }, { "stream": { - "name": "employees", + "name": "sequence_states", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["attributes/updatedAt"], + "source_defined_primary_key": [["id"]] }, "sync_mode": "incremental", - "destination_sync_mode": "append" + "destination_sync_mode": "overwrite", + "cursor_field": ["attributes", "updatedAt"] } ] } diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json index f3732995784f..58b879372702 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json @@ -1,3 +1,7 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" -} + "client_id": "thisisclientid", + "client_secret": "thisisclientsecret", + "refresh_token": "thisisnotavalidrefreshtoken", + "redirect_uri": "http://localhost:8000/callback", + "start_date": "2020-11-16T00:00:00Z" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_config.json index ecc4913b84c7..6b6f29ea9bf8 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_config.json @@ -1,3 +1,7 @@ { - "fix-me": "TODO" + "client_id": "jreFpNMZ8D_Y7O0Q0f1KmlREgKNwGDnJN9jtR0Wknj0", + "client_secret": "t0OgRWYq_FwjUaqxSJpURUrBFz9to15baSYWyyfCvE8", + "refresh_token": "tgFJoqbpkvzMtu47HbSgmmwyVAYzdAqGJMpzMSdA9Wg", + "redirect_uri": "https://api.staging.calixa.io/integrations/oauth/outreach/callback", + "start_date": "2020-11-16T00:00:00Z" } diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json index 3587e579822d..2d2bb507028e 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/sample_state.json @@ -1,5 +1,5 @@ { - "todo-stream-name": { - "todo-field-name": "value" - } + "prospects": { "updatedAt": "2020-11-16T00:00:00Z" }, + "sequences": { "updatedAt": "2021-02-16T00:00:00Z" }, + "sequenceStates": { "updatedAt": "2021-04-16T00:00:00Z" } } diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json index 00009c921fa6..e6674d932ebc 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json @@ -1078,7 +1078,7 @@ "engagedScore": { "type": [ "null", - "string" + "number" ] }, "eventName": { diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index 29f91ac348c3..959e282cb0c5 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -59,28 +59,28 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class IncrementalOutreachStream(OutreachStream, ABC): @property def cursor_field(self) -> str: - return "updatedAt" + return "attributes/properties/updatedAt" def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: current_stream_state = current_stream_state or {} - current_stream_state_date = current_stream_state.get(self.cursor_field, self.start_date) - latest_record_date = latest_record.get('attributes', {}).get(self.cursor_field, self.start_date) + current_stream_state_date = current_stream_state.get('updatedAt', self.start_date) + latest_record_date = latest_record.get('attributes', {}).get('updatedAt', self.start_date) - return {self.cursor_field: max(current_stream_state_date, latest_record_date)} + return {'updatedAt': max(current_stream_state_date, latest_record_date)} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - if self.cursor_field in stream_state: - params["filter[updatedAt]"] = stream_state[self.cursor_field] + '..inf' + if 'updatedAt' in stream_state: + params["filter[updatedAt]"] = stream_state['updatedAt'] + '..inf' return params class Prospects(IncrementalOutreachStream): primary_key = "id" - cursor_field = "updatedAt" + cursor_field = "attributes/properties/updatedAt" def path(self, **kwargs) -> str: return "prospects" @@ -88,7 +88,7 @@ def path(self, **kwargs) -> str: class Sequences(IncrementalOutreachStream): primary_key = "id" - cursor_field = "updatedAt" + cursor_field = "attributes/properties/updatedAt" def path(self, **kwargs) -> str: return "sequences" @@ -96,7 +96,7 @@ def path(self, **kwargs) -> str: class SequenceStates(IncrementalOutreachStream): primary_key = "id" - cursor_field = "updatedAt" + cursor_field = "attributes/properties/updatedAt" def path(self, **kwargs) -> str: return "sequenceStates" From 8dcfe86f34da7468306c4645fb4cdd99b97b8e7c Mon Sep 17 00:00:00 2001 From: lgomezm Date: Sun, 31 Oct 2021 21:29:24 -0500 Subject: [PATCH 07/10] Updated spec --- .../connectors/source-outreach/source_outreach/spec.json | 3 +-- .../source-outreach/unit_tests/test_incremental_streams.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json b/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json index 702ded909148..667226afad93 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json @@ -23,8 +23,7 @@ }, "redirect_uri": { "type": "string", - "description": "Outreach redirect URI.", - "airbyte_secret": true + "description": "Outreach redirect URI." }, "start_date": { "type": "string", diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py index 0df0f950a6a6..84acbe97907a 100644 --- a/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py @@ -19,7 +19,7 @@ def patch_incremental_base_class(mocker): def test_cursor_field(patch_incremental_base_class): stream = IncrementalOutreachStream(authenticator=MagicMock()) - expected_cursor_field = 'updatedAt' + expected_cursor_field = 'attributes/properties/updatedAt' assert stream.cursor_field == expected_cursor_field From 9a1b9c69d58b787bb40adca039ef72e8f6051305 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Tue, 2 Nov 2021 11:53:12 -0500 Subject: [PATCH 08/10] Ran gradlew format --- .../integration_tests/invalid_config.json | 2 +- .../source_outreach/schemas/prospects.json | 2179 ++++++----------- .../schemas/sequence_states.json | 233 +- .../source_outreach/schemas/sequences.json | 420 ++-- .../source-outreach/source_outreach/source.py | 36 +- .../source-outreach/source_outreach/spec.json | 8 +- .../unit_tests/test_incremental_streams.py | 3 +- .../unit_tests/test_streams.py | 12 +- 8 files changed, 993 insertions(+), 1900 deletions(-) diff --git a/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json index 58b879372702..ccc4a8190ab3 100644 --- a/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-outreach/integration_tests/invalid_config.json @@ -4,4 +4,4 @@ "refresh_token": "thisisnotavalidrefreshtoken", "redirect_uri": "http://localhost:8000/callback", "start_date": "2020-11-16T00:00:00Z" -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json index e6674d932ebc..5835bb70c988 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/prospects.json @@ -1,1445 +1,746 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "type": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "attributes": { - "type": [ - "null", - "object" - ], - "additionalProperties": true, - "properties": { - "addedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "addressCity": { - "type": [ - "null", - "string" - ] - }, - "addressCountry": { - "type": [ - "null", - "string" - ] - }, - "addressState": { - "type": [ - "null", - "string" - ] - }, - "addressStreet": { - "type": [ - "null", - "string" - ] - }, - "addressStreet2": { - "type": [ - "null", - "string" - ] - }, - "addressZip": { - "type": [ - "null", - "string" - ] - }, - "angelListUrl": { - "type": [ - "null", - "string" - ] - }, - "availableAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "callOptedOut": { - "type": [ - "null", - "boolean" - ] - }, - "callsOptStatus": { - "type": [ - "null", - "string" - ] - }, - "callsOptedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "campaignName": { - "type": [ - "null", - "string" - ] - }, - "clickCount": { - "type": [ - "null", - "integer" - ] - }, - "company": { - "type": [ - "null", - "string" - ] - }, - "contactHistogram": { - "type": [ - "null", - "array" - ], - "items": { - "type": "array", - "items": { - "type": "integer" - } - } - }, - "createdAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "custom1": { - "type": [ - "null", - "string" - ] - }, - "custom10": { - "type": [ - "null", - "string" - ] - }, - "custom100": { - "type": [ - "null", - "string" - ] - }, - "custom101": { - "type": [ - "null", - "string" - ] - }, - "custom102": { - "type": [ - "null", - "string" - ] - }, - "custom103": { - "type": [ - "null", - "string" - ] - }, - "custom104": { - "type": [ - "null", - "string" - ] - }, - "custom105": { - "type": [ - "null", - "string" - ] - }, - "custom106": { - "type": [ - "null", - "string" - ] - }, - "custom107": { - "type": [ - "null", - "string" - ] - }, - "custom108": { - "type": [ - "null", - "string" - ] - }, - "custom109": { - "type": [ - "null", - "string" - ] - }, - "custom11": { - "type": [ - "null", - "string" - ] - }, - "custom110": { - "type": [ - "null", - "string" - ] - }, - "custom111": { - "type": [ - "null", - "string" - ] - }, - "custom112": { - "type": [ - "null", - "string" - ] - }, - "custom113": { - "type": [ - "null", - "string" - ] - }, - "custom114": { - "type": [ - "null", - "string" - ] - }, - "custom115": { - "type": [ - "null", - "string" - ] - }, - "custom116": { - "type": [ - "null", - "string" - ] - }, - "custom117": { - "type": [ - "null", - "string" - ] - }, - "custom118": { - "type": [ - "null", - "string" - ] - }, - "custom119": { - "type": [ - "null", - "string" - ] - }, - "custom12": { - "type": [ - "null", - "string" - ] - }, - "custom120": { - "type": [ - "null", - "string" - ] - }, - "custom121": { - "type": [ - "null", - "string" - ] - }, - "custom122": { - "type": [ - "null", - "string" - ] - }, - "custom123": { - "type": [ - "null", - "string" - ] - }, - "custom124": { - "type": [ - "null", - "string" - ] - }, - "custom125": { - "type": [ - "null", - "string" - ] - }, - "custom126": { - "type": [ - "null", - "string" - ] - }, - "custom127": { - "type": [ - "null", - "string" - ] - }, - "custom128": { - "type": [ - "null", - "string" - ] - }, - "custom129": { - "type": [ - "null", - "string" - ] - }, - "custom13": { - "type": [ - "null", - "string" - ] - }, - "custom130": { - "type": [ - "null", - "string" - ] - }, - "custom131": { - "type": [ - "null", - "string" - ] - }, - "custom132": { - "type": [ - "null", - "string" - ] - }, - "custom133": { - "type": [ - "null", - "string" - ] - }, - "custom134": { - "type": [ - "null", - "string" - ] - }, - "custom135": { - "type": [ - "null", - "string" - ] - }, - "custom136": { - "type": [ - "null", - "string" - ] - }, - "custom137": { - "type": [ - "null", - "string" - ] - }, - "custom138": { - "type": [ - "null", - "string" - ] - }, - "custom139": { - "type": [ - "null", - "string" - ] - }, - "custom14": { - "type": [ - "null", - "string" - ] - }, - "custom140": { - "type": [ - "null", - "string" - ] - }, - "custom141": { - "type": [ - "null", - "string" - ] - }, - "custom142": { - "type": [ - "null", - "string" - ] - }, - "custom143": { - "type": [ - "null", - "string" - ] - }, - "custom144": { - "type": [ - "null", - "string" - ] - }, - "custom145": { - "type": [ - "null", - "string" - ] - }, - "custom146": { - "type": [ - "null", - "string" - ] - }, - "custom147": { - "type": [ - "null", - "string" - ] - }, - "custom148": { - "type": [ - "null", - "string" - ] - }, - "custom149": { - "type": [ - "null", - "string" - ] - }, - "custom15": { - "type": [ - "null", - "string" - ] - }, - "custom150": { - "type": [ - "null", - "string" - ] - }, - "custom16": { - "type": [ - "null", - "string" - ] - }, - "custom17": { - "type": [ - "null", - "string" - ] - }, - "custom18": { - "type": [ - "null", - "string" - ] - }, - "custom19": { - "type": [ - "null", - "string" - ] - }, - "custom2": { - "type": [ - "null", - "string" - ] - }, - "custom20": { - "type": [ - "null", - "string" - ] - }, - "custom21": { - "type": [ - "null", - "string" - ] - }, - "custom22": { - "type": [ - "null", - "string" - ] - }, - "custom23": { - "type": [ - "null", - "string" - ] - }, - "custom24": { - "type": [ - "null", - "string" - ] - }, - "custom25": { - "type": [ - "null", - "string" - ] - }, - "custom26": { - "type": [ - "null", - "string" - ] - }, - "custom27": { - "type": [ - "null", - "string" - ] - }, - "custom28": { - "type": [ - "null", - "string" - ] - }, - "custom29": { - "type": [ - "null", - "string" - ] - }, - "custom3": { - "type": [ - "null", - "string" - ] - }, - "custom30": { - "type": [ - "null", - "string" - ] - }, - "custom31": { - "type": [ - "null", - "string" - ] - }, - "custom32": { - "type": [ - "null", - "string" - ] - }, - "custom33": { - "type": [ - "null", - "string" - ] - }, - "custom34": { - "type": [ - "null", - "string" - ] - }, - "custom35": { - "type": [ - "null", - "string" - ] - }, - "custom36": { - "type": [ - "null", - "string" - ] - }, - "custom37": { - "type": [ - "null", - "string" - ] - }, - "custom38": { - "type": [ - "null", - "string" - ] - }, - "custom39": { - "type": [ - "null", - "string" - ] - }, - "custom4": { - "type": [ - "null", - "string" - ] - }, - "custom40": { - "type": [ - "null", - "string" - ] - }, - "custom41": { - "type": [ - "null", - "string" - ] - }, - "custom42": { - "type": [ - "null", - "string" - ] - }, - "custom43": { - "type": [ - "null", - "string" - ] - }, - "custom44": { - "type": [ - "null", - "string" - ] - }, - "custom45": { - "type": [ - "null", - "string" - ] - }, - "custom46": { - "type": [ - "null", - "string" - ] - }, - "custom47": { - "type": [ - "null", - "string" - ] - }, - "custom48": { - "type": [ - "null", - "string" - ] - }, - "custom49": { - "type": [ - "null", - "string" - ] - }, - "custom50": { - "type": [ - "null", - "string" - ] - }, - "custom51": { - "type": [ - "null", - "string" - ] - }, - "custom52": { - "type": [ - "null", - "string" - ] - }, - "custom53": { - "type": [ - "null", - "string" - ] - }, - "custom54": { - "type": [ - "null", - "string" - ] - }, - "custom55": { - "type": [ - "null", - "string" - ] - }, - "custom56": { - "type": [ - "null", - "string" - ] - }, - "custom57": { - "type": [ - "null", - "string" - ] - }, - "custom58": { - "type": [ - "null", - "string" - ] - }, - "custom59": { - "type": [ - "null", - "string" - ] - }, - "custom6": { - "type": [ - "null", - "string" - ] - }, - "custom60": { - "type": [ - "null", - "string" - ] - }, - "custom61": { - "type": [ - "null", - "string" - ] - }, - "custom62": { - "type": [ - "null", - "string" - ] - }, - "custom63": { - "type": [ - "null", - "string" - ] - }, - "custom64": { - "type": [ - "null", - "string" - ] - }, - "custom65": { - "type": [ - "null", - "string" - ] - }, - "custom66": { - "type": [ - "null", - "string" - ] - }, - "custom67": { - "type": [ - "null", - "string" - ] - }, - "custom68": { - "type": [ - "null", - "string" - ] - }, - "custom69": { - "type": [ - "null", - "string" - ] - }, - "custom7": { - "type": [ - "null", - "string" - ] - }, - "custom70": { - "type": [ - "null", - "string" - ] - }, - "custom71": { - "type": [ - "null", - "string" - ] - }, - "custom72": { - "type": [ - "null", - "string" - ] - }, - "custom73": { - "type": [ - "null", - "string" - ] - }, - "custom74": { - "type": [ - "null", - "string" - ] - }, - "custom75": { - "type": [ - "null", - "string" - ] - }, - "custom76": { - "type": [ - "null", - "string" - ] - }, - "custom77": { - "type": [ - "null", - "string" - ] - }, - "custom78": { - "type": [ - "null", - "string" - ] - }, - "custom79": { - "type": [ - "null", - "string" - ] - }, - "custom8": { - "type": [ - "null", - "string" - ] - }, - "custom80": { - "type": [ - "null", - "string" - ] - }, - "custom81": { - "type": [ - "null", - "string" - ] - }, - "custom82": { - "type": [ - "null", - "string" - ] - }, - "custom83": { - "type": [ - "null", - "string" - ] - }, - "custom84": { - "type": [ - "null", - "string" - ] - }, - "custom85": { - "type": [ - "null", - "string" - ] - }, - "custom86": { - "type": [ - "null", - "string" - ] - }, - "custom87": { - "type": [ - "null", - "string" - ] - }, - "custom88": { - "type": [ - "null", - "string" - ] - }, - "custom89": { - "type": [ - "null", - "string" - ] - }, - "custom9": { - "type": [ - "null", - "string" - ] - }, - "custom90": { - "type": [ - "null", - "string" - ] - }, - "custom91": { - "type": [ - "null", - "string" - ] - }, - "custom92": { - "type": [ - "null", - "string" - ] - }, - "custom93": { - "type": [ - "null", - "string" - ] - }, - "custom94": { - "type": [ - "null", - "string" - ] - }, - "custom95": { - "type": [ - "null", - "string" - ] - }, - "custom96": { - "type": [ - "null", - "string" - ] - }, - "custom97": { - "type": [ - "null", - "string" - ] - }, - "custom98": { - "type": [ - "null", - "string" - ] - }, - "custom99": { - "type": [ - "null", - "string" - ] - }, - "dateOfBirth": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "degree": { - "type": [ - "null", - "string" - ] - }, - "emailOptedOut": { - "type": [ - "null", - "boolean" - ] - }, - "emails": { - "type": [ - "null", - "array" - ], - "items": { - "type": "string" - } - }, - "emailsOptStatus": { - "type": [ - "null", - "string" - ] - }, - "emailsOptedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "engagedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "engagedScore": { - "type": [ - "null", - "number" - ] - }, - "eventName": { - "type": [ - "null", - "string" - ] - }, - "externalId": { - "type": [ - "null", - "string" - ] - }, - "externalOwner": { - "type": [ - "null", - "string" - ] - }, - "externalSource": { - "type": [ - "null", - "string" - ] - }, - "facebookUrl": { - "type": [ - "null", - "string" - ] - }, - "firstName": { - "type": [ - "null", - "string" - ] - }, - "gender": { - "type": [ - "null", - "string" - ] - }, - "githubUrl": { - "type": [ - "null", - "string" - ] - }, - "githubUsername": { - "type": [ - "null", - "string" - ] - }, - "googlePlusUrl": { - "type": [ - "null", - "string" - ] - }, - "graduationDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "homePhones": { - "type": [ - "null", - "array" - ], - "items": { - "type": "string" - } - }, - "jobStartDate": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "lastName": { - "type": [ - "null", - "string" - ] - }, - "linkedInConnections": { - "type": [ - "null", - "integer" - ] - }, - "linkedInId": { - "type": [ - "null", - "string" - ] - }, - "linkedInSlug": { - "type": [ - "null", - "string" - ] - }, - "linkedInUrl": { - "type": [ - "null", - "string" - ] - }, - "middleName": { - "type": [ - "null", - "string" - ] - }, - "mobilePhones": { - "type": [ - "null", - "array" - ], - "items": { - "type": "string" - } - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "nickname": { - "type": [ - "null", - "string" - ] - }, - "occupation": { - "type": [ - "null", - "string" - ] - }, - "openCount": { - "type": [ - "null", - "integer" - ] - }, - "optedOut": { - "type": [ - "null", - "boolean" - ] - }, - "optedOutAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "otherPhones": { - "type": [ - "null", - "array" - ], - "items": { - "type": "string" - } - }, - "personalNote1": { - "type": [ - "null", - "string" - ] - }, - "personalNote2": { - "type": [ - "null", - "string" - ] - }, - "preferredContact": { - "type": [ - "null", - "string" - ] - }, - "quoraUrl": { - "type": [ - "null", - "string" - ] - }, - "region": { - "type": [ - "null", - "string" - ] - }, - "replyCount": { - "type": [ - "null", - "integer" - ] - }, - "school": { - "type": [ - "null", - "string" - ] - }, - "score": { - "type": [ - "null", - "number" - ] - }, - "sharingTeamId": { - "type": [ - "null", - "string" - ] - }, - "source": { - "type": [ - "null", - "string" - ] - }, - "specialties": { - "type": [ - "null", - "string" - ] - }, - "stackOverflowId": { - "type": [ - "null", - "string" - ] - }, - "stackOverflowUrl": { - "type": [ - "null", - "string" - ] - }, - "tags": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "string" - ] - } - }, - "timeZone": { - "type": [ - "null", - "string" - ] - }, - "timeZoneIana": { - "type": [ - "null", - "string" - ] - }, - "timeZoneInferred": { - "type": [ - "null", - "string" - ] - }, - "title": { - "type": [ - "null", - "string" - ] - }, - "touchedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "trashedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "twitterUrl": { - "type": [ - "null", - "string" - ] - }, - "twitterUsername": { - "type": [ - "null", - "string" - ] - }, - "updatedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "voipPhones": { - "type": [ - "null", - "array" - ], - "items": { - "type": "string" - } - }, - "websiteUrl1": { - "type": [ - "null", - "string" - ] - }, - "websiteUrl2": { - "type": [ - "null", - "string" - ] - }, - "websiteUrl3": { - "type": [ - "null", - "string" - ] - }, - "workPhones": { - "type": [ - "null", - "array" - ], + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "attributes": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "addedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "addressCity": { + "type": ["null", "string"] + }, + "addressCountry": { + "type": ["null", "string"] + }, + "addressState": { + "type": ["null", "string"] + }, + "addressStreet": { + "type": ["null", "string"] + }, + "addressStreet2": { + "type": ["null", "string"] + }, + "addressZip": { + "type": ["null", "string"] + }, + "angelListUrl": { + "type": ["null", "string"] + }, + "availableAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "callOptedOut": { + "type": ["null", "boolean"] + }, + "callsOptStatus": { + "type": ["null", "string"] + }, + "callsOptedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "campaignName": { + "type": ["null", "string"] + }, + "clickCount": { + "type": ["null", "integer"] + }, + "company": { + "type": ["null", "string"] + }, + "contactHistogram": { + "type": ["null", "array"], + "items": { + "type": "array", "items": { - "type": "string" + "type": "integer" } } + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "custom1": { + "type": ["null", "string"] + }, + "custom10": { + "type": ["null", "string"] + }, + "custom100": { + "type": ["null", "string"] + }, + "custom101": { + "type": ["null", "string"] + }, + "custom102": { + "type": ["null", "string"] + }, + "custom103": { + "type": ["null", "string"] + }, + "custom104": { + "type": ["null", "string"] + }, + "custom105": { + "type": ["null", "string"] + }, + "custom106": { + "type": ["null", "string"] + }, + "custom107": { + "type": ["null", "string"] + }, + "custom108": { + "type": ["null", "string"] + }, + "custom109": { + "type": ["null", "string"] + }, + "custom11": { + "type": ["null", "string"] + }, + "custom110": { + "type": ["null", "string"] + }, + "custom111": { + "type": ["null", "string"] + }, + "custom112": { + "type": ["null", "string"] + }, + "custom113": { + "type": ["null", "string"] + }, + "custom114": { + "type": ["null", "string"] + }, + "custom115": { + "type": ["null", "string"] + }, + "custom116": { + "type": ["null", "string"] + }, + "custom117": { + "type": ["null", "string"] + }, + "custom118": { + "type": ["null", "string"] + }, + "custom119": { + "type": ["null", "string"] + }, + "custom12": { + "type": ["null", "string"] + }, + "custom120": { + "type": ["null", "string"] + }, + "custom121": { + "type": ["null", "string"] + }, + "custom122": { + "type": ["null", "string"] + }, + "custom123": { + "type": ["null", "string"] + }, + "custom124": { + "type": ["null", "string"] + }, + "custom125": { + "type": ["null", "string"] + }, + "custom126": { + "type": ["null", "string"] + }, + "custom127": { + "type": ["null", "string"] + }, + "custom128": { + "type": ["null", "string"] + }, + "custom129": { + "type": ["null", "string"] + }, + "custom13": { + "type": ["null", "string"] + }, + "custom130": { + "type": ["null", "string"] + }, + "custom131": { + "type": ["null", "string"] + }, + "custom132": { + "type": ["null", "string"] + }, + "custom133": { + "type": ["null", "string"] + }, + "custom134": { + "type": ["null", "string"] + }, + "custom135": { + "type": ["null", "string"] + }, + "custom136": { + "type": ["null", "string"] + }, + "custom137": { + "type": ["null", "string"] + }, + "custom138": { + "type": ["null", "string"] + }, + "custom139": { + "type": ["null", "string"] + }, + "custom14": { + "type": ["null", "string"] + }, + "custom140": { + "type": ["null", "string"] + }, + "custom141": { + "type": ["null", "string"] + }, + "custom142": { + "type": ["null", "string"] + }, + "custom143": { + "type": ["null", "string"] + }, + "custom144": { + "type": ["null", "string"] + }, + "custom145": { + "type": ["null", "string"] + }, + "custom146": { + "type": ["null", "string"] + }, + "custom147": { + "type": ["null", "string"] + }, + "custom148": { + "type": ["null", "string"] + }, + "custom149": { + "type": ["null", "string"] + }, + "custom15": { + "type": ["null", "string"] + }, + "custom150": { + "type": ["null", "string"] + }, + "custom16": { + "type": ["null", "string"] + }, + "custom17": { + "type": ["null", "string"] + }, + "custom18": { + "type": ["null", "string"] + }, + "custom19": { + "type": ["null", "string"] + }, + "custom2": { + "type": ["null", "string"] + }, + "custom20": { + "type": ["null", "string"] + }, + "custom21": { + "type": ["null", "string"] + }, + "custom22": { + "type": ["null", "string"] + }, + "custom23": { + "type": ["null", "string"] + }, + "custom24": { + "type": ["null", "string"] + }, + "custom25": { + "type": ["null", "string"] + }, + "custom26": { + "type": ["null", "string"] + }, + "custom27": { + "type": ["null", "string"] + }, + "custom28": { + "type": ["null", "string"] + }, + "custom29": { + "type": ["null", "string"] + }, + "custom3": { + "type": ["null", "string"] + }, + "custom30": { + "type": ["null", "string"] + }, + "custom31": { + "type": ["null", "string"] + }, + "custom32": { + "type": ["null", "string"] + }, + "custom33": { + "type": ["null", "string"] + }, + "custom34": { + "type": ["null", "string"] + }, + "custom35": { + "type": ["null", "string"] + }, + "custom36": { + "type": ["null", "string"] + }, + "custom37": { + "type": ["null", "string"] + }, + "custom38": { + "type": ["null", "string"] + }, + "custom39": { + "type": ["null", "string"] + }, + "custom4": { + "type": ["null", "string"] + }, + "custom40": { + "type": ["null", "string"] + }, + "custom41": { + "type": ["null", "string"] + }, + "custom42": { + "type": ["null", "string"] + }, + "custom43": { + "type": ["null", "string"] + }, + "custom44": { + "type": ["null", "string"] + }, + "custom45": { + "type": ["null", "string"] + }, + "custom46": { + "type": ["null", "string"] + }, + "custom47": { + "type": ["null", "string"] + }, + "custom48": { + "type": ["null", "string"] + }, + "custom49": { + "type": ["null", "string"] + }, + "custom50": { + "type": ["null", "string"] + }, + "custom51": { + "type": ["null", "string"] + }, + "custom52": { + "type": ["null", "string"] + }, + "custom53": { + "type": ["null", "string"] + }, + "custom54": { + "type": ["null", "string"] + }, + "custom55": { + "type": ["null", "string"] + }, + "custom56": { + "type": ["null", "string"] + }, + "custom57": { + "type": ["null", "string"] + }, + "custom58": { + "type": ["null", "string"] + }, + "custom59": { + "type": ["null", "string"] + }, + "custom6": { + "type": ["null", "string"] + }, + "custom60": { + "type": ["null", "string"] + }, + "custom61": { + "type": ["null", "string"] + }, + "custom62": { + "type": ["null", "string"] + }, + "custom63": { + "type": ["null", "string"] + }, + "custom64": { + "type": ["null", "string"] + }, + "custom65": { + "type": ["null", "string"] + }, + "custom66": { + "type": ["null", "string"] + }, + "custom67": { + "type": ["null", "string"] + }, + "custom68": { + "type": ["null", "string"] + }, + "custom69": { + "type": ["null", "string"] + }, + "custom7": { + "type": ["null", "string"] + }, + "custom70": { + "type": ["null", "string"] + }, + "custom71": { + "type": ["null", "string"] + }, + "custom72": { + "type": ["null", "string"] + }, + "custom73": { + "type": ["null", "string"] + }, + "custom74": { + "type": ["null", "string"] + }, + "custom75": { + "type": ["null", "string"] + }, + "custom76": { + "type": ["null", "string"] + }, + "custom77": { + "type": ["null", "string"] + }, + "custom78": { + "type": ["null", "string"] + }, + "custom79": { + "type": ["null", "string"] + }, + "custom8": { + "type": ["null", "string"] + }, + "custom80": { + "type": ["null", "string"] + }, + "custom81": { + "type": ["null", "string"] + }, + "custom82": { + "type": ["null", "string"] + }, + "custom83": { + "type": ["null", "string"] + }, + "custom84": { + "type": ["null", "string"] + }, + "custom85": { + "type": ["null", "string"] + }, + "custom86": { + "type": ["null", "string"] + }, + "custom87": { + "type": ["null", "string"] + }, + "custom88": { + "type": ["null", "string"] + }, + "custom89": { + "type": ["null", "string"] + }, + "custom9": { + "type": ["null", "string"] + }, + "custom90": { + "type": ["null", "string"] + }, + "custom91": { + "type": ["null", "string"] + }, + "custom92": { + "type": ["null", "string"] + }, + "custom93": { + "type": ["null", "string"] + }, + "custom94": { + "type": ["null", "string"] + }, + "custom95": { + "type": ["null", "string"] + }, + "custom96": { + "type": ["null", "string"] + }, + "custom97": { + "type": ["null", "string"] + }, + "custom98": { + "type": ["null", "string"] + }, + "custom99": { + "type": ["null", "string"] + }, + "dateOfBirth": { + "type": ["null", "string"], + "format": "date-time" + }, + "degree": { + "type": ["null", "string"] + }, + "emailOptedOut": { + "type": ["null", "boolean"] + }, + "emails": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "emailsOptStatus": { + "type": ["null", "string"] + }, + "emailsOptedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "engagedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "engagedScore": { + "type": ["null", "number"] + }, + "eventName": { + "type": ["null", "string"] + }, + "externalId": { + "type": ["null", "string"] + }, + "externalOwner": { + "type": ["null", "string"] + }, + "externalSource": { + "type": ["null", "string"] + }, + "facebookUrl": { + "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "gender": { + "type": ["null", "string"] + }, + "githubUrl": { + "type": ["null", "string"] + }, + "githubUsername": { + "type": ["null", "string"] + }, + "googlePlusUrl": { + "type": ["null", "string"] + }, + "graduationDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "homePhones": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "jobStartDate": { + "type": ["null", "string"], + "format": "date-time" + }, + "lastName": { + "type": ["null", "string"] + }, + "linkedInConnections": { + "type": ["null", "integer"] + }, + "linkedInId": { + "type": ["null", "string"] + }, + "linkedInSlug": { + "type": ["null", "string"] + }, + "linkedInUrl": { + "type": ["null", "string"] + }, + "middleName": { + "type": ["null", "string"] + }, + "mobilePhones": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "name": { + "type": ["null", "string"] + }, + "nickname": { + "type": ["null", "string"] + }, + "occupation": { + "type": ["null", "string"] + }, + "openCount": { + "type": ["null", "integer"] + }, + "optedOut": { + "type": ["null", "boolean"] + }, + "optedOutAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "otherPhones": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "personalNote1": { + "type": ["null", "string"] + }, + "personalNote2": { + "type": ["null", "string"] + }, + "preferredContact": { + "type": ["null", "string"] + }, + "quoraUrl": { + "type": ["null", "string"] + }, + "region": { + "type": ["null", "string"] + }, + "replyCount": { + "type": ["null", "integer"] + }, + "school": { + "type": ["null", "string"] + }, + "score": { + "type": ["null", "number"] + }, + "sharingTeamId": { + "type": ["null", "string"] + }, + "source": { + "type": ["null", "string"] + }, + "specialties": { + "type": ["null", "string"] + }, + "stackOverflowId": { + "type": ["null", "string"] + }, + "stackOverflowUrl": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "timeZone": { + "type": ["null", "string"] + }, + "timeZoneIana": { + "type": ["null", "string"] + }, + "timeZoneInferred": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "touchedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "trashedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "twitterUrl": { + "type": ["null", "string"] + }, + "twitterUsername": { + "type": ["null", "string"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "voipPhones": { + "type": ["null", "array"], + "items": { + "type": "string" + } + }, + "websiteUrl1": { + "type": ["null", "string"] + }, + "websiteUrl2": { + "type": ["null", "string"] + }, + "websiteUrl3": { + "type": ["null", "string"] + }, + "workPhones": { + "type": ["null", "array"], + "items": { + "type": "string" + } } } - }, - "additionalProperties": true - } \ No newline at end of file + } + }, + "additionalProperties": true +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequence_states.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequence_states.json index e4e89eb7c82c..bfce5ba1836a 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequence_states.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequence_states.json @@ -1,154 +1,85 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "type": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "attributes": { - "type": [ - "null", - "object" - ], - "additionalProperties": true, - "properties": { - "activeAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "bounceCount": { - "type": [ - "null", - "integer" - ] - }, - "callCompletedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "clickCount": { - "type": [ - "null", - "integer" - ] - }, - "createdAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "deliverCount": { - "type": [ - "null", - "integer" - ] - }, - "errorReason": { - "type": [ - "null", - "string" - ] - }, - "failureCount": { - "type": [ - "null", - "integer" - ] - }, - "negativeReplyCount": { - "type": [ - "null", - "integer" - ] - }, - "neutralReplyCount": { - "type": [ - "null", - "integer" - ] - }, - "openCount": { - "type": [ - "null", - "integer" - ] - }, - "optOutCount": { - "type": [ - "null", - "integer" - ] - }, - "pauseReason": { - "type": [ - "null", - "string" - ] - }, - "positiveReplyCount": { - "type": [ - "null", - "integer" - ] - }, - "repliedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "replyCount": { - "type": [ - "null", - "integer" - ] - }, - "scheduleCount": { - "type": [ - "null", - "integer" - ] - }, - "state": { - "type": [ - "null", - "string" - ] - }, - "stateChangedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "updatedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - } + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "attributes": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "activeAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "bounceCount": { + "type": ["null", "integer"] + }, + "callCompletedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "clickCount": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "deliverCount": { + "type": ["null", "integer"] + }, + "errorReason": { + "type": ["null", "string"] + }, + "failureCount": { + "type": ["null", "integer"] + }, + "negativeReplyCount": { + "type": ["null", "integer"] + }, + "neutralReplyCount": { + "type": ["null", "integer"] + }, + "openCount": { + "type": ["null", "integer"] + }, + "optOutCount": { + "type": ["null", "integer"] + }, + "pauseReason": { + "type": ["null", "string"] + }, + "positiveReplyCount": { + "type": ["null", "integer"] + }, + "repliedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "replyCount": { + "type": ["null", "integer"] + }, + "scheduleCount": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "stateChangedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" } } - }, - "additionalProperties": true - } \ No newline at end of file + } + }, + "additionalProperties": true +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json index c02f01c9fed3..fed1dfb27b37 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/schemas/sequences.json @@ -1,280 +1,148 @@ { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "type": { - "type": [ - "null", - "string" - ] - }, - "id": { - "type": [ - "null", - "integer" - ] - }, - "attributes": { - "type": [ - "null", - "object" - ], - "additionalProperties": true, - "properties": { - "automationPercentage": { - "type": [ - "null", - "number" - ] - }, - "bounceCount": { - "type": [ - "null", - "integer" - ] - }, - "clickCount": { - "type": [ - "null", - "integer" - ] - }, - "createdAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "deliverCount": { - "type": [ - "null", - "integer" - ] - }, - "description": { - "type": [ - "null", - "string" - ] - }, - "durationInDays": { - "type": [ - "null", - "integer" - ] - }, - "enabled": { - "type": [ - "null", - "boolean" - ] - }, - "enabledAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "failureCount": { - "type": [ - "null", - "integer" - ] - }, - "finishOnReply": { - "type": [ - "null", - "boolean" - ] - }, - "lastUsedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "locked": { - "type": [ - "null", - "boolean" - ] - }, - "lockedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "maxActivations": { - "type": [ - "null", - "integer" - ] - }, - "name": { - "type": [ - "null", - "string" - ] - }, - "negativeReplyCount": { - "type": [ - "null", - "integer" - ] - }, - "neutralReplyCount": { - "type": [ - "null", - "integer" - ] - }, - "numContactedProspects": { - "type": [ - "null", - "integer" - ] - }, - "numRepliedProspects": { - "type": [ - "null", - "integer" - ] - }, - "openCount": { - "type": [ - "null", - "integer" - ] - }, - "optOutCount": { - "type": [ - "null", - "integer" - ] - }, - "positiveReplyCount": { - "type": [ - "null", - "integer" - ] - }, - "primaryReplyAction": { - "type": [ - "null", - "string" - ] - }, - "primaryReplyPauseDuration": { - "type": [ - "null", - "integer" - ] - }, - "replyCount": { - "type": [ - "null", - "integer" - ] - }, - "scheduleCount": { - "type": [ - "null", - "integer" - ] - }, - "scheduleIntervalType": { - "type": [ - "null", - "string" - ] - }, - "secondaryReplyAction": { - "type": [ - "null", - "string" - ] - }, - "secondaryReplyPauseDuration": { - "type": [ - "null", - "integer" - ] - }, - "sequenceStepCount": { - "type": [ - "null", - "integer" - ] - }, - "sequenceType": { - "type": [ - "null", - "string" - ] - }, - "shareType": { - "type": [ - "null", - "string" - ] - }, - "tags": { - "type": [ - "null", - "array" - ], - "items": { - "type": [ - "null", - "string" - ] - } - }, - "throttleCapacity": { - "type": [ - "null", - "integer" - ] - }, - "throttleMaxAddsPerDay": { - "type": [ - "null", - "integer" - ] - }, - "throttlePaused": { - "type": [ - "null", - "boolean" - ] - }, - "throttlePausedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" - }, - "transactional": { - "type": [ - "null", - "boolean" - ] - }, - "updatedAt": { - "type": [ - "null", - "string" - ], - "format": "date-time" + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "type": { + "type": ["null", "string"] + }, + "id": { + "type": ["null", "integer"] + }, + "attributes": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "automationPercentage": { + "type": ["null", "number"] + }, + "bounceCount": { + "type": ["null", "integer"] + }, + "clickCount": { + "type": ["null", "integer"] + }, + "createdAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "deliverCount": { + "type": ["null", "integer"] + }, + "description": { + "type": ["null", "string"] + }, + "durationInDays": { + "type": ["null", "integer"] + }, + "enabled": { + "type": ["null", "boolean"] + }, + "enabledAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "failureCount": { + "type": ["null", "integer"] + }, + "finishOnReply": { + "type": ["null", "boolean"] + }, + "lastUsedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "locked": { + "type": ["null", "boolean"] + }, + "lockedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "maxActivations": { + "type": ["null", "integer"] + }, + "name": { + "type": ["null", "string"] + }, + "negativeReplyCount": { + "type": ["null", "integer"] + }, + "neutralReplyCount": { + "type": ["null", "integer"] + }, + "numContactedProspects": { + "type": ["null", "integer"] + }, + "numRepliedProspects": { + "type": ["null", "integer"] + }, + "openCount": { + "type": ["null", "integer"] + }, + "optOutCount": { + "type": ["null", "integer"] + }, + "positiveReplyCount": { + "type": ["null", "integer"] + }, + "primaryReplyAction": { + "type": ["null", "string"] + }, + "primaryReplyPauseDuration": { + "type": ["null", "integer"] + }, + "replyCount": { + "type": ["null", "integer"] + }, + "scheduleCount": { + "type": ["null", "integer"] + }, + "scheduleIntervalType": { + "type": ["null", "string"] + }, + "secondaryReplyAction": { + "type": ["null", "string"] + }, + "secondaryReplyPauseDuration": { + "type": ["null", "integer"] + }, + "sequenceStepCount": { + "type": ["null", "integer"] + }, + "sequenceType": { + "type": ["null", "string"] + }, + "shareType": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] } + }, + "throttleCapacity": { + "type": ["null", "integer"] + }, + "throttleMaxAddsPerDay": { + "type": ["null", "integer"] + }, + "throttlePaused": { + "type": ["null", "boolean"] + }, + "throttlePausedAt": { + "type": ["null", "string"], + "format": "date-time" + }, + "transactional": { + "type": ["null", "boolean"] + }, + "updatedAt": { + "type": ["null", "string"], + "format": "date-time" } } - }, - "additionalProperties": true - } \ No newline at end of file + } + }, + "additionalProperties": true +} diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index 959e282cb0c5..8b96847b42d2 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -19,7 +19,7 @@ class OutreachStream(HttpStream, ABC): url_base = "https://api.outreach.io/api/v2/" - + def __init__( self, authenticator: HttpAuthenticator, @@ -31,11 +31,11 @@ def __init__( def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: try: - next_page_url = response.json().get('links').get('next') + next_page_url = response.json().get("links").get("next") params = parse.parse_qs(parse.urlparse(next_page_url).query) - if not params or 'page[after]' not in params: + if not params or "page[after]" not in params: return {} - return {"after": params['page[after]'][0]} + return {"after": params["page[after]"][0]} except Exception as e: raise KeyError(f"error parsing next_page token: {e}") @@ -64,23 +64,22 @@ def cursor_field(self) -> str: def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: current_stream_state = current_stream_state or {} - current_stream_state_date = current_stream_state.get('updatedAt', self.start_date) - latest_record_date = latest_record.get('attributes', {}).get('updatedAt', self.start_date) + current_stream_state_date = current_stream_state.get("updatedAt", self.start_date) + latest_record_date = latest_record.get("attributes", {}).get("updatedAt", self.start_date) - return {'updatedAt': max(current_stream_state_date, latest_record_date)} + return {"updatedAt": max(current_stream_state_date, latest_record_date)} def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: params = super().request_params(stream_state=stream_state, stream_slice=stream_slice, next_page_token=next_page_token) - if 'updatedAt' in stream_state: - params["filter[updatedAt]"] = stream_state['updatedAt'] + '..inf' + if "updatedAt" in stream_state: + params["filter[updatedAt]"] = stream_state["updatedAt"] + "..inf" return params class Prospects(IncrementalOutreachStream): primary_key = "id" - cursor_field = "attributes/properties/updatedAt" def path(self, **kwargs) -> str: return "prospects" @@ -88,7 +87,6 @@ def path(self, **kwargs) -> str: class Sequences(IncrementalOutreachStream): primary_key = "id" - cursor_field = "attributes/properties/updatedAt" def path(self, **kwargs) -> str: return "sequences" @@ -96,29 +94,21 @@ def path(self, **kwargs) -> str: class SequenceStates(IncrementalOutreachStream): primary_key = "id" - cursor_field = "attributes/properties/updatedAt" def path(self, **kwargs) -> str: return "sequenceStates" class OutreachAuthenticator(Oauth2Authenticator): - def __init__(self, redirect_uri: str, - token_refresh_endpoint: str, - client_id: str, - client_secret: str, - refresh_token: str): + def __init__(self, redirect_uri: str, token_refresh_endpoint: str, client_id: str, client_secret: str, refresh_token: str): super().__init__( - token_refresh_endpoint=token_refresh_endpoint, - client_id=client_id, - client_secret=client_secret, - refresh_token=refresh_token + token_refresh_endpoint=token_refresh_endpoint, client_id=client_id, client_secret=client_secret, refresh_token=refresh_token ) self.redirect_uri = redirect_uri def get_refresh_request_body(self) -> Mapping[str, Any]: payload = super().get_refresh_request_body() - payload['redirect_uri'] = self.redirect_uri + payload["redirect_uri"] = self.redirect_uri return payload @@ -140,7 +130,7 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: response.raise_for_status() return True, None except Exception as e: - logger.error(f'Failed to check connection. Error: {e}') + logger.error(f"Failed to check connection. Error: {e}") return False, e def streams(self, config: Mapping[str, Any]) -> List[Stream]: diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json b/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json index 667226afad93..b6ec5b147982 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/spec.json @@ -4,7 +4,13 @@ "$schema": "http://json-schema.org/draft-07/schema#", "title": "Source Outreach Spec", "type": "object", - "required": ["client_id", "client_secret", "refresh_token", "redirect_uri", "start_date"], + "required": [ + "client_id", + "client_secret", + "refresh_token", + "redirect_uri", + "start_date" + ], "additionalProperties": false, "properties": { "client_id": { diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py index 84acbe97907a..b3116a9b7795 100644 --- a/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_incremental_streams.py @@ -4,6 +4,7 @@ from unittest.mock import MagicMock + from airbyte_cdk.models import SyncMode from pytest import fixture from source_outreach.source import IncrementalOutreachStream @@ -19,7 +20,7 @@ def patch_incremental_base_class(mocker): def test_cursor_field(patch_incremental_base_class): stream = IncrementalOutreachStream(authenticator=MagicMock()) - expected_cursor_field = 'attributes/properties/updatedAt' + expected_cursor_field = "attributes/properties/updatedAt" assert stream.cursor_field == expected_cursor_field diff --git a/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py index 0a96ed9f0988..ce3883ad9ec0 100644 --- a/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-outreach/unit_tests/test_streams.py @@ -20,7 +20,7 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): stream = OutreachStream(authenticator=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_params = {'count': 'false', 'page[size]': 100} + expected_params = {"count": "false", "page[size]": 100} assert stream.request_params(**inputs) == expected_params @@ -29,7 +29,7 @@ def test_next_page_token(patch_base_class): response = MagicMock() response.json.return_value = {"links": {"next": "http://api.outreach.io/api/v2/prospects?page[after]=100"}} inputs = {"response": response} - expected_token = {'after': '100'} + expected_token = {"after": "100"} assert stream.next_page_token(**inputs) == expected_token @@ -37,17 +37,13 @@ def test_parse_response(patch_base_class): stream = OutreachStream(authenticator=MagicMock()) response = MagicMock() response.json.return_value = { - "data": [{ - "id": 123, - "attributes": {"name": "John Doe"}, - "relationships": {"account": {"data": {"type": "account", "id": 4}}} - }] + "data": [{"id": 123, "attributes": {"name": "John Doe"}, "relationships": {"account": {"data": {"type": "account", "id": 4}}}}] } inputs = {"response": response} expected_parsed_object = { "id": 123, "attributes": {"name": "John Doe"}, - "relationships": {"account": {"data": {"type": "account", "id": 4}}} + "relationships": {"account": {"data": {"type": "account", "id": 4}}}, } assert next(stream.parse_response(**inputs)) == expected_parsed_object From b0242887da45f84e907651d813cec358455acb6e Mon Sep 17 00:00:00 2001 From: lgomezm Date: Tue, 2 Nov 2021 12:37:45 -0500 Subject: [PATCH 09/10] Added comments --- .../source-outreach/source_outreach/source.py | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py index 8b96847b42d2..9b457fdbee79 100644 --- a/airbyte-integrations/connectors/source-outreach/source_outreach/source.py +++ b/airbyte-integrations/connectors/source-outreach/source_outreach/source.py @@ -15,10 +15,14 @@ from airbyte_cdk.sources.streams.http.auth.oauth import Oauth2Authenticator +_TOKEN_REFRESH_ENDPOINT = "https://api.outreach.io/oauth/token" +_URL_BASE = "https://api.outreach.io/api/v2/" + + # Basic full refresh stream class OutreachStream(HttpStream, ABC): - url_base = "https://api.outreach.io/api/v2/" + url_base = _URL_BASE def __init__( self, @@ -30,6 +34,10 @@ def __init__( super().__init__(authenticator=authenticator) def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + Returns the token for the next page as per https://api.outreach.io/api/v2/docs#pagination. + It uses cursor-based pagination, by sending the 'page[size]' and 'page[after]' parameters. + """ try: next_page_url = response.json().get("links").get("next") params = parse.parse_qs(parse.urlparse(next_page_url).query) @@ -79,6 +87,11 @@ def request_params( class Prospects(IncrementalOutreachStream): + """ + Prospect stream. Yields data from the GET /prospects endpoint. + See https://api.outreach.io/api/v2/docs#prospect + """ + primary_key = "id" def path(self, **kwargs) -> str: @@ -86,6 +99,11 @@ def path(self, **kwargs) -> str: class Sequences(IncrementalOutreachStream): + """ + Sequence stream. Yields data from the GET /sequences endpoint. + See https://api.outreach.io/api/v2/docs#sequence + """ + primary_key = "id" def path(self, **kwargs) -> str: @@ -93,6 +111,11 @@ def path(self, **kwargs) -> str: class SequenceStates(IncrementalOutreachStream): + """ + Sequence stream. Yields data from the GET /sequences endpoint. + See https://api.outreach.io/api/v2/docs#sequenceState + """ + primary_key = "id" def path(self, **kwargs) -> str: @@ -117,7 +140,7 @@ class SourceOutreach(AbstractSource): def _create_authenticator(self, config): return OutreachAuthenticator( redirect_uri=config["redirect_uri"], - token_refresh_endpoint="https://api.outreach.io/oauth/token", + token_refresh_endpoint=_TOKEN_REFRESH_ENDPOINT, client_id=config["client_id"], client_secret=config["client_secret"], refresh_token=config["refresh_token"], @@ -126,7 +149,7 @@ def _create_authenticator(self, config): def check_connection(self, logger, config) -> Tuple[bool, any]: try: access_token, _ = self._create_authenticator(config).refresh_access_token() - response = requests.get("https://api.outreach.io/api/v2", headers={"Authorization": f"Bearer {access_token}"}) + response = requests.get(_URL_BASE, headers={"Authorization": f"Bearer {access_token}"}) response.raise_for_status() return True, None except Exception as e: From ae21229e5eec3cce2faa615c9ee95523af49e404 Mon Sep 17 00:00:00 2001 From: lgomezm Date: Wed, 3 Nov 2021 09:01:54 -0500 Subject: [PATCH 10/10] Added Outreach source documentation --- docs/SUMMARY.md | 1 + docs/integrations/README.md | 1 + docs/integrations/sources/outreach.md | 43 +++++++++++++++++++++++++++ 3 files changed, 45 insertions(+) create mode 100644 docs/integrations/sources/outreach.md diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 0b2e912cb581..e04b467e1d4b 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -94,6 +94,7 @@ * [Oracle DB](integrations/sources/oracle.md) * [Oracle Peoplesoft](integrations/sources/oracle-peoplesoft.md) * [Oracle Siebel CRM](integrations/sources/oracle-siebel-crm.md) + * [Outreach](integrations/sources/outreach.md) * [Paypal Transaction](integrations/sources/paypal-transaction.md) * [Plaid](integrations/sources/plaid.md) * [Pipedrive](integrations/sources/pipedrive.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 715d13a997c0..49ec3c90c549 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -77,6 +77,7 @@ Airbyte uses a grading system for connectors to help users understand what to ex | [Oracle DB](sources/oracle.md) | Certified | | [Oracle PeopleSoft](sources/oracle-peoplesoft.md) | Beta | | [Oracle Siebel CRM](sources/oracle-siebel-crm.md) | Beta | +| [Outreach](./sources/outreach.md)| Alpha | | [PayPal Transaction](sources/paypal-transaction.md) | Beta | | [Pipedrive](sources/pipedrive.md) | Alpha | | [Plaid](sources/plaid.md) | Alpha | diff --git a/docs/integrations/sources/outreach.md b/docs/integrations/sources/outreach.md new file mode 100644 index 000000000000..80956a6e9cc7 --- /dev/null +++ b/docs/integrations/sources/outreach.md @@ -0,0 +1,43 @@ +# Outreach + +## Overview + +The Outreach source supports both `Full Refresh` and `Incremental` syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +### Output schema + +Some output streams are available from this source. A list of these streams can be found below in the [Streams](outreach.md#streams) section. + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| SSL connection | Yes | +| Namespaces | No | + +## Getting started + +### Requirements + +* Outreach Account +* Outreach OAuth credentials + +### Setup guide + +Getting oauth credentials require contacting Outreach to request an account. Check out [here](https://www.outreach.io/lp/watch-demo#request-demo). + +## Streams + +List of available streams: + +* Prospects +* Sequences +* SequenceStates + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :-------- | :----- | :------ | +| 0.1.0 | 2021-11-03 | [7507](https://github.com/airbytehq/airbyte/pull/7507) | Outreach Connector |