From cc0613b283f7d3ac87d1ed9cd5922baf0fca0981 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Sun, 17 Oct 2021 00:46:07 +1100 Subject: [PATCH 01/10] add source notion --- airbyte-integrations/builds.md | 1 + .../connectors/source-notion/.dockerignore | 7 + .../connectors/source-notion/Dockerfile | 38 +++ .../connectors/source-notion/README.md | 132 ++++++++++ .../source-notion/acceptance-test-config.yml | 29 ++ .../source-notion/acceptance-test-docker.sh | 16 ++ .../connectors/source-notion/bootstrap.md | 32 +++ .../connectors/source-notion/build.gradle | 14 + .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 11 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/catalog.json | 31 +++ .../integration_tests/configured_catalog.json | 41 +++ .../integration_tests/expected_records.txt | 36 +++ .../integration_tests/invalid_config.json | 4 + .../integration_tests/sample_config.json | 4 + .../integration_tests/sample_state.json | 11 + .../connectors/source-notion/main.py | 13 + .../connectors/source-notion/requirements.txt | 2 + .../connectors/source-notion/setup.py | 30 +++ .../source-notion/source_notion/__init__.py | 8 + .../source_notion/schemas/blocks.json | 93 +++++++ .../source_notion/schemas/databases.json | 108 ++++++++ .../source_notion/schemas/pages.json | 241 +++++++++++++++++ .../source_notion/schemas/shared/child.json | 9 + .../source_notion/schemas/shared/date.json | 11 + .../source_notion/schemas/shared/emoji.json | 15 ++ .../source_notion/schemas/shared/file.json | 38 +++ .../source_notion/schemas/shared/heading.json | 9 + .../source_notion/schemas/shared/icon.json | 8 + .../source_notion/schemas/shared/options.json | 17 ++ .../source_notion/schemas/shared/parent.json | 21 ++ .../schemas/shared/rich_text.json | 62 +++++ .../schemas/shared/text_element.json | 10 + .../source_notion/schemas/shared/title.json | 62 +++++ .../source_notion/schemas/shared/user.json | 49 ++++ .../source_notion/schemas/users.json | 5 + .../source-notion/source_notion/source.py | 70 +++++ .../source-notion/source_notion/spec.json | 23 ++ .../source-notion/source_notion/streams.py | 247 ++++++++++++++++++ .../source-notion/unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 143 ++++++++++ .../source-notion/unit_tests/test_source.py | 25 ++ .../source-notion/unit_tests/test_streams.py | 80 ++++++ docs/SUMMARY.md | 1 + docs/integrations/README.md | 1 + docs/integrations/sources/notion.md | 62 +++++ 47 files changed, 1892 insertions(+) create mode 100644 airbyte-integrations/connectors/source-notion/.dockerignore create mode 100644 airbyte-integrations/connectors/source-notion/Dockerfile create mode 100644 airbyte-integrations/connectors/source-notion/README.md create mode 100644 airbyte-integrations/connectors/source-notion/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-notion/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-notion/build.gradle create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/expected_records.txt create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-notion/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-notion/main.py create mode 100644 airbyte-integrations/connectors/source-notion/requirements.txt create mode 100644 airbyte-integrations/connectors/source-notion/setup.py create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/__init__.py create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/child.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/date.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/emoji.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/icon.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/options.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/title.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/schemas/users.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/source.py create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/spec.json create mode 100644 airbyte-integrations/connectors/source-notion/source_notion/streams.py create mode 100644 airbyte-integrations/connectors/source-notion/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-notion/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/notion.md diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index fb11a5ba0a6e..366763284a18 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -53,6 +53,7 @@ | Mixpanel | [![source-mixpanel](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mixpanel%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mixpanel) | | Mongo DB | [![source-mongodb-v2](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mongodb-v2%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mongodb-v2) | | MySQL | [![source-mysql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mysql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mysql) | +| Notion | [![source-notion](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-notion%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-notion) | | Oracle DB | [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-oracle) | | Paypal Transaction | [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paypal-transaction) | | Pipedrive | [![source-pipedrive](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-pipedrive%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-pipedrive) | diff --git a/airbyte-integrations/connectors/source-notion/.dockerignore b/airbyte-integrations/connectors/source-notion/.dockerignore new file mode 100644 index 000000000000..955cfbdc08ec --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_notion +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-notion/Dockerfile b/airbyte-integrations/connectors/source-notion/Dockerfile new file mode 100644 index 000000000000..c5a925d1a3ed --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.7.11-alpine3.14 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_notion ./source_notion + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-notion diff --git a/airbyte-integrations/connectors/source-notion/README.md b/airbyte-integrations/connectors/source-notion/README.md new file mode 100644 index 000000000000..8d003a2c81f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/README.md @@ -0,0 +1,132 @@ +# Notion Source + +This is the repository for the Notion source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/notion). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-notion:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/notion) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_notion/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source notion test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-notion:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-notion:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-notion:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-notion:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-notion:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-notion:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-notion:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml new file mode 100644 index 000000000000..efac61848521 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml @@ -0,0 +1,29 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-notion:dev +tests: + spec: + - spec_path: "source_notion/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: no + exact_order: no + extra_records: yes + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-notion/bootstrap.md b/airbyte-integrations/connectors/source-notion/bootstrap.md new file mode 100644 index 000000000000..6d492039f873 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/bootstrap.md @@ -0,0 +1,32 @@ +# Notion + +## Overview + +Notion is an application that provides components such as notes, databases, kanban boards, wikis, calendars and reminders. Notion REST API allows a developer to retrieve pages, databases, blocks, and users on the Notion platform. + +## Endpoints + +Notion API consists of three endpoints which can be extracted data from: + +1. **User**: The User object represents a user in a Notion workspace. Users include guests, full workspace members, and bots. +2. **Block**: A block object represents content within Notion. Blocks can be text, lists, media, and more. Page and database is also a type of block. +3. **Search**: This endpoint is used to get list of pages and databases. + +## Quick Notes + +- Notion stores content in hierarchy, each node is called a 'block'. Block is a generic term which can be text, lists, media, even page and database are also block. + +- Due to this hierarchical structure, we use recursive request to get the full list of blocks. + +- Pages and databases can be extracted from the `Search` endpoint separately, so they are excluded from the block list request. + +- Airbyte CDK doesn't support recursive schema, so some elements of the block schema which can be recursive are replaced with empty objects. + +- Page and database must grant permission to the internal integration, otherwise API cannot extract data from them. See [https://developers.notion.com/docs/authorization#authorizing-internal-integrations](https://developers.notion.com/docs/authorization#authorizing-internal-integrations) + +- Rate limiting is a standard exponential backoff when a 429 HTTP status code returned. The rate limit for incoming requests is an average of 3 requests per second. Some bursts beyond the average rate are allowed. Notion API also has size limit, see [https://developers.notion.com/reference/errors#request-limits](https://developers.notion.com/reference/errors#request-limits) + +## API Reference + +The API reference documents: [https://developers.notion.com/reference/intro](https://developers.notion.com/reference) + diff --git a/airbyte-integrations/connectors/source-notion/build.gradle b/airbyte-integrations/connectors/source-notion/build.gradle new file mode 100644 index 000000000000..5b863c4f603b --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/build.gradle @@ -0,0 +1,14 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_notion' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) + implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) +} diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/__init__.py b/airbyte-integrations/connectors/source-notion/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-notion/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..835c09c625ba --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/abnormal_state.json @@ -0,0 +1,11 @@ +{ + "databases": { + "last_edited_time": "2099-10-10T04:40:00.000Z" + }, + "pages": { + "last_edited_time": "2099-10-10T04:40:00.000Z" + }, + "blocks": { + "last_edited_time": "2099-10-10T04:00:00.000Z" + } +} diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py new file mode 100644 index 000000000000..58c194c5d137 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """ This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/catalog.json b/airbyte-integrations/connectors/source-notion/integration_tests/catalog.json new file mode 100644 index 000000000000..24d5091f961a --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/catalog.json @@ -0,0 +1,31 @@ +{ + "streams": [ + { + "name": "users", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": true, + "json_schema": {} + }, + { + "name": "databases", + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": "last_edited_time", + "json_schema": {} + }, + { + "name": "pages", + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": "last_edited_time", + "json_schema": {} + }, + { + "name": "blocks", + "supported_sync_modes": ["incremental"], + "source_defined_cursor": true, + "default_cursor_field": "last_edited_time", + "json_schema": {} + } + ] +} diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..f89113659bbf --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json @@ -0,0 +1,41 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "databases", + "json_schema": {}, + "supported_sync_modes": ["incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "pages", + "json_schema": {}, + "supported_sync_modes": ["incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + }, + { + "stream": { + "name": "blocks", + "json_schema": {}, + "supported_sync_modes": ["incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} + diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.txt new file mode 100644 index 000000000000..a6f612bf67ec --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.txt @@ -0,0 +1,36 @@ +{"stream": "users", "data": {"object": "user", "id": "fd0ed76c-44bd-413a-9448-18ff4b1d6a5e", "name": "Bo Lu", "avatar_url": "https://lh3.googleusercontent.com/a-/AOh14Giq0P_BufoGYZJkwc5UciBcmcsLvNTumWzusm7ePg=s100", "type": "person", "person": {"email": "lv.patrick@gmail.com"}}, "emitted_at": 1634387505000} +{"stream": "users", "data": {"object": "user", "id": "352ef128-12da-44d2-91b6-d4a94d3cc899", "name": "test integration", "avatar_url": null, "type": "bot", "bot": {"owner": {"type": "workspace", "workspace": true}}}, "emitted_at": 1634387505000} +{"stream": "databases", "data": {"object": "database", "id": "ea141d69-b5b8-4a6b-ae17-828d5baf37ae", "cover": null, "icon": null, "created_time": "2021-10-14T02:18:00.000Z", "last_edited_time": "2021-10-14T05:38:00.000Z", "title": [{"type": "text", "text": {"content": "test page2", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "test page2", "href": null}], "properties": {"Status": {"id": "kZec", "name": "Status", "type": "select", "select": {"options": [{"id": "1", "name": "Not started", "color": "red"}, {"id": "2", "name": "In progress", "color": "yellow"}, {"id": "3", "name": "Completed", "color": "green"}]}}, "Assign": {"id": "mpGX", "name": "Assign", "type": "people", "people": {}}, "Name": {"id": "title", "name": "Name", "type": "title", "title": {}}}, "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/ea141d69b5b84a6bae17828d5baf37ae"}, "emitted_at": 1634387506000} +{"stream": "databases", "data": {"object": "database", "id": "71183a17-ecff-46cd-9da0-afc783ac81e8", "cover": null, "icon": {"type": "emoji", "emoji": "\u2714\ufe0f"}, "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-14T05:39:00.000Z", "title": [{"type": "text", "text": {"content": "Task List", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Task List", "href": null}], "properties": {"Date Created": {"id": "'Y6%3C", "name": "Date Created", "type": "created_time", "created_time": {}}, "Status": {"id": "%5EOE%40", "name": "Status", "type": "select", "select": {"options": [{"id": "1", "name": "To Do", "color": "red"}, {"id": "2", "name": "Doing", "color": "yellow"}, {"id": "3", "name": "Done \ud83d\ude4c", "color": "green"}]}}, "Name": {"id": "title", "name": "Name", "type": "title", "title": {}}}, "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/71183a17ecff46cd9da0afc783ac81e8"}, "emitted_at": 1634387506000} +{"stream": "databases", "data": {"object": "database", "id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b", "cover": null, "icon": null, "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-15T05:58:00.000Z", "title": [{"type": "text", "text": {"content": "test page", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "test page", "href": null}], "properties": {"test-property": {"id": "%3CCpP", "name": "test-property", "type": "date", "date": {}}, "Tags": {"id": "~fcg", "name": "Tags", "type": "multi_select", "multi_select": {"options": []}}, "Name": {"id": "title", "name": "Name", "type": "title", "title": {}}}, "parent": {"type": "workspace", "workspace": true}, "url": "https://www.notion.so/8d65239fa23e4dc5a5121e29f70e0c0b"}, "emitted_at": 1634387506000} +{"stream": "pages", "data": {"object": "page", "id": "560c83a7-6a66-445c-a138-034b66db863d", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "71183a17-ecff-46cd-9da0-afc783ac81e8"}, "archived": false, "properties": {"Date Created": {"id": "'Y6%3C", "type": "created_time", "created_time": "2021-10-10T08:09:00.000Z"}, "Status": {"id": "%5EOE%40", "type": "select", "select": null}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "New Task", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "New Task", "href": null}]}}, "url": "https://www.notion.so/New-Task-560c83a76a66445ca138034b66db863d"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "ed4eb196-48be-445c-ab2b-2e8e4ccd8937", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "71183a17-ecff-46cd-9da0-afc783ac81e8"}, "archived": false, "properties": {"Date Created": {"id": "'Y6%3C", "type": "created_time", "created_time": "2021-10-10T08:09:00.000Z"}, "Status": {"id": "%5EOE%40", "type": "select", "select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/ed4eb19648be445cab2b2e8e4ccd8937"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "43f33669-2700-4d6c-8ed5-12aea4455da6", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\udc36"}, "parent": {"type": "database_id", "database_id": "71183a17-ecff-46cd-9da0-afc783ac81e8"}, "archived": false, "properties": {"Date Created": {"id": "'Y6%3C", "type": "created_time", "created_time": "2021-10-10T08:09:00.000Z"}, "Status": {"id": "%5EOE%40", "type": "select", "select": {"id": "2", "name": "Doing", "color": "yellow"}}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Take Fig on a walk", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Take Fig on a walk", "href": null}]}}, "url": "https://www.notion.so/Take-Fig-on-a-walk-43f3366927004d6c8ed512aea4455da6"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "9332eb2e-7864-4209-9422-6028ca5af35a", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-14T02:15:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/9332eb2e7864420994226028ca5af35a"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "bc21d49f-3318-434e-8bc0-81981d40e78b", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-14T02:15:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "aaa", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "aaa", "href": null}]}}, "url": "https://www.notion.so/aaa-bc21d49f3318434e8bc081981d40e78b"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "26b09a78-1413-4a57-bb22-3f069017e9fb", "created_time": "2021-10-14T02:16:00.000Z", "last_edited_time": "2021-10-14T02:16:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/26b09a7814134a57bb223f069017e9fb"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "7f91fa0b-ea9b-49de-8036-7c71bfde7bd3", "created_time": "2021-10-14T02:16:00.000Z", "last_edited_time": "2021-10-14T02:16:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/7f91fa0bea9b49de80367c71bfde7bd3"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "52612945-9c07-4728-bd9f-6700372409f7", "created_time": "2021-10-14T02:18:00.000Z", "last_edited_time": "2021-10-14T02:18:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "ea141d69-b5b8-4a6b-ae17-828d5baf37ae"}, "archived": false, "properties": {"Status": {"id": "kZec", "type": "select", "select": null}, "Assign": {"id": "mpGX", "type": "people", "people": []}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Card 1", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Card 1", "href": null}]}}, "url": "https://www.notion.so/Card-1-526129459c074728bd9f6700372409f7"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "8a91862f-9386-452a-a64c-e06dcd4323d6", "created_time": "2021-10-14T02:18:00.000Z", "last_edited_time": "2021-10-14T02:18:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "ea141d69-b5b8-4a6b-ae17-828d5baf37ae"}, "archived": false, "properties": {"Status": {"id": "kZec", "type": "select", "select": null}, "Assign": {"id": "mpGX", "type": "people", "people": []}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Card 2", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Card 2", "href": null}]}}, "url": "https://www.notion.so/Card-2-8a91862f9386452aa64ce06dcd4323d6"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "ee86d233-543b-4d0b-b73d-6e674ee30421", "created_time": "2021-10-14T02:18:00.000Z", "last_edited_time": "2021-10-14T02:18:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "ea141d69-b5b8-4a6b-ae17-828d5baf37ae"}, "archived": false, "properties": {"Status": {"id": "kZec", "type": "select", "select": null}, "Assign": {"id": "mpGX", "type": "people", "people": []}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Card 3", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Card 3", "href": null}]}}, "url": "https://www.notion.so/Card-3-ee86d233543b4d0bb73d6e674ee30421"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "bcebbd8a-21e4-4359-8134-2bda86f59e2d", "created_time": "2021-10-15T04:51:00.000Z", "last_edited_time": "2021-10-15T04:52:00.000Z", "cover": null, "icon": null, "parent": {"type": "page_id", "page_id": "03924866-f383-4d39-a0bb-dec2f69b4e99"}, "archived": false, "properties": {"title": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "subpage", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "subpage", "href": null}]}}, "url": "https://www.notion.so/subpage-bcebbd8a21e4435981342bda86f59e2d"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "5a67c86f-d0da-4d0a-9dd7-f4cf164e6247", "created_time": "2021-10-15T05:41:00.000Z", "last_edited_time": "2021-10-15T05:49:00.000Z", "cover": null, "icon": null, "parent": {"type": "workspace", "workspace": true}, "archived": false, "properties": {"title": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "test page3", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "test page3", "href": null}]}}, "url": "https://www.notion.so/test-page3-5a67c86fd0da4d0a9dd7f4cf164e6247"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "7209939e-21cd-4975-9114-ca7717079422", "created_time": "2021-10-15T05:58:00.000Z", "last_edited_time": "2021-10-15T05:58:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/7209939e21cd49759114ca7717079422"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "03924866-f383-4d39-a0bb-dec2f69b4e99", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-16T12:30:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": {"start": "2021-10-14", "end": null}}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "bbb", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "bbb", "href": null}]}}, "url": "https://www.notion.so/bbb-03924866f3834d39a0bbdec2f69b4e99"}, "emitted_at": 1634387507000} +{"stream": "blocks", "data": {"object": "block", "id": "20b41f7f-7051-4c18-a7c3-2b63e2f54da2", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"text": [{"type": "text", "text": {"content": "To Do", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "To Do", "href": null}]}}, "emitted_at": 1634387508000} +{"stream": "blocks", "data": {"object": "block", "id": "e0e3fc30-c452-44c9-aa10-c3afc7f6fd29", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634387508000} +{"stream": "blocks", "data": {"object": "block", "id": "fb24d227-97db-4933-93cc-b2ea1ddfa41f", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634387508000} +{"stream": "blocks", "data": {"object": "block", "id": "d3a02e8a-b38e-4eb2-a0f4-8c55d099217d", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"text": [{"type": "text", "text": {"content": "Tasks", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Tasks", "href": null}]}}, "emitted_at": 1634387509000} +{"stream": "blocks", "data": {"object": "block", "id": "a75cc58d-2dba-4b90-bfc1-5af214af2200", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634387509000} +{"stream": "blocks", "data": {"object": "block", "id": "0bae298f-e378-4a8f-9f25-4c6feb8506be", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634387509000} +{"stream": "blocks", "data": {"object": "block", "id": "1af3ba11-8339-4c21-9025-9f96948b25dc", "created_time": "2021-10-15T04:52:00.000Z", "last_edited_time": "2021-10-15T04:52:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "subpage content", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "subpage content", "href": null}]}}, "emitted_at": 1634387515000} +{"stream": "blocks", "data": {"object": "block", "id": "fc248547-83ef-4069-b7c9-18897edb7150", "created_time": "2021-10-15T04:52:00.000Z", "last_edited_time": "2021-10-15T04:52:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634387515000} +{"stream": "blocks", "data": {"object": "block", "id": "740d4614-e2e7-4da1-9823-9942a842c944", "created_time": "2021-10-15T05:41:00.000Z", "last_edited_time": "2021-10-15T05:44:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "test content3", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "test content3", "href": null}]}}, "emitted_at": 1634387516000} +{"stream": "blocks", "data": {"object": "block", "id": "665fdcb0-da5a-42d8-a603-5719296ba929", "created_time": "2021-10-15T05:48:00.000Z", "last_edited_time": "2021-10-15T05:48:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634387516000} +{"stream": "blocks", "data": {"object": "block", "id": "05bee6fd-017e-4feb-9a7c-5c7dbf650504", "created_time": "2021-10-14T04:40:00.000Z", "last_edited_time": "2021-10-14T04:40:00.000Z", "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"text": [{"type": "text", "text": {"content": "this is header", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "this is header", "href": null}]}}, "emitted_at": 1634387517000} +{"stream": "blocks", "data": {"object": "block", "id": "3d1207a0-a73a-4475-b23e-9c8a4ca69c49", "created_time": "2021-10-14T04:40:00.000Z", "last_edited_time": "2021-10-15T07:09:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "this is some text2", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "this is some text2", "href": null}]}}, "emitted_at": 1634387517000} +{"stream": "blocks", "data": {"object": "block", "id": "eee06451-d354-48c9-b52f-27ddb5610256", "created_time": "2021-10-16T10:02:00.000Z", "last_edited_time": "2021-10-16T10:02:00.000Z", "has_children": false, "archived": false, "type": "code", "code": {"text": [{"type": "text", "text": {"content": "some code", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "some code", "href": null}], "language": "javascript"}}, "emitted_at": 1634387517000} +{"stream": "blocks", "data": {"object": "block", "id": "7557f377-c20d-4464-b50d-c3498f51e796", "created_time": "2021-10-16T10:04:00.000Z", "last_edited_time": "2021-10-16T10:04:00.000Z", "has_children": false, "archived": false, "type": "callout", "callout": {"text": [{"type": "text", "text": {"content": "some call out", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "some call out", "href": null}], "icon": {"type": "emoji", "emoji": "\ud83d\udca1"}}}, "emitted_at": 1634387517000} +{"stream": "blocks", "data": {"object": "block", "id": "473803b3-0bd6-45b6-a3f0-53535041f98c", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "dadsfadsf", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "dadsfadsf", "href": null}]}}, "emitted_at": 1634387517000} +{"stream": "blocks", "data": {"object": "block", "id": "db8c1f18-28f9-4230-8af3-9377d2e52aaf", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "slaksjdfl", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "slaksjdfl", "href": null}]}}, "emitted_at": 1634387517000} +{"stream": "blocks", "data": {"object": "block", "id": "effd0166-724d-4a1a-8e41-87ee4aaf2499", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634387517000} diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-notion/integration_tests/invalid_config.json new file mode 100644 index 000000000000..57bbe69e1d62 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "access_token": "wrong token", + "start_date": "2021-01-01T00:00:00.000Z" +} diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-notion/integration_tests/sample_config.json new file mode 100644 index 000000000000..65cdc5420984 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "access_token": "your Notion access token", + "start_date": "2021-01-01T00:00:00.000Z" +} diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-notion/integration_tests/sample_state.json new file mode 100644 index 000000000000..d8e9c49584c8 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/integration_tests/sample_state.json @@ -0,0 +1,11 @@ +{ + "databases": { + "last_edited_time": "2021-10-10T04:40:00.000Z" + }, + "pages": { + "last_edited_time": "2021-10-10T04:40:00.000Z" + }, + "blocks": { + "last_edited_time": "2021-10-10T04:00:00.000Z" + } +} diff --git a/airbyte-integrations/connectors/source-notion/main.py b/airbyte-integrations/connectors/source-notion/main.py new file mode 100644 index 000000000000..424f1da313e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_notion import SourceNotion + +if __name__ == "__main__": + source = SourceNotion() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-notion/requirements.txt b/airbyte-integrations/connectors/source-notion/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-notion/setup.py b/airbyte-integrations/connectors/source-notion/setup.py new file mode 100644 index 000000000000..2ab7c196ba42 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/setup.py @@ -0,0 +1,30 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", + "requests-mock", +] + +setup( + name="source_notion", + description="Source implementation for Notion.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-notion/source_notion/__init__.py b/airbyte-integrations/connectors/source-notion/source_notion/__init__.py new file mode 100644 index 000000000000..6b47201e717f --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceNotion + +__all__ = ["SourceNotion"] diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json new file mode 100644 index 000000000000..6ab880a4f0b4 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json @@ -0,0 +1,93 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "object": { + "enum": ["block"] + }, + "id": { + "type": "string" + }, + "created_time": { + "type": "string" + }, + "last_edited_time": { + "type": "string" + }, + "archived": { + "type": "boolean" + }, + "has_children": { + "type": "boolean" + }, + "type": { + "enum": ["paragraph", "heading_1", "heading_2", "heading_3", "callout", "bulleted_list_item", "numbered_list_item", "to_do", "toggle", "code", "child_page", "child_database", "embed", "image", "video", "file", "pdf", "bookmark", "equation", "unsupported"] + }, + "paragraph": { "$ref": "text_element.json" }, + "quote": { "$ref": "text_element.json" }, + "bulleted_list_item": { "$ref": "text_element.json" }, + "numbered_list_item": { "$ref": "text_element.json" }, + "toggle": { "$ref": "text_element.json" }, + "heading_1": { "$ref": "heading.json" }, + "heading_2": { "$ref": "heading.json" }, + "heading_3": { "$ref": "heading.json" }, + "callout": { + "type": "object", + "additionalProperties": false, + "properties": { + "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "icon": { "$ref": "icon.json" }, + "children": { "type": "array", "items": { "type": "object" } } + } + }, + "to_do": { + "type": "object", + "additionalProperties": false, + "properties": { + "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "checked": { "type": ["null", "boolean"] }, + "children": { "type": "array", "items": { "type": "object" } } + } + }, + "code": { + "type": "object", + "additionalProperties": false, + "properties": { + "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "language": { + "enum": ["abap", "arduino", "bash", "basic", "c", "clojure", "coffeescript", "c++", "c#", "css", "dart", "diff", "docker", "elixir", "elm", "erlang", "flow", "fortran", "f#", "gherkin", "glsl", "go", "graphql", "groovy", "haskell", "html", "java", "javascript", "json", "julia", "kotlin", "latex", "less", "lisp", "livescript", "lua", "makefile", "markdown", "markup", "matlab", "mermaid", "nix", "objective-c", "ocaml", "pascal", "perl", "php", "plain text", "powershell", "prolog", "protobuf", "python", "r", "reason", "ruby", "rust", "sass", "scala", "scheme", "scss", "shell", "sql", "swift", "typescript", "vb.net", "verilog", "vhdl", "visual basic", "webassembly", "xml", "yaml", "java/c/c++/c#"] + } + } + }, + "child_page": { "$ref": "child.json" }, + "child_database": { "$ref": "child.json" }, + "embed": { + "type": "object", + "additionalProperties": false, + "properties": { + "url": { "type": "string" } + } + }, + "image": { "$ref": "file.json" }, + "vidoe": { "$ref": "file.json" }, + "file": { "$ref": "file.json" }, + "pdf": { "$ref": "file.json" }, + "bookmark": { + "type": "object", + "additionalProperties": false, + "properties": { + "url": { "type": "string" }, + "caption": { "type": "array", "items": { "$ref": "rich_text.json" } } + } + }, + "equation": { + "type": "object", + "additionalProperties": false, + "properties": { + "expression": { "type": "string" } + } + } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json new file mode 100644 index 000000000000..adaaddfab2ea --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json @@ -0,0 +1,108 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "object": { + "enum": ["database"] + }, + "id": { + "type": "string" + }, + "created_time": { + "type": "string" + }, + "last_edited_time": { + "type": "string" + }, + "title": { + "type": "array", + "items": { + "$ref": "rich_text.json" + } + }, + "icon": { + "$ref": "icon.json" + }, + "cover": { + "$ref": "file.json" + }, + "parent": { + "$ref": "parent.json" + }, + "url": { + "type": "string" + }, + "properties": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "anyOf": [ + { + "type": "object", + "additionalProperties": true, + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["title", "rich_text", "date", "people", "files", "checkbox", "url", "email", "phone_number", "created_time", "created_by", "last_edited_time", "last_edited_by"] }, + "name": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["number"] }, + "name": { "type": "string" }, + "format": { "enum": ["number", "number_with_commas", "percent", "dollar", "canadian_dollar", "euro", "pound", "yen", "ruble", "rupee", "won", "yuan", "real", "lira", "rupiah", "franc", "hong_kong_dollar", "new_zealand_dollar", "krona", "norwegian_krone", "mexican_peso", "rand", "new_taiwan_dollar", "danish_krone", "zloty", "baht", "forint", "koruna", "shekel", "chilean_peso", "philippine_peso", "dirham", "colombian_peso", "riyal", "ringgit", "leu"] } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["select", "multi_select"] }, + "name": { "type": "string" }, + "options": { "type": "array", "items": { "$ref": "options.json" } } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["formula"] }, + "name": { "type": "string" }, + "expression": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["relation"] }, + "name": { "type": "string" }, + "database_id": { "type": "string" }, + "synced_property_name": { "type": ["null", "string"] }, + "synced_property_id": { "type": ["null", "string"] } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["rollup"] }, + "name": { "type": "string" }, + "relation_property_name": { "type": "string" }, + "relation_property_id": { "type": "string" }, + "rollup_property_name": { "type": "string" }, + "rollup_property_id": { "type": "string" }, + "function": { "enum": ["count_all", "count_values", "count_unique_values", "count_empty", "count_not_empty", "percent_empty", "percent_not_empty", "sum", "average", "median", "min", "max", "range", "show_original"] } + } + } + ] + } + } + } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json new file mode 100644 index 000000000000..ed2b8a9c2409 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json @@ -0,0 +1,241 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "object": { + "enum": ["page"] + }, + "id": { + "type": "string" + }, + "created_time": { + "type": "string" + }, + "last_edited_time": { + "type": "string" + }, + "archived": { + "type": "boolean" + }, + "icon": { + "$ref": "icon.json" + }, + "cover": { + "$ref": "file.json" + }, + "parent": { + "$ref": "parent.json" + }, + "url": { + "type": "string" + }, + "properties": { + "type": "object", + "additionalProperties": false, + "patternProperties": { + ".*": { + "anyOf": [ + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^id$": { "type": "string" }, + "^type$": { "enum": ["title", "rich_text"] }, + "^title$|^rich_text$": { "type": "array", "items": { "$ref": "rich_text.json" } } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["number"] }, + "number": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["select"] }, + "select": { "$ref": "options.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["multi_select"] }, + "multi_select": { "type": ["null", "array"], "items": { "$ref": "options.json" } } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["date"] }, + "date": { "$ref": "date.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["formula"] }, + "formula": { + "type": ["null", "object"], + "properties": { + "type": { "enum": ["string", "number", "boolean", "date"] }, + "string": { "type": ["null", "string"] }, + "number": { "type": ["null", "number"] }, + "boolean": { "type": ["null", "boolean"] }, + "date": { "$ref": "date.json" } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["relation"] }, + "relation": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "id": { "type": "string" } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["rollup"] }, + "rollup": { + "type": ["null", "object"], + "properties": { + "type": { "enum": ["number", "date", "array"] }, + "number": { "type": ["null", "number"] }, + "date": { "$ref": "date.json" }, + "array": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "type": { "type": "string" } + } + } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["people"] }, + "people": { + "type": ["null", "array"], + "items": { + "$ref": "user.json" + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["files"] }, + "files": { + "type": ["null", "array"], + "items": { + "type": "object", + "properties": { + "type": { "enum": ["external", "file"] }, + "url": { "type": "string" }, + "expiry_time": { "type": "string" }, + "name": { "type": "string" } + } + } + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["checkbox"] }, + "checkout": { + "type": ["null", "boolean"] + } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["url"] }, + "url": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["email"] }, + "email": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["phone_number"] }, + "phone_number": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["created_time"] }, + "created_time": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["created_by"] }, + "created_by": { "$ref": "user.json" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["last_edited_time"] }, + "last_edited_time": { "type": "string" } + } + }, + { + "type": "object", + "properties": { + "id": { "type": "string" }, + "type": { "enum": ["last_edited_by"] }, + "last_edited_by": { "$ref": "user.json" } + } + } + ] + } + } + } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/child.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/child.json new file mode 100644 index 000000000000..c92be2eb8af2 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/child.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "title": { "type": "string" } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/date.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/date.json new file mode 100644 index 000000000000..07ebba50bc23 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/date.json @@ -0,0 +1,11 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "start": { "type": "string" }, + "end": { "type": ["null", "string"] } + } +} + + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/emoji.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/emoji.json new file mode 100644 index 000000000000..5ecd33417f6a --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/emoji.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "type": { + "type": "string" + }, + "emoji": { + "type": "string" + } + } +} + + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json new file mode 100644 index 000000000000..4a842353887d --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json @@ -0,0 +1,38 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "type": { + "enum": ["file", "external"] + }, + "caption": { + "type": "array", + "items": { + "$ref": "rich_text.json" + } + }, + "external": { + "type": "object", + "additionalProperties": false, + "properties": { + "url": { + "type": "string" + } + } + }, + "file": { + "type": "object", + "additionalProperties": false, + "properties": { + "url": { + "type": "string" + }, + "expiry_time": { + "type": "string" + } + } + } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json new file mode 100644 index 000000000000..9de9b913a479 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json @@ -0,0 +1,9 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "text": { "type": "array", "items": { "$ref": "rich_text.json" } } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/icon.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/icon.json new file mode 100644 index 000000000000..25018eb562dd --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/icon.json @@ -0,0 +1,8 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "anyOf": [ + { "$ref": "file.json" }, + { "$ref": "emoji.json" } + ] +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/options.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/options.json new file mode 100644 index 000000000000..f0bc86d30d5a --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/options.json @@ -0,0 +1,17 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "color": { + "enum": ["default", "gray", "brown", "orange", "yellow", "green", "blue", "purple", "pink", "red"] + } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json new file mode 100644 index 000000000000..2c67b3ef3b52 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "enum": ["database_id", "page_id", "workspace"] + }, + "database_id": { + "type": "string" + }, + "page_id": { + "type": "string" + }, + "workspace": { + "type": "boolean" + } + } +} + + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json new file mode 100644 index 000000000000..294d881e427e --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "enum": ["text", "mention", "equation"] + }, + "text": { + "type": "object", + "additionalProperties": false, + "properties": { + "content": { + "type": "string" + }, + "link": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "type": { + "enum": ["url"] + }, + "url": { + "type": "string" + } + } + } + } + }, + "annotations": { + "type": "object", + "additionalProperties": false, + "properties": { + "bold": { + "type": "boolean" + }, + "italic": { + "type": "boolean" + }, + "strikethrough": { + "type": "boolean" + }, + "underline": { + "type": "boolean" + }, + "code": { + "type": "boolean" + }, + "color": { + "enum": ["default", "gray", "brown", "orange", "yellow", "green", "blue", "purple", "pink", "red", "gray_background", "brown_background", "orange_background", "yellow_background", "green_background", "blue_background", "purple_background", "pink_background", "red_background"] + } + } + }, + "plain_text": { + "type": "string" + }, + "href": { + "type": ["null", "string"] + } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json new file mode 100644 index 000000000000..dbc3aa89b2bc --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json @@ -0,0 +1,10 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, + "children": { "type": "array", "items": { "type": "object" } } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/title.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/title.json new file mode 100644 index 000000000000..294d881e427e --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/title.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "type": { + "enum": ["text", "mention", "equation"] + }, + "text": { + "type": "object", + "additionalProperties": false, + "properties": { + "content": { + "type": "string" + }, + "link": { + "type": ["null", "object"], + "additionalProperties": false, + "properties": { + "type": { + "enum": ["url"] + }, + "url": { + "type": "string" + } + } + } + } + }, + "annotations": { + "type": "object", + "additionalProperties": false, + "properties": { + "bold": { + "type": "boolean" + }, + "italic": { + "type": "boolean" + }, + "strikethrough": { + "type": "boolean" + }, + "underline": { + "type": "boolean" + }, + "code": { + "type": "boolean" + }, + "color": { + "enum": ["default", "gray", "brown", "orange", "yellow", "green", "blue", "purple", "pink", "red", "gray_background", "brown_background", "orange_background", "yellow_background", "green_background", "blue_background", "purple_background", "pink_background", "red_background"] + } + } + }, + "plain_text": { + "type": "string" + }, + "href": { + "type": ["null", "string"] + } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json new file mode 100644 index 000000000000..47cd292d517b --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": false, + "properties": { + "object": { + "enum": ["user"] + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "avatar_url": { + "type": ["null", "string"] + }, + "type": { + "enum": ["person", "bot"] + }, + "person": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "email": { + "type": "string" + } + } + }, + "bot": { + "type": ["null", "object"], + "additionalProperties": true, + "properties": { + "owner": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "workspace": { + "type": ["null", "boolean"] + } + } + } + } + } + } +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/users.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/users.json new file mode 100644 index 000000000000..0df83c2a7618 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/users.json @@ -0,0 +1,5 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "$ref": "user.json" +} + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/source.py b/airbyte-integrations/connectors/source-notion/source_notion/source.py new file mode 100644 index 000000000000..d1ec3db24b8c --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/source.py @@ -0,0 +1,70 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterator, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import SyncMode, ConfiguredAirbyteStream, AirbyteMessage +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator +from airbyte_cdk.sources.utils.schema_helpers import InternalConfig + +from .streams import Databases, Pages, Blocks, Users + + +class SourceNotion(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + authenticator = TokenAuthenticator(config["access_token"]) + stream = Users(authenticator=authenticator, config=config) + records = stream.read_records(sync_mode=SyncMode.full_refresh) + next(records) + return True, None + except requests.exceptions.RequestException as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") + + authenticator = TokenAuthenticator(config["access_token"]) + args = { "authenticator": authenticator, "config": config } + + pages = Pages(**args) + blocks = Blocks(parent=pages, **args) + + return [Users(**args), Databases(**args), pages, blocks] + + # Override AbstractSource._read_incremental() function to emit an extra state + # message after finishing syncing the whole stream. + # It is for updating the maximum cursor field date state, because Notion's + # block isn't sorted between stream slices. There is no way to get the + # maximum cursor field date unless reading until the end of the stream. + # This is a dirty hack and might be removed if HttpStream provided an + # end-of-synching-stream event hook in the future. + def _read_incremental( + self, + logger: AirbyteLogger, + stream: Stream, + configured_stream: ConfiguredAirbyteStream, + connector_state: MutableMapping[str, Any], + internal_config: InternalConfig, + ) -> Iterator[AirbyteMessage]: + yield from super()._read_incremental(logger, stream, configured_stream, connector_state, internal_config) + + # only do this hack for Blocks stream + if not isinstance(stream, Blocks): + return + + stream_name = configured_stream.stream.name + stream_state = connector_state.get(stream_name, {}) + state_date = stream_state.get(stream.cursor_field, stream.start_date) + stream_state = { stream.cursor_field: max(state_date, stream.max_cursor_time) } + + yield self._checkpoint_state(stream_name, stream_state, connector_state, logger) + diff --git a/airbyte-integrations/connectors/source-notion/source_notion/spec.json b/airbyte-integrations/connectors/source-notion/source_notion/spec.json new file mode 100644 index 000000000000..4ab7b6db4d20 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/spec.json @@ -0,0 +1,23 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Notion Source Spec", + "type": "object", + "required": ["access_token", "start_date"], + "additionalProperties": false, + "properties": { + "access_token": { + "type": "string", + "description": "Notion API access token, see the docs for more information on how to obtain this token.", + "airbyte_secret": true + }, + "start_date": { + "type": "string", + "description": "The date from which you'd like to replicate data for Notion API, in the format YYYY-MM-DDT00:00:00.000Z. All data generated after this date will be replicated.", + "examples": ["2020-11-16T00:00:00.000Z"], + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py new file mode 100644 index 000000000000..c753cb3222af --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -0,0 +1,247 @@ +import time +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from datetime import datetime + +import requests +from airbyte_cdk.models import SyncMode +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream +from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator + +# maximum block recursive hierarchy depth +MAX_BLOCK_DEPTH = 30 + + +class NotionStream(HttpStream, ABC): + + url_base = "https://api.notion.com/v1/" + + primary_key = "id" + + page_size = 100 # set by Notion API + + def __init__(self, config: Mapping[str, Any], **kwargs): + super().__init__(**kwargs) + self.start_date = config["start_date"] + + def request_headers( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + params = super().request_headers(stream_state, stream_slice, next_page_token) + # Notion API version, see https://developers.notion.com/reference/versioning + params["Notion-Version"] = "2021-08-16" + return params + + def next_page_token( + self, + response: requests.Response, + ) -> Optional[Mapping[str, Any]]: + next_cursor = response.json()["next_cursor"] + if next_cursor: + return { "next_cursor": next_cursor } + + # default record filter, do nothing + def filter_by_state( + self, + stream_state: Mapping[str, Any] = None, + record: Mapping[str, Any] = None + ) -> Iterable: + yield record + + def parse_response( + self, + response: requests.Response, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Iterable[Mapping]: + data = response.json().get("results") + for record in data: + yield from self.filter_by_state(stream_state=stream_state, record=record) + + +class IncrementalNotionStream(NotionStream, ABC): + + cursor_field = "last_edited_time" + + http_method = "POST" + + def __init__(self, obj_type: Optional[str] = None, **kwargs): + super().__init__(**kwargs) + self.obj_type = obj_type + self.filter_on = True + + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + return "search" + + def request_body_json( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> Optional[Mapping]: + if not self.obj_type: + return + + body = { + "sort": { "direction": "ascending", "timestamp": "last_edited_time" }, + "filter": { "property": "object", "value": self.obj_type }, + "page_size": self.page_size + } + if next_page_token: + body["start_cursor"] = next_page_token["next_cursor"] + + return body + + def filter_by_state( + self, + stream_state: Mapping[str, Any] = None, + record: Mapping[str, Any] = None + ) -> Iterable: + if not self.filter_on: + yield record + return + + value = "" + if record: + value = record.get(self.cursor_field, value) + if not stream_state or value >= stream_state.get(self.cursor_field, ""): + yield record + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ) -> Mapping[str, Any]: + state_date = current_stream_state.get(self.cursor_field, self.start_date) + record_date = latest_record.get(self.cursor_field, self.start_date) + return { self.cursor_field: max(state_date, record_date) } + + +class Users(NotionStream): + + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + return "users" + + +class Databases(IncrementalNotionStream): + + def __init__(self, **kwargs): + super().__init__(obj_type = "database", **kwargs) + + +class Pages(IncrementalNotionStream): + + def __init__(self, **kwargs): + super().__init__(obj_type = "page", **kwargs) + + +class Blocks(HttpSubStream, IncrementalNotionStream): + + http_method = "GET" + + def __init__(self, **kwargs): + super().__init__(**kwargs) + + # block id stack for block hierarchy traversal + self.block_id_stack = [] + + # largest time in cursor field across all stream slices + self.max_cursor_time = self.start_date + + def path( + self, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + return f"blocks/{self.block_id_stack[-1]}/children" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + params = { "page_size": self.page_size } + if next_page_token: + params["start_cursor"] = next_page_token["next_cursor"] + return params + + def stream_slices( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Optional[Mapping[str, Any]]]: + # turn off parent's record filter to get full list of parents, because + # children's changed time may not propagated to parent + self.parent.filter_on = False + + parents = super().stream_slices(sync_mode, cursor_field, stream_state) + for item in parents: + parent_id = item["parent"]["id"] + self.block_id_stack.append(parent_id) + yield { "page_id": parent_id } + yield from [] + + def filter_by_state( + self, + stream_state: Mapping[str, Any] = None, + record: Mapping[str, Any] = None + ) -> Iterable: + # pages and databases blocks are already fetched in their streams, no + # need to do it again + for item in super().filter_by_state(stream_state, record): + if item["type"] not in ("child_page", "child_database"): + yield item + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + # if reached recursive limit, don't read any more + if len(self.block_id_stack) > MAX_BLOCK_DEPTH: + yield from [] + return + + records = super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + for record in records: + if record["has_children"]: + self.block_id_stack.append(record["id"]) + yield from self.read_records(sync_mode, cursor_field, stream_slice, stream_state) + else: + yield record + + self.block_id_stack.pop() + + yield from [] + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ) -> Mapping[str, Any]: + # we don't update state here, just keep a record of maximum cursor field + # date, state will be actually updated after syncing the whole stream + record_date = latest_record.get(self.cursor_field, self.start_date) + self.max_cursor_time = max(self.max_cursor_time, record_date) + return current_stream_state + diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/__init__.py b/airbyte-integrations/connectors/source-notion/unit_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..fffe15ca15d6 --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py @@ -0,0 +1,143 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import requests +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_notion.streams import IncrementalNotionStream, Pages, Blocks + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalNotionStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalNotionStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalNotionStream, "__abstractmethods__", set()) + + +@fixture +def args(): + return { + "authenticator": None, + "config": { + "access_token": "", + "start_date": "2021-01-01T00:00:00.000Z" + } + } + + +@fixture +def parent(args): + return Pages(**args) + + +@fixture +def stream(patch_incremental_base_class, args): + return IncrementalNotionStream(**args) + + +@fixture +def blocks(parent, args): + return Blocks(parent=parent, **args) + + +def test_cursor_field(stream): + expected_cursor_field = "last_edited_time" + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(stream): + inputs = { + "current_stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, + "latest_record": { "last_edited_time": "2021-10-20T00:00:00.000Z" } + } + expected_state = { "last_edited_time": "2021-10-20T00:00:00.000Z" } + assert stream.get_updated_state(**inputs) == expected_state + + inputs = { + "current_stream_state": { "last_edited_time": "2021-10-20T00:00:00.000Z" }, + "latest_record": { "last_edited_time": "2021-10-10T00:00:00.000Z" } + } + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(blocks, requests_mock): + stream = blocks + requests_mock.post("https://api.notion.com/v1/search", json={ + "results": [ { "id": "aaa" }, { "id": "bbb" } ], + "next_cursor": None + }) + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + expected_stream_slice = [ { "page_id": "aaa" }, { "page_id": "bbb" } ] + assert list(stream.stream_slices(**inputs)) == expected_stream_slice + + +def test_supports_incremental(stream, mocker): + mocker.patch.object(IncrementalNotionStream, "cursor_field", "dummy_field") + assert stream.supports_incremental + + +def test_source_defined_cursor(stream): + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(stream): + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval + + +def test_request_params(blocks): + stream = blocks + inputs = { "stream_state": {}, "next_page_token": { "next_cursor": "aaa" } } + expected_request_params = { "page_size": 100, "start_cursor": "aaa" } + assert stream.request_params(**inputs) == expected_request_params + + +def test_filter_by_state(stream): + inputs = { + "stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, + "record": { "last_edited_time": "2021-10-20T00:00:00.000Z" } + } + expected_filter_by_state = [{ "last_edited_time": "2021-10-20T00:00:00.000Z" }] + assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state + + inputs = { + "stream_state": { "last_edited_time": "2021-10-20T00:00:00.000Z" }, + "record": { "last_edited_time": "2021-10-10T00:00:00.000Z" } + } + expected_filter_by_state = [] + assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state + + +def test_filter_by_state_blocks(blocks): + stream = blocks + + inputs = { + "stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, + "record": { "last_edited_time": "2021-10-20T00:00:00.000Z", "type": "aaa" } + } + expected_filter_by_state = [{ "last_edited_time": "2021-10-20T00:00:00.000Z", "type": "aaa" }] + assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state + + inputs = { + "stream_state": { "last_edited_time": "2021-10-20T00:00:00.000Z" }, + "record": { "last_edited_time": "2021-10-10T00:00:00.000Z", "type": "aaa" } + } + expected_filter_by_state = [] + assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state + + # 'child_page' and 'child_database' should not be included + inputs = { + "stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, + "record": { "last_edited_time": "2021-10-20T00:00:00.000Z", "type": "child_page" } + } + expected_filter_by_state = [] + assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state + inputs = { + "stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, + "record": { "last_edited_time": "2021-10-20T00:00:00.000Z", "type": "child_database" } + } + expected_filter_by_state = [] + assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py new file mode 100644 index 000000000000..ced560c4540f --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py @@ -0,0 +1,25 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_notion.source import SourceNotion + + +def test_check_connection(mocker, requests_mock): + source = SourceNotion() + logger_mock, config_mock = MagicMock(), MagicMock() + requests_mock.get("https://api.notion.com/v1/users", json={ + "results": [ { "id": "aaa" } ], + "next_cursor": None + }) + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceNotion() + config_mock = MagicMock() + streams = source.streams(config_mock) + expected_streams_number = 4 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py new file mode 100644 index 000000000000..5a1960eafeeb --- /dev/null +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -0,0 +1,80 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import requests +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_notion.streams import NotionStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(NotionStream, "path", "v0/example_endpoint") + mocker.patch.object(NotionStream, "primary_key", "test_primary_key") + mocker.patch.object(NotionStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = NotionStream(config=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class, requests_mock): + stream = NotionStream(config=MagicMock()) + requests_mock.get("https://dummy", json={ "next_cursor": "aaa" }) + inputs = { "response": requests.get("https://dummy") } + expected_token = { "next_cursor": "aaa" } + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class, requests_mock): + stream = NotionStream(config=MagicMock()) + requests_mock.get("https://dummy", json={ + "results": [{ "a": 123 }, { "b": "xx" }] + }) + resp = requests.get("https://dummy") + inputs = { "response": resp, "stream_state": MagicMock() } + expected_parsed_object = [{ "a": 123 }, { "b": "xx" }] + assert list(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = NotionStream(config=MagicMock()) + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + expected_headers = { "Notion-Version": "2021-08-16" } + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = NotionStream(config=MagicMock()) + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = NotionStream(config=MagicMock()) + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = NotionStream(config=MagicMock()) + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 9c3a25a8c8fd..42c683a10ad2 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -89,6 +89,7 @@ * [Mixpanel](integrations/sources/mixpanel.md) * [Mongo DB](integrations/sources/mongodb-v2.md) * [MySQL](integrations/sources/mysql.md) + * [Notion](integrations/sources/notion.md) * [Okta](integrations/sources/okta.md) * [Oracle DB](integrations/sources/oracle.md) * [Oracle Peoplesoft](integrations/sources/oracle-peoplesoft.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 0ce0f7212f37..03ec68bbfb33 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -72,6 +72,7 @@ Airbyte uses a grading system for connectors to help users understand what to ex | [Mixpanel](sources/mixpanel.md) | Beta | | [Mongo DB](sources/mongodb-v2.md) | Beta | | [MySQL](sources/mysql.md) | Certified | +| [Notion](sources/notion.md) | Alpha | | [Okta](sources/okta.md) | Beta | | [Oracle DB](sources/oracle.md) | Certified | | [Oracle PeopleSoft](sources/oracle-peoplesoft.md) | Beta | diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md new file mode 100644 index 000000000000..98c835ec39c3 --- /dev/null +++ b/docs/integrations/sources/notion.md @@ -0,0 +1,62 @@ +# Notion + +## Sync overview + +This source can sync data for the [Notion API](https://developers.notion.com/reference/intro). It supports both Full Refresh and Incremental syncs. You can choose if this connector will copy only the new or updated data, or all rows in the tables and columns you set up for replication, every time a sync is run. + +### Output schema + +This Source is capable of syncing the following core Streams: + +* [Users](https://developers.notion.com/reference/get-users) +* [Databases](https://developers.notion.com/reference/post-search) \(Incremental\) +* [Pages](https://developers.notion.com/reference/post-search) \(Incremental\) +* [Blocks](https://developers.notion.com/reference/get-block-children) \(Incremental\) + +The `Databases` and `Pages` streams are using same `Search` endpoint. + +Notion stores `Blocks` in hierarchical structure, so we use recursive request to get list of blocks. + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `integer` | `integer` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Namespaces | No | | + +### Performance considerations + +The connector is restricted by normal Notion [rate limits and size limits](https://developers.notion.com/reference/errors#request-limits). + +The Notion connector should not run into Notion API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Notion account +* An internal integration in Notion workspace +* Notion internal integration access key + +### Setup guide + +Please register on Notion and follow this [docs](https://developers.notion.com/docs#getting-started) to create an integration, and then grant pages or databases permission to that integration so that API can access their data. + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2021-10-17 | [9999](https://github.com/airbytehq/airbyte/pull/9999) | Initial Release | + + From ebe59e84c890135e95389d1a4bcbc0b5b36bfe44 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Sun, 17 Oct 2021 01:02:46 +1100 Subject: [PATCH 02/10] update PR number in change log --- docs/integrations/sources/notion.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/integrations/sources/notion.md b/docs/integrations/sources/notion.md index 98c835ec39c3..006bf9a02476 100644 --- a/docs/integrations/sources/notion.md +++ b/docs/integrations/sources/notion.md @@ -57,6 +57,6 @@ Please register on Notion and follow this [docs](https://developers.notion.com/d | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.0 | 2021-10-17 | [9999](https://github.com/airbytehq/airbyte/pull/9999) | Initial Release | +| 0.1.0 | 2021-10-17 | [7092](https://github.com/airbytehq/airbyte/pull/7092) | Initial Release | From b19b5ac52c88856ca64b97d058e7f565177840f9 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Thu, 21 Oct 2021 16:49:01 +1100 Subject: [PATCH 03/10] bug fix and code improvement as code review suggestions --- .../integration_tests/acceptance.py | 2 - .../integration_tests/configured_catalog.json | 1 - .../integration_tests/expected_records.txt | 47 ++-- .../source_notion/schemas/blocks.json | 99 ++++++- .../source_notion/schemas/databases.json | 84 +++++- .../source_notion/schemas/pages.json | 11 +- .../source_notion/schemas/shared/child.json | 1 - .../source_notion/schemas/shared/date.json | 2 - .../source_notion/schemas/shared/emoji.json | 2 - .../source_notion/schemas/shared/file.json | 1 - .../source_notion/schemas/shared/heading.json | 1 - .../source_notion/schemas/shared/icon.json | 6 +- .../source_notion/schemas/shared/options.json | 14 +- .../source_notion/schemas/shared/parent.json | 2 - .../schemas/shared/rich_text.json | 35 ++- .../schemas/shared/text_element.json | 1 - .../source_notion/schemas/shared/title.json | 35 ++- .../source_notion/schemas/shared/user.json | 1 - .../source_notion/schemas/users.json | 1 - .../source-notion/source_notion/source.py | 40 +-- .../source-notion/source_notion/streams.py | 256 +++++++++--------- .../unit_tests/test_incremental_streams.py | 168 ++++++++---- .../source-notion/unit_tests/test_source.py | 5 +- .../source-notion/unit_tests/test_streams.py | 18 +- 24 files changed, 517 insertions(+), 316 deletions(-) diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py index 58c194c5d137..108075487440 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py @@ -11,6 +11,4 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """ This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json index f89113659bbf..af9d971465b8 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-notion/integration_tests/configured_catalog.json @@ -38,4 +38,3 @@ } ] } - diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.txt index a6f612bf67ec..b4e1ede42237 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-notion/integration_tests/expected_records.txt @@ -6,31 +6,32 @@ {"stream": "pages", "data": {"object": "page", "id": "560c83a7-6a66-445c-a138-034b66db863d", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "71183a17-ecff-46cd-9da0-afc783ac81e8"}, "archived": false, "properties": {"Date Created": {"id": "'Y6%3C", "type": "created_time", "created_time": "2021-10-10T08:09:00.000Z"}, "Status": {"id": "%5EOE%40", "type": "select", "select": null}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "New Task", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "New Task", "href": null}]}}, "url": "https://www.notion.so/New-Task-560c83a76a66445ca138034b66db863d"}, "emitted_at": 1634387507000} {"stream": "pages", "data": {"object": "page", "id": "ed4eb196-48be-445c-ab2b-2e8e4ccd8937", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "71183a17-ecff-46cd-9da0-afc783ac81e8"}, "archived": false, "properties": {"Date Created": {"id": "'Y6%3C", "type": "created_time", "created_time": "2021-10-10T08:09:00.000Z"}, "Status": {"id": "%5EOE%40", "type": "select", "select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/ed4eb19648be445cab2b2e8e4ccd8937"}, "emitted_at": 1634387507000} {"stream": "pages", "data": {"object": "page", "id": "43f33669-2700-4d6c-8ed5-12aea4455da6", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "cover": null, "icon": {"type": "emoji", "emoji": "\ud83d\udc36"}, "parent": {"type": "database_id", "database_id": "71183a17-ecff-46cd-9da0-afc783ac81e8"}, "archived": false, "properties": {"Date Created": {"id": "'Y6%3C", "type": "created_time", "created_time": "2021-10-10T08:09:00.000Z"}, "Status": {"id": "%5EOE%40", "type": "select", "select": {"id": "2", "name": "Doing", "color": "yellow"}}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Take Fig on a walk", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Take Fig on a walk", "href": null}]}}, "url": "https://www.notion.so/Take-Fig-on-a-walk-43f3366927004d6c8ed512aea4455da6"}, "emitted_at": 1634387507000} -{"stream": "pages", "data": {"object": "page", "id": "9332eb2e-7864-4209-9422-6028ca5af35a", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-14T02:15:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/9332eb2e7864420994226028ca5af35a"}, "emitted_at": 1634387507000} -{"stream": "pages", "data": {"object": "page", "id": "bc21d49f-3318-434e-8bc0-81981d40e78b", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-14T02:15:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "aaa", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "aaa", "href": null}]}}, "url": "https://www.notion.so/aaa-bc21d49f3318434e8bc081981d40e78b"}, "emitted_at": 1634387507000} -{"stream": "pages", "data": {"object": "page", "id": "26b09a78-1413-4a57-bb22-3f069017e9fb", "created_time": "2021-10-14T02:16:00.000Z", "last_edited_time": "2021-10-14T02:16:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/26b09a7814134a57bb223f069017e9fb"}, "emitted_at": 1634387507000} -{"stream": "pages", "data": {"object": "page", "id": "7f91fa0b-ea9b-49de-8036-7c71bfde7bd3", "created_time": "2021-10-14T02:16:00.000Z", "last_edited_time": "2021-10-14T02:16:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/7f91fa0bea9b49de80367c71bfde7bd3"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "9332eb2e-7864-4209-9422-6028ca5af35a", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-14T02:15:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": []}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/9332eb2e7864420994226028ca5af35a"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "bc21d49f-3318-434e-8bc0-81981d40e78b", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-14T02:15:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": []}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "aaa", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "aaa", "href": null}]}}, "url": "https://www.notion.so/aaa-bc21d49f3318434e8bc081981d40e78b"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "26b09a78-1413-4a57-bb22-3f069017e9fb", "created_time": "2021-10-14T02:16:00.000Z", "last_edited_time": "2021-10-14T02:16:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": []}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/26b09a7814134a57bb223f069017e9fb"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "7f91fa0b-ea9b-49de-8036-7c71bfde7bd3", "created_time": "2021-10-14T02:16:00.000Z", "last_edited_time": "2021-10-14T02:16:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": []}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/7f91fa0bea9b49de80367c71bfde7bd3"}, "emitted_at": 1634387507000} {"stream": "pages", "data": {"object": "page", "id": "52612945-9c07-4728-bd9f-6700372409f7", "created_time": "2021-10-14T02:18:00.000Z", "last_edited_time": "2021-10-14T02:18:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "ea141d69-b5b8-4a6b-ae17-828d5baf37ae"}, "archived": false, "properties": {"Status": {"id": "kZec", "type": "select", "select": null}, "Assign": {"id": "mpGX", "type": "people", "people": []}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Card 1", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Card 1", "href": null}]}}, "url": "https://www.notion.so/Card-1-526129459c074728bd9f6700372409f7"}, "emitted_at": 1634387507000} {"stream": "pages", "data": {"object": "page", "id": "8a91862f-9386-452a-a64c-e06dcd4323d6", "created_time": "2021-10-14T02:18:00.000Z", "last_edited_time": "2021-10-14T02:18:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "ea141d69-b5b8-4a6b-ae17-828d5baf37ae"}, "archived": false, "properties": {"Status": {"id": "kZec", "type": "select", "select": null}, "Assign": {"id": "mpGX", "type": "people", "people": []}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Card 2", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Card 2", "href": null}]}}, "url": "https://www.notion.so/Card-2-8a91862f9386452aa64ce06dcd4323d6"}, "emitted_at": 1634387507000} {"stream": "pages", "data": {"object": "page", "id": "ee86d233-543b-4d0b-b73d-6e674ee30421", "created_time": "2021-10-14T02:18:00.000Z", "last_edited_time": "2021-10-14T02:18:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "ea141d69-b5b8-4a6b-ae17-828d5baf37ae"}, "archived": false, "properties": {"Status": {"id": "kZec", "type": "select", "select": null}, "Assign": {"id": "mpGX", "type": "people", "people": []}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "Card 3", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Card 3", "href": null}]}}, "url": "https://www.notion.so/Card-3-ee86d233543b4d0bb73d6e674ee30421"}, "emitted_at": 1634387507000} {"stream": "pages", "data": {"object": "page", "id": "bcebbd8a-21e4-4359-8134-2bda86f59e2d", "created_time": "2021-10-15T04:51:00.000Z", "last_edited_time": "2021-10-15T04:52:00.000Z", "cover": null, "icon": null, "parent": {"type": "page_id", "page_id": "03924866-f383-4d39-a0bb-dec2f69b4e99"}, "archived": false, "properties": {"title": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "subpage", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "subpage", "href": null}]}}, "url": "https://www.notion.so/subpage-bcebbd8a21e4435981342bda86f59e2d"}, "emitted_at": 1634387507000} {"stream": "pages", "data": {"object": "page", "id": "5a67c86f-d0da-4d0a-9dd7-f4cf164e6247", "created_time": "2021-10-15T05:41:00.000Z", "last_edited_time": "2021-10-15T05:49:00.000Z", "cover": null, "icon": null, "parent": {"type": "workspace", "workspace": true}, "archived": false, "properties": {"title": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "test page3", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "test page3", "href": null}]}}, "url": "https://www.notion.so/test-page3-5a67c86fd0da4d0a9dd7f4cf164e6247"}, "emitted_at": 1634387507000} -{"stream": "pages", "data": {"object": "page", "id": "7209939e-21cd-4975-9114-ca7717079422", "created_time": "2021-10-15T05:58:00.000Z", "last_edited_time": "2021-10-15T05:58:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/7209939e21cd49759114ca7717079422"}, "emitted_at": 1634387507000} -{"stream": "pages", "data": {"object": "page", "id": "03924866-f383-4d39-a0bb-dec2f69b4e99", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-16T12:30:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": {"start": "2021-10-14", "end": null}}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": null}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "bbb", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "bbb", "href": null}]}}, "url": "https://www.notion.so/bbb-03924866f3834d39a0bbdec2f69b4e99"}, "emitted_at": 1634387507000} -{"stream": "blocks", "data": {"object": "block", "id": "20b41f7f-7051-4c18-a7c3-2b63e2f54da2", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"text": [{"type": "text", "text": {"content": "To Do", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "To Do", "href": null}]}}, "emitted_at": 1634387508000} -{"stream": "blocks", "data": {"object": "block", "id": "e0e3fc30-c452-44c9-aa10-c3afc7f6fd29", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634387508000} -{"stream": "blocks", "data": {"object": "block", "id": "fb24d227-97db-4933-93cc-b2ea1ddfa41f", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634387508000} -{"stream": "blocks", "data": {"object": "block", "id": "d3a02e8a-b38e-4eb2-a0f4-8c55d099217d", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"text": [{"type": "text", "text": {"content": "Tasks", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Tasks", "href": null}]}}, "emitted_at": 1634387509000} -{"stream": "blocks", "data": {"object": "block", "id": "a75cc58d-2dba-4b90-bfc1-5af214af2200", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634387509000} -{"stream": "blocks", "data": {"object": "block", "id": "0bae298f-e378-4a8f-9f25-4c6feb8506be", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634387509000} -{"stream": "blocks", "data": {"object": "block", "id": "1af3ba11-8339-4c21-9025-9f96948b25dc", "created_time": "2021-10-15T04:52:00.000Z", "last_edited_time": "2021-10-15T04:52:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "subpage content", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "subpage content", "href": null}]}}, "emitted_at": 1634387515000} -{"stream": "blocks", "data": {"object": "block", "id": "fc248547-83ef-4069-b7c9-18897edb7150", "created_time": "2021-10-15T04:52:00.000Z", "last_edited_time": "2021-10-15T04:52:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634387515000} -{"stream": "blocks", "data": {"object": "block", "id": "740d4614-e2e7-4da1-9823-9942a842c944", "created_time": "2021-10-15T05:41:00.000Z", "last_edited_time": "2021-10-15T05:44:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "test content3", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "test content3", "href": null}]}}, "emitted_at": 1634387516000} -{"stream": "blocks", "data": {"object": "block", "id": "665fdcb0-da5a-42d8-a603-5719296ba929", "created_time": "2021-10-15T05:48:00.000Z", "last_edited_time": "2021-10-15T05:48:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634387516000} -{"stream": "blocks", "data": {"object": "block", "id": "05bee6fd-017e-4feb-9a7c-5c7dbf650504", "created_time": "2021-10-14T04:40:00.000Z", "last_edited_time": "2021-10-14T04:40:00.000Z", "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"text": [{"type": "text", "text": {"content": "this is header", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "this is header", "href": null}]}}, "emitted_at": 1634387517000} -{"stream": "blocks", "data": {"object": "block", "id": "3d1207a0-a73a-4475-b23e-9c8a4ca69c49", "created_time": "2021-10-14T04:40:00.000Z", "last_edited_time": "2021-10-15T07:09:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "this is some text2", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "this is some text2", "href": null}]}}, "emitted_at": 1634387517000} -{"stream": "blocks", "data": {"object": "block", "id": "eee06451-d354-48c9-b52f-27ddb5610256", "created_time": "2021-10-16T10:02:00.000Z", "last_edited_time": "2021-10-16T10:02:00.000Z", "has_children": false, "archived": false, "type": "code", "code": {"text": [{"type": "text", "text": {"content": "some code", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "some code", "href": null}], "language": "javascript"}}, "emitted_at": 1634387517000} -{"stream": "blocks", "data": {"object": "block", "id": "7557f377-c20d-4464-b50d-c3498f51e796", "created_time": "2021-10-16T10:04:00.000Z", "last_edited_time": "2021-10-16T10:04:00.000Z", "has_children": false, "archived": false, "type": "callout", "callout": {"text": [{"type": "text", "text": {"content": "some call out", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "some call out", "href": null}], "icon": {"type": "emoji", "emoji": "\ud83d\udca1"}}}, "emitted_at": 1634387517000} -{"stream": "blocks", "data": {"object": "block", "id": "473803b3-0bd6-45b6-a3f0-53535041f98c", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "dadsfadsf", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "dadsfadsf", "href": null}]}}, "emitted_at": 1634387517000} -{"stream": "blocks", "data": {"object": "block", "id": "db8c1f18-28f9-4230-8af3-9377d2e52aaf", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "slaksjdfl", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "slaksjdfl", "href": null}]}}, "emitted_at": 1634387517000} -{"stream": "blocks", "data": {"object": "block", "id": "effd0166-724d-4a1a-8e41-87ee4aaf2499", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634387517000} +{"stream": "pages", "data": {"object": "page", "id": "7209939e-21cd-4975-9114-ca7717079422", "created_time": "2021-10-15T05:58:00.000Z", "last_edited_time": "2021-10-15T05:58:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": null}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": []}, "Name": {"id": "title", "type": "title", "title": []}}, "url": "https://www.notion.so/7209939e21cd49759114ca7717079422"}, "emitted_at": 1634387507000} +{"stream": "pages", "data": {"object": "page", "id": "03924866-f383-4d39-a0bb-dec2f69b4e99", "created_time": "2021-10-14T02:15:00.000Z", "last_edited_time": "2021-10-16T12:30:00.000Z", "cover": null, "icon": null, "parent": {"type": "database_id", "database_id": "8d65239f-a23e-4dc5-a512-1e29f70e0c0b"}, "archived": false, "properties": {"test-property": {"id": "%3CCpP", "type": "date", "date": {"start": "2021-10-14", "end": null}}, "Tags": {"id": "~fcg", "type": "multi_select", "multi_select": []}, "Name": {"id": "title", "type": "title", "title": [{"type": "text", "text": {"content": "bbb", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "bbb", "href": null}]}}, "url": "https://www.notion.so/bbb-03924866f3834d39a0bbdec2f69b4e99"}, "emitted_at": 1634387507000} +{"stream": "blocks", "data": {"object": "block", "id": "20b41f7f-7051-4c18-a7c3-2b63e2f54da2", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"text": [{"type": "text", "text": {"content": "To Do", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "To Do", "href": null}]}}, "emitted_at": 1634793427000} +{"stream": "blocks", "data": {"object": "block", "id": "e0e3fc30-c452-44c9-aa10-c3afc7f6fd29", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634793427000} +{"stream": "blocks", "data": {"object": "block", "id": "fb24d227-97db-4933-93cc-b2ea1ddfa41f", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634793427000} +{"stream": "blocks", "data": {"object": "block", "id": "d3a02e8a-b38e-4eb2-a0f4-8c55d099217d", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "heading_1", "heading_1": {"text": [{"type": "text", "text": {"content": "Tasks", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "Tasks", "href": null}]}}, "emitted_at": 1634793428000} +{"stream": "blocks", "data": {"object": "block", "id": "a75cc58d-2dba-4b90-bfc1-5af214af2200", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634793428000} +{"stream": "blocks", "data": {"object": "block", "id": "0bae298f-e378-4a8f-9f25-4c6feb8506be", "created_time": "2021-10-10T08:09:00.000Z", "last_edited_time": "2021-10-10T08:09:00.000Z", "has_children": false, "archived": false, "type": "to_do", "to_do": {"text": [{"type": "text", "text": {"content": "...", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "...", "href": null}], "checked": false}}, "emitted_at": 1634793428000} +{"stream": "blocks", "data": {"object": "block", "id": "1af3ba11-8339-4c21-9025-9f96948b25dc", "created_time": "2021-10-15T04:52:00.000Z", "last_edited_time": "2021-10-15T04:52:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "subpage content", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "subpage content", "href": null}]}}, "emitted_at": 1634793433000} +{"stream": "blocks", "data": {"object": "block", "id": "fc248547-83ef-4069-b7c9-18897edb7150", "created_time": "2021-10-15T04:52:00.000Z", "last_edited_time": "2021-10-15T04:52:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634793433000} +{"stream": "blocks", "data": {"object": "block", "id": "740d4614-e2e7-4da1-9823-9942a842c944", "created_time": "2021-10-15T05:41:00.000Z", "last_edited_time": "2021-10-15T05:44:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "test content3", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "test content3", "href": null}]}}, "emitted_at": 1634793433000} +{"stream": "blocks", "data": {"object": "block", "id": "665fdcb0-da5a-42d8-a603-5719296ba929", "created_time": "2021-10-15T05:48:00.000Z", "last_edited_time": "2021-10-15T05:48:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634793433000} +{"stream": "blocks", "data": {"object": "block", "id": "3d1207a0-a73a-4475-b23e-9c8a4ca69c49", "created_time": "2021-10-14T04:40:00.000Z", "last_edited_time": "2021-10-15T07:09:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "this is some text2", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "this is some text2", "href": null}]}}, "emitted_at": 1634793434000} +{"stream": "blocks", "data": {"object": "block", "id": "eee06451-d354-48c9-b52f-27ddb5610256", "created_time": "2021-10-16T10:02:00.000Z", "last_edited_time": "2021-10-16T10:02:00.000Z", "has_children": false, "archived": false, "type": "code", "code": {"text": [{"type": "text", "text": {"content": "some code", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "some code", "href": null}], "language": "javascript"}}, "emitted_at": 1634793434000} +{"stream": "blocks", "data": {"object": "block", "id": "7557f377-c20d-4464-b50d-c3498f51e796", "created_time": "2021-10-16T10:04:00.000Z", "last_edited_time": "2021-10-16T10:04:00.000Z", "has_children": false, "archived": false, "type": "callout", "callout": {"text": [{"type": "text", "text": {"content": "some call out", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "some call out", "href": null}], "icon": {"type": "emoji", "emoji": "\ud83d\udca1"}}}, "emitted_at": 1634793434000} +{"stream": "blocks", "data": {"object": "block", "id": "473803b3-0bd6-45b6-a3f0-53535041f98c", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "dadsfadsf", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "dadsfadsf", "href": null}]}}, "emitted_at": 1634793435000} +{"stream": "blocks", "data": {"object": "block", "id": "d472b580-667f-4699-8674-52901ef402b5", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": true, "archived": false, "type": "toggle", "toggle": {"text": [{"type": "text", "text": {"content": "asdfasdf", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "asdfasdf", "href": null}]}}, "emitted_at": 1634793435000} +{"stream": "blocks", "data": {"object": "block", "id": "db8c1f18-28f9-4230-8af3-9377d2e52aaf", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": [{"type": "text", "text": {"content": "slaksjdfl", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "slaksjdfl", "href": null}]}}, "emitted_at": 1634793435000} +{"stream": "blocks", "data": {"object": "block", "id": "effd0166-724d-4a1a-8e41-87ee4aaf2499", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": false, "archived": false, "type": "paragraph", "paragraph": {"text": []}}, "emitted_at": 1634793435000} +{"stream": "blocks", "data": {"object": "block", "id": "5c8a1e06-5050-458d-9e5a-bc47d4a8c5f4", "created_time": "2021-10-16T10:06:00.000Z", "last_edited_time": "2021-10-16T10:06:00.000Z", "has_children": true, "archived": false, "type": "toggle", "toggle": {"text": [{"type": "text", "text": {"content": "sdfsadf", "link": null}, "annotations": {"bold": false, "italic": false, "strikethrough": false, "underline": false, "code": false, "color": "default"}, "plain_text": "sdfsadf", "href": null}]}}, "emitted_at": 1634793435000} diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json index 6ab880a4f0b4..2fb2c36fa83a 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json @@ -22,7 +22,28 @@ "type": "boolean" }, "type": { - "enum": ["paragraph", "heading_1", "heading_2", "heading_3", "callout", "bulleted_list_item", "numbered_list_item", "to_do", "toggle", "code", "child_page", "child_database", "embed", "image", "video", "file", "pdf", "bookmark", "equation", "unsupported"] + "enum": [ + "paragraph", + "heading_1", + "heading_2", + "heading_3", + "callout", + "bulleted_list_item", + "numbered_list_item", + "to_do", + "toggle", + "code", + "child_page", + "child_database", + "embed", + "image", + "video", + "file", + "pdf", + "bookmark", + "equation", + "unsupported" + ] }, "paragraph": { "$ref": "text_element.json" }, "quote": { "$ref": "text_element.json" }, @@ -56,7 +77,80 @@ "properties": { "text": { "type": "array", "items": { "$ref": "rich_text.json" } }, "language": { - "enum": ["abap", "arduino", "bash", "basic", "c", "clojure", "coffeescript", "c++", "c#", "css", "dart", "diff", "docker", "elixir", "elm", "erlang", "flow", "fortran", "f#", "gherkin", "glsl", "go", "graphql", "groovy", "haskell", "html", "java", "javascript", "json", "julia", "kotlin", "latex", "less", "lisp", "livescript", "lua", "makefile", "markdown", "markup", "matlab", "mermaid", "nix", "objective-c", "ocaml", "pascal", "perl", "php", "plain text", "powershell", "prolog", "protobuf", "python", "r", "reason", "ruby", "rust", "sass", "scala", "scheme", "scss", "shell", "sql", "swift", "typescript", "vb.net", "verilog", "vhdl", "visual basic", "webassembly", "xml", "yaml", "java/c/c++/c#"] + "enum": [ + "abap", + "arduino", + "bash", + "basic", + "c", + "clojure", + "coffeescript", + "c++", + "c#", + "css", + "dart", + "diff", + "docker", + "elixir", + "elm", + "erlang", + "flow", + "fortran", + "f#", + "gherkin", + "glsl", + "go", + "graphql", + "groovy", + "haskell", + "html", + "java", + "javascript", + "json", + "julia", + "kotlin", + "latex", + "less", + "lisp", + "livescript", + "lua", + "makefile", + "markdown", + "markup", + "matlab", + "mermaid", + "nix", + "objective-c", + "ocaml", + "pascal", + "perl", + "php", + "plain text", + "powershell", + "prolog", + "protobuf", + "python", + "r", + "reason", + "ruby", + "rust", + "sass", + "scala", + "scheme", + "scss", + "shell", + "sql", + "swift", + "typescript", + "vb.net", + "verilog", + "vhdl", + "visual basic", + "webassembly", + "xml", + "yaml", + "java/c/c++/c#" + ] } } }, @@ -90,4 +184,3 @@ } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json index adaaddfab2ea..d24616a27558 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/databases.json @@ -44,7 +44,23 @@ "additionalProperties": true, "properties": { "id": { "type": "string" }, - "type": { "enum": ["title", "rich_text", "date", "people", "files", "checkbox", "url", "email", "phone_number", "created_time", "created_by", "last_edited_time", "last_edited_by"] }, + "type": { + "enum": [ + "title", + "rich_text", + "date", + "people", + "files", + "checkbox", + "url", + "email", + "phone_number", + "created_time", + "created_by", + "last_edited_time", + "last_edited_by" + ] + }, "name": { "type": "string" } } }, @@ -54,7 +70,46 @@ "id": { "type": "string" }, "type": { "enum": ["number"] }, "name": { "type": "string" }, - "format": { "enum": ["number", "number_with_commas", "percent", "dollar", "canadian_dollar", "euro", "pound", "yen", "ruble", "rupee", "won", "yuan", "real", "lira", "rupiah", "franc", "hong_kong_dollar", "new_zealand_dollar", "krona", "norwegian_krone", "mexican_peso", "rand", "new_taiwan_dollar", "danish_krone", "zloty", "baht", "forint", "koruna", "shekel", "chilean_peso", "philippine_peso", "dirham", "colombian_peso", "riyal", "ringgit", "leu"] } + "format": { + "enum": [ + "number", + "number_with_commas", + "percent", + "dollar", + "canadian_dollar", + "euro", + "pound", + "yen", + "ruble", + "rupee", + "won", + "yuan", + "real", + "lira", + "rupiah", + "franc", + "hong_kong_dollar", + "new_zealand_dollar", + "krona", + "norwegian_krone", + "mexican_peso", + "rand", + "new_taiwan_dollar", + "danish_krone", + "zloty", + "baht", + "forint", + "koruna", + "shekel", + "chilean_peso", + "philippine_peso", + "dirham", + "colombian_peso", + "riyal", + "ringgit", + "leu" + ] + } } }, { @@ -63,7 +118,10 @@ "id": { "type": "string" }, "type": { "enum": ["select", "multi_select"] }, "name": { "type": "string" }, - "options": { "type": "array", "items": { "$ref": "options.json" } } + "options": { + "type": "array", + "items": { "$ref": "options.json" } + } } }, { @@ -96,7 +154,24 @@ "relation_property_id": { "type": "string" }, "rollup_property_name": { "type": "string" }, "rollup_property_id": { "type": "string" }, - "function": { "enum": ["count_all", "count_values", "count_unique_values", "count_empty", "count_not_empty", "percent_empty", "percent_not_empty", "sum", "average", "median", "min", "max", "range", "show_original"] } + "function": { + "enum": [ + "count_all", + "count_values", + "count_unique_values", + "count_empty", + "count_not_empty", + "percent_empty", + "percent_not_empty", + "sum", + "average", + "median", + "min", + "max", + "range", + "show_original" + ] + } } } ] @@ -105,4 +180,3 @@ } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json index ed2b8a9c2409..13df44d0676a 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json @@ -42,7 +42,10 @@ "patternProperties": { "^id$": { "type": "string" }, "^type$": { "enum": ["title", "rich_text"] }, - "^title$|^rich_text$": { "type": "array", "items": { "$ref": "rich_text.json" } } + "^title$|^rich_text$": { + "type": "array", + "items": { "$ref": "rich_text.json" } + } } }, { @@ -66,7 +69,10 @@ "properties": { "id": { "type": "string" }, "type": { "enum": ["multi_select"] }, - "multi_select": { "type": ["null", "array"], "items": { "$ref": "options.json" } } + "multi_select": { + "type": ["null", "array"], + "items": { "$ref": "options.json" } + } } }, { @@ -238,4 +244,3 @@ } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/child.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/child.json index c92be2eb8af2..8d7e475ef9b6 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/child.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/child.json @@ -6,4 +6,3 @@ "title": { "type": "string" } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/date.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/date.json index 07ebba50bc23..94f1e299c8bc 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/date.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/date.json @@ -7,5 +7,3 @@ "end": { "type": ["null", "string"] } } } - - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/emoji.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/emoji.json index 5ecd33417f6a..65658b580945 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/emoji.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/emoji.json @@ -11,5 +11,3 @@ } } } - - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json index 4a842353887d..62a1ffeb0f16 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/file.json @@ -35,4 +35,3 @@ } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json index 9de9b913a479..9bf06eb9189c 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/heading.json @@ -6,4 +6,3 @@ "text": { "type": "array", "items": { "$ref": "rich_text.json" } } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/icon.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/icon.json index 25018eb562dd..afda26b0f776 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/icon.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/icon.json @@ -1,8 +1,4 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "anyOf": [ - { "$ref": "file.json" }, - { "$ref": "emoji.json" } - ] + "anyOf": [{ "$ref": "file.json" }, { "$ref": "emoji.json" }] } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/options.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/options.json index f0bc86d30d5a..eceed867c8de 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/options.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/options.json @@ -10,8 +10,18 @@ "type": "string" }, "color": { - "enum": ["default", "gray", "brown", "orange", "yellow", "green", "blue", "purple", "pink", "red"] + "enum": [ + "default", + "gray", + "brown", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "red" + ] } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json index 2c67b3ef3b52..d792c93fd4c9 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/parent.json @@ -17,5 +17,3 @@ } } } - - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json index 294d881e427e..006b988c658c 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/rich_text.json @@ -17,12 +17,12 @@ "type": ["null", "object"], "additionalProperties": false, "properties": { - "type": { - "enum": ["url"] - }, - "url": { - "type": "string" - } + "type": { + "enum": ["url"] + }, + "url": { + "type": "string" + } } } } @@ -47,7 +47,27 @@ "type": "boolean" }, "color": { - "enum": ["default", "gray", "brown", "orange", "yellow", "green", "blue", "purple", "pink", "red", "gray_background", "brown_background", "orange_background", "yellow_background", "green_background", "blue_background", "purple_background", "pink_background", "red_background"] + "enum": [ + "default", + "gray", + "brown", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "red", + "gray_background", + "brown_background", + "orange_background", + "yellow_background", + "green_background", + "blue_background", + "purple_background", + "pink_background", + "red_background" + ] } } }, @@ -59,4 +79,3 @@ } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json index dbc3aa89b2bc..7f122b53b9c2 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/text_element.json @@ -7,4 +7,3 @@ "children": { "type": "array", "items": { "type": "object" } } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/title.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/title.json index 294d881e427e..006b988c658c 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/title.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/title.json @@ -17,12 +17,12 @@ "type": ["null", "object"], "additionalProperties": false, "properties": { - "type": { - "enum": ["url"] - }, - "url": { - "type": "string" - } + "type": { + "enum": ["url"] + }, + "url": { + "type": "string" + } } } } @@ -47,7 +47,27 @@ "type": "boolean" }, "color": { - "enum": ["default", "gray", "brown", "orange", "yellow", "green", "blue", "purple", "pink", "red", "gray_background", "brown_background", "orange_background", "yellow_background", "green_background", "blue_background", "purple_background", "pink_background", "red_background"] + "enum": [ + "default", + "gray", + "brown", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "red", + "gray_background", + "brown_background", + "orange_background", + "yellow_background", + "green_background", + "blue_background", + "purple_background", + "pink_background", + "red_background" + ] } } }, @@ -59,4 +79,3 @@ } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json index 47cd292d517b..5177ce7dfbe6 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/shared/user.json @@ -46,4 +46,3 @@ } } } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/users.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/users.json index 0df83c2a7618..4cba16c2e1e0 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/users.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/users.json @@ -2,4 +2,3 @@ "$schema": "http://json-schema.org/draft-04/schema#", "$ref": "user.json" } - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/source.py b/airbyte-integrations/connectors/source-notion/source_notion/source.py index d1ec3db24b8c..22900dbbccfe 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/source.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/source.py @@ -3,19 +3,16 @@ # -from abc import ABC -from typing import Any, Iterator, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, List, Mapping, Tuple import requests from airbyte_cdk.logger import AirbyteLogger -from airbyte_cdk.models import SyncMode, ConfiguredAirbyteStream, AirbyteMessage +from airbyte_cdk.models import SyncMode from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -from airbyte_cdk.sources.utils.schema_helpers import InternalConfig -from .streams import Databases, Pages, Blocks, Users +from .streams import Blocks, Databases, Pages, Users class SourceNotion(AbstractSource): @@ -33,38 +30,9 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: AirbyteLogger().log("INFO", f"Using start_date: {config['start_date']}") authenticator = TokenAuthenticator(config["access_token"]) - args = { "authenticator": authenticator, "config": config } + args = {"authenticator": authenticator, "config": config} pages = Pages(**args) blocks = Blocks(parent=pages, **args) return [Users(**args), Databases(**args), pages, blocks] - - # Override AbstractSource._read_incremental() function to emit an extra state - # message after finishing syncing the whole stream. - # It is for updating the maximum cursor field date state, because Notion's - # block isn't sorted between stream slices. There is no way to get the - # maximum cursor field date unless reading until the end of the stream. - # This is a dirty hack and might be removed if HttpStream provided an - # end-of-synching-stream event hook in the future. - def _read_incremental( - self, - logger: AirbyteLogger, - stream: Stream, - configured_stream: ConfiguredAirbyteStream, - connector_state: MutableMapping[str, Any], - internal_config: InternalConfig, - ) -> Iterator[AirbyteMessage]: - yield from super()._read_incremental(logger, stream, configured_stream, connector_state, internal_config) - - # only do this hack for Blocks stream - if not isinstance(stream, Blocks): - return - - stream_name = configured_stream.stream.name - stream_state = connector_state.get(stream_name, {}) - state_date = stream_state.get(stream.cursor_field, stream.start_date) - stream_state = { stream.cursor_field: max(state_date, stream.max_cursor_time) } - - yield self._checkpoint_state(stream_name, stream_state, connector_state, logger) - diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index c753cb3222af..821c9ddafc28 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -1,15 +1,16 @@ -import time +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from datetime import datetime +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, TypeVar +import pydantic import requests from airbyte_cdk.models import SyncMode -from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream -from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator -# maximum block recursive hierarchy depth +# maximum block hierarchy recursive request depth MAX_BLOCK_DEPTH = 30 @@ -19,19 +20,14 @@ class NotionStream(HttpStream, ABC): primary_key = "id" - page_size = 100 # set by Notion API + page_size = 100 # set by Notion API spec def __init__(self, config: Mapping[str, Any], **kwargs): super().__init__(**kwargs) self.start_date = config["start_date"] - def request_headers( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None - ) -> Mapping[str, Any]: - params = super().request_headers(stream_state, stream_slice, next_page_token) + def request_headers(self, **kwargs) -> Mapping[str, Any]: + params = super().request_headers(**kwargs) # Notion API version, see https://developers.notion.com/reference/versioning params["Notion-Version"] = "2021-08-16" return params @@ -40,28 +36,45 @@ def next_page_token( self, response: requests.Response, ) -> Optional[Mapping[str, Any]]: + """ + An example of response: + { + "next_cursor": "fe2cc560-036c-44cd-90e8-294d5a74cebc", + "has_more": true, + "results": [ ... ] + } + Doc: https://developers.notion.com/reference/pagination + """ next_cursor = response.json()["next_cursor"] if next_cursor: - return { "next_cursor": next_cursor } + return {"next_cursor": next_cursor} - # default record filter, do nothing - def filter_by_state( - self, - stream_state: Mapping[str, Any] = None, - record: Mapping[str, Any] = None - ) -> Iterable: - yield record - - def parse_response( - self, - response: requests.Response, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: data = response.json().get("results") - for record in data: - yield from self.filter_by_state(stream_state=stream_state, record=record) + yield from data + + +T = TypeVar("T") + + +class StateValueWrapper(pydantic.BaseModel): + + stream: T + state_value: str + max_cursor_time = "" + + def __repr__(self): + """Overrides print view""" + return self.value + + @property + def value(self) -> str: + """Return max cursor time after stream sync is finished.""" + return self.max_cursor_time if self.stream.is_finished else self.state_value + + def dict(self, **kwargs): + """Overrides default logic to return current value only.""" + return {pydantic.utils.ROOT_KEY: self.value} class IncrementalNotionStream(NotionStream, ABC): @@ -70,64 +83,68 @@ class IncrementalNotionStream(NotionStream, ABC): http_method = "POST" + # whether the whole stream sync is finished + is_finished = True + def __init__(self, obj_type: Optional[str] = None, **kwargs): super().__init__(**kwargs) + + # object type for search filtering, either "page" or "database" if not None self.obj_type = obj_type - self.filter_on = True - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: + def path(self, **kwargs) -> str: return "search" - def request_body_json( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Optional[Mapping]: + def request_body_json(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> Optional[Mapping]: if not self.obj_type: return + # search request body + # Docs: https://developers.notion.com/reference/post-search body = { - "sort": { "direction": "ascending", "timestamp": "last_edited_time" }, - "filter": { "property": "object", "value": self.obj_type }, - "page_size": self.page_size + "sort": {"direction": "ascending", "timestamp": "last_edited_time"}, + "filter": {"property": "object", "value": self.obj_type}, + "page_size": self.page_size, } if next_page_token: body["start_cursor"] = next_page_token["next_cursor"] return body - def filter_by_state( - self, - stream_state: Mapping[str, Any] = None, - record: Mapping[str, Any] = None - ) -> Iterable: - if not self.filter_on: - yield record - return + def read_records(self, sync_mode: SyncMode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Mapping[str, Any]]: + if sync_mode == SyncMode.full_refresh: + stream_state = None + return super().read_records(sync_mode, stream_state=stream_state, **kwargs) - value = "" - if record: - value = record.get(self.cursor_field, value) - if not stream_state or value >= stream_state.get(self.cursor_field, ""): - yield record + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + records = super().parse_response(response, stream_state=stream_state, **kwargs) + for record in records: + record_lmd = record.get(self.cursor_field, "") + state_lmd = stream_state.get(self.cursor_field, "") + if isinstance(state_lmd, StateValueWrapper): + state_lmd = state_lmd.value + if not stream_state or record_lmd >= state_lmd: + yield record def get_updated_state( self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any], ) -> Mapping[str, Any]: - state_date = current_stream_state.get(self.cursor_field, self.start_date) - record_date = latest_record.get(self.cursor_field, self.start_date) - return { self.cursor_field: max(state_date, record_date) } + state_value = (current_stream_state or {}).get(self.cursor_field, "") + if not isinstance(state_value, StateValueWrapper): + state_value = StateValueWrapper(stream=self, state_value=state_value) + + record_time = latest_record.get(self.cursor_field, self.start_date) + state_value.max_cursor_time = max(state_value.max_cursor_time, record_time) + + return {self.cursor_field: state_value} class Users(NotionStream): + """ + Docs: https://developers.notion.com/reference/get-users + """ def path( self, @@ -135,49 +152,42 @@ def path( stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None, ) -> str: - return "users" + return self.name class Databases(IncrementalNotionStream): + """ + Docs: https://developers.notion.com/reference/post-search + """ def __init__(self, **kwargs): - super().__init__(obj_type = "database", **kwargs) + super().__init__(obj_type="database", **kwargs) class Pages(IncrementalNotionStream): + """ + Docs: https://developers.notion.com/reference/post-search + """ def __init__(self, **kwargs): - super().__init__(obj_type = "page", **kwargs) + super().__init__(obj_type="page", **kwargs) class Blocks(HttpSubStream, IncrementalNotionStream): + """ + Docs: https://developers.notion.com/reference/get-block-children + """ http_method = "GET" - def __init__(self, **kwargs): - super().__init__(**kwargs) - - # block id stack for block hierarchy traversal - self.block_id_stack = [] + # block id stack for block hierarchy traversal + block_id_stack = [] - # largest time in cursor field across all stream slices - self.max_cursor_time = self.start_date - - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: + def path(self, **kwargs) -> str: return f"blocks/{self.block_id_stack[-1]}/children" - def request_params( - self, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> MutableMapping[str, Any]: - params = { "page_size": self.page_size } + def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: + params = {"page_size": self.page_size} if next_page_token: params["start_cursor"] = next_page_token["next_cursor"] return params @@ -188,60 +198,38 @@ def stream_slices( cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, ) -> Iterable[Optional[Mapping[str, Any]]]: - # turn off parent's record filter to get full list of parents, because - # children's changed time may not propagated to parent - self.parent.filter_on = False - - parents = super().stream_slices(sync_mode, cursor_field, stream_state) - for item in parents: - parent_id = item["parent"]["id"] - self.block_id_stack.append(parent_id) - yield { "page_id": parent_id } - yield from [] - - def filter_by_state( - self, - stream_state: Mapping[str, Any] = None, - record: Mapping[str, Any] = None - ) -> Iterable: - # pages and databases blocks are already fetched in their streams, no + # gather parent stream records in full + slices = list(super().stream_slices(SyncMode.full_refresh, cursor_field, stream_state)) + + self.is_finished = False + for page in slices: + page_id = page["parent"]["id"] + self.block_id_stack.append(page_id) + + # stream sync is finished when it is on the last slice + self.is_finished = page_id == slices[-1]["parent"]["id"] + + yield {"page_id": page_id} + + def parse_response(self, response: requests.Response, stream_state: Mapping[str, Any], **kwargs) -> Iterable[Mapping]: + # pages and databases blocks are already fetched in their streams, so no # need to do it again - for item in super().filter_by_state(stream_state, record): - if item["type"] not in ("child_page", "child_database"): - yield item + records = super().parse_response(response, stream_state=stream_state, **kwargs) + for record in records: + if record["type"] not in ("child_page", "child_database"): + yield record - def read_records( - self, - sync_mode: SyncMode, - cursor_field: List[str] = None, - stream_slice: Mapping[str, Any] = None, - stream_state: Mapping[str, Any] = None, - ) -> Iterable[Mapping[str, Any]]: + def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: # if reached recursive limit, don't read any more if len(self.block_id_stack) > MAX_BLOCK_DEPTH: - yield from [] return - records = super().read_records(sync_mode, cursor_field, stream_slice, stream_state) + records = super().read_records(**kwargs) for record in records: if record["has_children"]: + # do the depth first traversal recursive call, get children blocks self.block_id_stack.append(record["id"]) - yield from self.read_records(sync_mode, cursor_field, stream_slice, stream_state) - else: - yield record + yield from self.read_records(**kwargs) + yield record self.block_id_stack.pop() - - yield from [] - - def get_updated_state( - self, - current_stream_state: MutableMapping[str, Any], - latest_record: Mapping[str, Any], - ) -> Mapping[str, Any]: - # we don't update state here, just keep a record of maximum cursor field - # date, state will be actually updated after syncing the whole stream - record_date = latest_record.get(self.cursor_field, self.start_date) - self.max_cursor_time = max(self.max_cursor_time, record_date) - return current_stream_state - diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py index fffe15ca15d6..36ebd7cd36b1 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_incremental_streams.py @@ -2,11 +2,11 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # +from unittest.mock import MagicMock -import requests from airbyte_cdk.models import SyncMode from pytest import fixture -from source_notion.streams import IncrementalNotionStream, Pages, Blocks +from source_notion.streams import Blocks, IncrementalNotionStream, Pages @fixture @@ -19,13 +19,7 @@ def patch_incremental_base_class(mocker): @fixture def args(): - return { - "authenticator": None, - "config": { - "access_token": "", - "start_date": "2021-01-01T00:00:00.000Z" - } - } + return {"authenticator": None, "config": {"access_token": "", "start_date": "2021-01-01T00:00:00.000Z"}} @fixture @@ -49,31 +43,76 @@ def test_cursor_field(stream): def test_get_updated_state(stream): - inputs = { - "current_stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, - "latest_record": { "last_edited_time": "2021-10-20T00:00:00.000Z" } - } - expected_state = { "last_edited_time": "2021-10-20T00:00:00.000Z" } - assert stream.get_updated_state(**inputs) == expected_state + stream.is_finished = False inputs = { - "current_stream_state": { "last_edited_time": "2021-10-20T00:00:00.000Z" }, - "latest_record": { "last_edited_time": "2021-10-10T00:00:00.000Z" } + "current_stream_state": {"last_edited_time": "2021-10-10T00:00:00.000Z"}, + "latest_record": {"last_edited_time": "2021-10-20T00:00:00.000Z"}, } - assert stream.get_updated_state(**inputs) == expected_state + expected_state = "2021-10-10T00:00:00.000Z" + state = stream.get_updated_state(**inputs) + assert state["last_edited_time"].value == expected_state + + inputs = {"current_stream_state": state, "latest_record": {"last_edited_time": "2021-10-30T00:00:00.000Z"}} + state = stream.get_updated_state(**inputs) + assert state["last_edited_time"].value == expected_state + + # after stream sync is finished, state should output the max cursor time + stream.is_finished = True + inputs = {"current_stream_state": state, "latest_record": {"last_edited_time": "2021-10-10T00:00:00.000Z"}} + expected_state = "2021-10-30T00:00:00.000Z" + state = stream.get_updated_state(**inputs) + assert state["last_edited_time"].value == expected_state def test_stream_slices(blocks, requests_mock): stream = blocks - requests_mock.post("https://api.notion.com/v1/search", json={ - "results": [ { "id": "aaa" }, { "id": "bbb" } ], - "next_cursor": None - }) + requests_mock.post("https://api.notion.com/v1/search", json={"results": [{"id": "aaa"}, {"id": "bbb"}], "next_cursor": None}) inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} - expected_stream_slice = [ { "page_id": "aaa" }, { "page_id": "bbb" } ] + expected_stream_slice = [{"page_id": "aaa"}, {"page_id": "bbb"}] assert list(stream.stream_slices(**inputs)) == expected_stream_slice +def test_end_of_stream_state(blocks, requests_mock): + stream = blocks + requests_mock.post( + "https://api.notion.com/v1/search", json={"results": [{"id": "aaa"}, {"id": "bbb"}, {"id": "ccc"}], "next_cursor": None} + ) + requests_mock.get( + "https://api.notion.com/v1/blocks/aaa/children", + json={ + "results": [{"id": "block 1", "type": "heading_1", "has_children": False, "last_edited_time": "2021-10-30T00:00:00.000Z"}], + "next_cursor": None, + }, + ) + requests_mock.get( + "https://api.notion.com/v1/blocks/bbb/children", + json={ + "results": [{"id": "block 2", "type": "heading_1", "has_children": False, "last_edited_time": "2021-10-20T00:00:00.000Z"}], + "next_cursor": None, + }, + ) + requests_mock.get( + "https://api.notion.com/v1/blocks/ccc/children", + json={ + "results": [{"id": "block 3", "type": "heading_1", "has_children": False, "last_edited_time": "2021-10-10T00:00:00.000Z"}], + "next_cursor": None, + }, + ) + + state = {"last_edited_time": "2021-10-01T00:00:00.000Z"} + sync_mode = SyncMode.incremental + + for idx, app_slice in enumerate(stream.stream_slices(sync_mode, **MagicMock())): + for record in stream.read_records(sync_mode=sync_mode, stream_slice=app_slice): + state = stream.get_updated_state(state, record) + state_value = state["last_edited_time"].value + if idx == 2: # the last slice + assert state_value == "2021-10-30T00:00:00.000Z" + else: + assert state_value == "2021-10-01T00:00:00.000Z" + + def test_supports_incremental(stream, mocker): mocker.patch.object(IncrementalNotionStream, "cursor_field", "dummy_field") assert stream.supports_incremental @@ -90,54 +129,63 @@ def test_stream_checkpoint_interval(stream): def test_request_params(blocks): stream = blocks - inputs = { "stream_state": {}, "next_page_token": { "next_cursor": "aaa" } } - expected_request_params = { "page_size": 100, "start_cursor": "aaa" } + inputs = {"stream_state": {}, "next_page_token": {"next_cursor": "aaa"}} + expected_request_params = {"page_size": 100, "start_cursor": "aaa"} assert stream.request_params(**inputs) == expected_request_params -def test_filter_by_state(stream): - inputs = { - "stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, - "record": { "last_edited_time": "2021-10-20T00:00:00.000Z" } - } - expected_filter_by_state = [{ "last_edited_time": "2021-10-20T00:00:00.000Z" }] - assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state - - inputs = { - "stream_state": { "last_edited_time": "2021-10-20T00:00:00.000Z" }, - "record": { "last_edited_time": "2021-10-10T00:00:00.000Z" } - } - expected_filter_by_state = [] - assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state - - -def test_filter_by_state_blocks(blocks): +def test_record_filter(blocks, requests_mock): stream = blocks + sync_mode = SyncMode.incremental + + root = "aaa" + record = {"id": "id1", "type": "heading_1", "has_children": False, "last_edited_time": "2021-10-20T00:00:00.000Z"} + requests_mock.get(f"https://api.notion.com/v1/blocks/{root}/children", json={"results": [record], "next_cursor": None}) inputs = { - "stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, - "record": { "last_edited_time": "2021-10-20T00:00:00.000Z", "type": "aaa" } + "sync_mode": sync_mode, + "stream_state": {"last_edited_time": "2021-10-10T00:00:00.000Z"}, } - expected_filter_by_state = [{ "last_edited_time": "2021-10-20T00:00:00.000Z", "type": "aaa" }] - assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state + stream.block_id_stack = [root] + assert next(stream.read_records(**inputs)) == record inputs = { - "stream_state": { "last_edited_time": "2021-10-20T00:00:00.000Z" }, - "record": { "last_edited_time": "2021-10-10T00:00:00.000Z", "type": "aaa" } + "sync_mode": sync_mode, + "stream_state": {"last_edited_time": "2021-10-30T00:00:00.000Z"}, } - expected_filter_by_state = [] - assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state + stream.block_id_stack = [root] + assert list(stream.read_records(**inputs)) == [] # 'child_page' and 'child_database' should not be included + record["type"] = "child_page" inputs = { - "stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, - "record": { "last_edited_time": "2021-10-20T00:00:00.000Z", "type": "child_page" } + "sync_mode": sync_mode, + "stream_state": {"last_edited_time": "2021-10-10T00:00:00.000Z"}, } - expected_filter_by_state = [] - assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state - inputs = { - "stream_state": { "last_edited_time": "2021-10-10T00:00:00.000Z" }, - "record": { "last_edited_time": "2021-10-20T00:00:00.000Z", "type": "child_database" } - } - expected_filter_by_state = [] - assert list(stream.filter_by_state(**inputs)) == expected_filter_by_state + stream.block_id_stack = [root] + assert list(stream.read_records(**inputs)) == [] + record["type"] = "child_database" + stream.block_id_stack = [root] + assert list(stream.read_records(**inputs)) == [] + + +def test_recursive_read(blocks, requests_mock): + stream = blocks + + # block records tree: + # + # root |-> record1 -> record2 -> record3 + # |-> record4 + + root = "aaa" + record1 = {"id": "id1", "type": "heading_1", "has_children": True, "last_edited_time": ""} + record2 = {"id": "id2", "type": "heading_1", "has_children": True, "last_edited_time": ""} + record3 = {"id": "id3", "type": "heading_1", "has_children": False, "last_edited_time": ""} + record4 = {"id": "id4", "type": "heading_1", "has_children": False, "last_edited_time": ""} + requests_mock.get(f"https://api.notion.com/v1/blocks/{root}/children", json={"results": [record1, record4], "next_cursor": None}) + requests_mock.get(f"https://api.notion.com/v1/blocks/{record1['id']}/children", json={"results": [record2], "next_cursor": None}) + requests_mock.get(f"https://api.notion.com/v1/blocks/{record2['id']}/children", json={"results": [record3], "next_cursor": None}) + + inputs = {"sync_mode": SyncMode.incremental} + stream.block_id_stack = [root] + assert list(stream.read_records(**inputs)) == [record3, record2, record1, record4] diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py index ced560c4540f..b2074e503d39 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_source.py @@ -10,10 +10,7 @@ def test_check_connection(mocker, requests_mock): source = SourceNotion() logger_mock, config_mock = MagicMock(), MagicMock() - requests_mock.get("https://api.notion.com/v1/users", json={ - "results": [ { "id": "aaa" } ], - "next_cursor": None - }) + requests_mock.get("https://api.notion.com/v1/users", json={"results": [{"id": "aaa"}], "next_cursor": None}) assert source.check_connection(logger_mock, config_mock) == (True, None) diff --git a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py index 5a1960eafeeb..61fd6f16ed48 100644 --- a/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-notion/unit_tests/test_streams.py @@ -2,11 +2,11 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -import requests from http import HTTPStatus from unittest.mock import MagicMock import pytest +import requests from source_notion.streams import NotionStream @@ -27,27 +27,25 @@ def test_request_params(patch_base_class): def test_next_page_token(patch_base_class, requests_mock): stream = NotionStream(config=MagicMock()) - requests_mock.get("https://dummy", json={ "next_cursor": "aaa" }) - inputs = { "response": requests.get("https://dummy") } - expected_token = { "next_cursor": "aaa" } + requests_mock.get("https://dummy", json={"next_cursor": "aaa"}) + inputs = {"response": requests.get("https://dummy")} + expected_token = {"next_cursor": "aaa"} assert stream.next_page_token(**inputs) == expected_token def test_parse_response(patch_base_class, requests_mock): stream = NotionStream(config=MagicMock()) - requests_mock.get("https://dummy", json={ - "results": [{ "a": 123 }, { "b": "xx" }] - }) + requests_mock.get("https://dummy", json={"results": [{"a": 123}, {"b": "xx"}]}) resp = requests.get("https://dummy") - inputs = { "response": resp, "stream_state": MagicMock() } - expected_parsed_object = [{ "a": 123 }, { "b": "xx" }] + inputs = {"response": resp, "stream_state": MagicMock()} + expected_parsed_object = [{"a": 123}, {"b": "xx"}] assert list(stream.parse_response(**inputs)) == expected_parsed_object def test_request_headers(patch_base_class): stream = NotionStream(config=MagicMock()) inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - expected_headers = { "Notion-Version": "2021-08-16" } + expected_headers = {"Notion-Version": "2021-08-16"} assert stream.request_headers(**inputs) == expected_headers From 35a3261b936cb6fe915483ce0797538b8ea803e5 Mon Sep 17 00:00:00 2001 From: Bo Lu Date: Sat, 23 Oct 2021 12:17:12 +1100 Subject: [PATCH 04/10] code improvement as review advices --- .../source-notion/acceptance-test-config.yml | 5 ----- .../source-notion/source_notion/schemas/blocks.json | 2 +- .../connectors/source-notion/source_notion/streams.py | 11 +++-------- 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml index efac61848521..5875984c6edb 100644 --- a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml @@ -15,11 +15,6 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - expect_records: - path: "integration_tests/expected_records.txt" - extra_fields: no - exact_order: no - extra_records: yes incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json index 2fb2c36fa83a..4bc8809113fd 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/blocks.json @@ -19,7 +19,7 @@ "type": "boolean" }, "has_children": { - "type": "boolean" + "type": ["null", "boolean"] }, "type": { "enum": [ diff --git a/airbyte-integrations/connectors/source-notion/source_notion/streams.py b/airbyte-integrations/connectors/source-notion/source_notion/streams.py index 821c9ddafc28..3643152b94cf 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/streams.py +++ b/airbyte-integrations/connectors/source-notion/source_notion/streams.py @@ -146,13 +146,8 @@ class Users(NotionStream): Docs: https://developers.notion.com/reference/get-users """ - def path( - self, - stream_state: Mapping[str, Any] = None, - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> str: - return self.name + def path(self, **kwargs) -> str: + return "users" class Databases(IncrementalNotionStream): @@ -226,7 +221,7 @@ def read_records(self, **kwargs) -> Iterable[Mapping[str, Any]]: records = super().read_records(**kwargs) for record in records: - if record["has_children"]: + if record.get("has_children", False): # do the depth first traversal recursive call, get children blocks self.block_id_stack.append(record["id"]) yield from self.read_records(**kwargs) From 8b39df1ad8172f5e2459383b022fb5f62358558c Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 17 Nov 2021 23:13:14 -0300 Subject: [PATCH 05/10] new connector notion --- .../source_acceptance_test/tests/test_full_refresh.py | 11 ----------- .../source-notion/acceptance-test-config.yml | 2 ++ .../source-notion/integration_tests/acceptance.py | 2 +- .../source-notion/source_notion/schemas/pages.json | 2 +- tools/bin/ci_credentials.sh | 1 + 5 files changed, 5 insertions(+), 13 deletions(-) diff --git a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py index 279a451ea1d5..1d78264ef026 100644 --- a/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py +++ b/airbyte-integrations/bases/source-acceptance-test/source_acceptance_test/tests/test_full_refresh.py @@ -37,16 +37,6 @@ def test_sequential_reads( for record in records_2: records_by_stream_2[record.stream].append(record.data) -<<<<<<< HEAD - output_diff = set(map(serialize, records_1)) - set(map(serialize, records_2)) - if output_diff: - detailed_logger.info(f"record_1: {records_1}") - detailed_logger.info(f"record_2: {records_2}") - msg = "The two sequential reads should produce either equal set of records or one of them is a strict subset of the other" - detailed_logger.info(msg) - detailed_logger.log_json_list(output_diff) - pytest.fail(msg) -======= for stream in records_by_stream_1.keys(): serializer = partial(make_hashable, exclude_fields=ignored_fields.get(stream)) stream_records_1 = records_by_stream_1.get(stream) @@ -62,4 +52,3 @@ def test_sequential_reads( detailed_logger.info("Difference") detailed_logger.log_json_list(output_diff) pytest.fail(msg) ->>>>>>> 1dcd525eb812fee389c50802d275a1c223a8a8f0 diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml index 5875984c6edb..ba1e0164d53c 100644 --- a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml @@ -22,3 +22,5 @@ tests: full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + "pages": ["icon"] \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py index 108075487440..0347f2a0b143 100644 --- a/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-notion/integration_tests/acceptance.py @@ -10,5 +10,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): - """ This fixture is a placeholder for external resources that acceptance test might require.""" + """This fixture is a placeholder for external resources that acceptance test might require.""" yield diff --git a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json index 9f299084d13f..ce59b0037eae 100644 --- a/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json +++ b/airbyte-integrations/connectors/source-notion/source_notion/schemas/pages.json @@ -1,7 +1,7 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "additionalProperties": false, + "additionalProperties": true, "properties": { "object": { "enum": ["page"] diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index cd586e0070fe..14690043fb86 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -246,6 +246,7 @@ read_secrets source-monday "$SOURCE_MONDAY_TEST_CREDS" read_secrets source-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json" read_secrets source-mongodb-v2 "$MONGODB_TEST_CREDS" "credentials.json" read_secrets source-mssql "$MSSQL_RDS_TEST_CREDS" +read_secrets source-mssql "$SOURCE_NOTION" read_secrets source-okta "$SOURCE_OKTA_TEST_CREDS" read_secrets source-onesignal "$SOURCE_ONESIGNAL_TEST_CREDS" read_secrets source-plaid "$PLAID_INTEGRATION_TEST_CREDS" From 22cd2dc0ac04a4e63781061722a20e57fcd96d84 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 17 Nov 2021 23:18:25 -0300 Subject: [PATCH 06/10] format --- airbyte-integrations/builds.md | 5 ----- .../destination/e2e_test/FailAfterNDestination.java | 11 +++++++---- .../source-salesforce/source_salesforce/streams.py | 1 - 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/airbyte-integrations/builds.md b/airbyte-integrations/builds.md index 744cf592e29c..977331c8de4f 100644 --- a/airbyte-integrations/builds.md +++ b/airbyte-integrations/builds.md @@ -59,13 +59,8 @@ | Mongo DB | [![source-mongodb-v2](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mongodb-v2%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mongodb-v2) | | Monday | [![source-monday](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-monday%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-monday) | | MySQL | [![source-mysql](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-mysql%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-mysql) | -<<<<<<< HEAD -| OneSignal | [![source-onesignal](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-onesignal%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-onesignal) | -| Notion | [![source-notion](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-notion%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-notion) | -======= | Notion | [![source-notion](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-notion%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-notion) | | OneSignal | [![source-onesignal](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-onesignal%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-onesignal) | ->>>>>>> 1dcd525eb812fee389c50802d275a1c223a8a8f0 | Oracle DB | [![source-oracle](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-oracle%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-oracle) | | Paypal Transaction | [![paypal-transaction](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paypal-transaction%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paypal-transaction) | | Paystack | [![source-paystack](https://img.shields.io/endpoint?url=https%3A%2F%2Fdnsgjos7lj2fu.cloudfront.net%2Ftests%2Fsummary%2Fsource-paystack%2Fbadge.json)](https://dnsgjos7lj2fu.cloudfront.net/tests/summary/source-paystack) | diff --git a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java index e45fcdaf06f3..4a4a577b5eba 100644 --- a/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java +++ b/airbyte-integrations/connectors/destination-e2e-test/src/main/java/io/airbyte/integrations/destination/e2e_test/FailAfterNDestination.java @@ -1,6 +1,8 @@ -package io.airbyte.integrations.destination.e2e_test; +/* + * Copyright (c) 2021 Airbyte, Inc., all rights reserved. + */ -import static java.lang.Thread.sleep; +package io.airbyte.integrations.destination.e2e_test; import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.integrations.BaseConnector; @@ -26,8 +28,8 @@ public AirbyteConnectionStatus check(final JsonNode config) { @Override public AirbyteMessageConsumer getConsumer(final JsonNode config, - final ConfiguredAirbyteCatalog catalog, - final Consumer outputRecordCollector) { + final ConfiguredAirbyteCatalog catalog, + final Consumer outputRecordCollector) { return new FailAfterNConsumer(config.get("num_messages").asLong(), outputRecordCollector); } @@ -66,4 +68,5 @@ public void accept(final AirbyteMessage message) throws Exception { public void close() {} } + } diff --git a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py index 26a12c5a051d..8f542a755ffc 100644 --- a/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py +++ b/airbyte-integrations/connectors/source-salesforce/source_salesforce/streams.py @@ -257,7 +257,6 @@ def read_records( break - class IncrementalSalesforceStream(SalesforceStream, ABC): state_checkpoint_interval = 500 From 16a33aa7952347e919f710926be71431c5515c76 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Wed, 17 Nov 2021 23:29:32 -0300 Subject: [PATCH 07/10] correct creds file --- tools/bin/ci_credentials.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/bin/ci_credentials.sh b/tools/bin/ci_credentials.sh index 14690043fb86..d8b21fb0ebac 100755 --- a/tools/bin/ci_credentials.sh +++ b/tools/bin/ci_credentials.sh @@ -246,7 +246,7 @@ read_secrets source-monday "$SOURCE_MONDAY_TEST_CREDS" read_secrets source-mongodb-strict-encrypt "$MONGODB_TEST_CREDS" "credentials.json" read_secrets source-mongodb-v2 "$MONGODB_TEST_CREDS" "credentials.json" read_secrets source-mssql "$MSSQL_RDS_TEST_CREDS" -read_secrets source-mssql "$SOURCE_NOTION" +read_secrets source-notion "$SOURCE_NOTION_TEST_CREDS" read_secrets source-okta "$SOURCE_OKTA_TEST_CREDS" read_secrets source-onesignal "$SOURCE_ONESIGNAL_TEST_CREDS" read_secrets source-plaid "$PLAID_INTEGRATION_TEST_CREDS" From 7e1f64a5028d2f3c62b4741460cc13772839fce4 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 18 Nov 2021 00:17:15 -0300 Subject: [PATCH 08/10] run seed --- .../connectors/source-notion/acceptance-test-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml index ba1e0164d53c..df6161d2262b 100644 --- a/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-notion/acceptance-test-config.yml @@ -23,4 +23,4 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" ignored_fields: - "pages": ["icon"] \ No newline at end of file + "pages": ["icon"] From 3edfdef199b2c1ce64f5225391071c3bc901c542 Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 18 Nov 2021 01:02:11 -0300 Subject: [PATCH 09/10] bump connector version --- .../6e00b415-b02e-4160-bf02-58176a0ae687.json | 8 ++++++ .../resources/seed/source_definitions.yaml | 6 ++++ .../src/main/resources/seed/source_specs.yaml | 28 +++++++++++++++++++ 3 files changed, 42 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6e00b415-b02e-4160-bf02-58176a0ae687.json diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6e00b415-b02e-4160-bf02-58176a0ae687.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6e00b415-b02e-4160-bf02-58176a0ae687.json new file mode 100644 index 000000000000..e2525b28e136 --- /dev/null +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6e00b415-b02e-4160-bf02-58176a0ae687.json @@ -0,0 +1,8 @@ +{ + "sourceDefinitionId": "6e00b415-b02e-4160-bf02-58176a0ae687", + "name": "Notion", + "dockerRepository": "airbyte/source-notion", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-notion" + } + \ No newline at end of file diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c3e8317dc3e7..ba21a21aa0b6 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -370,6 +370,12 @@ documentationUrl: https://docs.airbyte.io/integrations/sources/mysql icon: mysql.svg sourceType: database +- name: Notion + sourceDefinitionId: 6e00b415-b02e-4160-bf02-58176a0ae687 + dockerRepository: airbyte/source-notion + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/notion + sourceType: api - name: Okta sourceDefinitionId: 1d4fdb25-64fc-4569-92da-fcdca79a8372 dockerRepository: airbyte/source-okta diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index eefc469674fa..e0efbc5f6640 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3841,6 +3841,34 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-notion:0.1.0" + spec: + documentationUrl: "https://docsurl.com" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Notion Source Spec" + type: "object" + required: + - "access_token" + - "start_date" + additionalProperties: false + properties: + access_token: + type: "string" + description: "Notion API access token, see the docs for more information on how to obtain this token." + airbyte_secret: true + start_date: + type: "string" + description: "The date from which you'd like to replicate data for Notion\ + \ API, in the format YYYY-MM-DDT00:00:00.000Z. All data generated after\ + \ this date will be replicated." + examples: + - "2020-11-16T00:00:00.000Z" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z$" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-okta:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/okta" From 1ece88d42a30690e536bd3848dd2467a9bee2abf Mon Sep 17 00:00:00 2001 From: Marcos Marx Date: Thu, 18 Nov 2021 01:04:19 -0300 Subject: [PATCH 10/10] format --- .../6e00b415-b02e-4160-bf02-58176a0ae687.json | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6e00b415-b02e-4160-bf02-58176a0ae687.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6e00b415-b02e-4160-bf02-58176a0ae687.json index e2525b28e136..974a267c7520 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6e00b415-b02e-4160-bf02-58176a0ae687.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/6e00b415-b02e-4160-bf02-58176a0ae687.json @@ -1,8 +1,7 @@ { - "sourceDefinitionId": "6e00b415-b02e-4160-bf02-58176a0ae687", - "name": "Notion", - "dockerRepository": "airbyte/source-notion", - "dockerImageTag": "0.1.0", - "documentationUrl": "https://hub.docker.com/r/airbyte/source-notion" - } - \ No newline at end of file + "sourceDefinitionId": "6e00b415-b02e-4160-bf02-58176a0ae687", + "name": "Notion", + "dockerRepository": "airbyte/source-notion", + "dockerImageTag": "0.1.0", + "documentationUrl": "https://hub.docker.com/r/airbyte/source-notion" +}