diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index e8dc4f7f4217..eeb50da6f185 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -221,6 +221,12 @@ documentationUrl: https://hub.docker.com/r/airbyte/source-facebook-pages icon: facebook.svg sourceType: api +- name: Faker + sourceDefinitionId: dfd88b22-b603-4c3d-aad7-3701784586b1 + dockerRepository: airbyte/source-faker + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/source-faker + sourceType: api - name: File sourceDefinitionId: 778daa7c-feaf-4db6-96f3-70fd645acc77 dockerRepository: airbyte/source-file diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 13cefe4ff414..b063be8d7b72 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2068,6 +2068,34 @@ oauthFlowInitParameters: [] oauthFlowOutputParameters: - - "access_token" +- dockerImage: "airbyte/source-faker:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/faker" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Faker Source Spec" + type: "object" + required: + - "count" + additionalProperties: false + properties: + count: + title: "Count" + description: "How many fake records should be generated" + type: "integer" + minimum: 1 + default: 100 + order: 0 + seed: + title: "Seed" + description: "Manually control the faker random seed to return the same\ + \ values on subsequent runs (leave -1 for random)" + type: "integer" + default: -1 + order: 1 + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-file:0.2.10" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/file" diff --git a/airbyte-integrations/connectors/source-faker/.dockerignore b/airbyte-integrations/connectors/source-faker/.dockerignore new file mode 100644 index 000000000000..ec1364a9707a --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_faker +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-faker/CHANGELOG.md b/airbyte-integrations/connectors/source-faker/CHANGELOG.md new file mode 100644 index 000000000000..e25ce17e7908 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/CHANGELOG.md @@ -0,0 +1,3 @@ +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------- | +| 0.1.0 | 2022-04-12 | [11738](https://github.com/airbytehq/airbyte/pull/11738) | The Faker Source is created | diff --git a/airbyte-integrations/connectors/source-faker/Dockerfile b/airbyte-integrations/connectors/source-faker/Dockerfile new file mode 100644 index 000000000000..77fcee276c17 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_faker ./source_faker + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-faker diff --git a/airbyte-integrations/connectors/source-faker/README.md b/airbyte-integrations/connectors/source-faker/README.md new file mode 100644 index 000000000000..544140802cd2 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/README.md @@ -0,0 +1,166 @@ +# Faker Source + +This is the repository for the Faker source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/faker). + +## Local development + +### Prerequisites + +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies + +From this connector directory, create a virtual environment: + +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: + +``` +source .venv/bin/activate +pip install -r requirements.txt +``` + +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle + +From the Airbyte repository root, run: + +``` +./gradlew :airbyte-integrations:connectors:source-faker:build +``` + +#### Create credentials + +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/faker) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_faker/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source faker test creds` +and place them into `secrets/config.json`. + +### Locally running the connector + +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build + +First, make sure you build the latest Docker image: + +``` +docker build . -t airbyte/source-faker:dev +``` + +You can also build the connector image via Gradle: + +``` +./gradlew :airbyte-integrations:connectors:source-faker:airbyteDocker +``` + +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run + +Then run any of the connector commands as follows: + +``` +docker run --rm airbyte/source-faker:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-faker:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-faker:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-faker:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +## Testing + +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: + +``` +pip install .[tests] +``` + +### Unit Tests + +To run unit tests locally, from the connector directory run: + +``` +python -m pytest unit_tests +``` + +### Integration Tests + +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). + +#### Custom Integration tests + +Place custom tests inside `integration_tests/` folder, then, from the connector root, run + +``` +python -m pytest integration_tests +``` + +#### Acceptance Tests + +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run + +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` + +To run your integration tests with docker + +### Using gradle to run tests + +All commands should be run from airbyte project root. +To run unit tests: + +``` +./gradlew :airbyte-integrations:connectors:source-faker:unitTest +``` + +To run acceptance and custom integration tests: + +``` +./gradlew :airbyte-integrations:connectors:source-faker:integrationTest +``` + +## Dependency Management + +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: + +- required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +- required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector + +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? + +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml new file mode 100644 index 000000000000..1484247c4d7c --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/acceptance-test-config.yml @@ -0,0 +1,29 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-faker:dev +tests: + spec: + - spec_path: "source_faker/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "exception" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: no + exact_order: yes + extra_records: no + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + ignored_fields: + "Users": + - created_at + - updated_at diff --git a/airbyte-integrations/connectors/source-faker/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-faker/acceptance-test-docker.sh new file mode 100755 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-faker/build.gradle b/airbyte-integrations/connectors/source-faker/build.gradle new file mode 100644 index 000000000000..af227c969082 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_faker' +} diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/__init__.py b/airbyte-integrations/connectors/source-faker/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..caa5926c3c55 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "Users": { + "cursor": -1 + } +} diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py new file mode 100644 index 000000000000..0347f2a0b143 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json b/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json new file mode 100644 index 000000000000..ec7b3a0f98ee --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "name": "Users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": "number" }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "job": { "type": "string" }, + "company": { "type": "string" }, + "ssn": { "type": "string" }, + "residence": { "type": "string" }, + "current_location": { "type": "array" }, + "blood_group": { "type": "string" }, + "website": { "type": "array" }, + "username": { "type": "string" }, + "name": { "type": "string" }, + "sex": { "type": "string" }, + "address": { "type": "string" }, + "mail": { "type": "string" }, + "birthdate": { "type": "string", "format": "date" } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"] + } + ] +} diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..6cfd99a5fc02 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/configured_catalog.json @@ -0,0 +1,36 @@ +{ + "streams": [ + { + "stream": { + "name": "Users", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": "number" }, + "created_at": { "type": "date" }, + "updated_at": { "type": "date" }, + "job": { "type": "string" }, + "company": { "type": "string" }, + "ssn": { "type": "string" }, + "residence": { "type": "string" }, + "current_location": { "type": "array" }, + "blood_group": { "type": "string" }, + "website": { "type": "array" }, + "username": { "type": "string" }, + "name": { "type": "string" }, + "sex": { "type": "string" }, + "address": { "type": "string" }, + "mail": { "type": "string" }, + "birthdate": { "type": "date" } + } + }, + "supported_sync_modes": ["incremental", "full_refresh"], + "source_defined_cursor": true, + "default_cursor_field": ["created_at"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.txt new file mode 100644 index 000000000000..5960c79779b8 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/expected_records.txt @@ -0,0 +1,10 @@ +{"stream": "Users", "data": {"job": "Musician", "company": "Williams-Sheppard", "ssn": "498-52-4970", "residence": "Unit 5938 Box 2421\nDPO AP 33335", "current_location": [52.958961, 143.143712], "blood_group": "B+", "website": ["http://www.rivera.com/", "http://grimes-green.net/", "http://www.larsen.com/"], "username": "leeashley", "name": "Gary Cross", "sex": "M", "address": "711 Golden Overpass\nWest Andreaville, MA 71317", "mail": "tamaramorrison@hotmail.com", "birthdate": "1945-06-07", "id": 0, "created_at": "1976-03-27T12:40:22", "updated_at": "2000-04-13T06:17:38"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Therapist, occupational", "company": "Stewart-Allen", "ssn": "189-25-3060", "residence": "1122 Megan Squares Suite 848\nPort Jason, OR 55475", "current_location": [65.610695, -32.24732], "blood_group": "O-", "website": ["https://www.salazar-tucker.com/", "http://www.dennis.com/", "https://www.simmons-brown.com/", "http://www.walters.com/"], "username": "myersmitchell", "name": "Chelsea Greer", "sex": "F", "address": "Unit 0903 Box 2173\nDPO AP 08507", "mail": "stephenschristine@yahoo.com", "birthdate": "1915-10-23", "id": 1, "created_at": "1974-11-24T20:47:03", "updated_at": "1985-01-10T17:47:40"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Illustrator", "company": "Graham-Brown", "ssn": "479-06-9773", "residence": "45792 Tammy Centers Apt. 258\nDavidmouth, HI 02231", "current_location": [36.600983, 40.066283], "blood_group": "AB+", "website": ["https://castro.info/", "http://williams.info/", "https://howard.org/"], "username": "smithjames", "name": "Jasmine Perry", "sex": "F", "address": "76960 Savage Port\nBartonton, NV 08874", "mail": "jacqueline78@yahoo.com", "birthdate": "1982-09-09", "id": 2, "created_at": "2011-02-06T22:32:40", "updated_at": "2021-05-30T16:13:19"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Academic librarian", "company": "Simpson LLC", "ssn": "858-60-0817", "residence": "81206 Stewart Forest Apt. 089\nEast Davidborough, ME 37198", "current_location": [-41.0853825, -126.215901], "blood_group": "O-", "website": ["http://www.sloan-marsh.com/", "https://www.meyer.com/", "https://www.williams.com/", "https://browning.org/"], "username": "monica23", "name": "Steven Bowman", "sex": "M", "address": "22455 Higgins Junction Apt. 042\nNew Keith, OH 17493", "mail": "danny30@yahoo.com", "birthdate": "1930-09-22", "id": 3, "created_at": "2005-08-05T11:36:52", "updated_at": "2019-08-02T23:04:14"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Building services engineer", "company": "Powell-Murphy", "ssn": "425-14-1629", "residence": "5517 Holly Meadow Apt. 452\nLake Anne, SC 11894", "current_location": [-72.327639, -134.543372], "blood_group": "B-", "website": ["http://garcia.net/", "https://kramer-klein.com/"], "username": "ryanhoward", "name": "Daniel Duarte", "sex": "M", "address": "731 Sanders Fords\nPort Jasonberg, ID 60585", "mail": "jward@gmail.com", "birthdate": "1987-06-07", "id": 4, "created_at": "1980-02-06T13:15:11", "updated_at": "2003-03-15T20:31:32"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Surveyor, rural practice", "company": "Little-Henderson", "ssn": "534-64-5284", "residence": "389 Alec Squares Suite 508\nPort Jonathan, FL 50177", "current_location": [66.4839605, -21.954682], "blood_group": "O-", "website": ["http://www.perez.com/"], "username": "mistymurray", "name": "Joan Atkins", "sex": "F", "address": "809 Erika Valley Apt. 634\nPetersenfort, WY 51431", "mail": "melissayates@hotmail.com", "birthdate": "1906-04-25", "id": 5, "created_at": "1982-05-20T20:41:25", "updated_at": "2020-05-29T07:50:41"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Immigration officer", "company": "Hudson, Cook and Snyder", "ssn": "269-39-6686", "residence": "22345 Sheri Orchard Suite 279\nLake Hollystad, AZ 80687", "current_location": [35.165804, 27.598858], "blood_group": "O-", "website": ["http://smith.org/"], "username": "james71", "name": "Ashley Dunn", "sex": "F", "address": "519 Kramer Crossroad Suite 418\nNorth Kimberly, MN 99672", "mail": "stanleyclarke@gmail.com", "birthdate": "1912-05-11", "id": 6, "created_at": "1972-09-20T17:24:07", "updated_at": "2003-01-31T16:46:49"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Freight forwarder", "company": "Duffy Ltd", "ssn": "583-74-3539", "residence": "412 Snow Manors Apt. 161\nSouth Kimtown, NV 57004", "current_location": [21.942823, -163.821807], "blood_group": "O-", "website": ["https://www.jones-howard.com/", "https://smith.com/", "http://www.smith.com/"], "username": "michelelopez", "name": "Melissa Cantu", "sex": "F", "address": "420 Michael Mountains Suite 485\nNew Victoria, ND 76634", "mail": "eric22@gmail.com", "birthdate": "1949-04-11", "id": 7, "created_at": "1986-07-11T18:55:27", "updated_at": "2004-08-04T16:06:58"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Fast food restaurant manager", "company": "Carter Group", "ssn": "680-06-2167", "residence": "12648 Yang Divide Suite 451\nSouth Cynthia, NC 08084", "current_location": [11.047629, 39.379532], "blood_group": "A+", "website": ["https://watson.com/", "http://www.rodriguez-jacobs.com/", "https://saunders.com/", "http://giles-thomas.info/"], "username": "amontgomery", "name": "Christina Smith", "sex": "F", "address": "489 Roger Terrace\nDavisfort, IN 71770", "mail": "bramirez@gmail.com", "birthdate": "1943-01-15", "id": 8, "created_at": "1972-10-04T19:18:09", "updated_at": "2015-06-29T09:01:24"}, "emitted_at": 1649892395000} +{"stream": "Users", "data": {"job": "Academic librarian", "company": "Ross-Zamora", "ssn": "728-51-7285", "residence": "4391 Chad Greens Suite 851\nPort Frank, LA 37561", "current_location": [49.5419055, -107.833532], "blood_group": "B-", "website": ["https://osborne.com/"], "username": "qtaylor", "name": "Melissa James", "sex": "F", "address": "637 Neal Island Suite 074\nLake Tyler, RI 28775", "mail": "kellypeter@gmail.com", "birthdate": "2021-06-15", "id": 9, "created_at": "1974-07-10T19:02:25", "updated_at": "2005-11-07T09:09:06"}, "emitted_at": 1649892395000} diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-faker/integration_tests/invalid_config.json new file mode 100644 index 000000000000..e688d4573525 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "count": "foo" +} diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-faker/integration_tests/sample_config.json new file mode 100644 index 000000000000..4d3cd81d0d23 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "count": 10, + "seed": 0 +} diff --git a/airbyte-integrations/connectors/source-faker/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-faker/integration_tests/sample_state.json new file mode 100644 index 000000000000..d102abe320da --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "Users": { + "cursor": 100 + } +} diff --git a/airbyte-integrations/connectors/source-faker/main.py b/airbyte-integrations/connectors/source-faker/main.py new file mode 100644 index 000000000000..f7fdeaa94cc7 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_faker import SourceFaker + +if __name__ == "__main__": + source = SourceFaker() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-faker/requirements.txt b/airbyte-integrations/connectors/source-faker/requirements.txt new file mode 100644 index 000000000000..7be17a56d745 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/requirements.txt @@ -0,0 +1,3 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-faker/setup.py b/airbyte-integrations/connectors/source-faker/setup.py new file mode 100644 index 000000000000..19781fda2983 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/setup.py @@ -0,0 +1,26 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "Faker==13.3.1"] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "source-acceptance-test", +] + +setup( + name="source_faker", + description="Source implementation for Faker.", + author="Airbyte", + author_email="evan@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/__init__.py b/airbyte-integrations/connectors/source-faker/source_faker/__init__.py new file mode 100644 index 000000000000..f5a530bea965 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceFaker + +__all__ = ["SourceFaker"] diff --git a/airbyte-integrations/connectors/source-faker/source_faker/catalog.json b/airbyte-integrations/connectors/source-faker/source_faker/catalog.json new file mode 100644 index 000000000000..6f2670530cc5 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/catalog.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { "type": "number" }, + "created_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "updated_at": { + "type": "string", + "format": "date-time", + "airbyte_type": "timestamp_without_timezone" + }, + "job": { "type": "string" }, + "company": { "type": "string" }, + "ssn": { "type": "string" }, + "residence": { "type": "string" }, + "current_location": { "type": "array" }, + "blood_group": { "type": "string" }, + "website": { "type": "array" }, + "username": { "type": "string" }, + "name": { "type": "string" }, + "sex": { "type": "string" }, + "address": { "type": "string" }, + "mail": { "type": "string" }, + "birthdate": { "type": "string", "format": "date" } + } +} diff --git a/airbyte-integrations/connectors/source-faker/source_faker/source.py b/airbyte-integrations/connectors/source-faker/source_faker/source.py new file mode 100644 index 000000000000..933005e16341 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/source.py @@ -0,0 +1,142 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import json +import os +from datetime import datetime +from typing import Dict, Generator + +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import ( + AirbyteCatalog, + AirbyteConnectionStatus, + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStateMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + Status, + Type, +) +from airbyte_cdk.sources import Source +from faker import Faker + + +class SourceFaker(Source): + def check(self, logger: AirbyteLogger, config: Dict[str, any]) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to the integration + e.g: if a provided Stripe API token can be used to connect to the Stripe API. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + + # As this is an in-memory source, it always succeeds + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + + def discover(self, logger: AirbyteLogger, config: Dict[str, any]) -> AirbyteCatalog: + """ + Returns an AirbyteCatalog representing the available streams and fields in this integration. + For example, given valid credentials to a Postgres database, + returns an Airbyte catalog where each postgres table is a stream, and each table column is a field. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteCatalog is an object describing a list of all available streams in this source. + A stream is an AirbyteStream object that includes: + - its stream name (or table name in the case of Postgres) + - json_schema providing the specifications of expected schema for this stream (a list of columns described + by their names and types) + """ + streams = [] + + # Fake Users + dirname = os.path.dirname(os.path.realpath(__file__)) + spec_path = os.path.join(dirname, "catalog.json") + catalog = read_json(spec_path) + streams.append(AirbyteStream(name="Users", json_schema=catalog, supported_sync_modes=["full_refresh", "incremental"])) + return AirbyteCatalog(streams=streams) + + def read( + self, logger: AirbyteLogger, config: Dict[str, any], catalog: ConfiguredAirbyteCatalog, state: Dict[str, any] + ) -> Generator[AirbyteMessage, None, None]: + """ + Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, + catalog, and state. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.json file + :param catalog: The input catalog is a ConfiguredAirbyteCatalog which is almost the same as AirbyteCatalog + returned by discover(), but + in addition, it's been configured in the UI! For each particular stream and field, there may have been provided + with extra modifications such as: filtering streams and/or columns out, renaming some entities, etc + :param state: When a Airbyte reads data from a source, it might need to keep a checkpoint cursor to resume + replication in the future from that saved checkpoint. + This is the object that is provided with state from previous runs and avoid replicating the entire set of + data everytime. + + :return: A generator that produces a stream of AirbyteRecordMessage contained in AirbyteMessage object. + """ + + count: int = config["count"] if "count" in config else 0 + seed: int = config["seed"] if "seed" in config else state["seed"] if "seed" in state else None + Faker.seed(seed) + fake = Faker() + + for stream in catalog.streams: + if stream.stream.name == "Users": + cursor = get_stream_cursor(state, stream.stream.name) + total_records = cursor + + for i in range(cursor, count): + yield AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=stream.stream.name, data=emit_user(fake, i), emitted_at=int(datetime.now().timestamp()) * 1000 + ), + ) + total_records = total_records + 1 + + yield emit_state(stream.stream.name, total_records, seed) + else: + raise ValueError(stream.stream.name) + + +def get_stream_cursor(state: Dict[str, any], stream: str) -> int: + cursor = (state[stream]["cursor"] or 0) if stream in state else 0 + return cursor + + +def emit_state(stream: str, value: int, seed: int): + message = AirbyteMessage(type=Type.STATE, state=AirbyteStateMessage(data={stream: {"cursor": value, "seed": seed}})) + return message + + +def emit_user(fake: Faker, idx: int): + profile = fake.profile() + time_a = fake.date_time() + time_b = fake.date_time() + metadata = { + "id": idx, + "created_at": time_a if time_a <= time_b else time_b, + "updated_at": time_a if time_a > time_b else time_b, + } + profile.update(metadata) + return profile + + +def read_json(filepath): + with open(filepath, "r") as f: + return json.loads(f.read()) diff --git a/airbyte-integrations/connectors/source-faker/source_faker/spec.json b/airbyte-integrations/connectors/source-faker/source_faker/spec.json new file mode 100644 index 000000000000..34d0eec517c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/source_faker/spec.json @@ -0,0 +1,27 @@ +{ + "documentationUrl": "https://docs.airbyte.com/integrations/sources/faker", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Faker Source Spec", + "type": "object", + "required": ["count"], + "additionalProperties": false, + "properties": { + "count": { + "title": "Count", + "description": "How many fake records should be generated", + "type": "integer", + "minimum": 1, + "default": 100, + "order": 0 + }, + "seed": { + "title": "Seed", + "description": "Manually control the faker random seed to return the same values on subsequent runs (leave -1 for random)", + "type": "integer", + "default": -1, + "order": 1 + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py new file mode 100644 index 000000000000..93648451e62d --- /dev/null +++ b/airbyte-integrations/connectors/source-faker/unit_tests/unit_test.py @@ -0,0 +1,79 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +import jsonschema +from airbyte_cdk.models import AirbyteMessage, ConfiguredAirbyteCatalog, Type +from source_faker import SourceFaker + + +def test_source_streams(): + source = SourceFaker() + config = {"count": 1} + catalog = source.discover(None, config) + catalog = AirbyteMessage(type=Type.CATALOG, catalog=catalog).dict(exclude_unset=True) + schemas = [stream["json_schema"] for stream in catalog["catalog"]["streams"]] + + assert len(schemas) == 1 + assert schemas[0]["properties"] == { + "id": {"type": "number"}, + "created_at": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, + "updated_at": {"type": "string", "format": "date-time", "airbyte_type": "timestamp_without_timezone"}, + "job": {"type": "string"}, + "company": {"type": "string"}, + "ssn": {"type": "string"}, + "residence": {"type": "string"}, + "current_location": {"type": "array"}, + "blood_group": {"type": "string"}, + "website": {"type": "array"}, + "username": {"type": "string"}, + "name": {"type": "string"}, + "sex": {"type": "string"}, + "address": {"type": "string"}, + "mail": {"type": "string"}, + "birthdate": {"type": "string", "format": "date"}, + } + + for schema in schemas: + jsonschema.Draft7Validator.check_schema(schema) + + +def test_read_random_data(): + source = SourceFaker() + logger = None + config = {"count": 10} + catalog = ConfiguredAirbyteCatalog( + streams=[{"stream": {"name": "Users", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}] + ) + state = {} + iterator = source.read(logger, config, catalog, state) + + record_rows_count = 0 + state_rows_count = 0 + for row in iterator: + if row.type is Type.RECORD: + record_rows_count = record_rows_count + 1 + if row.type is Type.STATE: + state_rows_count = state_rows_count + 1 + + assert record_rows_count == 10 + assert state_rows_count == 1 + + +def test_read_with_seed(): + """ + This test asserts that setting a seed always returns the same values + """ + + source = SourceFaker() + logger = None + config = {"count": 1, "seed": 100} + catalog = ConfiguredAirbyteCatalog( + streams=[{"stream": {"name": "Users", "json_schema": {}}, "sync_mode": "full_refresh", "destination_sync_mode": "overwrite"}] + ) + state = {} + iterator = source.read(logger, config, catalog, state) + + records = [row for row in iterator if row.type is Type.RECORD] + assert records[0].record.data["company"] == "Gibson-Townsend" + assert records[0].record.data["mail"] == "zamoradenise@yahoo.com" diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 61ed01c463bc..8632b81013bf 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -1,302 +1,303 @@ # Table of contents -* [Introduction](../README.md) -* [Airbyte Cloud QuickStart](cloud/getting-started-with-airbyte-cloud.md) - * [Core Concepts](cloud/core-concepts.md) - * [Managing Airbyte Cloud](cloud/managing-airbyte-cloud.md) -* [Airbyte Open Source Quickstart](quickstart/README.md) - * [Deploy Airbyte](quickstart/deploy-airbyte.md) - * [Add a Source](quickstart/add-a-source.md) - * [Add a Destination](quickstart/add-a-destination.md) - * [Set up a Connection](quickstart/set-up-a-connection.md) -* [Deploying Airbyte Open Source](deploying-airbyte/README.md) - * [Local Deployment](deploying-airbyte/local-deployment.md) - * [On AWS (EC2)](deploying-airbyte/on-aws-ec2.md) - * [On AWS ECS (Coming Soon)](deploying-airbyte/on-aws-ecs.md) - * [On Azure(VM)](deploying-airbyte/on-azure-vm-cloud-shell.md) - * [On GCP (Compute Engine)](deploying-airbyte/on-gcp-compute-engine.md) - * [On Kubernetes (Beta)](deploying-airbyte/on-kubernetes.md) - * [On Plural (Beta)](deploying-airbyte/on-plural.md) - * [On Oracle Cloud Infrastructure VM](deploying-airbyte/on-oci-vm.md) - * [On Digital Ocean Droplet](deploying-airbyte/on-digitalocean-droplet.md) -* [Operator Guides](operator-guides/README.md) - * [Upgrading Airbyte](operator-guides/upgrading-airbyte.md) - * [Resetting Your Data](operator-guides/reset.md) - * [Configuring the Airbyte Database](operator-guides/configuring-airbyte-db.md) - * [Browsing Output Logs](operator-guides/browsing-output-logs.md) - * [Using the Airflow Airbyte Operator](operator-guides/using-the-airflow-airbyte-operator.md) - * [Using the Prefect Task](operator-guides/using-prefect-task.md) - * [Using the Dagster Integration](operator-guides/using-dagster-integration.md) - * [Windows - Browsing Local File Output](operator-guides/locating-files-local-destination.md) - * [Transformations and Normalization](operator-guides/transformation-and-normalization/README.md) - * [Transformations with SQL (Part 1/3)](operator-guides/transformation-and-normalization/transformations-with-sql.md) - * [Transformations with dbt (Part 2/3)](operator-guides/transformation-and-normalization/transformations-with-dbt.md) - * [Transformations with Airbyte (Part 3/3)](operator-guides/transformation-and-normalization/transformations-with-airbyte.md) - * [Configuring Airbyte](operator-guides/configuring-airbyte.md) - * [Sentry Integration](operator-guides/sentry-integration.md) - * [Using Custom Connectors](operator-guides/using-custom-connectors.md) - * [Scaling Airbyte](operator-guides/scaling-airbyte.md) - * [Securing Airbyte](operator-guides/securing-airbyte.md) -* [Connector Catalog](integrations/README.md) - * [Sources](integrations/sources/README.md) - * [3PL Central](integrations/sources/tplcentral.md) - * [Airtable](integrations/sources/airtable.md) - * [Amazon SQS](integrations/sources/amazon-sqs.md) - * [Amazon Seller Partner](integrations/sources/amazon-seller-partner.md) - * [Amazon Ads](integrations/sources/amazon-ads.md) - * [Amplitude](integrations/sources/amplitude.md) - * [Apify Dataset](integrations/sources/apify-dataset.md) - * [Appstore](integrations/sources/appstore.md) - * [Asana](integrations/sources/asana.md) - * [AWS CloudTrail](integrations/sources/aws-cloudtrail.md) - * [Azure Table Storage](integrations/sources/azure-table.md) - * [Bamboo HR](integrations/sources/bamboo-hr.md) - * [Bing Ads](integrations/sources/bing-ads.md) - * [BigCommerce](integrations/sources/bigcommerce.md) - * [BigQuery](integrations/sources/bigquery.md) - * [Braintree](integrations/sources/braintree.md) - * [Cart](integrations/sources/cart.md) - * [Chargebee](integrations/sources/chargebee.md) - * [Chartmogul](integrations/sources/chartmogul.md) - * [ClickHouse](integrations/sources/clickhouse.md) - * [Close.com](integrations/sources/close-com.md) - * [CockroachDB](integrations/sources/cockroachdb.md) - * [Confluence](integrations/sources/confluence.md) - * [Customer.io (Sponsored by Faros AI)](integrations/sources/customer-io.md) - * [Delighted](integrations/sources/delighted.md) - * [Db2](integrations/sources/db2.md) - * [Dixa](integrations/sources/dixa.md) - * [Drift](integrations/sources/drift.md) - * [Drupal](integrations/sources/drupal.md) - * [End-to-End Testing](integrations/sources/e2e-test.md) - * [Exchange Rates API](integrations/sources/exchangeratesapi.md) - * [Facebook Marketing](integrations/sources/facebook-marketing.md) - * [Facebook Pages](integrations/sources/facebook-pages.md) - * [Files](integrations/sources/file.md) - * [Flexport](integrations/sources/flexport.md) - * [Freshdesk](integrations/sources/freshdesk.md) - * [Freshsales](integrations/sources/freshsales.md) - * [Freshservice](integrations/sources/freshservice.md) - * [GitHub](integrations/sources/github.md) - * [GitLab](integrations/sources/gitlab.md) - * [Google Ads](integrations/sources/google-ads.md) - * [Google Analytics](integrations/sources/google-analytics-v4.md) - * [Google Directory](integrations/sources/google-directory.md) - * [Google Search Console](integrations/sources/google-search-console.md) - * [Google Sheets](integrations/sources/google-sheets.md) - * [Google Workspace Admin Reports](integrations/sources/google-workspace-admin-reports.md) - * [Greenhouse](integrations/sources/greenhouse.md) - * [Harvest](integrations/sources/harvest.md) - * [Harness (Sponsored by Faros AI)](integrations/sources/harness.md) - * [HTTP Request (Graveyarded)](integrations/sources/http-request.md) - * [HubSpot](integrations/sources/hubspot.md) - * [Instagram](integrations/sources/instagram.md) - * [Intercom](integrations/sources/intercom.md) - * [Iterable](integrations/sources/iterable.md) - * [Jenkins (Sponsored by Faros AI)](integrations/sources/jenkins.md) - * [Jira](integrations/sources/jira.md) - * [Kafka](integrations/sources/kafka.md) - * [Klaviyo](integrations/sources/klaviyo.md) - * [Kustomer](integrations/sources/kustomer.md) - * [Lemlist](integrations/sources/lemlist.md) - * [LinkedIn Ads](integrations/sources/linkedin-ads.md) - * [Linnworks](integrations/sources/linnworks.md) - * [Lever Hiring](integrations/sources/lever-hiring.md) - * [Looker](integrations/sources/looker.md) - * [Magento](integrations/sources/magento.md) - * [Mailchimp](integrations/sources/mailchimp.md) - * [Marketo](integrations/sources/marketo.md) - * [Microsoft Dynamics AX](integrations/sources/microsoft-dynamics-ax.md) - * [Microsoft Dynamics Customer Engagement](integrations/sources/microsoft-dynamics-customer-engagement.md) - * [Microsoft Dynamics GP](integrations/sources/microsoft-dynamics-gp.md) - * [Microsoft Dynamics NAV](integrations/sources/microsoft-dynamics-nav.md) - * [Microsoft SQL Server (MSSQL)](integrations/sources/mssql.md) - * [Microsoft Teams](integrations/sources/microsoft-teams.md) - * [Mixpanel](integrations/sources/mixpanel.md) - * [Monday](integrations/sources/monday.md) - * [Mongo DB](integrations/sources/mongodb-v2.md) - * [My Hours](integrations/sources/my-hours.md) - * [MySQL](integrations/sources/mysql.md) - * [Notion](integrations/sources/notion.md) - * [Okta](integrations/sources/okta.md) - * [OneSignal](integrations/sources/onesignal.md) - * [OpenWeather](integrations/sources/openweather.md) - * [Oracle DB](integrations/sources/oracle.md) - * [Oracle Peoplesoft](integrations/sources/oracle-peoplesoft.md) - * [Oracle Siebel CRM](integrations/sources/oracle-siebel-crm.md) - * [Orb](integrations/sources/orb.md) - * [Outreach](integrations/sources/outreach.md) - * [PagerDuty (Sponsored by Faros AI)](integrations/sources/pagerduty.md) - * [Paypal Transaction](integrations/sources/paypal-transaction.md) - * [Paystack](integrations/sources/paystack.md) - * [Persistiq](integrations/sources/persistiq.md) - * [Plaid](integrations/sources/plaid.md) - * [Pinterest](integrations/sources/pinterest.md) - * [Pipedrive](integrations/sources/pipedrive.md) - * [PokéAPI](integrations/sources/pokeapi.md) - * [Postgres](integrations/sources/postgres.md) - * [PostHog](integrations/sources/posthog.md) - * [PrestaShop](integrations/sources/presta-shop.md) - * [Qualaroo](integrations/sources/qualaroo.md) - * [QuickBooks](integrations/sources/quickbooks.md) - * [Recharge](integrations/sources/recharge.md) - * [Recurly](integrations/sources/recurly.md) - * [Redshift](integrations/sources/redshift.md) - * [S3](integrations/sources/s3.md) - * [SAP Business One](integrations/sources/sap-business-one.md) - * [SearchMetrics](integrations/sources/search-metrics.md) - * [Salesforce](integrations/sources/salesforce.md) - * [SalesLoft](integrations/sources/salesloft.md) - * [Sendgrid](integrations/sources/sendgrid.md) - * [Sentry](integrations/sources/sentry.md) - * [Shopify](integrations/sources/shopify.md) - * [Shortio](integrations/sources/shortio.md) - * [Slack](integrations/sources/slack.md) - * [Smartsheets](integrations/sources/smartsheets.md) - * [Snapchat Marketing](integrations/sources/snapchat-marketing.md) - * [Snowflake](integrations/sources/snowflake.md) - * [Spree Commerce](integrations/sources/spree-commerce.md) - * [Square](integrations/sources/square.md) - * [Strava](integrations/sources/strava.md) - * [Stripe](integrations/sources/stripe.md) - * [Sugar CRM](integrations/sources/sugar-crm.md) - * [SurveyMonkey](integrations/sources/surveymonkey.md) - * [Tempo](integrations/sources/tempo.md) - * [TikTok Marketing](integrations/sources/tiktok-marketing.md) - * [Trello](integrations/sources/trello.md) - * [Twilio](integrations/sources/twilio.md) - * [Typeform](integrations/sources/typeform.md) - * [US Census API](integrations/sources/us-census.md) - * [VictorOps (Sponsored by Faros AI)](integrations/sources/victorops.md) - * [Woo Commerce](integrations/sources/woocommerce.md) - * [Wordpress](integrations/sources/wordpress.md) - * [YouTube Analytics](integrations/sources/youtube-analytics.md) - * [Zencart](integrations/sources/zencart.md) - * [Zendesk Chat](integrations/sources/zendesk-chat.md) - * [Zendesk Sunshine](integrations/sources/zendesk-sunshine.md) - * [Zendesk Support](integrations/sources/zendesk-support.md) - * [Zendesk Talk](integrations/sources/zendesk-talk.md) - * [Zenloop](integrations/sources/zenloop.md) - * [Zoho CRM](integrations/sources/zoho-crm.md) - * [Zoom](integrations/sources/zoom.md) - * [Zuora](integrations/sources/zuora.md) - * [Destinations](integrations/destinations/README.md) - * [Amazon SQS](integrations/destinations/amazon-sqs.md) - * [AzureBlobStorage](integrations/destinations/azureblobstorage.md) - * [BigQuery](integrations/destinations/bigquery.md) - * [ClickHouse](integrations/destinations/clickhouse.md) - * [Databricks](integrations/destinations/databricks.md) - * [DynamoDB](integrations/destinations/dynamodb.md) - * [Elasticsearch](integrations/destinations/elasticsearch.md) - * [End-to-End Testing](integrations/destinations/e2e-test.md) - * [Chargify](integrations/destinations/chargify.md) - * [Google Cloud Storage (GCS)](integrations/destinations/gcs.md) - * [Google Firestore](integrations/destinations/firestore.md) - * [Google PubSub](integrations/destinations/pubsub.md) - * [Kafka](integrations/destinations/kafka.md) - * [Keen](integrations/destinations/keen.md) - * [Local CSV](integrations/destinations/local-csv.md) - * [Local JSON](integrations/destinations/local-json.md) - * [MariaDB ColumnStore](integrations/destinations/mariadb-columnstore.md) - * [MeiliSearch](integrations/destinations/meilisearch.md) - * [MongoDB](integrations/destinations/mongodb.md) - * [MQTT](integrations/destinations/mqtt.md) - * [MSSQL](integrations/destinations/mssql.md) - * [MySQL](integrations/destinations/mysql.md) - * [Oracle DB](integrations/destinations/oracle.md) - * [Postgres](integrations/destinations/postgres.md) - * [Pulsar](integrations/destinations/pulsar.md) - * [RabbitMQ](integrations/destinations/rabbitmq.md) - * [Redshift](integrations/destinations/redshift.md) - * [Rockset](integrations/destinations/rockset.md) - * [S3](integrations/destinations/s3.md) - * [SFTP JSON](integrations/destinations/sftp-json.md) - * [Snowflake](integrations/destinations/snowflake.md) - * [Cassandra](integrations/destinations/cassandra.md) - * [Scylla](integrations/destinations/scylla.md) - * [Redis](integrations/destinations/redis.md) - * [Kinesis](integrations/destinations/kinesis.md) - * [Streamr](integrations/destinations/streamr.md) - * [Custom or New Connector](integrations/custom-connectors.md) -* [Connector Development](connector-development/README.md) - * [Tutorials](connector-development/tutorials/README.md) - * [Python CDK Speedrun: Creating a Source](connector-development/tutorials/cdk-speedrun.md) - * [Python CDK: Creating a HTTP API Source](connector-development/tutorials/cdk-tutorial-python-http/README.md) - * [Getting Started](connector-development/tutorials/cdk-tutorial-python-http/0-getting-started.md) - * [Step 1: Creating the Source](connector-development/tutorials/cdk-tutorial-python-http/1-creating-the-source.md) - * [Step 2: Install Dependencies](connector-development/tutorials/cdk-tutorial-python-http/2-install-dependencies.md) - * [Step 3: Define Inputs](connector-development/tutorials/cdk-tutorial-python-http/3-define-inputs.md) - * [Step 4: Connection Checking](connector-development/tutorials/cdk-tutorial-python-http/4-connection-checking.md) - * [Step 5: Declare the Schema](connector-development/tutorials/cdk-tutorial-python-http/5-declare-schema.md) - * [Step 6: Read Data](connector-development/tutorials/cdk-tutorial-python-http/6-read-data.md) - * [Step 7: Use the Connector in Airbyte](connector-development/tutorials/cdk-tutorial-python-http/7-use-connector-in-airbyte.md) - * [Step 8: Test Connector](connector-development/tutorials/cdk-tutorial-python-http/8-test-your-connector.md) - * [Building a Python Source](connector-development/tutorials/building-a-python-source.md) - * [Building a Python Destination](connector-development/tutorials/building-a-python-destination.md) - * [Building a Java Destination](connector-development/tutorials/building-a-java-destination.md) - * [Profile Java Connector Memory](connector-development/tutorials/profile-java-connector-memory.md) - * [Connector Development Kit (Python)](connector-development/cdk-python/README.md) - * [Basic Concepts](connector-development/cdk-python/basic-concepts.md) - * [Defining Stream Schemas](connector-development/cdk-python/schemas.md) - * [Full Refresh Streams](connector-development/cdk-python/full-refresh-stream.md) - * [Incremental Streams](connector-development/cdk-python/incremental-stream.md) - * [HTTP-API-based Connectors](connector-development/cdk-python/http-streams.md) - * [Python Concepts](connector-development/cdk-python/python-concepts.md) - * [Stream Slices](connector-development/cdk-python/stream-slices.md) - * [Connector Development Kit (Javascript)](connector-development/cdk-faros-js.md) - * [Airbyte 101 for Connector Development](connector-development/airbyte101.md) - * [Testing Connectors](connector-development/testing-connectors/README.md) - * [Source Acceptance Tests Reference](connector-development/testing-connectors/source-acceptance-tests-reference.md) - * [Connector Specification Reference](connector-development/connector-specification-reference.md) - * [Best Practices](connector-development/best-practices.md) - * [UX Handbook](connector-development/ux-handbook.md) -* [Contributing to Airbyte](contributing-to-airbyte/README.md) - * [Code of Conduct](contributing-to-airbyte/code-of-conduct.md) - * [Developing Locally](contributing-to-airbyte/developing-locally.md) - * [Developing on Docker](contributing-to-airbyte/developing-on-docker.md) - * [Developing on Kubernetes](contributing-to-airbyte/developing-on-kubernetes.md) - * [Monorepo Python Development](contributing-to-airbyte/monorepo-python-development.md) - * [Code Style](contributing-to-airbyte/code-style.md) - * [Gradle Cheatsheet](contributing-to-airbyte/gradle-cheatsheet.md) - * [Updating Documentation](contributing-to-airbyte/updating-documentation.md) - * [Templates](contributing-to-airbyte/templates/README.md) - * [Connector Doc Template](contributing-to-airbyte/templates/integration-documentation-template.md) -* [Understanding Airbyte](understanding-airbyte/README.md) - * [A Beginner's Guide to the AirbyteCatalog](understanding-airbyte/beginners-guide-to-catalog.md) - * [AirbyteCatalog Reference](understanding-airbyte/catalog.md) - * [Airbyte Specification](understanding-airbyte/airbyte-specification.md) - * [Basic Normalization](understanding-airbyte/basic-normalization.md) - * [Connections and Sync Modes](understanding-airbyte/connections/README.md) - * [Full Refresh - Overwrite](understanding-airbyte/connections/full-refresh-overwrite.md) - * [Full Refresh - Append](understanding-airbyte/connections/full-refresh-append.md) - * [Incremental Sync - Append](understanding-airbyte/connections/incremental-append.md) - * [Incremental Sync - Deduped History](understanding-airbyte/connections/incremental-deduped-history.md) - * [Operations](understanding-airbyte/operations.md) - * [High-level View](understanding-airbyte/high-level-view.md) - * [Workers & Jobs](understanding-airbyte/jobs.md) - * [Technical Stack](understanding-airbyte/tech-stack.md) - * [Change Data Capture (CDC)](understanding-airbyte/cdc.md) - * [Namespaces](understanding-airbyte/namespaces.md) - * [Supported Data Types](understanding-airbyte/supported-data-types.md) - * [Json to Avro Conversion](understanding-airbyte/json-avro-conversion.md) - * [Glossary of Terms](understanding-airbyte/glossary.md) -* [API documentation](api-documentation.md) -* [CLI documentation](https://github.com/airbytehq/airbyte/tree/master/octavia-cli) -* [Project Overview](project-overview/README.md) - * [Roadmap](project-overview/roadmap.md) - * [Changelog](project-overview/changelog/README.md) - * [Platform](project-overview/changelog/platform.md) - * [Connectors](project-overview/changelog/connectors.md) - * [Slack Code of Conduct](project-overview/slack-code-of-conduct.md) - * [Security and Data Privacy](project-overview/security.md) - * [Licenses](project-overview/licenses/README.md) - * [License FAQ](project-overview/licenses/license-faq.md) - * [ELv2](project-overview/licenses/elv2-license.md) - * [MIT](project-overview/licenses/mit-license.md) - * [Examples](project-overview/licenses/examples.md) - * [Product Release Stages](project-overview/product-release-stages.md) -* [Troubleshooting & FAQ](troubleshooting/README.md) - * [On Deploying](troubleshooting/on-deploying.md) - * [On Setting up a New Connection](troubleshooting/new-connection.md) - * [On Running a Sync](troubleshooting/running-sync.md) - * [On Upgrading](troubleshooting/on-upgrading.md) +- [Introduction](../README.md) +- [Airbyte Cloud QuickStart](cloud/getting-started-with-airbyte-cloud.md) + - [Core Concepts](cloud/core-concepts.md) + - [Managing Airbyte Cloud](cloud/managing-airbyte-cloud.md) +- [Airbyte Open Source Quickstart](quickstart/README.md) + - [Deploy Airbyte](quickstart/deploy-airbyte.md) + - [Add a Source](quickstart/add-a-source.md) + - [Add a Destination](quickstart/add-a-destination.md) + - [Set up a Connection](quickstart/set-up-a-connection.md) +- [Deploying Airbyte Open Source](deploying-airbyte/README.md) + - [Local Deployment](deploying-airbyte/local-deployment.md) + - [On AWS (EC2)](deploying-airbyte/on-aws-ec2.md) + - [On AWS ECS (Coming Soon)](deploying-airbyte/on-aws-ecs.md) + - [On Azure(VM)](deploying-airbyte/on-azure-vm-cloud-shell.md) + - [On GCP (Compute Engine)](deploying-airbyte/on-gcp-compute-engine.md) + - [On Kubernetes (Beta)](deploying-airbyte/on-kubernetes.md) + - [On Plural (Beta)](deploying-airbyte/on-plural.md) + - [On Oracle Cloud Infrastructure VM](deploying-airbyte/on-oci-vm.md) + - [On Digital Ocean Droplet](deploying-airbyte/on-digitalocean-droplet.md) +- [Operator Guides](operator-guides/README.md) + - [Upgrading Airbyte](operator-guides/upgrading-airbyte.md) + - [Resetting Your Data](operator-guides/reset.md) + - [Configuring the Airbyte Database](operator-guides/configuring-airbyte-db.md) + - [Browsing Output Logs](operator-guides/browsing-output-logs.md) + - [Using the Airflow Airbyte Operator](operator-guides/using-the-airflow-airbyte-operator.md) + - [Using the Prefect Task](operator-guides/using-prefect-task.md) + - [Using the Dagster Integration](operator-guides/using-dagster-integration.md) + - [Windows - Browsing Local File Output](operator-guides/locating-files-local-destination.md) + - [Transformations and Normalization](operator-guides/transformation-and-normalization/README.md) + - [Transformations with SQL (Part 1/3)](operator-guides/transformation-and-normalization/transformations-with-sql.md) + - [Transformations with dbt (Part 2/3)](operator-guides/transformation-and-normalization/transformations-with-dbt.md) + - [Transformations with Airbyte (Part 3/3)](operator-guides/transformation-and-normalization/transformations-with-airbyte.md) + - [Configuring Airbyte](operator-guides/configuring-airbyte.md) + - [Sentry Integration](operator-guides/sentry-integration.md) + - [Using Custom Connectors](operator-guides/using-custom-connectors.md) + - [Scaling Airbyte](operator-guides/scaling-airbyte.md) + - [Securing Airbyte](operator-guides/securing-airbyte.md) +- [Connector Catalog](integrations/README.md) + - [Sources](integrations/sources/README.md) + - [3PL Central](integrations/sources/tplcentral.md) + - [Airtable](integrations/sources/airtable.md) + - [Amazon SQS](integrations/sources/amazon-sqs.md) + - [Amazon Seller Partner](integrations/sources/amazon-seller-partner.md) + - [Amazon Ads](integrations/sources/amazon-ads.md) + - [Amplitude](integrations/sources/amplitude.md) + - [Apify Dataset](integrations/sources/apify-dataset.md) + - [Appstore](integrations/sources/appstore.md) + - [Asana](integrations/sources/asana.md) + - [AWS CloudTrail](integrations/sources/aws-cloudtrail.md) + - [Azure Table Storage](integrations/sources/azure-table.md) + - [Bamboo HR](integrations/sources/bamboo-hr.md) + - [Bing Ads](integrations/sources/bing-ads.md) + - [BigCommerce](integrations/sources/bigcommerce.md) + - [BigQuery](integrations/sources/bigquery.md) + - [Braintree](integrations/sources/braintree.md) + - [Cart](integrations/sources/cart.md) + - [Chargebee](integrations/sources/chargebee.md) + - [Chartmogul](integrations/sources/chartmogul.md) + - [ClickHouse](integrations/sources/clickhouse.md) + - [Close.com](integrations/sources/close-com.md) + - [CockroachDB](integrations/sources/cockroachdb.md) + - [Confluence](integrations/sources/confluence.md) + - [Customer.io (Sponsored by Faros AI)](integrations/sources/customer-io.md) + - [Delighted](integrations/sources/delighted.md) + - [Db2](integrations/sources/db2.md) + - [Dixa](integrations/sources/dixa.md) + - [Drift](integrations/sources/drift.md) + - [Drupal](integrations/sources/drupal.md) + - [End-to-End Testing](integrations/sources/e2e-test.md) + - [Exchange Rates API](integrations/sources/exchangeratesapi.md) + - [Facebook Marketing](integrations/sources/facebook-marketing.md) + - [Facebook Pages](integrations/sources/facebook-pages.md) + - [Faker](integrations/sources/faker.md) + - [Files](integrations/sources/file.md) + - [Flexport](integrations/sources/flexport.md) + - [Freshdesk](integrations/sources/freshdesk.md) + - [Freshsales](integrations/sources/freshsales.md) + - [Freshservice](integrations/sources/freshservice.md) + - [GitHub](integrations/sources/github.md) + - [GitLab](integrations/sources/gitlab.md) + - [Google Ads](integrations/sources/google-ads.md) + - [Google Analytics](integrations/sources/google-analytics-v4.md) + - [Google Directory](integrations/sources/google-directory.md) + - [Google Search Console](integrations/sources/google-search-console.md) + - [Google Sheets](integrations/sources/google-sheets.md) + - [Google Workspace Admin Reports](integrations/sources/google-workspace-admin-reports.md) + - [Greenhouse](integrations/sources/greenhouse.md) + - [Harvest](integrations/sources/harvest.md) + - [Harness (Sponsored by Faros AI)](integrations/sources/harness.md) + - [HTTP Request (Graveyarded)](integrations/sources/http-request.md) + - [HubSpot](integrations/sources/hubspot.md) + - [Instagram](integrations/sources/instagram.md) + - [Intercom](integrations/sources/intercom.md) + - [Iterable](integrations/sources/iterable.md) + - [Jenkins (Sponsored by Faros AI)](integrations/sources/jenkins.md) + - [Jira](integrations/sources/jira.md) + - [Kafka](integrations/sources/kafka.md) + - [Klaviyo](integrations/sources/klaviyo.md) + - [Kustomer](integrations/sources/kustomer.md) + - [Lemlist](integrations/sources/lemlist.md) + - [LinkedIn Ads](integrations/sources/linkedin-ads.md) + - [Linnworks](integrations/sources/linnworks.md) + - [Lever Hiring](integrations/sources/lever-hiring.md) + - [Looker](integrations/sources/looker.md) + - [Magento](integrations/sources/magento.md) + - [Mailchimp](integrations/sources/mailchimp.md) + - [Marketo](integrations/sources/marketo.md) + - [Microsoft Dynamics AX](integrations/sources/microsoft-dynamics-ax.md) + - [Microsoft Dynamics Customer Engagement](integrations/sources/microsoft-dynamics-customer-engagement.md) + - [Microsoft Dynamics GP](integrations/sources/microsoft-dynamics-gp.md) + - [Microsoft Dynamics NAV](integrations/sources/microsoft-dynamics-nav.md) + - [Microsoft SQL Server (MSSQL)](integrations/sources/mssql.md) + - [Microsoft Teams](integrations/sources/microsoft-teams.md) + - [Mixpanel](integrations/sources/mixpanel.md) + - [Monday](integrations/sources/monday.md) + - [Mongo DB](integrations/sources/mongodb-v2.md) + - [My Hours](integrations/sources/my-hours.md) + - [MySQL](integrations/sources/mysql.md) + - [Notion](integrations/sources/notion.md) + - [Okta](integrations/sources/okta.md) + - [OneSignal](integrations/sources/onesignal.md) + - [OpenWeather](integrations/sources/openweather.md) + - [Oracle DB](integrations/sources/oracle.md) + - [Oracle Peoplesoft](integrations/sources/oracle-peoplesoft.md) + - [Oracle Siebel CRM](integrations/sources/oracle-siebel-crm.md) + - [Orb](integrations/sources/orb.md) + - [Outreach](integrations/sources/outreach.md) + - [PagerDuty (Sponsored by Faros AI)](integrations/sources/pagerduty.md) + - [Paypal Transaction](integrations/sources/paypal-transaction.md) + - [Paystack](integrations/sources/paystack.md) + - [Persistiq](integrations/sources/persistiq.md) + - [Plaid](integrations/sources/plaid.md) + - [Pinterest](integrations/sources/pinterest.md) + - [Pipedrive](integrations/sources/pipedrive.md) + - [PokéAPI](integrations/sources/pokeapi.md) + - [Postgres](integrations/sources/postgres.md) + - [PostHog](integrations/sources/posthog.md) + - [PrestaShop](integrations/sources/presta-shop.md) + - [Qualaroo](integrations/sources/qualaroo.md) + - [QuickBooks](integrations/sources/quickbooks.md) + - [Recharge](integrations/sources/recharge.md) + - [Recurly](integrations/sources/recurly.md) + - [Redshift](integrations/sources/redshift.md) + - [S3](integrations/sources/s3.md) + - [SAP Business One](integrations/sources/sap-business-one.md) + - [SearchMetrics](integrations/sources/search-metrics.md) + - [Salesforce](integrations/sources/salesforce.md) + - [SalesLoft](integrations/sources/salesloft.md) + - [Sendgrid](integrations/sources/sendgrid.md) + - [Sentry](integrations/sources/sentry.md) + - [Shopify](integrations/sources/shopify.md) + - [Shortio](integrations/sources/shortio.md) + - [Slack](integrations/sources/slack.md) + - [Smartsheets](integrations/sources/smartsheets.md) + - [Snapchat Marketing](integrations/sources/snapchat-marketing.md) + - [Snowflake](integrations/sources/snowflake.md) + - [Spree Commerce](integrations/sources/spree-commerce.md) + - [Square](integrations/sources/square.md) + - [Strava](integrations/sources/strava.md) + - [Stripe](integrations/sources/stripe.md) + - [Sugar CRM](integrations/sources/sugar-crm.md) + - [SurveyMonkey](integrations/sources/surveymonkey.md) + - [Tempo](integrations/sources/tempo.md) + - [TikTok Marketing](integrations/sources/tiktok-marketing.md) + - [Trello](integrations/sources/trello.md) + - [Twilio](integrations/sources/twilio.md) + - [Typeform](integrations/sources/typeform.md) + - [US Census API](integrations/sources/us-census.md) + - [VictorOps (Sponsored by Faros AI)](integrations/sources/victorops.md) + - [Woo Commerce](integrations/sources/woocommerce.md) + - [Wordpress](integrations/sources/wordpress.md) + - [YouTube Analytics](integrations/sources/youtube-analytics.md) + - [Zencart](integrations/sources/zencart.md) + - [Zendesk Chat](integrations/sources/zendesk-chat.md) + - [Zendesk Sunshine](integrations/sources/zendesk-sunshine.md) + - [Zendesk Support](integrations/sources/zendesk-support.md) + - [Zendesk Talk](integrations/sources/zendesk-talk.md) + - [Zenloop](integrations/sources/zenloop.md) + - [Zoho CRM](integrations/sources/zoho-crm.md) + - [Zoom](integrations/sources/zoom.md) + - [Zuora](integrations/sources/zuora.md) + - [Destinations](integrations/destinations/README.md) + - [Amazon SQS](integrations/destinations/amazon-sqs.md) + - [AzureBlobStorage](integrations/destinations/azureblobstorage.md) + - [BigQuery](integrations/destinations/bigquery.md) + - [ClickHouse](integrations/destinations/clickhouse.md) + - [Databricks](integrations/destinations/databricks.md) + - [DynamoDB](integrations/destinations/dynamodb.md) + - [Elasticsearch](integrations/destinations/elasticsearch.md) + - [End-to-End Testing](integrations/destinations/e2e-test.md) + - [Chargify](integrations/destinations/chargify.md) + - [Google Cloud Storage (GCS)](integrations/destinations/gcs.md) + - [Google Firestore](integrations/destinations/firestore.md) + - [Google PubSub](integrations/destinations/pubsub.md) + - [Kafka](integrations/destinations/kafka.md) + - [Keen](integrations/destinations/keen.md) + - [Local CSV](integrations/destinations/local-csv.md) + - [Local JSON](integrations/destinations/local-json.md) + - [MariaDB ColumnStore](integrations/destinations/mariadb-columnstore.md) + - [MeiliSearch](integrations/destinations/meilisearch.md) + - [MongoDB](integrations/destinations/mongodb.md) + - [MQTT](integrations/destinations/mqtt.md) + - [MSSQL](integrations/destinations/mssql.md) + - [MySQL](integrations/destinations/mysql.md) + - [Oracle DB](integrations/destinations/oracle.md) + - [Postgres](integrations/destinations/postgres.md) + - [Pulsar](integrations/destinations/pulsar.md) + - [RabbitMQ](integrations/destinations/rabbitmq.md) + - [Redshift](integrations/destinations/redshift.md) + - [Rockset](integrations/destinations/rockset.md) + - [S3](integrations/destinations/s3.md) + - [SFTP JSON](integrations/destinations/sftp-json.md) + - [Snowflake](integrations/destinations/snowflake.md) + - [Cassandra](integrations/destinations/cassandra.md) + - [Scylla](integrations/destinations/scylla.md) + - [Redis](integrations/destinations/redis.md) + - [Kinesis](integrations/destinations/kinesis.md) + - [Streamr](integrations/destinations/streamr.md) + - [Custom or New Connector](integrations/custom-connectors.md) +- [Connector Development](connector-development/README.md) + - [Tutorials](connector-development/tutorials/README.md) + - [Python CDK Speedrun: Creating a Source](connector-development/tutorials/cdk-speedrun.md) + - [Python CDK: Creating a HTTP API Source](connector-development/tutorials/cdk-tutorial-python-http/README.md) + - [Getting Started](connector-development/tutorials/cdk-tutorial-python-http/0-getting-started.md) + - [Step 1: Creating the Source](connector-development/tutorials/cdk-tutorial-python-http/1-creating-the-source.md) + - [Step 2: Install Dependencies](connector-development/tutorials/cdk-tutorial-python-http/2-install-dependencies.md) + - [Step 3: Define Inputs](connector-development/tutorials/cdk-tutorial-python-http/3-define-inputs.md) + - [Step 4: Connection Checking](connector-development/tutorials/cdk-tutorial-python-http/4-connection-checking.md) + - [Step 5: Declare the Schema](connector-development/tutorials/cdk-tutorial-python-http/5-declare-schema.md) + - [Step 6: Read Data](connector-development/tutorials/cdk-tutorial-python-http/6-read-data.md) + - [Step 7: Use the Connector in Airbyte](connector-development/tutorials/cdk-tutorial-python-http/7-use-connector-in-airbyte.md) + - [Step 8: Test Connector](connector-development/tutorials/cdk-tutorial-python-http/8-test-your-connector.md) + - [Building a Python Source](connector-development/tutorials/building-a-python-source.md) + - [Building a Python Destination](connector-development/tutorials/building-a-python-destination.md) + - [Building a Java Destination](connector-development/tutorials/building-a-java-destination.md) + - [Profile Java Connector Memory](connector-development/tutorials/profile-java-connector-memory.md) + - [Connector Development Kit (Python)](connector-development/cdk-python/README.md) + - [Basic Concepts](connector-development/cdk-python/basic-concepts.md) + - [Defining Stream Schemas](connector-development/cdk-python/schemas.md) + - [Full Refresh Streams](connector-development/cdk-python/full-refresh-stream.md) + - [Incremental Streams](connector-development/cdk-python/incremental-stream.md) + - [HTTP-API-based Connectors](connector-development/cdk-python/http-streams.md) + - [Python Concepts](connector-development/cdk-python/python-concepts.md) + - [Stream Slices](connector-development/cdk-python/stream-slices.md) + - [Connector Development Kit (Javascript)](connector-development/cdk-faros-js.md) + - [Airbyte 101 for Connector Development](connector-development/airbyte101.md) + - [Testing Connectors](connector-development/testing-connectors/README.md) + - [Source Acceptance Tests Reference](connector-development/testing-connectors/source-acceptance-tests-reference.md) + - [Connector Specification Reference](connector-development/connector-specification-reference.md) + - [Best Practices](connector-development/best-practices.md) + - [UX Handbook](connector-development/ux-handbook.md) +- [Contributing to Airbyte](contributing-to-airbyte/README.md) + - [Code of Conduct](contributing-to-airbyte/code-of-conduct.md) + - [Developing Locally](contributing-to-airbyte/developing-locally.md) + - [Developing on Docker](contributing-to-airbyte/developing-on-docker.md) + - [Developing on Kubernetes](contributing-to-airbyte/developing-on-kubernetes.md) + - [Monorepo Python Development](contributing-to-airbyte/monorepo-python-development.md) + - [Code Style](contributing-to-airbyte/code-style.md) + - [Gradle Cheatsheet](contributing-to-airbyte/gradle-cheatsheet.md) + - [Updating Documentation](contributing-to-airbyte/updating-documentation.md) + - [Templates](contributing-to-airbyte/templates/README.md) + - [Connector Doc Template](contributing-to-airbyte/templates/integration-documentation-template.md) +- [Understanding Airbyte](understanding-airbyte/README.md) + - [A Beginner's Guide to the AirbyteCatalog](understanding-airbyte/beginners-guide-to-catalog.md) + - [AirbyteCatalog Reference](understanding-airbyte/catalog.md) + - [Airbyte Specification](understanding-airbyte/airbyte-specification.md) + - [Basic Normalization](understanding-airbyte/basic-normalization.md) + - [Connections and Sync Modes](understanding-airbyte/connections/README.md) + - [Full Refresh - Overwrite](understanding-airbyte/connections/full-refresh-overwrite.md) + - [Full Refresh - Append](understanding-airbyte/connections/full-refresh-append.md) + - [Incremental Sync - Append](understanding-airbyte/connections/incremental-append.md) + - [Incremental Sync - Deduped History](understanding-airbyte/connections/incremental-deduped-history.md) + - [Operations](understanding-airbyte/operations.md) + - [High-level View](understanding-airbyte/high-level-view.md) + - [Workers & Jobs](understanding-airbyte/jobs.md) + - [Technical Stack](understanding-airbyte/tech-stack.md) + - [Change Data Capture (CDC)](understanding-airbyte/cdc.md) + - [Namespaces](understanding-airbyte/namespaces.md) + - [Supported Data Types](understanding-airbyte/supported-data-types.md) + - [Json to Avro Conversion](understanding-airbyte/json-avro-conversion.md) + - [Glossary of Terms](understanding-airbyte/glossary.md) +- [API documentation](api-documentation.md) +- [CLI documentation](https://github.com/airbytehq/airbyte/tree/master/octavia-cli) +- [Project Overview](project-overview/README.md) + - [Roadmap](project-overview/roadmap.md) + - [Changelog](project-overview/changelog/README.md) + - [Platform](project-overview/changelog/platform.md) + - [Connectors](project-overview/changelog/connectors.md) + - [Slack Code of Conduct](project-overview/slack-code-of-conduct.md) + - [Security and Data Privacy](project-overview/security.md) + - [Licenses](project-overview/licenses/README.md) + - [License FAQ](project-overview/licenses/license-faq.md) + - [ELv2](project-overview/licenses/elv2-license.md) + - [MIT](project-overview/licenses/mit-license.md) + - [Examples](project-overview/licenses/examples.md) + - [Product Release Stages](project-overview/product-release-stages.md) +- [Troubleshooting & FAQ](troubleshooting/README.md) + - [On Deploying](troubleshooting/on-deploying.md) + - [On Setting up a New Connection](troubleshooting/new-connection.md) + - [On Running a Sync](troubleshooting/running-sync.md) + - [On Upgrading](troubleshooting/on-upgrading.md) diff --git a/docs/integrations/README.md b/docs/integrations/README.md index ed17d625cf7f..44436b029038 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -20,184 +20,185 @@ For more information about the grading system, see [Product Release Stages](http ## Sources -| Connector | Stage | -| :--- | :--- | -| [3PL Central](sources/tplcentral.md) | Alpha | -| [Airtable](sources/airtable.md) | Alpha | -| [Amazon SQS](sources/amazon-sqs.md) | Alpha | -| [Amazon Seller Partner](sources/amazon-seller-partner.md) | Alpha | -| [Amplitude](sources/amplitude.md) | Alpha | -| [Apify Dataset](sources/apify-dataset.md) | Alpha | -| [Appstore](sources/appstore.md) | Alpha | -| [Asana](sources/asana.md) | Alpha | -| [AWS CloudTrail](sources/aws-cloudtrail.md) | Alpha | -| [Azure Table Storage](sources/azure-table.md) | Alpha | -| [BambooHR](sources/bamboo-hr.md) | Alpha | -| [Braintree](sources/braintree.md) | Alpha | -| [BigCommerce](sources/bigcommerce.md) | Alpha | -| [BigQuery](sources/bigquery.md) | Alpha | -| [Bing Ads](sources/bing-ads.md) | Alpha | -| [Cart.com](sources/cart.md) | Alpha | -| [Chargebee](sources/chargebee.md) | Alpha | -| [Chartmogul](sources/chartmogul.md) | Alpha | -| [ClickHouse](sources/clickhouse.md) | Alpha | -| [Close.com](sources/close-com.md) | Alpha | -| [CockroachDB](sources/cockroachdb.md) | Alpha | -| [Customer.io](sources/customer-io.md) | Alpha | -| [Db2](sources/db2.md) | Alpha | -| [Delighted](sources/delighted.md) | Alpha | -| [Dixa](sources/dixa.md) | Alpha | -| [Drift](sources/drift.md) | Alpha | -| [Drupal](sources/drupal.md) | Alpha | -| [End-to-End Testing](sources/e2e-test.md) | Alpha | -| [Exchange Rates API](sources/exchangeratesapi.md) | Alpha | -| [Facebook Marketing](sources/facebook-marketing.md) | Generally Available | -| [Facebook Pages](sources/facebook-pages.md) | Alpha | -| [Files](sources/file.md) | Alpha | -| [Flexport](sources/flexport.md) | Alpha | -| [Freshdesk](sources/freshdesk.md) | Alpha | -| [GitHub](sources/github.md) | Beta | -| [GitLab](sources/gitlab.md) | Alpha | -| [Google Ads](sources/google-ads.md) | Beta | -| [Google Adwords](sources/google-adwords.md) | Alpha | -| [Google Analytics v4](sources/google-analytics-v4.md) | Beta | -| [Google Directory](sources/google-directory.md) | Alpha | -| [Google Search Console](sources/google-search-console.md) | Alpha | -| [Google Sheets](sources/google-sheets.md) | Generally Available | -| [Google Workspace Admin Reports](sources/google-workspace-admin-reports.md) | Alpha | -| [Greenhouse](sources/greenhouse.md) | Alpha | -| [Harness](sources/harness.md) | Alpha | -| [HubSpot](sources/hubspot.md) | Beta | -| [Instagram](sources/instagram.md) | Beta | -| [Intercom](sources/intercom.md) | Beta | -| [Iterable](sources/iterable.md) | Alpha | -| [Jenkins](sources/jenkins.md) | Alpha | -| [Jira](sources/jira.md) | Alpha | -| [Klaviyo](sources/klaviyo.md) | Alpha | -| [Kustomer](sources/kustomer.md) | Alpha | -| [Lemlist](sources/lemlist.md) | Alpha | -| [LinkedIn Ads](sources/linkedin-ads.md) | Alpha | -| [Linnworks](sources/linnworks.md) | Alpha | -| [Kustomer](sources/kustomer.md) | Alpha | -| [Lever Hiring](sources/lever-hiring.md) | Alpha | -| [Looker](sources/looker.md) | Alpha | -| [Magento](sources/magento.md) | Alpha | -| [Mailchimp](sources/mailchimp.md) | Alpha | -| [Marketo](sources/marketo.md) | Alpha | -| [Microsoft SQL Server \(MSSQL\)](sources/mssql.md) | Alpha | -| [Microsoft Dynamics AX](sources/microsoft-dynamics-ax.md) | Alpha | -| [Microsoft Dynamics Customer Engagement](sources/microsoft-dynamics-customer-engagement.md) | Alpha | -| [Microsoft Dynamics GP](sources/microsoft-dynamics-gp.md) | Alpha | -| [Microsoft Dynamics NAV](sources/microsoft-dynamics-nav.md) | Alpha | -| [Microsoft Teams](sources/microsoft-teams.md) | Alpha | -| [Mixpanel](sources/mixpanel.md) | Alpha | -| [Monday](sources/monday.md) | Alpha | -| [Mongo DB](sources/mongodb-v2.md) | Alpha | -| [My Hours](sources/my-hours.md) | Alpha | -| [MySQL](sources/mysql.md) | Alpha | -| [Notion](sources/notion.md) | Alpha | -| [Okta](sources/okta.md) | Alpha | -| [OneSignal](sources/onesignal.md) | Alpha | -| [OpenWeather](sources/openweather.md) | Alpha | -| [Oracle DB](sources/oracle.md) | Alpha | -| [Oracle PeopleSoft](sources/oracle-peoplesoft.md) | Alpha | -| [Oracle Siebel CRM](sources/oracle-siebel-crm.md) | Alpha | -| [Outreach](./sources/outreach.md)| Alpha | -| [PagerDuty](sources/pagerduty.md) | Alpha | -| [PayPal Transaction](sources/paypal-transaction.md) | Alpha | -| [Paystack](sources/paystack.md) | Alpha | -| [PersistIq](sources/persistiq.md) | Alpha | -| [Pinterest](sources/pinterest.md) | Alpha | -| [Pipedrive](sources/pipedrive.md) | Alpha | -| [Plaid](sources/plaid.md) | Alpha | -| [PokéAPI](sources/pokeapi.md) | Alpha | -| [Postgres](sources/postgres.md) | Alpha | -| [PostHog](sources/posthog.md) | Alpha | -| [PrestaShop](sources/presta-shop.md) | Alpha | -| [Qualaroo](sources/qualaroo.md) | Alpha | -| [QuickBooks](sources/quickbooks.md) | Alpha | -| [Recharge](sources/recharge.md) | Alpha | -| [Recurly](sources/recurly.md) | Alpha | -| [Redshift](sources/redshift.md) | Alpha | -| [Retently](sources/retently.md) | Alpha | -| [S3](sources/s3.md) | Beta | -| [Salesforce](sources/salesforce.md) | Generally Available | -| [Salesloft](./sources/salesloft.md)| Alpha | -| [SAP Business One](sources/sap-business-one.md) | Alpha | -| [SearchMetrics](./sources/search-metrics.md)| Alpha | -| [Sendgrid](sources/sendgrid.md) | Alpha | -| [Sentry](sources/sentry.md) | Alpha | -| [Shopify](sources/shopify.md) | Alpha | -| [Short.io](sources/shortio.md) | Alpha | -| [Slack](sources/slack.md) | Alpha | -| [Spree Commerce](sources/spree-commerce.md) | Alpha | -| [Smartsheets](sources/smartsheets.md) | Alpha | -| [Snowflake](sources/snowflake.md) | Alpha | -| [Square](sources/square.md) | Alpha | -| [Strava](sources/strava.md) | Alpha | -| [Stripe](sources/stripe.md) | Beta | -| [Sugar CRM](sources/sugar-crm.md) | Alpha | -| [SurveyMonkey](sources/surveymonkey.md) | Alpha | -| [Tempo](sources/tempo.md) | Alpha | -| [TikTok Marketing](./sources/tiktok-marketing.md)| Alpha | -| [Trello](sources/trello.md) | Alpha | -| [Twilio](sources/twilio.md) | Alpha | -| [Typeform](sources/typeform.md) | Alpha | -| [US Census](sources/us-census.md) | Alpha | -| [VictorOps](sources/victorops.md) | Alpha | -| [WooCommerce](sources/woocommerce.md) | Alpha | -| [Wordpress](sources/wordpress.md) | Alpha | -| [YouTube Analytics](sources/youtube-analytics.md) | Alpha | -| [Zencart](sources/zencart.md) | Alpha | -| [Zendesk Chat](sources/zendesk-chat.md) | Alpha | -| [Zendesk Sunshine](sources/zendesk-sunshine.md) | Alpha | -| [Zendesk Support](sources/zendesk-support.md) | Alpha | -| [Zendesk Talk](sources/zendesk-talk.md) | Alpha | -| [Zenloop](sources/zenloop.md)| Alpha | -| [Zoom](sources/zoom.md) | Alpha | -| [Zuora](sources/zuora.md) | Alpha | +| Connector | Stage | +| :------------------------------------------------------------------------------------------ | :------------------ | +| [3PL Central](sources/tplcentral.md) | Alpha | +| [Airtable](sources/airtable.md) | Alpha | +| [Amazon SQS](sources/amazon-sqs.md) | Alpha | +| [Amazon Seller Partner](sources/amazon-seller-partner.md) | Alpha | +| [Amplitude](sources/amplitude.md) | Alpha | +| [Apify Dataset](sources/apify-dataset.md) | Alpha | +| [Appstore](sources/appstore.md) | Alpha | +| [Asana](sources/asana.md) | Alpha | +| [AWS CloudTrail](sources/aws-cloudtrail.md) | Alpha | +| [Azure Table Storage](sources/azure-table.md) | Alpha | +| [BambooHR](sources/bamboo-hr.md) | Alpha | +| [Braintree](sources/braintree.md) | Alpha | +| [BigCommerce](sources/bigcommerce.md) | Alpha | +| [BigQuery](sources/bigquery.md) | Alpha | +| [Bing Ads](sources/bing-ads.md) | Alpha | +| [Cart.com](sources/cart.md) | Alpha | +| [Chargebee](sources/chargebee.md) | Alpha | +| [Chartmogul](sources/chartmogul.md) | Alpha | +| [ClickHouse](sources/clickhouse.md) | Alpha | +| [Close.com](sources/close-com.md) | Alpha | +| [CockroachDB](sources/cockroachdb.md) | Alpha | +| [Customer.io](sources/customer-io.md) | Alpha | +| [Db2](sources/db2.md) | Alpha | +| [Delighted](sources/delighted.md) | Alpha | +| [Dixa](sources/dixa.md) | Alpha | +| [Drift](sources/drift.md) | Alpha | +| [Drupal](sources/drupal.md) | Alpha | +| [End-to-End Testing](sources/e2e-test.md) | Alpha | +| [Exchange Rates API](sources/exchangeratesapi.md) | Alpha | +| [Facebook Marketing](sources/facebook-marketing.md) | Generally Available | +| [Facebook Pages](sources/facebook-pages.md) | Alpha | +| [Faker](sources/faker.md) | Alpha | +| [Files](sources/file.md) | Alpha | +| [Flexport](sources/flexport.md) | Alpha | +| [Freshdesk](sources/freshdesk.md) | Alpha | +| [GitHub](sources/github.md) | Beta | +| [GitLab](sources/gitlab.md) | Alpha | +| [Google Ads](sources/google-ads.md) | Beta | +| [Google Adwords](sources/google-adwords.md) | Alpha | +| [Google Analytics v4](sources/google-analytics-v4.md) | Beta | +| [Google Directory](sources/google-directory.md) | Alpha | +| [Google Search Console](sources/google-search-console.md) | Alpha | +| [Google Sheets](sources/google-sheets.md) | Generally Available | +| [Google Workspace Admin Reports](sources/google-workspace-admin-reports.md) | Alpha | +| [Greenhouse](sources/greenhouse.md) | Alpha | +| [Harness](sources/harness.md) | Alpha | +| [HubSpot](sources/hubspot.md) | Beta | +| [Instagram](sources/instagram.md) | Beta | +| [Intercom](sources/intercom.md) | Beta | +| [Iterable](sources/iterable.md) | Alpha | +| [Jenkins](sources/jenkins.md) | Alpha | +| [Jira](sources/jira.md) | Alpha | +| [Klaviyo](sources/klaviyo.md) | Alpha | +| [Kustomer](sources/kustomer.md) | Alpha | +| [Lemlist](sources/lemlist.md) | Alpha | +| [LinkedIn Ads](sources/linkedin-ads.md) | Alpha | +| [Linnworks](sources/linnworks.md) | Alpha | +| [Kustomer](sources/kustomer.md) | Alpha | +| [Lever Hiring](sources/lever-hiring.md) | Alpha | +| [Looker](sources/looker.md) | Alpha | +| [Magento](sources/magento.md) | Alpha | +| [Mailchimp](sources/mailchimp.md) | Alpha | +| [Marketo](sources/marketo.md) | Alpha | +| [Microsoft SQL Server \(MSSQL\)](sources/mssql.md) | Alpha | +| [Microsoft Dynamics AX](sources/microsoft-dynamics-ax.md) | Alpha | +| [Microsoft Dynamics Customer Engagement](sources/microsoft-dynamics-customer-engagement.md) | Alpha | +| [Microsoft Dynamics GP](sources/microsoft-dynamics-gp.md) | Alpha | +| [Microsoft Dynamics NAV](sources/microsoft-dynamics-nav.md) | Alpha | +| [Microsoft Teams](sources/microsoft-teams.md) | Alpha | +| [Mixpanel](sources/mixpanel.md) | Alpha | +| [Monday](sources/monday.md) | Alpha | +| [Mongo DB](sources/mongodb-v2.md) | Alpha | +| [My Hours](sources/my-hours.md) | Alpha | +| [MySQL](sources/mysql.md) | Alpha | +| [Notion](sources/notion.md) | Alpha | +| [Okta](sources/okta.md) | Alpha | +| [OneSignal](sources/onesignal.md) | Alpha | +| [OpenWeather](sources/openweather.md) | Alpha | +| [Oracle DB](sources/oracle.md) | Alpha | +| [Oracle PeopleSoft](sources/oracle-peoplesoft.md) | Alpha | +| [Oracle Siebel CRM](sources/oracle-siebel-crm.md) | Alpha | +| [Outreach](./sources/outreach.md) | Alpha | +| [PagerDuty](sources/pagerduty.md) | Alpha | +| [PayPal Transaction](sources/paypal-transaction.md) | Alpha | +| [Paystack](sources/paystack.md) | Alpha | +| [PersistIq](sources/persistiq.md) | Alpha | +| [Pinterest](sources/pinterest.md) | Alpha | +| [Pipedrive](sources/pipedrive.md) | Alpha | +| [Plaid](sources/plaid.md) | Alpha | +| [PokéAPI](sources/pokeapi.md) | Alpha | +| [Postgres](sources/postgres.md) | Alpha | +| [PostHog](sources/posthog.md) | Alpha | +| [PrestaShop](sources/presta-shop.md) | Alpha | +| [Qualaroo](sources/qualaroo.md) | Alpha | +| [QuickBooks](sources/quickbooks.md) | Alpha | +| [Recharge](sources/recharge.md) | Alpha | +| [Recurly](sources/recurly.md) | Alpha | +| [Redshift](sources/redshift.md) | Alpha | +| [Retently](sources/retently.md) | Alpha | +| [S3](sources/s3.md) | Beta | +| [Salesforce](sources/salesforce.md) | Generally Available | +| [Salesloft](./sources/salesloft.md) | Alpha | +| [SAP Business One](sources/sap-business-one.md) | Alpha | +| [SearchMetrics](./sources/search-metrics.md) | Alpha | +| [Sendgrid](sources/sendgrid.md) | Alpha | +| [Sentry](sources/sentry.md) | Alpha | +| [Shopify](sources/shopify.md) | Alpha | +| [Short.io](sources/shortio.md) | Alpha | +| [Slack](sources/slack.md) | Alpha | +| [Spree Commerce](sources/spree-commerce.md) | Alpha | +| [Smartsheets](sources/smartsheets.md) | Alpha | +| [Snowflake](sources/snowflake.md) | Alpha | +| [Square](sources/square.md) | Alpha | +| [Strava](sources/strava.md) | Alpha | +| [Stripe](sources/stripe.md) | Beta | +| [Sugar CRM](sources/sugar-crm.md) | Alpha | +| [SurveyMonkey](sources/surveymonkey.md) | Alpha | +| [Tempo](sources/tempo.md) | Alpha | +| [TikTok Marketing](./sources/tiktok-marketing.md) | Alpha | +| [Trello](sources/trello.md) | Alpha | +| [Twilio](sources/twilio.md) | Alpha | +| [Typeform](sources/typeform.md) | Alpha | +| [US Census](sources/us-census.md) | Alpha | +| [VictorOps](sources/victorops.md) | Alpha | +| [WooCommerce](sources/woocommerce.md) | Alpha | +| [Wordpress](sources/wordpress.md) | Alpha | +| [YouTube Analytics](sources/youtube-analytics.md) | Alpha | +| [Zencart](sources/zencart.md) | Alpha | +| [Zendesk Chat](sources/zendesk-chat.md) | Alpha | +| [Zendesk Sunshine](sources/zendesk-sunshine.md) | Alpha | +| [Zendesk Support](sources/zendesk-support.md) | Alpha | +| [Zendesk Talk](sources/zendesk-talk.md) | Alpha | +| [Zenloop](sources/zenloop.md) | Alpha | +| [Zoom](sources/zoom.md) | Alpha | +| [Zuora](sources/zuora.md) | Alpha | ## Destinations -| Connector | Stage | -| :--- | :--- | -| [Amazon SQS](destinations/amazon-sqs.md) | Alpha | -| [AzureBlobStorage](destinations/azureblobstorage.md) | Alpha | -| [BigQuery](destinations/bigquery.md) | Beta | -| [Cassandra](sources/cassandra.md) | Alpha | -| [Chargify \(Keen\)](destinations/chargify.md) | Alpha | -| [ClickHouse](destinations/clickhouse.md) | Alpha | -| [Databricks](destinations/databricks.md) | Alpha | -| [DynamoDB](sources/dynamodb.md) | Alpha | -| [Elasticsearch](destinations/elasticsearch.md) | Alpha | -| [End-to-End Testing](destinations/e2e-test.md) | Alpha | -| [Google Cloud Storage \(GCS\)](destinations/gcs.md) | Beta | -| [Google Firestore](destinations/firestore.md) | Alpha | -| [Google Pubsub](destinations/pubsub.md) | Alpha | -| [Kafka](destinations/kafka.md) | Alpha | -| [Keen](destinations/keen.md) | Alpha | -| [Local CSV](destinations/local-csv.md) | Alpha | -| [Local JSON](destinations/local-json.md) | Alpha | -| [MariaDB ColumnStore](destinations/mariadb-columnstore.md) | Alpha | -| [MeiliSearch](destinations/meilisearch.md) | Alpha | -| [MongoDB](destinations/mongodb.md) | Alpha | -| [MQTT](destinations/mqtt.md) | Alpha | -| [MS SQL Server](sources/mssql.md) | Alpha | -| [MySQL](destinations/mysql.md) | Alpha | -| [Oracle](destinations/oracle.md) | Alpha | -| [Postgres](destinations/postgres.md) | Alpha | -| [Pulsar](destinations/pulsar.md) | Alpha | -| [RabbitMQ](destinations/rabbitmq.md) | Alpha | -| [Redis](sources/redis.md) | Alpha | -| [Redshift](destinations/redshift.md) | Beta | -| [Rockset](destinations/rockset.md) | Alpha | -| [S3](destinations/s3.md) | Beta | -| [Scylla](sources/scylla.md) | Alpha | -| [SFTP JSON](./destinations/sftp-json.md)| Alpha | -| [SQL Server \(MSSQL\)](destinations/mssql.md) | Alpha | -| [Snowflake](destinations/snowflake.md) | Generally Available | -| [Cassandra](destinations/cassandra.md) | Alpha | -| [Scylla](destinations/scylla.md) | Alpha | -| [Redis](destinations/redis.md) | Alpha | -| [Kinesis](destinations/kinesis.md) | Alpha | -| [Streamr](destinations/streamr.md) | Alpha | +| Connector | Stage | +| :--------------------------------------------------------- | :------------------ | +| [Amazon SQS](destinations/amazon-sqs.md) | Alpha | +| [AzureBlobStorage](destinations/azureblobstorage.md) | Alpha | +| [BigQuery](destinations/bigquery.md) | Beta | +| [Cassandra](sources/cassandra.md) | Alpha | +| [Chargify \(Keen\)](destinations/chargify.md) | Alpha | +| [ClickHouse](destinations/clickhouse.md) | Alpha | +| [Databricks](destinations/databricks.md) | Alpha | +| [DynamoDB](sources/dynamodb.md) | Alpha | +| [Elasticsearch](destinations/elasticsearch.md) | Alpha | +| [End-to-End Testing](destinations/e2e-test.md) | Alpha | +| [Google Cloud Storage \(GCS\)](destinations/gcs.md) | Beta | +| [Google Firestore](destinations/firestore.md) | Alpha | +| [Google Pubsub](destinations/pubsub.md) | Alpha | +| [Kafka](destinations/kafka.md) | Alpha | +| [Keen](destinations/keen.md) | Alpha | +| [Local CSV](destinations/local-csv.md) | Alpha | +| [Local JSON](destinations/local-json.md) | Alpha | +| [MariaDB ColumnStore](destinations/mariadb-columnstore.md) | Alpha | +| [MeiliSearch](destinations/meilisearch.md) | Alpha | +| [MongoDB](destinations/mongodb.md) | Alpha | +| [MQTT](destinations/mqtt.md) | Alpha | +| [MS SQL Server](sources/mssql.md) | Alpha | +| [MySQL](destinations/mysql.md) | Alpha | +| [Oracle](destinations/oracle.md) | Alpha | +| [Postgres](destinations/postgres.md) | Alpha | +| [Pulsar](destinations/pulsar.md) | Alpha | +| [RabbitMQ](destinations/rabbitmq.md) | Alpha | +| [Redis](sources/redis.md) | Alpha | +| [Redshift](destinations/redshift.md) | Beta | +| [Rockset](destinations/rockset.md) | Alpha | +| [S3](destinations/s3.md) | Beta | +| [Scylla](sources/scylla.md) | Alpha | +| [SFTP JSON](./destinations/sftp-json.md) | Alpha | +| [SQL Server \(MSSQL\)](destinations/mssql.md) | Alpha | +| [Snowflake](destinations/snowflake.md) | Generally Available | +| [Cassandra](destinations/cassandra.md) | Alpha | +| [Scylla](destinations/scylla.md) | Alpha | +| [Redis](destinations/redis.md) | Alpha | +| [Kinesis](destinations/kinesis.md) | Alpha | +| [Streamr](destinations/streamr.md) | Alpha | diff --git a/docs/integrations/sources/faker.md b/docs/integrations/sources/faker.md new file mode 100644 index 000000000000..bf270c9f5931 --- /dev/null +++ b/docs/integrations/sources/faker.md @@ -0,0 +1,44 @@ +# Faker + +## Sync overview + +The Faker source generates sample data using the python [`faker`](https://faker.readthedocs.io/) package. Specifically, we generate data that looks like an e-commerce company's `users` table with the [`faker.profile()`](https://faker.readthedocs.io/en/master/providers/faker.providers.profile.html) method. + +### Output schema + +Only `Users` is supported. + +### Data type mapping + +Native Airbyte types (string, number, date, etc) + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :---------------- | :------------------- | :---- | +| Full Refresh Sync | Yes | | +| Incremental Sync | Yes | | +| Namespaces | No | | + +Of note, if you choose `Incremental Sync`, state will be maintained between syncs, and once you hit `count` records, no new records will be added. +You can choose a specific `seed` (integer) as an option for this connector which will guarantee that the same fake records are generated each time. Otherwise, random data will be created on each subsequent sync. + +### Rate Limiting & Performance Considerations + +N/A + +## Getting started + +### Requirements + +None! + +### Setup guide + +N/A + +## Changelog + +| Version | Date | Pull Request | Subject | +| :------ | :--------- | :------------------------------------------------------- | :-------------------------- | +| 0.1.0 | 2022-04-12 | [11738](https://github.com/airbytehq/airbyte/pull/11738) | The Faker Source is created |