From 074adefcdc46e5b15cd61c68343bd99de8bad2f3 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Fri, 17 Jun 2022 17:47:08 +0530 Subject: [PATCH] Harshith/test pr 12536 (#13874) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🎉 New Source: Firebolt (#1) * fix: Boolean type cast * test: Improve format testing and doc * refactor: Move some db functionality * docs: Adding types doc link in utils * feat: Use future-proof Auth in SDK * fix: integration tests are failing * chore: update seed file Co-authored-by: Petro Tiurin <93913847+ptiurin@users.noreply.github.com> Co-authored-by: ptiurin --- .../resources/seed/source_definitions.yaml | 7 + .../src/main/resources/seed/source_specs.yaml | 44 +++ .../connectors/source-firebolt/.dockerignore | 6 + .../connectors/source-firebolt/Dockerfile | 39 +++ .../connectors/source-firebolt/README.md | 132 ++++++++ .../acceptance-test-config.yml | 26 ++ .../source-firebolt/acceptance-test-docker.sh | 16 + .../connectors/source-firebolt/bootstrap.md | 22 ++ .../connectors/source-firebolt/build.gradle | 9 + .../integration_tests/__init__.py | 3 + .../integration_tests/acceptance.py | 16 + .../integration_tests/catalog.json | 35 ++ .../integration_tests/configured_catalog.json | 38 +++ .../integration_tests/expected_records.txt | 2 + .../integration_tests/integration_test.py | 141 ++++++++ .../integration_tests/invalid_config.json | 6 + .../integration_tests/sample_config.json | 6 + .../connectors/source-firebolt/main.py | 13 + .../source-firebolt/requirements.txt | 3 + .../connectors/source-firebolt/setup.py | 27 ++ .../source_firebolt/__init__.py | 8 + .../source_firebolt/database.py | 94 ++++++ .../source-firebolt/source_firebolt/source.py | 147 +++++++++ .../source-firebolt/source_firebolt/spec.json | 44 +++ .../source-firebolt/source_firebolt/utils.py | 113 +++++++ .../unit_tests/test_firebolt_source.py | 294 +++++++++++++++++ docs/SUMMARY.md | 305 ++++++++++++++++++ docs/integrations/README.md | 1 + docs/integrations/sources/firebolt.md | 61 ++++ 29 files changed, 1658 insertions(+) create mode 100644 airbyte-integrations/connectors/source-firebolt/.dockerignore create mode 100644 airbyte-integrations/connectors/source-firebolt/Dockerfile create mode 100644 airbyte-integrations/connectors/source-firebolt/README.md create mode 100644 airbyte-integrations/connectors/source-firebolt/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-firebolt/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-firebolt/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-firebolt/build.gradle create mode 100644 airbyte-integrations/connectors/source-firebolt/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-firebolt/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-firebolt/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-firebolt/integration_tests/expected_records.txt create mode 100644 airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py create mode 100644 airbyte-integrations/connectors/source-firebolt/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-firebolt/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-firebolt/main.py create mode 100644 airbyte-integrations/connectors/source-firebolt/requirements.txt create mode 100644 airbyte-integrations/connectors/source-firebolt/setup.py create mode 100644 airbyte-integrations/connectors/source-firebolt/source_firebolt/__init__.py create mode 100644 airbyte-integrations/connectors/source-firebolt/source_firebolt/database.py create mode 100644 airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py create mode 100644 airbyte-integrations/connectors/source-firebolt/source_firebolt/spec.json create mode 100644 airbyte-integrations/connectors/source-firebolt/source_firebolt/utils.py create mode 100644 airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py create mode 100644 docs/SUMMARY.md create mode 100644 docs/integrations/sources/firebolt.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c98b9edc4058..05860d916e33 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -1065,3 +1065,10 @@ documentationUrl: https://docs.airbyte.com/integrations/sources/sftp sourceType: file releaseStage: alpha +- name: Firebolt + sourceDefinitionId: 6f2ac653-8623-43c4-8950-19218c7caf3d + dockerRepository: airbyte/source-firebolt + dockerImageTag: 0.1.0 + documentationUrl: https://docs.firebolt.io/ + sourceType: database + releaseStage: alpha diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 87af2986067b..aa913b0ff5d9 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -10222,3 +10222,47 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-firebolt:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/firebolt" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Firebolt Spec" + type: "object" + required: + - "username" + - "password" + - "database" + additionalProperties: false + properties: + username: + type: "string" + title: "Username" + description: "Firebolt email address you use to login." + examples: + - "username@email.com" + password: + type: "string" + title: "Password" + description: "Firebolt password." + account: + type: "string" + title: "Account" + description: "Firebolt account to login." + host: + type: "string" + title: "Host" + description: "The host name of your Firebolt database." + examples: + - "api.app.firebolt.io" + database: + type: "string" + title: "Database" + description: "The database to connect to." + engine: + type: "string" + title: "Engine" + description: "Engine name or url to connect to." + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] diff --git a/airbyte-integrations/connectors/source-firebolt/.dockerignore b/airbyte-integrations/connectors/source-firebolt/.dockerignore new file mode 100644 index 000000000000..087dfcae5269 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_firebolt +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-firebolt/Dockerfile b/airbyte-integrations/connectors/source-firebolt/Dockerfile new file mode 100644 index 000000000000..c02cb10a19f2 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + +RUN apk add libffi-dev + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_firebolt ./source_firebolt + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-firebolt diff --git a/airbyte-integrations/connectors/source-firebolt/README.md b/airbyte-integrations/connectors/source-firebolt/README.md new file mode 100644 index 000000000000..14282a3bf812 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/README.md @@ -0,0 +1,132 @@ +# Firebolt Source + +This is the repository for the Firebolt source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/firebolt). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +From the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-firebolt:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/firebolt) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_firebolt/spec.json` file. +Note that the `secrets` directory is gitignored by default, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source firebolt test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-firebolt:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-firebolt:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-firebolt:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-firebolt:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-firebolt:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-firebolt:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-firebolt:unitTest +``` +To run acceptance and custom integration tests: + +Make sure you have a running Firebolt engine that was specified in the config.json. It is needed to run the test queries. + +``` +./gradlew :airbyte-integrations:connectors:source-firebolt:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-firebolt/acceptance-test-config.yml b/airbyte-integrations/connectors/source-firebolt/acceptance-test-config.yml new file mode 100644 index 000000000000..5888449a7fc4 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/acceptance-test-config.yml @@ -0,0 +1,26 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-firebolt:dev +tests: + spec: + - spec_path: "source_firebolt/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + timeout_seconds: 120 + expect_records: + path: "integration_tests/expected_records.txt" + extra_fields: no + exact_order: yes + extra_records: no + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-firebolt/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-firebolt/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-firebolt/bootstrap.md b/airbyte-integrations/connectors/source-firebolt/bootstrap.md new file mode 100644 index 000000000000..fccc42134ac6 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/bootstrap.md @@ -0,0 +1,22 @@ +# Firebolt Source + +## Overview + +Firebolt is a cloud data warehouse purpose-built to provide sub-second analytics performance on massive, terabyte-scale data sets. + +Firebolt has two main concepts: Databases, which denote the storage of data and Engines, which describe the compute layer on top of a Database. + +Firebolt has three types of tables: External, Fact and Dimension. External tables, which represent a raw file structure in storage. Dimension tables, which are optimised for fetching and store data on each node in an Engine. Fact tables are similar to Dimension, but they shard the data across the nodes. The usual workload is to write source data into a set of files on S3, wrap them with an External table and write this data to a fetch-optimised Fact or Dimension table. + +## Connector + +This connector uses [firebolt-sdk](https://pypi.org/project/firebolt-sdk/), which is a [PEP-249](https://peps.python.org/pep-0249/) DB API implementation. +`Connection` object is used to connect to a specified Engine, wich runs subsequent queries against the data stored in the Database using the `Cursor` object. + +## Notes + +* External tables are not available as a source for performance reasons. +* Views are not available as a source due to possible complicated structure and non-obvious data types. +* Only Full reads are supported for now. +* Integration/Acceptance testing requires the user to have a running engine. Spinning up an engine can take a while so this ensures a faster iteration on the connector. +* Pagination is not available at the moment so large enough data sets might cause out of memory errors \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-firebolt/build.gradle b/airbyte-integrations/connectors/source-firebolt/build.gradle new file mode 100644 index 000000000000..f92bd34f7fb3 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_firebolt_singer' +} diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/__init__.py b/airbyte-integrations/connectors/source-firebolt/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py new file mode 100644 index 000000000000..34f2f625e15b --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/catalog.json b/airbyte-integrations/connectors/source-firebolt/integration_tests/catalog.json new file mode 100644 index 000000000000..8fed5b41ccd6 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/catalog.json @@ -0,0 +1,35 @@ +{ + "streams": [ + { + "name": "airbyte_acceptance_table", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + }, + "column3": { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_without_timezone" + }, + "column4": { + "type": "number" + }, + "column5": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-firebolt/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..b64ba92617a3 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/configured_catalog.json @@ -0,0 +1,38 @@ +{ + "streams": [ + { + "stream": { + "name": "airbyte_acceptance_table", + "supported_sync_modes": ["full_refresh"], + "source_defined_cursor": false, + "json_schema": { + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + }, + "column3": { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_without_timezone" + }, + "column4": { + "type": "number" + }, + "column5": { + "type": "array", + "items": { + "type": "integer" + } + } + } + } + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-firebolt/integration_tests/expected_records.txt new file mode 100644 index 000000000000..828fa3ce95d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/expected_records.txt @@ -0,0 +1,2 @@ +{"stream": "airbyte_acceptance_table", "data": {"column1": "my_value", "column2": 221, "column3": "2021-01-01T20:10:22", "column4": 1.214, "column5": [1,2,3]}, "emitted_at": 1626172757000} +{"stream": "airbyte_acceptance_table", "data": {"column1": "my_value2", "column2": 222, "column3": "2021-01-02T22:10:22", "column5": [1,2,null]}, "emitted_at": 1626172757000} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py b/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py new file mode 100644 index 000000000000..e03a428f5794 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/integration_test.py @@ -0,0 +1,141 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import random +import string +from json import load +from typing import Dict, Generator +from unittest.mock import MagicMock + +from airbyte_cdk.models import Status +from airbyte_cdk.models.airbyte_protocol import ( + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + SyncMode, +) +from firebolt.db import Connection +from pytest import fixture +from source_firebolt.source import SourceFirebolt, establish_connection + + +@fixture(scope="module") +def test_table_name() -> str: + letters = string.ascii_lowercase + rnd_string = "".join(random.choice(letters) for i in range(10)) + return f"airbyte_integration_{rnd_string}" + + +@fixture(scope="module") +def create_test_data(config: Dict[str, str], test_table_name: str) -> Generator[Connection, None, None]: + with establish_connection(config, MagicMock()) as connection: + with connection.cursor() as cursor: + cursor.execute( + f"CREATE DIMENSION TABLE {test_table_name} (column1 STRING NULL, column2 INT NULL, column3 DATE NULL, column4 DATETIME NULL, column5 DECIMAL(38, 31) NULL, column6 ARRAY(INT), column7 BOOLEAN NULL)" + ) + cursor.execute( + f"INSERT INTO {test_table_name} VALUES ('my_value',221,'2021-01-01','2021-01-01 12:00:01', Null, [1,2,3], true), ('my_value2',null,'2021-01-02','2021-01-02 12:00:02','1231232.123459999990457054844258706536', [1,2,3], null)" + ) + yield connection + cursor.execute(f"DROP TABLE {test_table_name}") + + +@fixture +def table_schema() -> str: + schema = { + "type": "object", + "properties": { + "column1": {"type": ["null", "string"]}, + "column2": {"type": ["null", "integer"]}, + "column3": {"type": ["null", "string"], "format": "date"}, + "column4": { + "type": ["null", "string"], + "format": "datetime", + "airbyte_type": "timestamp_without_timezone", + }, + "column5": {"type": ["null", "number"]}, # TODO: change once Decimal hits production + "column6": {"type": "array", "items": {"type": ["null", "integer"]}}, + "column7": {"type": ["null", "integer"]}, + }, + } + return schema + + +@fixture +def test_stream(test_table_name: str, table_schema: str) -> AirbyteStream: + return AirbyteStream(name=test_table_name, json_schema=table_schema, supported_sync_modes=[SyncMode.full_refresh]) + + +@fixture +def test_configured_catalogue(test_table_name: str, table_schema: str) -> ConfiguredAirbyteCatalog: + # Deleting one column to simulate manipulation in UI + del table_schema["properties"]["column1"] + append_stream = ConfiguredAirbyteStream( + stream=AirbyteStream(name=test_table_name, json_schema=table_schema), + sync_mode=SyncMode.incremental, + destination_sync_mode=DestinationSyncMode.append, + ) + return ConfiguredAirbyteCatalog(streams=[append_stream]) + + +@fixture(scope="module") +def config() -> Dict[str, str]: + with open( + "secrets/config.json", + ) as f: + yield load(f) + + +@fixture(scope="module") +def invalid_config() -> Dict[str, str]: + with open( + "integration_tests/invalid_config.json", + ) as f: + yield load(f) + + +def test_check_fails(invalid_config: Dict[str, str]): + source = SourceFirebolt() + status = source.check(logger=MagicMock(), config=invalid_config) + assert status.status == Status.FAILED + + +def test_check_succeeds(config: Dict[str, str]): + source = SourceFirebolt() + status = source.check(logger=MagicMock(), config=config) + assert status.status == Status.SUCCEEDED + + +def test_discover( + config: Dict[str, str], create_test_data: Generator[Connection, None, None], test_table_name: str, test_stream: AirbyteStream +): + source = SourceFirebolt() + catalog = source.discover(MagicMock(), config) + assert any(stream.name == test_table_name for stream in catalog.streams), "Test table not found" + for stream in catalog.streams: + if stream.name == test_table_name: + assert stream == test_stream + + +def test_read( + config: Dict[str, str], + create_test_data: Generator[Connection, None, None], + test_table_name: str, + test_configured_catalogue: ConfiguredAirbyteCatalog, +): + expected_data = [ + {"column2": 221, "column3": "2021-01-01", "column4": "2021-01-01T12:00:01", "column6": [1, 2, 3], "column7": 1}, + { + "column3": "2021-01-02", + "column4": "2021-01-02T12:00:02", + "column5": 1231232.12346, # TODO: change once Decimal is in production + "column6": [1, 2, 3], + }, + ] + source = SourceFirebolt() + result = source.read(logger=MagicMock(), config=config, catalog=test_configured_catalogue, state={}) + data = list(result) + assert all([x.record.stream == test_table_name for x in data]), "Table name is incorrect" + assert [x.record.data for x in data] == expected_data, "Test data is not matching" diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-firebolt/integration_tests/invalid_config.json new file mode 100644 index 000000000000..a7c5e13c48ff --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/invalid_config.json @@ -0,0 +1,6 @@ +{ + "username": "xxx", + "password": "xxx", + "database": "non_existing_database_name", + "engine": "database_name_Analytics" +} diff --git a/airbyte-integrations/connectors/source-firebolt/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-firebolt/integration_tests/sample_config.json new file mode 100644 index 000000000000..5f3ba0feb631 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/integration_tests/sample_config.json @@ -0,0 +1,6 @@ +{ + "database": "database_name", + "username": "xxx", + "password": "xxx", + "engine": "database_name_Analytics" +} diff --git a/airbyte-integrations/connectors/source-firebolt/main.py b/airbyte-integrations/connectors/source-firebolt/main.py new file mode 100644 index 000000000000..c7fdca83a929 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_firebolt import SourceFirebolt + +if __name__ == "__main__": + source = SourceFirebolt() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-firebolt/requirements.txt b/airbyte-integrations/connectors/source-firebolt/requirements.txt new file mode 100644 index 000000000000..7be17a56d745 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/requirements.txt @@ -0,0 +1,3 @@ +# This file is autogenerated -- only edit if you know what you are doing. Use setup.py for declaring dependencies. +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-firebolt/setup.py b/airbyte-integrations/connectors/source-firebolt/setup.py new file mode 100644 index 000000000000..c2d9148fe28d --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/setup.py @@ -0,0 +1,27 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = ["airbyte-cdk", "firebolt-sdk>=0.8.0"] + +TEST_REQUIREMENTS = [ + "pytest>=6.2.5", # 6.2.5 has python10 compatibility fixes + "pytest-asyncio>=0.18.0", + "source-acceptance-test", +] + +setup( + name="source_firebolt", + description="Source implementation for Firebolt.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/__init__.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/__init__.py new file mode 100644 index 000000000000..51a0626dad03 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceFirebolt + +__all__ = ["SourceFirebolt"] diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/database.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/database.py new file mode 100644 index 000000000000..6d133fb3cd6c --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/database.py @@ -0,0 +1,94 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import json +from typing import Any, Dict, List + +from airbyte_cdk.logger import AirbyteLogger +from firebolt.async_db import Connection as AsyncConnection +from firebolt.async_db import connect as async_connect +from firebolt.client import DEFAULT_API_URL +from firebolt.client.auth import UsernamePassword +from firebolt.db import Connection, connect + + +def parse_config(config: json, logger: AirbyteLogger) -> Dict[str, Any]: + """ + Convert dict of config values to firebolt.db.Connection arguments + + :param config: json-compatible dict of settings + :param logger: AirbyteLogger instance to print logs. + + :return: dictionary of firebolt.db.Connection-compatible kwargs + """ + connection_args = { + "database": config["database"], + "auth": UsernamePassword(config["username"], config["password"]), + "api_endpoint": config.get("host", DEFAULT_API_URL), + "account_name": config.get("account"), + } + # engine can be a name or a full URL of a cluster + engine = config.get("engine") + if engine: + if "." in engine: + connection_args["engine_url"] = engine + else: + connection_args["engine_name"] = engine + else: + logger.info("Engine parameter was not provided. Connecting to the default engine.") + return connection_args + + +def establish_connection(config: json, logger: AirbyteLogger) -> Connection: + """ + Creates a connection to Firebolt database using the parameters provided. + + :param config: Json object containing db credentials. + :param logger: AirbyteLogger instance to print logs. + + :return: PEP-249 compliant database Connection object. + """ + logger.debug("Connecting to Firebolt.") + connection = connect(**parse_config(config, logger)) + logger.debug("Connection to Firebolt established.") + return connection + + +async def establish_async_connection(config: json, logger: AirbyteLogger) -> AsyncConnection: + """ + Creates an async connection to Firebolt database using the parameters provided. + This connection can be used for parallel operations. + + :param config: Json object containing db credentials. + :param logger: AirbyteLogger instance to print logs. + + :return: PEP-249 compliant database Connection object. + """ + logger.debug("Connecting to Firebolt.") + connection = await async_connect(**parse_config(config, logger)) + logger.debug("Connection to Firebolt established.") + return connection + + +async def get_firebolt_tables(connection: AsyncConnection) -> List[str]: + """ + Fetch a list of tables that are compatible with Airbyte. + Currently this includes Fact and Dimension tables + + :param connection: Connection object connected to a database + + :return: List of table names + """ + query = """ + SELECT + table_name + FROM + information_schema.tables + WHERE + "table_type" IN ('FACT', 'DIMENSION') + """ + cursor = connection.cursor() + await cursor.execute(query) + return [table[0] for table in await cursor.fetchall()] diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py new file mode 100644 index 000000000000..b534ffb734b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/source.py @@ -0,0 +1,147 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import json +from asyncio import gather, get_event_loop +from typing import Dict, Generator + +from airbyte_cdk.logger import AirbyteLogger +from airbyte_cdk.models import ( + AirbyteCatalog, + AirbyteConnectionStatus, + AirbyteMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + Status, + SyncMode, +) +from airbyte_cdk.sources import Source +from firebolt.async_db import Connection as AsyncConnection + +from .database import establish_async_connection, establish_connection, get_firebolt_tables +from .utils import airbyte_message_from_data, convert_type + +SUPPORTED_SYNC_MODES = [SyncMode.full_refresh] + + +async def get_table_stream(connection: AsyncConnection, table: str) -> AirbyteStream: + """ + Get AirbyteStream for a particular table with table structure defined. + + :param connection: Connection object connected to a database + + :return: AirbyteStream object containing the table structure + """ + column_mapping = {} + cursor = connection.cursor() + await cursor.execute(f"SHOW COLUMNS {table}") + for t_name, c_name, c_type, nullable in await cursor.fetchall(): + airbyte_type = convert_type(c_type, nullable) + column_mapping[c_name] = airbyte_type + cursor.close() + json_schema = { + "type": "object", + "properties": column_mapping, + } + return AirbyteStream(name=table, json_schema=json_schema, supported_sync_modes=SUPPORTED_SYNC_MODES) + + +class SourceFirebolt(Source): + def check(self, logger: AirbyteLogger, config: json) -> AirbyteConnectionStatus: + """ + Tests if the input configuration can be used to successfully connect to the integration + e.g: if a provided Stripe API token can be used to connect to the Stripe API. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteConnectionStatus indicating a Success or Failure + """ + try: + with establish_connection(config, logger) as connection: + # We can only verify correctness of connection parameters on execution + with connection.cursor() as cursor: + cursor.execute("SELECT 1") + return AirbyteConnectionStatus(status=Status.SUCCEEDED) + except Exception as e: + return AirbyteConnectionStatus(status=Status.FAILED, message=f"An exception occurred: {str(e)}") + + def discover(self, logger: AirbyteLogger, config: json) -> AirbyteCatalog: + """ + Returns an AirbyteCatalog representing the available streams and fields in this integration. + For example, given valid credentials to a Postgres database, + returns an Airbyte catalog where each postgres table is a stream, and each table column is a field. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.json file + + :return: AirbyteCatalog is an object describing a list of all available streams in this source. + A stream is an AirbyteStream object that includes: + - its stream name (or table name in the case of Postgres) + - json_schema providing the specifications of expected schema for this stream (a list of columns described + by their names and types) + """ + + async def get_streams(): + async with await establish_async_connection(config, logger) as connection: + tables = await get_firebolt_tables(connection) + logger.info(f"Found {len(tables)} available tables.") + return await gather(*[get_table_stream(connection, table) for table in tables]) + + loop = get_event_loop() + streams = loop.run_until_complete(get_streams()) + logger.info(f"Provided {len(streams)} streams to the Aribyte Catalog.") + return AirbyteCatalog(streams=streams) + + def read( + self, + logger: AirbyteLogger, + config: json, + catalog: ConfiguredAirbyteCatalog, + state: Dict[str, any], + ) -> Generator[AirbyteMessage, None, None]: + """ + Returns a generator of the AirbyteMessages generated by reading the source with the given configuration, + catalog, and state. + + :param logger: Logging object to display debug/info/error to the logs + (logs will not be accessible via airbyte UI if they are not passed to this logger) + :param config: Json object containing the configuration of this source, content of this json is as specified in + the properties of the spec.json file + :param catalog: The input catalog is a ConfiguredAirbyteCatalog which is almost the same as AirbyteCatalog + returned by discover(), but + in addition, it's been configured in the UI! For each particular stream and field, there may have been provided + with extra modifications such as: filtering streams and/or columns out, renaming some entities, etc + :param state: When a Airbyte reads data from a source, it might need to keep a checkpoint cursor to resume + replication in the future from that saved checkpoint. + This is the object that is provided with state from previous runs and avoid replicating the entire set of + data everytime. + + :return: A generator that produces a stream of AirbyteRecordMessage contained in AirbyteMessage object. + """ + + logger.info(f"Reading data from {len(catalog.streams)} Firebolt tables.") + with establish_connection(config, logger) as connection: + with connection.cursor() as cursor: + for c_stream in catalog.streams: + table_name = c_stream.stream.name + table_properties = c_stream.stream.json_schema["properties"] + columns = list(table_properties.keys()) + + # Escape columns with " to avoid reserved keywords e.g. id + escaped_columns = ['"{}"'.format(col) for col in columns] + + query = "SELECT {columns} FROM {table}".format(columns=",".join(escaped_columns), table=table_name) + cursor.execute(query) + + logger.info(f"Fetched {cursor.rowcount} rows from table {table_name}.") + for result in cursor.fetchall(): + message = airbyte_message_from_data(result, columns, table_name) + if message: + yield message + logger.info("Data read complete.") diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/spec.json b/airbyte-integrations/connectors/source-firebolt/source_firebolt/spec.json new file mode 100644 index 000000000000..b3423e1eb4c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/spec.json @@ -0,0 +1,44 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/firebolt", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Firebolt Spec", + "type": "object", + "required": ["username", "password", "database"], + "additionalProperties": false, + "properties": { + "username": { + "type": "string", + "title": "Username", + "description": "Firebolt email address you use to login.", + "examples": ["username@email.com"] + }, + "password": { + "type": "string", + "title": "Password", + "description": "Firebolt password." + }, + "account": { + "type": "string", + "title": "Account", + "description": "Firebolt account to login." + }, + "host": { + "type": "string", + "title": "Host", + "description": "The host name of your Firebolt database.", + "examples": ["api.app.firebolt.io"] + }, + "database": { + "type": "string", + "title": "Database", + "description": "The database to connect to." + }, + "engine": { + "type": "string", + "title": "Engine", + "description": "Engine name or url to connect to." + } + } + } +} diff --git a/airbyte-integrations/connectors/source-firebolt/source_firebolt/utils.py b/airbyte-integrations/connectors/source-firebolt/source_firebolt/utils.py new file mode 100644 index 000000000000..f5f9e832b199 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/source_firebolt/utils.py @@ -0,0 +1,113 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import date, datetime +from decimal import Decimal +from typing import Any, Dict, List, Optional, Union + +from airbyte_cdk.models import AirbyteMessage, AirbyteRecordMessage, Type + + +def convert_type(fb_type: str, nullable: bool) -> Dict[str, Union[str, Dict]]: + """ + Convert from Firebolt type to Airbyte. If type is not defined in + Firebolt then it will be set to string, as per Airbyte reccommendation. + More on Firebolt types can be found in docs: + https://docs.firebolt.io/general-reference/data-types.html + + :param fb_type: Firebolt type. + + :return: Dict containing Airbyte type specification. + """ + map = { + "VARCHAR": {"type": "string"}, + "TEXT": {"type": "string"}, + "STRING": {"type": "string"}, + "INTEGER": {"type": "integer"}, + "INT": {"type": "integer"}, + "FLOAT": {"type": "number"}, + "DOUBLE": {"type": "number"}, + "DOUBLE PRECISION": {"type": "number"}, + "BOOLEAN": {"type": "integer"}, + # Firebolt bigint is max 8 byte so it fits in Airbyte's "integer" + "BIGINT": {"type": "integer"}, + "LONG": {"type": "integer"}, + "DECIMAL": {"type": "string", "airbyte_type": "big_number"}, + "DATE": {"type": "string", "format": "date"}, + "TIMESTAMP": { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_without_timezone", + }, + "DATETIME": { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_without_timezone", + }, + } + if fb_type.upper().startswith("ARRAY"): + inner_type = fb_type[6:-1] # Strip ARRAY() + # Array can't be nullable, but items can + airbyte_type = convert_type(inner_type, nullable=True) + result = {"type": "array", "items": airbyte_type} + else: + result = map.get(fb_type.upper(), {"type": "string"}) + if nullable: + result["type"] = ["null", result["type"]] + return result + + +def format_fetch_result(data: List[Any]) -> List[List[Any]]: + """ + Format data from a firebolt query to be compatible with Airbyte, + convert Firebolt timestamp string to Airbyte. + Firebolt stores dates in YYYY-MM-DD HH:mm:SS format. + Airbyte requires YYYY-MM-DDTHH:mm:SS. + + :param data: list of data items that may require conversion. + Example: [Decimal("22.1000921"), [2,3,4], datetime.datetime('2021-01-01 10:11:02')] + + :return: List of the same data as passed that's been converted to compatible types. + https://docs.airbyte.com/understanding-airbyte/supported-data-types/#the-types + """ + + for idx, item in enumerate(data): + if type(item) == datetime: + data[idx] = item.isoformat() + elif type(item) == date: + data[idx] = str(item) + elif type(item) == list: + data[idx] = format_fetch_result(item) + elif type(item) == Decimal: + data[idx] = str(item) + return data + + +def airbyte_message_from_data(raw_data: List[Any], columns: List[str], table_name: str) -> Optional[AirbyteMessage]: + """ + Wrap data into an AirbyteMessage. + + :param raw_data: Raw data row returned from a fetch query. Each item in the list + represents a row of data. + Example: [10, "Oranges"] + :param columns: List of column names + Example: ["Quantity", "Fruit"] + :param table_name: Name of a table where data was fetched from + + :return: AirbyteMessage containing parsed data + """ + raw_data = format_fetch_result(raw_data) + data = dict(zip(columns, raw_data)) + # Remove empty values + data = {k: v for k, v in data.items() if v is not None} + if not data: + return None + return AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream=table_name, + data=data, + emitted_at=int(datetime.now().timestamp()) * 1000, + ), + ) diff --git a/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py new file mode 100644 index 000000000000..19cf1c8d2232 --- /dev/null +++ b/airbyte-integrations/connectors/source-firebolt/unit_tests/test_firebolt_source.py @@ -0,0 +1,294 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from datetime import date, datetime +from decimal import Decimal +from unittest.mock import AsyncMock, MagicMock, patch + +from airbyte_cdk.models import ( + AirbyteMessage, + AirbyteRecordMessage, + AirbyteStream, + ConfiguredAirbyteCatalog, + ConfiguredAirbyteStream, + DestinationSyncMode, + Status, + SyncMode, + Type, +) +from pytest import fixture, mark +from source_firebolt.database import parse_config +from source_firebolt.source import ( + SUPPORTED_SYNC_MODES, + SourceFirebolt, + convert_type, + establish_connection, + get_firebolt_tables, + get_table_stream, +) +from source_firebolt.utils import airbyte_message_from_data, format_fetch_result + + +@fixture(params=["my_engine", "my_engine.api.firebolt.io"]) +def config(request): + args = { + "database": "my_database", + "username": "my_username", + "password": "my_password", + "engine": request.param, + } + return args + + +@fixture() +def config_no_engine(): + args = { + "database": "my_database", + "username": "my_username", + "password": "my_password", + } + return args + + +@fixture +def stream1() -> AirbyteStream: + stream1 = AirbyteStream( + name="table1", + supported_sync_modes=SUPPORTED_SYNC_MODES, + json_schema={ + "type": "object", + "properties": {"col1": {"type": "string"}, "col2": {"type": "integer"}}, + }, + ) + return stream1 + + +@fixture +def stream2() -> AirbyteStream: + stream2 = AirbyteStream( + name="table2", + supported_sync_modes=SUPPORTED_SYNC_MODES, + json_schema={ + "type": "object", + "properties": { + "col3": {"type": "array", "items": {"type": ["null", "string"]}}, + "col4": {"type": "string", "airbyte_type": "big_number"}, + }, + }, + ) + return stream2 + + +@fixture +def table1_structure(): + return [("table1", "col1", "STRING", 0), ("table1", "col2", "INT", 0)] + + +@fixture +def table2_structure(): + return [("table2", "col3", "ARRAY", 0), ("table2", "col4", "DECIMAL", 0)] + + +@fixture +def logger(): + return MagicMock() + + +@fixture(name="mock_connection") +def async_connection_cursor_mock(): + connection = MagicMock() + cursor = AsyncMock() + connection.cursor.return_value = cursor + return connection, cursor + + +def test_parse_config(config, logger): + config["engine"] = "override_engine" + result = parse_config(config, logger) + assert result["database"] == "my_database" + assert result["engine_name"] == "override_engine" + assert result["auth"].username == "my_username" + assert result["auth"].password == "my_password" + config["engine"] = "override_engine.api.firebolt.io" + result = parse_config(config, logger) + assert result["engine_url"] == "override_engine.api.firebolt.io" + + +@patch("source_firebolt.database.connect") +def test_connection(mock_connection, config, config_no_engine, logger): + establish_connection(config, logger) + logger.reset_mock() + establish_connection(config_no_engine, logger) + assert any(["default engine" in msg.args[0] for msg in logger.info.mock_calls]), "No message on using default engine" + + +@mark.parametrize( + "type,nullable,result", + [ + ("VARCHAR", False, {"type": "string"}), + ("INT", False, {"type": "integer"}), + ("int", False, {"type": "integer"}), + ("LONG", False, {"type": "integer"}), + ( + "TIMESTAMP", + False, + { + "type": "string", + "format": "datetime", + "airbyte_type": "timestamp_without_timezone", + }, + ), + ("ARRAY(ARRAY(INT))", False, {"type": "array", "items": {"type": "array", "items": {"type": ["null", "integer"]}}}), + ("int", True, {"type": ["null", "integer"]}), + ("DUMMY", False, {"type": "string"}), + ("boolean", False, {"type": "integer"}), + ], +) +def test_convert_type(type, nullable, result): + assert convert_type(type, nullable) == result + + +@mark.parametrize( + "data,expected", + [ + ( + ["a", 1], + ["a", 1], + ), + ([datetime.fromisoformat("2019-01-01 20:12:02"), 2], ["2019-01-01T20:12:02", 2]), + ([[date.fromisoformat("2019-01-01"), 2], 0.2214], [["2019-01-01", 2], 0.2214]), + ([[None, 2], None], [[None, 2], None]), + ([Decimal("1231232.123459999990457054844258706536")], ["1231232.123459999990457054844258706536"]), + ], +) +def test_format_fetch_result(data, expected): + assert format_fetch_result(data) == expected + + +@patch("source_firebolt.utils.datetime") +def test_airbyte_message_from_data(mock_datetime): + mock_datetime.now.return_value.timestamp.return_value = 10 + raw_data = [1, "a", [1, 2, 3]] + columns = ["Col1", "Col2", "Col3"] + table_name = "dummy" + expected = AirbyteMessage( + type=Type.RECORD, + record=AirbyteRecordMessage( + stream="dummy", + data={"Col1": 1, "Col2": "a", "Col3": [1, 2, 3]}, + emitted_at=10000, + ), + ) + result = airbyte_message_from_data(raw_data, columns, table_name) + assert result == expected + + +def test_airbyte_message_from_data_no_data(): + raw_data = [] + columns = ["Col1", "Col2"] + table_name = "dummy" + result = airbyte_message_from_data(raw_data, columns, table_name) + assert result is None + + +@mark.asyncio +async def test_get_firebolt_tables(mock_connection): + connection, cursor = mock_connection + cursor.fetchall.return_value = [("table1",), ("table2",)] + result = await get_firebolt_tables(connection) + assert result == ["table1", "table2"] + + +@mark.asyncio +async def test_get_table_stream(mock_connection, table1_structure, stream1): + connection, cursor = mock_connection + cursor.fetchall.return_value = table1_structure + result = await get_table_stream(connection, "table1") + assert result == stream1 + + +@patch("source_firebolt.source.establish_connection") +def test_check(mock_connection, config, logger): + source = SourceFirebolt() + status = source.check(logger, config) + assert status.status == Status.SUCCEEDED + mock_connection().__enter__().cursor().__enter__().execute.side_effect = Exception("my exception") + status = source.check(logger, config) + assert status.status == Status.FAILED + + +@patch("source_firebolt.source.get_table_stream") +@patch("source_firebolt.source.establish_async_connection") +def test_discover( + mock_establish_connection, + mock_get_stream, + mock_connection, + config, + stream1, + stream2, + logger, +): + connection, cursor = mock_connection + cursor.fetchall.return_value = ["table1", "table2"] + mock_establish_connection.return_value.__aenter__.return_value = connection + mock_get_stream.side_effect = [stream1, stream2] + + source = SourceFirebolt() + catalog = source.discover(logger, config) + assert catalog.streams[0].name == "table1" + assert catalog.streams[1].name == "table2" + assert catalog.streams[0].json_schema == stream1.json_schema + assert catalog.streams[1].json_schema == stream2.json_schema + mock_establish_connection.assert_awaited_once_with(config, logger) + + +@patch("source_firebolt.source.establish_connection") +def test_read_no_state(mock_connection, config, stream1, logger): + source = SourceFirebolt() + + c_stream = ConfiguredAirbyteStream( + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + stream=stream1, + ) + catalog = ConfiguredAirbyteCatalog(streams=[c_stream]) + mock_connection().__enter__().cursor().__enter__().fetchall().__iter__.return_value = iter( + [ + ["s_value1", 1], + ["s_value2", 2], + ] + ) + message1 = next(source.read(logger, config, catalog, {})) + assert message1.record.stream == stream1.name + assert message1.record.data == {"col1": "s_value1", "col2": 1} + message2 = next(source.read(logger, config, catalog, {})) + assert message2.record.stream == stream1.name + assert message2.record.data == {"col1": "s_value2", "col2": 2} + + +@patch("source_firebolt.source.establish_connection") +def test_read_special_types_no_state(mock_connection, config, stream2, logger): + source = SourceFirebolt() + + c_stream = ConfiguredAirbyteStream( + sync_mode=SyncMode.full_refresh, + destination_sync_mode=DestinationSyncMode.overwrite, + stream=stream2, + ) + catalog = ConfiguredAirbyteCatalog(streams=[c_stream]) + mock_connection().__enter__().cursor().__enter__().fetchall().__iter__.return_value = iter( + [ + [ + [datetime.fromisoformat("2019-01-01 20:12:02"), datetime.fromisoformat("2019-02-01 20:12:02")], + Decimal("1231232.123459999990457054844258706536"), + ], + ] + ) + + message1 = next(source.read(logger, config, catalog, {})) + assert message1.record.stream == stream2.name + assert message1.record.data == { + "col3": ["2019-01-01T20:12:02", "2019-02-01T20:12:02"], + "col4": "1231232.123459999990457054844258706536", + } diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md new file mode 100644 index 000000000000..774f04cb8036 --- /dev/null +++ b/docs/SUMMARY.md @@ -0,0 +1,305 @@ +# Table of contents + +- [Introduction](../README.md) +- [Airbyte Cloud QuickStart](cloud/getting-started-with-airbyte-cloud.md) + - [Core Concepts](cloud/core-concepts.md) + - [Managing Airbyte Cloud](cloud/managing-airbyte-cloud.md) +- [Airbyte Open Source Quickstart](quickstart/README.md) + - [Deploy Airbyte](quickstart/deploy-airbyte.md) + - [Add a Source](quickstart/add-a-source.md) + - [Add a Destination](quickstart/add-a-destination.md) + - [Set up a Connection](quickstart/set-up-a-connection.md) +- [Deploying Airbyte Open Source](deploying-airbyte/README.md) + - [Local Deployment](deploying-airbyte/local-deployment.md) + - [On AWS (EC2)](deploying-airbyte/on-aws-ec2.md) + - [On AWS ECS (Coming Soon)](deploying-airbyte/on-aws-ecs.md) + - [On Azure(VM)](deploying-airbyte/on-azure-vm-cloud-shell.md) + - [On GCP (Compute Engine)](deploying-airbyte/on-gcp-compute-engine.md) + - [On Kubernetes (Beta)](deploying-airbyte/on-kubernetes.md) + - [On Plural (Beta)](deploying-airbyte/on-plural.md) + - [On Oracle Cloud Infrastructure VM](deploying-airbyte/on-oci-vm.md) + - [On Digital Ocean Droplet](deploying-airbyte/on-digitalocean-droplet.md) +- [Operator Guides](operator-guides/README.md) + - [Upgrading Airbyte](operator-guides/upgrading-airbyte.md) + - [Resetting Your Data](operator-guides/reset.md) + - [Configuring the Airbyte Database](operator-guides/configuring-airbyte-db.md) + - [Browsing Output Logs](operator-guides/browsing-output-logs.md) + - [Using the Airflow Airbyte Operator](operator-guides/using-the-airflow-airbyte-operator.md) + - [Using the Prefect Task](operator-guides/using-prefect-task.md) + - [Using the Dagster Integration](operator-guides/using-dagster-integration.md) + - [Windows - Browsing Local File Output](operator-guides/locating-files-local-destination.md) + - [Transformations and Normalization](operator-guides/transformation-and-normalization/README.md) + - [Transformations with SQL (Part 1/3)](operator-guides/transformation-and-normalization/transformations-with-sql.md) + - [Transformations with dbt (Part 2/3)](operator-guides/transformation-and-normalization/transformations-with-dbt.md) + - [Transformations with Airbyte (Part 3/3)](operator-guides/transformation-and-normalization/transformations-with-airbyte.md) + - [Configuring Airbyte](operator-guides/configuring-airbyte.md) + - [Sentry Integration](operator-guides/sentry-integration.md) + - [Using Custom Connectors](operator-guides/using-custom-connectors.md) + - [Scaling Airbyte](operator-guides/scaling-airbyte.md) + - [Securing Airbyte](operator-guides/securing-airbyte.md) +- [Connector Catalog](integrations/README.md) + - [Sources](integrations/sources/README.md) + - [3PL Central](integrations/sources/tplcentral.md) + - [Airtable](integrations/sources/airtable.md) + - [Amazon SQS](integrations/sources/amazon-sqs.md) + - [Amazon Seller Partner](integrations/sources/amazon-seller-partner.md) + - [Amazon Ads](integrations/sources/amazon-ads.md) + - [Amplitude](integrations/sources/amplitude.md) + - [Apify Dataset](integrations/sources/apify-dataset.md) + - [Appstore](integrations/sources/appstore.md) + - [Asana](integrations/sources/asana.md) + - [AWS CloudTrail](integrations/sources/aws-cloudtrail.md) + - [Azure Table Storage](integrations/sources/azure-table.md) + - [Bamboo HR](integrations/sources/bamboo-hr.md) + - [Bing Ads](integrations/sources/bing-ads.md) + - [BigCommerce](integrations/sources/bigcommerce.md) + - [BigQuery](integrations/sources/bigquery.md) + - [Braintree](integrations/sources/braintree.md) + - [Cart](integrations/sources/cart.md) + - [Chargebee](integrations/sources/chargebee.md) + - [Chartmogul](integrations/sources/chartmogul.md) + - [ClickHouse](integrations/sources/clickhouse.md) + - [Close.com](integrations/sources/close-com.md) + - [CockroachDB](integrations/sources/cockroachdb.md) + - [Confluence](integrations/sources/confluence.md) + - [Customer.io (Sponsored by Faros AI)](integrations/sources/customer-io.md) + - [Delighted](integrations/sources/delighted.md) + - [Db2](integrations/sources/db2.md) + - [Dixa](integrations/sources/dixa.md) + - [Drift](integrations/sources/drift.md) + - [Drupal](integrations/sources/drupal.md) + - [End-to-End Testing](integrations/sources/e2e-test.md) + - [Exchange Rates API](integrations/sources/exchangeratesapi.md) + - [Facebook Marketing](integrations/sources/facebook-marketing.md) + - [Facebook Pages](integrations/sources/facebook-pages.md) + - [Faker](integrations/sources/faker.md) + - [Files](integrations/sources/file.md) + - [Firebolt](integrations/sources/firebolt.md) + - [Flexport](integrations/sources/flexport.md) + - [Freshdesk](integrations/sources/freshdesk.md) + - [Freshsales](integrations/sources/freshsales.md) + - [Freshservice](integrations/sources/freshservice.md) + - [GitHub](integrations/sources/github.md) + - [GitLab](integrations/sources/gitlab.md) + - [Google Ads](integrations/sources/google-ads.md) + - [Google Analytics](integrations/sources/google-analytics-v4.md) + - [Google Directory](integrations/sources/google-directory.md) + - [Google Search Console](integrations/sources/google-search-console.md) + - [Google Sheets](integrations/sources/google-sheets.md) + - [Google Workspace Admin Reports](integrations/sources/google-workspace-admin-reports.md) + - [Greenhouse](integrations/sources/greenhouse.md) + - [Harvest](integrations/sources/harvest.md) + - [Harness (Sponsored by Faros AI)](integrations/sources/harness.md) + - [HTTP Request (Graveyarded)](integrations/sources/http-request.md) + - [HubSpot](integrations/sources/hubspot.md) + - [Instagram](integrations/sources/instagram.md) + - [Intercom](integrations/sources/intercom.md) + - [Iterable](integrations/sources/iterable.md) + - [Jenkins (Sponsored by Faros AI)](integrations/sources/jenkins.md) + - [Jira](integrations/sources/jira.md) + - [Kafka](integrations/sources/kafka.md) + - [Klaviyo](integrations/sources/klaviyo.md) + - [Kustomer](integrations/sources/kustomer.md) + - [Lemlist](integrations/sources/lemlist.md) + - [LinkedIn Ads](integrations/sources/linkedin-ads.md) + - [Linnworks](integrations/sources/linnworks.md) + - [Lever Hiring](integrations/sources/lever-hiring.md) + - [Looker](integrations/sources/looker.md) + - [Magento](integrations/sources/magento.md) + - [Mailchimp](integrations/sources/mailchimp.md) + - [Marketo](integrations/sources/marketo.md) + - [Microsoft Dynamics AX](integrations/sources/microsoft-dynamics-ax.md) + - [Microsoft Dynamics Customer Engagement](integrations/sources/microsoft-dynamics-customer-engagement.md) + - [Microsoft Dynamics GP](integrations/sources/microsoft-dynamics-gp.md) + - [Microsoft Dynamics NAV](integrations/sources/microsoft-dynamics-nav.md) + - [Microsoft SQL Server (MSSQL)](integrations/sources/mssql.md) + - [Microsoft Teams](integrations/sources/microsoft-teams.md) + - [Mixpanel](integrations/sources/mixpanel.md) + - [Monday](integrations/sources/monday.md) + - [Mongo DB](integrations/sources/mongodb-v2.md) + - [My Hours](integrations/sources/my-hours.md) + - [MySQL](integrations/sources/mysql.md) + - [Notion](integrations/sources/notion.md) + - [Okta](integrations/sources/okta.md) + - [OneSignal](integrations/sources/onesignal.md) + - [OpenWeather](integrations/sources/openweather.md) + - [Oracle DB](integrations/sources/oracle.md) + - [Oracle Peoplesoft](integrations/sources/oracle-peoplesoft.md) + - [Oracle Siebel CRM](integrations/sources/oracle-siebel-crm.md) + - [Orb](integrations/sources/orb.md) + - [Outreach](integrations/sources/outreach.md) + - [PagerDuty (Sponsored by Faros AI)](integrations/sources/pagerduty.md) + - [Paypal Transaction](integrations/sources/paypal-transaction.md) + - [Paystack](integrations/sources/paystack.md) + - [Persistiq](integrations/sources/persistiq.md) + - [Plaid](integrations/sources/plaid.md) + - [Pinterest](integrations/sources/pinterest.md) + - [Pipedrive](integrations/sources/pipedrive.md) + - [PokéAPI](integrations/sources/pokeapi.md) + - [Postgres](integrations/sources/postgres.md) + - [PostHog](integrations/sources/posthog.md) + - [PrestaShop](integrations/sources/presta-shop.md) + - [Qualaroo](integrations/sources/qualaroo.md) + - [QuickBooks](integrations/sources/quickbooks.md) + - [Recharge](integrations/sources/recharge.md) + - [Recurly](integrations/sources/recurly.md) + - [Redshift](integrations/sources/redshift.md) + - [S3](integrations/sources/s3.md) + - [SAP Business One](integrations/sources/sap-business-one.md) + - [SearchMetrics](integrations/sources/search-metrics.md) + - [Salesforce](integrations/sources/salesforce.md) + - [SalesLoft](integrations/sources/salesloft.md) + - [Sendgrid](integrations/sources/sendgrid.md) + - [Sentry](integrations/sources/sentry.md) + - [Shopify](integrations/sources/shopify.md) + - [Shortio](integrations/sources/shortio.md) + - [Slack](integrations/sources/slack.md) + - [Smartsheets](integrations/sources/smartsheets.md) + - [Snapchat Marketing](integrations/sources/snapchat-marketing.md) + - [Snowflake](integrations/sources/snowflake.md) + - [Spree Commerce](integrations/sources/spree-commerce.md) + - [Square](integrations/sources/square.md) + - [Strava](integrations/sources/strava.md) + - [Stripe](integrations/sources/stripe.md) + - [Sugar CRM](integrations/sources/sugar-crm.md) + - [SurveyMonkey](integrations/sources/surveymonkey.md) + - [Tempo](integrations/sources/tempo.md) + - [TikTok Marketing](integrations/sources/tiktok-marketing.md) + - [Trello](integrations/sources/trello.md) + - [Twilio](integrations/sources/twilio.md) + - [TiDB](integrations/sources/tidb.md) + - [Typeform](integrations/sources/typeform.md) + - [US Census API](integrations/sources/us-census.md) + - [VictorOps (Sponsored by Faros AI)](integrations/sources/victorops.md) + - [Woo Commerce](integrations/sources/woocommerce.md) + - [Wordpress](integrations/sources/wordpress.md) + - [YouTube Analytics](integrations/sources/youtube-analytics.md) + - [Zencart](integrations/sources/zencart.md) + - [Zendesk Chat](integrations/sources/zendesk-chat.md) + - [Zendesk Sunshine](integrations/sources/zendesk-sunshine.md) + - [Zendesk Support](integrations/sources/zendesk-support.md) + - [Zendesk Talk](integrations/sources/zendesk-talk.md) + - [Zenloop](integrations/sources/zenloop.md) + - [Zoho CRM](integrations/sources/zoho-crm.md) + - [Zoom](integrations/sources/zoom.md) + - [Zuora](integrations/sources/zuora.md) + - [Destinations](integrations/destinations/README.md) + - [Amazon SQS](integrations/destinations/amazon-sqs.md) + - [AzureBlobStorage](integrations/destinations/azureblobstorage.md) + - [BigQuery](integrations/destinations/bigquery.md) + - [ClickHouse](integrations/destinations/clickhouse.md) + - [Databricks](integrations/destinations/databricks.md) + - [DynamoDB](integrations/destinations/dynamodb.md) + - [Elasticsearch](integrations/destinations/elasticsearch.md) + - [End-to-End Testing](integrations/destinations/e2e-test.md) + - [Chargify](integrations/destinations/chargify.md) + - [Google Cloud Storage (GCS)](integrations/destinations/gcs.md) + - [Google Firestore](integrations/destinations/firestore.md) + - [Google PubSub](integrations/destinations/pubsub.md) + - [Kafka](integrations/destinations/kafka.md) + - [Keen](integrations/destinations/keen.md) + - [Local CSV](integrations/destinations/local-csv.md) + - [Local JSON](integrations/destinations/local-json.md) + - [MariaDB ColumnStore](integrations/destinations/mariadb-columnstore.md) + - [MeiliSearch](integrations/destinations/meilisearch.md) + - [MongoDB](integrations/destinations/mongodb.md) + - [MQTT](integrations/destinations/mqtt.md) + - [MSSQL](integrations/destinations/mssql.md) + - [MySQL](integrations/destinations/mysql.md) + - [Oracle DB](integrations/destinations/oracle.md) + - [Postgres](integrations/destinations/postgres.md) + - [Pulsar](integrations/destinations/pulsar.md) + - [RabbitMQ](integrations/destinations/rabbitmq.md) + - [Redshift](integrations/destinations/redshift.md) + - [Rockset](integrations/destinations/rockset.md) + - [S3](integrations/destinations/s3.md) + - [SFTP JSON](integrations/destinations/sftp-json.md) + - [Snowflake](integrations/destinations/snowflake.md) + - [Cassandra](integrations/destinations/cassandra.md) + - [Scylla](integrations/destinations/scylla.md) + - [Redis](integrations/destinations/redis.md) + - [Kinesis](integrations/destinations/kinesis.md) + - [Streamr](integrations/destinations/streamr.md) + - [Custom or New Connector](integrations/custom-connectors.md) +- [Connector Development](connector-development/README.md) + - [Tutorials](connector-development/tutorials/README.md) + - [Python CDK Speedrun: Creating a Source](connector-development/tutorials/cdk-speedrun.md) + - [Python CDK: Creating a HTTP API Source](connector-development/tutorials/cdk-tutorial-python-http/README.md) + - [Getting Started](connector-development/tutorials/cdk-tutorial-python-http/0-getting-started.md) + - [Step 1: Creating the Source](connector-development/tutorials/cdk-tutorial-python-http/1-creating-the-source.md) + - [Step 2: Install Dependencies](connector-development/tutorials/cdk-tutorial-python-http/2-install-dependencies.md) + - [Step 3: Define Inputs](connector-development/tutorials/cdk-tutorial-python-http/3-define-inputs.md) + - [Step 4: Connection Checking](connector-development/tutorials/cdk-tutorial-python-http/4-connection-checking.md) + - [Step 5: Declare the Schema](connector-development/tutorials/cdk-tutorial-python-http/5-declare-schema.md) + - [Step 6: Read Data](connector-development/tutorials/cdk-tutorial-python-http/6-read-data.md) + - [Step 7: Use the Connector in Airbyte](connector-development/tutorials/cdk-tutorial-python-http/7-use-connector-in-airbyte.md) + - [Step 8: Test Connector](connector-development/tutorials/cdk-tutorial-python-http/8-test-your-connector.md) + - [Building a Python Source](connector-development/tutorials/building-a-python-source.md) + - [Building a Python Destination](connector-development/tutorials/building-a-python-destination.md) + - [Building a Java Destination](connector-development/tutorials/building-a-java-destination.md) + - [Profile Java Connector Memory](connector-development/tutorials/profile-java-connector-memory.md) + - [Connector Development Kit (Python)](connector-development/cdk-python/README.md) + - [Basic Concepts](connector-development/cdk-python/basic-concepts.md) + - [Defining Stream Schemas](connector-development/cdk-python/schemas.md) + - [Full Refresh Streams](connector-development/cdk-python/full-refresh-stream.md) + - [Incremental Streams](connector-development/cdk-python/incremental-stream.md) + - [HTTP-API-based Connectors](connector-development/cdk-python/http-streams.md) + - [Python Concepts](connector-development/cdk-python/python-concepts.md) + - [Stream Slices](connector-development/cdk-python/stream-slices.md) + - [Connector Development Kit (Javascript)](connector-development/cdk-faros-js.md) + - [Airbyte 101 for Connector Development](connector-development/airbyte101.md) + - [Testing Connectors](connector-development/testing-connectors/README.md) + - [Source Acceptance Tests Reference](connector-development/testing-connectors/source-acceptance-tests-reference.md) + - [Connector Specification Reference](connector-development/connector-specification-reference.md) + - [Best Practices](connector-development/best-practices.md) + - [UX Handbook](connector-development/ux-handbook.md) +- [Contributing to Airbyte](contributing-to-airbyte/README.md) + - [Code of Conduct](contributing-to-airbyte/code-of-conduct.md) + - [Developing Locally](contributing-to-airbyte/developing-locally.md) + - [Developing on Docker](contributing-to-airbyte/developing-on-docker.md) + - [Developing on Kubernetes](contributing-to-airbyte/developing-on-kubernetes.md) + - [Monorepo Python Development](contributing-to-airbyte/monorepo-python-development.md) + - [Code Style](contributing-to-airbyte/code-style.md) + - [Gradle Cheatsheet](contributing-to-airbyte/gradle-cheatsheet.md) + - [Updating Documentation](contributing-to-airbyte/updating-documentation.md) + - [Templates](contributing-to-airbyte/templates/README.md) + - [Connector Doc Template](contributing-to-airbyte/templates/integration-documentation-template.md) +- [Understanding Airbyte](understanding-airbyte/README.md) + - [A Beginner's Guide to the AirbyteCatalog](understanding-airbyte/beginners-guide-to-catalog.md) + - [AirbyteCatalog Reference](understanding-airbyte/catalog.md) + - [Airbyte Specification](understanding-airbyte/airbyte-specification.md) + - [Basic Normalization](understanding-airbyte/basic-normalization.md) + - [Connections and Sync Modes](understanding-airbyte/connections/README.md) + - [Full Refresh - Overwrite](understanding-airbyte/connections/full-refresh-overwrite.md) + - [Full Refresh - Append](understanding-airbyte/connections/full-refresh-append.md) + - [Incremental Sync - Append](understanding-airbyte/connections/incremental-append.md) + - [Incremental Sync - Deduped History](understanding-airbyte/connections/incremental-deduped-history.md) + - [Operations](understanding-airbyte/operations.md) + - [High-level View](understanding-airbyte/high-level-view.md) + - [Workers & Jobs](understanding-airbyte/jobs.md) + - [Technical Stack](understanding-airbyte/tech-stack.md) + - [Change Data Capture (CDC)](understanding-airbyte/cdc.md) + - [Namespaces](understanding-airbyte/namespaces.md) + - [Supported Data Types](understanding-airbyte/supported-data-types.md) + - [Json to Avro Conversion](understanding-airbyte/json-avro-conversion.md) + - [Glossary of Terms](understanding-airbyte/glossary.md) +- [API documentation](api-documentation.md) +- [CLI documentation](https://github.com/airbytehq/airbyte/tree/master/octavia-cli) +- [Project Overview](project-overview/README.md) + - [Roadmap](project-overview/roadmap.md) + - [Changelog](project-overview/changelog/README.md) + - [Platform](project-overview/changelog/platform.md) + - [Connectors](project-overview/changelog/connectors.md) + - [Slack Code of Conduct](project-overview/slack-code-of-conduct.md) + - [Security and Data Privacy](project-overview/security.md) + - [Licenses](project-overview/licenses/README.md) + - [License FAQ](project-overview/licenses/license-faq.md) + - [ELv2](project-overview/licenses/elv2-license.md) + - [MIT](project-overview/licenses/mit-license.md) + - [Examples](project-overview/licenses/examples.md) + - [Product Release Stages](project-overview/product-release-stages.md) +- [Troubleshooting & FAQ](troubleshooting/README.md) + - [On Deploying](troubleshooting/on-deploying.md) + - [On Setting up a New Connection](troubleshooting/new-connection.md) + - [On Running a Sync](troubleshooting/running-sync.md) + - [On Upgrading](troubleshooting/on-upgrading.md) \ No newline at end of file diff --git a/docs/integrations/README.md b/docs/integrations/README.md index c8beb01ca914..4d75c4144aef 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -54,6 +54,7 @@ For more information about the grading system, see [Product Release Stages](http | [Facebook Pages](sources/facebook-pages.md) | Alpha | No | | [Faker](sources/faker.md) | Alpha | Yes | | [File](sources/file.md) | Alpha | Yes | +| [Firebolt](sources/firebolt.md) | Alpha | Yes | | [Flexport](sources/flexport.md) | Alpha | No | | [Freshdesk](sources/freshdesk.md) | Alpha | Yes | | [Freshsales](sources/freshsales.md) | Alpha | No | diff --git a/docs/integrations/sources/firebolt.md b/docs/integrations/sources/firebolt.md new file mode 100644 index 000000000000..35650bcb179b --- /dev/null +++ b/docs/integrations/sources/firebolt.md @@ -0,0 +1,61 @@ +# Firebolt + +## Overview + +The Firebolt source allows you to sync your data from [Firebolt](https://www.firebolt.io/). Only Full refresh is supported at the moment. + +The connector is built on top of a pure Python [firebolt-sdk](https://pypi.org/project/firebolt-sdk/) and does not require additonal dependencies. + +#### Resulting schema + +The Firebolt source does not alter schema present in your database. Depending on the destination connected to this source, however, the result schema may be altered. See the destination's documentation for more details. + +#### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | No | | + +## Getting started + +### Requirements + +1. An existing AWS account + + +### Setup guide + +1. Create a Firebolt account following the [guide](https://docs.firebolt.io/managing-your-account/creating-an-account.html) + +1. Follow the getting started [tutorial](https://docs.firebolt.io/getting-started.html) to setup a database + +1. [Load data](https://docs.firebolt.io/loading-data/loading-data.html) + +1. Create an Analytics (read-only) engine as described in [here](https://docs.firebolt.io/working-with-engines/working-with-engines-using-the-firebolt-manager.html) + + +#### You should now have the following + +1. An existing Firebolt account +1. Connection parameters handy + 1. Username + 1. Password + 1. Account, in case of a multi-account setup (Optional) + 1. Host (Optional) + 1. Engine (Optional), preferably Analytics/read-only +1. A running engine (if an engine is stopped or booting up you won't be able to connect to it) +1. Your data in either [Fact or Dimension](https://docs.firebolt.io/working-with-tables.html#fact-and-dimension-tables) tables. + + +You can now use the Airbyte Firebolt source. + + + + +## Changelog + + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2022-04-28 | TBD | Create Firebolt source | \ No newline at end of file