From 929ab6b5236c314a11a347cd4fd235ff9a3105cf Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Thu, 3 Nov 2022 12:13:23 +0530 Subject: [PATCH 01/12] =?UTF-8?q?=F0=9F=8E=89=20New=20Source:=20Aha?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-aha/.dockerignore | 6 ++ .../connectors/source-aha/Dockerfile | 38 +++++++++ .../connectors/source-aha/README.md | 79 +++++++++++++++++++ .../connectors/source-aha/__init__.py | 3 + .../source-aha/acceptance-test-config.yml | 38 +++++++++ .../source-aha/acceptance-test-docker.sh | 16 ++++ .../connectors/source-aha/build.gradle | 9 +++ .../source-aha/integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 ++ .../integration_tests/acceptance.py | 16 ++++ .../integration_tests/configured_catalog.json | 22 ++++++ .../integration_tests/invalid_config.json | 4 + .../integration_tests/sample_config.json | 4 + .../connectors/source-aha/main.py | 13 +++ .../connectors/source-aha/requirements.txt | 2 + .../connectors/source-aha/setup.py | 29 +++++++ .../source-aha/source_aha/__init__.py | 8 ++ .../connectors/source-aha/source_aha/aha.yaml | 54 +++++++++++++ .../source-aha/source_aha/schemas/TODO.md | 16 ++++ .../source_aha/schemas/features.json | 57 +++++++++++++ .../source_aha/schemas/products.json | 53 +++++++++++++ .../source-aha/source_aha/source.py | 18 +++++ .../source-aha/source_aha/spec.yaml | 16 ++++ 23 files changed, 509 insertions(+) create mode 100644 airbyte-integrations/connectors/source-aha/.dockerignore create mode 100644 airbyte-integrations/connectors/source-aha/Dockerfile create mode 100644 airbyte-integrations/connectors/source-aha/README.md create mode 100644 airbyte-integrations/connectors/source-aha/__init__.py create mode 100644 airbyte-integrations/connectors/source-aha/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-aha/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-aha/build.gradle create mode 100644 airbyte-integrations/connectors/source-aha/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-aha/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-aha/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-aha/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-aha/main.py create mode 100644 airbyte-integrations/connectors/source-aha/requirements.txt create mode 100644 airbyte-integrations/connectors/source-aha/setup.py create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/__init__.py create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/aha.yaml create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/source.py create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/spec.yaml diff --git a/airbyte-integrations/connectors/source-aha/.dockerignore b/airbyte-integrations/connectors/source-aha/.dockerignore new file mode 100644 index 0000000000000..b2f40d77950bf --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_aha +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-aha/Dockerfile b/airbyte-integrations/connectors/source-aha/Dockerfile new file mode 100644 index 0000000000000..4e35b33f850fb --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_aha ./source_aha + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-aha diff --git a/airbyte-integrations/connectors/source-aha/README.md b/airbyte-integrations/connectors/source-aha/README.md new file mode 100644 index 0000000000000..16158b89d2db9 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/README.md @@ -0,0 +1,79 @@ +# Aha Source + +This is the repository for the Aha configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/aha). + +## Local development + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-aha:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/aha) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_aha/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source aha test creds` +and place them into `secrets/config.json`. + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-aha:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-aha:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-aha:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aha:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-aha:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-aha:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing + +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-aha:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-aha:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-aha/__init__.py b/airbyte-integrations/connectors/source-aha/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-aha/acceptance-test-config.yml b/airbyte-integrations/connectors/source-aha/acceptance-test-config.yml new file mode 100644 index 0000000000000..ccf5036734e4a --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/acceptance-test-config.yml @@ -0,0 +1,38 @@ +# See [Source Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-aha:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_aha/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-aha/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-aha/acceptance-test-docker.sh new file mode 100644 index 0000000000000..c51577d10690c --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-aha/build.gradle b/airbyte-integrations/connectors/source-aha/build.gradle new file mode 100644 index 0000000000000..a4363e64843c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_aha' +} diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/__init__.py b/airbyte-integrations/connectors/source-aha/integration_tests/__init__.py new file mode 100644 index 0000000000000..1100c1c58cf51 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-aha/integration_tests/abnormal_state.json new file mode 100644 index 0000000000000..52b0f2c2118f4 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py new file mode 100644 index 0000000000000..1302b2f57e10e --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-aha/integration_tests/configured_catalog.json new file mode 100644 index 0000000000000..dda2659e960dd --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "features", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "products", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-aha/integration_tests/invalid_config.json new file mode 100644 index 0000000000000..198fd05930e31 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/integration_tests/invalid_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "abb74d48e6d2dadcc1f0bb", + "url" : "https://905c7285-30d2-4419-a9e4-8eef125ce6c1.aha.io" +} diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json new file mode 100644 index 0000000000000..2417dfeaebf2e --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json @@ -0,0 +1,4 @@ +{ + "api_key": "abb74d48e6d2371242eb2a84d1ac2198a64a60b3396d76c35bf8a6dadcc1f0bb", + "url" : "https://905c7285-30d2-4419-a9e4-8eef125ce6c1.aha.io" +} diff --git a/airbyte-integrations/connectors/source-aha/main.py b/airbyte-integrations/connectors/source-aha/main.py new file mode 100644 index 0000000000000..feff5fa4fea7e --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_aha import SourceAha + +if __name__ == "__main__": + source = SourceAha() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-aha/requirements.txt b/airbyte-integrations/connectors/source-aha/requirements.txt new file mode 100644 index 0000000000000..0411042aa0911 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-aha/setup.py b/airbyte-integrations/connectors/source-aha/setup.py new file mode 100644 index 0000000000000..58773a233b863 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_aha", + description="Source implementation for Aha.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-aha/source_aha/__init__.py b/airbyte-integrations/connectors/source-aha/source_aha/__init__.py new file mode 100644 index 0000000000000..8d8a64a1ce882 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceAha + +__all__ = ["SourceAha"] diff --git a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml new file mode 100644 index 0000000000000..46b435d49b248 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml @@ -0,0 +1,54 @@ +version: "0.1.0" + +definitions: + selector: + extractor: + field_pointer: [] + requester: + url_base: "{{ config['url'] }}" + http_method: "GET" + authenticator: + type: "BearerAuthenticator" + api_token: "{{ config['api_key'] }}" + increment_paginator: + type: DefaultPaginator + url_base: "*ref(definitions.requester.url_base)" + page_size_option: + inject_into: "request_parameter" + field_name: "limit" + pagination_strategy: + type: PageIncrement + page_size: 5 + page_token_option: + inject_into: "request_parameter" + field_name: "page" + retriever: + record_selector: + $ref: "*ref(definitions.selector)" + paginator: + type: NoPagination + requester: + $ref: "*ref(definitions.requester)" + + base_stream: + retriever: + $ref: "*ref(definitions.retriever)" + features_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "features" + path: "/api/v1/features" + products_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "products" + path: "/api/v1/products" + +streams: + - "*ref(definitions.features_stream)" + - "*ref(definitions.products_stream)" + +check: + stream_names: + - "features" + - "products" diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/TODO.md b/airbyte-integrations/connectors/source-aha/source_aha/schemas/TODO.md new file mode 100644 index 0000000000000..4f37ca5f87dea --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/TODO.md @@ -0,0 +1,16 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +You can describe the schema of your streams using one `.json` file per stream. + +## Static schemas +From the `aha.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: +``` +schema_loader: + type: JsonSchema + file_path: "./source_aha/schemas/customers.json" +``` +Every stream specified in the configuration file should have a corresponding `.json` schema file. + +Delete this file once you're done. Or don't. Up to you :) + diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json new file mode 100644 index 0000000000000..f71b42793d044 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json @@ -0,0 +1,57 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "features": { + "type": "object", + "properties": { + "id":{ + "type": ["null", "string"] + + }, + "reference_num":{ + "type": ["null", "string"] + + }, + "name":{ + "type": ["null", "string"] + + }, + "created_at":{ + "type": ["null", "string"] + + }, + "url":{ + "type": ["null", "string"] + + }, + "resource":{ + "type": ["null", "string"] + + }, + "product_id":{ + "type": ["null", "string"] + + } + } + }, + "pagination": { + "type": "object", + "properties": { + "total_records":{ + "type": ["null", "integer"] + + }, + "total_pages":{ + "type": ["null", "integer"] + + }, + "current_page":{ + "type": ["null", "integer"] + + } + } + + } + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json new file mode 100644 index 0000000000000..d01bc93e5eff3 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json @@ -0,0 +1,53 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "products": { + "type": "object", + "properties": { + "id":{ + "type": ["null", "string"] + + }, + "reference_prefix":{ + "type": ["null", "string"] + + }, + "name":{ + "type": ["null", "string"] + + }, + "product_line":{ + "type": ["null", "string"] + + }, + "created_at":{ + "type": ["null", "string"] + + }, + "workspace_type":{ + "type": ["null", "string"] + + } + } + }, + "pagination": { + "type": "object", + "properties": { + "total_records":{ + "type": ["null", "integer"] + + }, + "total_pages":{ + "type": ["null", "integer"] + + }, + "current_page":{ + "type": ["null", "integer"] + + } + } + + } + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/source.py b/airbyte-integrations/connectors/source-aha/source_aha/source.py new file mode 100644 index 0000000000000..e91575fa8cd90 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceAha(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "aha.yaml"}) diff --git a/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml b/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml new file mode 100644 index 0000000000000..faf1ccddf2e36 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml @@ -0,0 +1,16 @@ +documentationUrl: https://docsurl.com +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Aha Spec + type: object + required: + - api_key + - url + additionalProperties: true + properties: + api_key: + type: string + description: API Key + url: + type: string + description: URL From c3b8e44e23516ac98445fb6f98801cfaa85a709b Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Mon, 7 Nov 2022 08:31:04 +0530 Subject: [PATCH 02/12] addressing the comments --- .../source-aha/source_aha/spec.yaml | 4 +- docs/integrations/sources/aha.md | 40 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) create mode 100644 docs/integrations/sources/aha.md diff --git a/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml b/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml index faf1ccddf2e36..7fd7aca562153 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml @@ -1,4 +1,4 @@ -documentationUrl: https://docsurl.com +documentationUrl: https://docs.airbyte.com/integrations/sources/aha connectionSpecification: $schema: http://json-schema.org/draft-07/schema# title: Aha Spec @@ -11,6 +11,8 @@ connectionSpecification: api_key: type: string description: API Key + title: API BEARER TOKEN url: type: string description: URL + title: AHA URL INSTANCE diff --git a/docs/integrations/sources/aha.md b/docs/integrations/sources/aha.md new file mode 100644 index 0000000000000..f9dbb329649e5 --- /dev/null +++ b/docs/integrations/sources/aha.md @@ -0,0 +1,40 @@ +# Aha API +API Documentation link [here](https://www.aha.io/api) +## Overview + +The Aha API source supports full refresh syncs + +### Output schema + +Two output streams are available from this source: + +*[features](https://www.aha.io/api/resources/features/list_features). +*[products](https://www.aha.io/api/resources/products/list_products_in_the_account). + +### Features + +| Feature | Supported? | +|:------------------|:-----------| +| Full Refresh Sync | Yes | +| Incremental Sync | No | + +### Performance considerations + +Rate Limiting information is updated [here](https://www.aha.io/api#rate-limiting). + +## Getting started + +### Requirements + +* Aha API Key. + +### Connect using `API Key`: + +1. Generate an API Key as described [here](https://www.aha.io/api#authentication). +2. Use the generated `API Key` in the Airbyte connection. + +## Changelog + +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:------------------------------------------------| +| 0.1.0 | 2022-11-02 | [18883](https://github.com/airbytehq/airbyte/pull/18893) | 🎉 New Source: Aha | \ No newline at end of file From 38c68c428f0111a95419ccc039383cb4aa271bdc Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Tue, 8 Nov 2022 13:18:43 +0530 Subject: [PATCH 03/12] addressing review comments --- .../connectors/source-aha/source_aha/aha.yaml | 22 +++- .../source-aha/source_aha/schemas/TODO.md | 16 --- .../source_aha/schemas/features.json | 2 +- .../source-aha/source_aha/schemas/goals.json | 119 ++++++++++++++++++ .../source-aha/source_aha/schemas/ideas.json | 102 +++++++++++++++ .../source-aha/source_aha/schemas/users.json | 77 ++++++++++++ .../source-aha/source_aha/spec.yaml | 4 +- 7 files changed, 318 insertions(+), 24 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-aha/source_aha/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json create mode 100644 airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json diff --git a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml index 46b435d49b248..67132d43eac1b 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml @@ -5,7 +5,7 @@ definitions: extractor: field_pointer: [] requester: - url_base: "{{ config['url'] }}" + url_base: "{{ config['url'] }}/api/v1" http_method: "GET" authenticator: type: "BearerAuthenticator" @@ -25,8 +25,6 @@ definitions: retriever: record_selector: $ref: "*ref(definitions.selector)" - paginator: - type: NoPagination requester: $ref: "*ref(definitions.requester)" @@ -37,18 +35,32 @@ definitions: $ref: "*ref(definitions.base_stream)" $options: name: "features" - path: "/api/v1/features" + path: "/features" products_stream: $ref: "*ref(definitions.base_stream)" $options: name: "products" - path: "/api/v1/products" + path: "/products" + ideas_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "ideas" + path: "/ideas" + users_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "users" + path: "/users" streams: - "*ref(definitions.features_stream)" - "*ref(definitions.products_stream)" + - "*ref(definitions.ideas_stream)" + - "*ref(definitions.users_stream)" check: stream_names: - "features" - "products" + - "ideas" + - "users" \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/TODO.md b/airbyte-integrations/connectors/source-aha/source_aha/schemas/TODO.md deleted file mode 100644 index 4f37ca5f87dea..0000000000000 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/TODO.md +++ /dev/null @@ -1,16 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -You can describe the schema of your streams using one `.json` file per stream. - -## Static schemas -From the `aha.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: -``` -schema_loader: - type: JsonSchema - file_path: "./source_aha/schemas/customers.json" -``` -Every stream specified in the configuration file should have a corresponding `.json` schema file. - -Delete this file once you're done. Or don't. Up to you :) - diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json index f71b42793d044..00ba80a37fe01 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json @@ -54,4 +54,4 @@ } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json new file mode 100644 index 0000000000000..189599823826a --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json @@ -0,0 +1,119 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "goals": { + "type": "object", + "properties": { + "id":{ + "type": ["null", "string"] + + }, + "name":{ + "type": ["null", "string"] + + }, + "reference_num":{ + "type": ["null", "string"] + + }, + "effort":{ + "type": ["null", "string"] + + }, + "value":{ + "type": ["null", "string"] + + }, + "position":{ + "type": ["null", "string"] + + }, + "created_at":{ + "type": ["null", "string"] + + }, + "updated_at":{ + "type": ["null", "string"] + + }, + "product_id":{ + "type": ["null", "string"] + + }, + "progress":{ + "type": ["null", "string"] + + }, + "progress_source":{ + "type": ["null", "string"] + + }, + "url":{ + "type": ["null", "string"] + + }, + "resource":{ + "type": ["null", "string"] + + }, + "description":{ + "type": ["null", "object"] + + }, + "success_metric":{ + "type": ["null", "object"] + + }, + "project":{ + "type": ["null", "object"] + + }, + "timeframe":{ + "type": ["null", "object"] + + }, + "initiatives":{ + "type": ["null", "array"] + + }, + "comments_count":{ + "type": ["null", "integer"] + + }, + "features":{ + "type": ["null", "array"] + + }, + "releases":{ + "type": ["null", "string"] + + }, + "custom_fields":{ + "type": ["null", "string"] + + } + + + } + }, + "pagination": { + "type": "object", + "properties": { + "total_records":{ + "type": ["null", "integer"] + + }, + "total_pages":{ + "type": ["null", "integer"] + + }, + "current_page":{ + "type": ["null", "integer"] + + } + } + + } + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json new file mode 100644 index 0000000000000..8160dcafda448 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json @@ -0,0 +1,102 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "ideas": { + "type": "object", + "properties": { + "id":{ + "type": ["null", "string"] + + }, + "reference_num":{ + "type": ["null", "string"] + + }, + "created_at":{ + "type": ["null", "string"] + + }, + "updated_at":{ + "type": ["null", "string"] + + }, + "workflow_status":{ + "type":"object", + "properties": { + "id":{ + "type": ["null", "string"] + + }, + "name":{ + "type": ["null", "string"] + + }, + "position":{ + "type": ["null", "integer"] + + }, + "complete":{ + "type": ["null", "boolean"] + + }, + "color":{ + "type": ["null", "string"] + + } + + } + + }, + "description":{ + "type": "object", + "properties": { + "id":{ + "type": ["null", "string"] + + }, + "body":{ + "type": ["null", "string"] + + }, + "created_at":{ + "type": ["null", "string"] + + }, + "attachments":{ + "type": ["null", "array"] + + } + } + + }, + "url":{ + "type": ["null", "string"] + + }, + "resource":{ + "type": ["null", "string"] + + } + } + }, + "pagination": { + "type": "object", + "properties": { + "total_records":{ + "type": ["null", "integer"] + + }, + "total_pages":{ + "type": ["null", "integer"] + + }, + "current_page":{ + "type": ["null", "integer"] + + } + } + + } + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json new file mode 100644 index 0000000000000..1ac5e44a71be9 --- /dev/null +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json @@ -0,0 +1,77 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "users": { + "type": "object", + "properties": { + "id":{ + "type": ["null", "string"] + + }, + "name":{ + "type": ["null", "string"] + + }, + "email":{ + "type": ["null", "string"] + + }, + "created_at":{ + "type": ["null", "string"] + + }, + "updated_at":{ + "type": ["null", "string"] + + }, + "accessed_at":{ + "type": ["null", "string"] + + }, + "product_roles":{ + "type": ["null", "object"] + + }, + "enabled":{ + "type": ["null", "boolean"] + + }, + "paid_seat":{ + "type": ["null", "boolean"] + + }, + "administrator":{ + "type": ["null", "boolean"] + + }, + "administrator_roles":{ + "type": ["null", "object"] + + }, + "identity_provider":{ + "type": ["null", "object"] + + } + } + }, + "pagination": { + "type": "object", + "properties": { + "total_records":{ + "type": ["null", "integer"] + + }, + "total_pages":{ + "type": ["null", "integer"] + + }, + "current_page":{ + "type": ["null", "integer"] + + } + } + + } + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml b/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml index 7fd7aca562153..fc55fc9176f71 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/spec.yaml @@ -11,8 +11,8 @@ connectionSpecification: api_key: type: string description: API Key - title: API BEARER TOKEN + title: API Bearer Token url: type: string description: URL - title: AHA URL INSTANCE + title: Aha Url Instance From 44bce397dbfee9ed7656c9be78fba8c18bb08a49 Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Tue, 8 Nov 2022 19:27:20 +0530 Subject: [PATCH 04/12] addressing review comments & removing newly added EoL --- .../integration_tests/configured_catalog.json | 33 +++++++++++++++++-- .../connectors/source-aha/source_aha/aha.yaml | 9 ++++- 2 files changed, 38 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-aha/integration_tests/configured_catalog.json index dda2659e960dd..5e433039fa1de 100644 --- a/airbyte-integrations/connectors/source-aha/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-aha/integration_tests/configured_catalog.json @@ -13,10 +13,37 @@ "stream": { "name": "products", "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "ideas", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "goals", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" } ] } diff --git a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml index 67132d43eac1b..a09bc010227e5 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml @@ -51,16 +51,23 @@ definitions: $options: name: "users" path: "/users" + goals_stream: + $ref: "*ref(definitions.base_stream)" + $options: + name: "goals" + path: "/goals" streams: - "*ref(definitions.features_stream)" - "*ref(definitions.products_stream)" - "*ref(definitions.ideas_stream)" - "*ref(definitions.users_stream)" + - "*ref(definitions.goals_stream)" check: stream_names: - "features" - "products" - "ideas" - - "users" \ No newline at end of file + - "users" + - "goals" \ No newline at end of file From f73a07f7b5406e425633a81598f93f5c22208e8c Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Tue, 8 Nov 2022 21:49:47 +0530 Subject: [PATCH 05/12] fixing end of file line --- airbyte-integrations/connectors/source-aha/source_aha/aha.yaml | 3 ++- .../connectors/source-aha/source_aha/schemas/features.json | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml index a09bc010227e5..0afe508bc9f4d 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml @@ -70,4 +70,5 @@ check: - "products" - "ideas" - "users" - - "goals" \ No newline at end of file + - "goals" + \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json index 00ba80a37fe01..f71b42793d044 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json @@ -54,4 +54,4 @@ } } -} \ No newline at end of file +} From cec12ef23a6e059289f6092cc4656c0eb12b0f9d Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Tue, 8 Nov 2022 21:51:12 +0530 Subject: [PATCH 06/12] fixing end of file line --- airbyte-integrations/connectors/source-aha/source_aha/aha.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml index 0afe508bc9f4d..fa07510e5abba 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml @@ -71,4 +71,3 @@ check: - "ideas" - "users" - "goals" - \ No newline at end of file From 94bb17378ad773ea0db3c50715e81bd3390805aa Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Thu, 10 Nov 2022 09:08:36 +0530 Subject: [PATCH 07/12] including paginator --- .../connectors/source-aha/source_aha/aha.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml index fa07510e5abba..966eeb4304280 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml @@ -11,13 +11,13 @@ definitions: type: "BearerAuthenticator" api_token: "{{ config['api_key'] }}" increment_paginator: - type: DefaultPaginator + type: "DefaultPaginator" url_base: "*ref(definitions.requester.url_base)" page_size_option: inject_into: "request_parameter" - field_name: "limit" + field_name: "per_page" pagination_strategy: - type: PageIncrement + type: "PageIncrement" page_size: 5 page_token_option: inject_into: "request_parameter" @@ -27,7 +27,8 @@ definitions: $ref: "*ref(definitions.selector)" requester: $ref: "*ref(definitions.requester)" - + paginator: + $ref: "*ref(definitions.increment_paginator)" base_stream: retriever: $ref: "*ref(definitions.retriever)" From 30ee9521b386cc8b36d7e1e90c62adba0f9faf68 Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Sat, 12 Nov 2022 08:25:01 +0530 Subject: [PATCH 08/12] Fixing the pagination issue --- .../connectors/source-aha/source_aha/aha.yaml | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml index 966eeb4304280..9dd3fd470f69f 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml +++ b/airbyte-integrations/connectors/source-aha/source_aha/aha.yaml @@ -4,6 +4,21 @@ definitions: selector: extractor: field_pointer: [] + selector_features: + extractor: + field_pointer: ["features"] + selector_products: + extractor: + field_pointer: ["products"] + selector_ideas: + extractor: + field_pointer: ["ideas"] + selector_users: + extractor: + field_pointer: ["users"] + selector_goals: + extractor: + field_pointer: ["goals"] requester: url_base: "{{ config['url'] }}/api/v1" http_method: "GET" @@ -34,26 +49,46 @@ definitions: $ref: "*ref(definitions.retriever)" features_stream: $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_features)" $options: name: "features" path: "/features" products_stream: $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_products)" $options: name: "products" path: "/products" ideas_stream: $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_ideas)" $options: name: "ideas" path: "/ideas" users_stream: $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_users)" $options: name: "users" path: "/users" goals_stream: $ref: "*ref(definitions.base_stream)" + retriever: + $ref: "*ref(definitions.retriever)" + record_selector: + $ref: "*ref(definitions.selector_goals)" $options: name: "goals" path: "/goals" From a022c6c60e405e4dc211d12b8aa9c69542ffa4cf Mon Sep 17 00:00:00 2001 From: bala_ceg Date: Sat, 12 Nov 2022 08:39:37 +0530 Subject: [PATCH 09/12] Removing the contents from sample_config.json --- .../source-aha/integration_tests/sample_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json index 2417dfeaebf2e..4b5cca495fa8d 100644 --- a/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { - "api_key": "abb74d48e6d2371242eb2a84d1ac2198a64a60b3396d76c35bf8a6dadcc1f0bb", - "url" : "https://905c7285-30d2-4419-a9e4-8eef125ce6c1.aha.io" + "api_key": "Your API key", + "url" : "Your Aha URL Instance" } From 321bf351a7e6ca3c06d267b732ab018ffd7efc78 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 14 Nov 2022 22:37:42 -0300 Subject: [PATCH 10/12] fix schema, tests and add aha to source def --- .../resources/seed/source_definitions.yaml | 7 ++++ .../source_aha/schemas/features.json | 25 +----------- .../source-aha/source_aha/schemas/goals.json | 38 ++++--------------- .../source-aha/source_aha/schemas/ideas.json | 23 ----------- .../source_aha/schemas/products.json | 27 +------------ .../source-aha/source_aha/schemas/users.json | 27 +------------ 6 files changed, 20 insertions(+), 127 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 599898ef06466..20ffa18176737 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -21,6 +21,13 @@ icon: airtable.svg sourceType: api releaseStage: alpha +- name: Aha + sourceDefinitionId: 81ca39dc-4534-4dd2-b848-b0cfd2c11fce + dockerRepository: airbyte/source-aha + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.com/integrations/sources/aha + sourceType: api + releaseStage: alpha - name: Alpha Vantage sourceDefinitionId: db385323-9333-4fec-bec3-9e0ca9326c90 dockerRepository: airbyte/source-alpha-vantage diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json index f71b42793d044..6c06e28918a6c 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json @@ -2,9 +2,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "features": { - "type": "object", - "properties": { "id":{ "type": ["null", "string"] @@ -34,24 +31,6 @@ } } - }, - "pagination": { - "type": "object", - "properties": { - "total_records":{ - "type": ["null", "integer"] - - }, - "total_pages":{ - "type": ["null", "integer"] - - }, - "current_page":{ - "type": ["null", "integer"] - - } - } - } - } -} + + diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json index 189599823826a..3c3de356bf1b1 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json @@ -2,9 +2,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "goals": { - "type": "object", - "properties": { "id":{ "type": ["null", "string"] @@ -18,15 +15,15 @@ }, "effort":{ - "type": ["null", "string"] + "type": ["null", "number"] }, "value":{ - "type": ["null", "string"] + "type": ["null", "number"] }, "position":{ - "type": ["null", "string"] + "type": ["null", "number"] }, "created_at":{ @@ -86,34 +83,13 @@ }, "releases":{ - "type": ["null", "string"] + "type": ["null", "array"] }, "custom_fields":{ - "type": ["null", "string"] - - } - - - } - }, - "pagination": { - "type": "object", - "properties": { - "total_records":{ - "type": ["null", "integer"] - - }, - "total_pages":{ - "type": ["null", "integer"] - - }, - "current_page":{ - "type": ["null", "integer"] + "type": ["null", "array"] - } + } } - } - } -} + diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json index 8160dcafda448..17ee2f8bc32c2 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json @@ -2,9 +2,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "ideas": { - "type": "object", - "properties": { "id":{ "type": ["null", "string"] @@ -79,24 +76,4 @@ } } - }, - "pagination": { - "type": "object", - "properties": { - "total_records":{ - "type": ["null", "integer"] - - }, - "total_pages":{ - "type": ["null", "integer"] - - }, - "current_page":{ - "type": ["null", "integer"] - - } - } - } - } -} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json index d01bc93e5eff3..3209e77f5aec6 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json @@ -2,9 +2,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "products": { - "type": "object", - "properties": { "id":{ "type": ["null", "string"] @@ -18,7 +15,7 @@ }, "product_line":{ - "type": ["null", "string"] + "type": ["null", "boolean"] }, "created_at":{ @@ -30,24 +27,4 @@ } } - }, - "pagination": { - "type": "object", - "properties": { - "total_records":{ - "type": ["null", "integer"] - - }, - "total_pages":{ - "type": ["null", "integer"] - - }, - "current_page":{ - "type": ["null", "integer"] - - } - } - - } - } -} + } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json index 1ac5e44a71be9..e91681f12416f 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json @@ -2,9 +2,6 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "users": { - "type": "object", - "properties": { "id":{ "type": ["null", "string"] @@ -30,7 +27,7 @@ }, "product_roles":{ - "type": ["null", "object"] + "type": ["null", "array"] }, "enabled":{ @@ -54,24 +51,4 @@ } } - }, - "pagination": { - "type": "object", - "properties": { - "total_records":{ - "type": ["null", "integer"] - - }, - "total_pages":{ - "type": ["null", "integer"] - - }, - "current_page":{ - "type": ["null", "integer"] - - } - } - - } - } -} + } \ No newline at end of file From 2dcff75fa932055a931eca410ad8158e0fd62e24 Mon Sep 17 00:00:00 2001 From: marcosmarxm Date: Mon, 14 Nov 2022 22:44:30 -0300 Subject: [PATCH 11/12] format files --- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../source_aha/schemas/features.json | 53 +++--- .../source-aha/source_aha/schemas/goals.json | 157 ++++++++---------- .../source-aha/source_aha/schemas/ideas.json | 106 +++++------- .../source_aha/schemas/products.json | 46 +++-- .../source-aha/source_aha/schemas/users.json | 88 +++++----- 7 files changed, 193 insertions(+), 261 deletions(-) diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-aha/integration_tests/invalid_config.json index 198fd05930e31..e41b4cc7edc79 100644 --- a/airbyte-integrations/connectors/source-aha/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-aha/integration_tests/invalid_config.json @@ -1,4 +1,4 @@ { "api_key": "abb74d48e6d2dadcc1f0bb", - "url" : "https://905c7285-30d2-4419-a9e4-8eef125ce6c1.aha.io" + "url": "https://905c7285-30d2-4419-a9e4-8eef125ce6c1.aha.io" } diff --git a/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json index 4b5cca495fa8d..c13f52d73c36a 100644 --- a/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-aha/integration_tests/sample_config.json @@ -1,4 +1,4 @@ { "api_key": "Your API key", - "url" : "Your Aha URL Instance" + "url": "Your Aha URL Instance" } diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json index 6c06e28918a6c..191cda4b85d70 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/features.json @@ -2,35 +2,26 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id":{ - "type": ["null", "string"] - - }, - "reference_num":{ - "type": ["null", "string"] - - }, - "name":{ - "type": ["null", "string"] - - }, - "created_at":{ - "type": ["null", "string"] - - }, - "url":{ - "type": ["null", "string"] - - }, - "resource":{ - "type": ["null", "string"] - - }, - "product_id":{ - "type": ["null", "string"] - - } - } + "id": { + "type": ["null", "string"] + }, + "reference_num": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "resource": { + "type": ["null", "string"] + }, + "product_id": { + "type": ["null", "string"] } - - + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json index 3c3de356bf1b1..f790fd5f716f1 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/goals.json @@ -2,94 +2,71 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id":{ - "type": ["null", "string"] - - }, - "name":{ - "type": ["null", "string"] - - }, - "reference_num":{ - "type": ["null", "string"] - - }, - "effort":{ - "type": ["null", "number"] - - }, - "value":{ - "type": ["null", "number"] - - }, - "position":{ - "type": ["null", "number"] - - }, - "created_at":{ - "type": ["null", "string"] - - }, - "updated_at":{ - "type": ["null", "string"] - - }, - "product_id":{ - "type": ["null", "string"] - - }, - "progress":{ - "type": ["null", "string"] - - }, - "progress_source":{ - "type": ["null", "string"] - - }, - "url":{ - "type": ["null", "string"] - - }, - "resource":{ - "type": ["null", "string"] - - }, - "description":{ - "type": ["null", "object"] - - }, - "success_metric":{ - "type": ["null", "object"] - - }, - "project":{ - "type": ["null", "object"] - - }, - "timeframe":{ - "type": ["null", "object"] - - }, - "initiatives":{ - "type": ["null", "array"] - - }, - "comments_count":{ - "type": ["null", "integer"] - - }, - "features":{ - "type": ["null", "array"] - - }, - "releases":{ - "type": ["null", "array"] - - }, - "custom_fields":{ - "type": ["null", "array"] - - } - } + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "reference_num": { + "type": ["null", "string"] + }, + "effort": { + "type": ["null", "number"] + }, + "value": { + "type": ["null", "number"] + }, + "position": { + "type": ["null", "number"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "product_id": { + "type": ["null", "string"] + }, + "progress": { + "type": ["null", "string"] + }, + "progress_source": { + "type": ["null", "string"] + }, + "url": { + "type": ["null", "string"] + }, + "resource": { + "type": ["null", "string"] + }, + "description": { + "type": ["null", "object"] + }, + "success_metric": { + "type": ["null", "object"] + }, + "project": { + "type": ["null", "object"] + }, + "timeframe": { + "type": ["null", "object"] + }, + "initiatives": { + "type": ["null", "array"] + }, + "comments_count": { + "type": ["null", "integer"] + }, + "features": { + "type": ["null", "array"] + }, + "releases": { + "type": ["null", "array"] + }, + "custom_fields": { + "type": ["null", "array"] } - + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json index 17ee2f8bc32c2..92e79e7e8d7dd 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/ideas.json @@ -2,78 +2,60 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id":{ + "id": { + "type": ["null", "string"] + }, + "reference_num": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "workflow_status": { + "type": "object", + "properties": { + "id": { "type": ["null", "string"] - }, - "reference_num":{ + "name": { "type": ["null", "string"] - }, - "created_at":{ - "type": ["null", "string"] - - }, - "updated_at":{ - "type": ["null", "string"] - + "position": { + "type": ["null", "integer"] }, - "workflow_status":{ - "type":"object", - "properties": { - "id":{ - "type": ["null", "string"] - - }, - "name":{ - "type": ["null", "string"] - - }, - "position":{ - "type": ["null", "integer"] - - }, - "complete":{ - "type": ["null", "boolean"] - - }, - "color":{ - "type": ["null", "string"] - - } - - } - + "complete": { + "type": ["null", "boolean"] }, - "description":{ - "type": "object", - "properties": { - "id":{ - "type": ["null", "string"] - - }, - "body":{ - "type": ["null", "string"] - - }, - "created_at":{ - "type": ["null", "string"] - - }, - "attachments":{ - "type": ["null", "array"] - - } - } - + "color": { + "type": ["null", "string"] + } + } + }, + "description": { + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] }, - "url":{ + "body": { "type": ["null", "string"] - }, - "resource":{ + "created_at": { "type": ["null", "string"] - + }, + "attachments": { + "type": ["null", "array"] } } + }, + "url": { + "type": ["null", "string"] + }, + "resource": { + "type": ["null", "string"] } + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json index 3209e77f5aec6..69e4f907710bd 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/products.json @@ -2,29 +2,23 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id":{ - "type": ["null", "string"] - - }, - "reference_prefix":{ - "type": ["null", "string"] - - }, - "name":{ - "type": ["null", "string"] - - }, - "product_line":{ - "type": ["null", "boolean"] - - }, - "created_at":{ - "type": ["null", "string"] - - }, - "workspace_type":{ - "type": ["null", "string"] - - } - } - } \ No newline at end of file + "id": { + "type": ["null", "string"] + }, + "reference_prefix": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "product_line": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"] + }, + "workspace_type": { + "type": ["null", "string"] + } + } +} diff --git a/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json b/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json index e91681f12416f..df54138e9695b 100644 --- a/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json +++ b/airbyte-integrations/connectors/source-aha/source_aha/schemas/users.json @@ -2,53 +2,41 @@ "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "id":{ - "type": ["null", "string"] - - }, - "name":{ - "type": ["null", "string"] - - }, - "email":{ - "type": ["null", "string"] - - }, - "created_at":{ - "type": ["null", "string"] - - }, - "updated_at":{ - "type": ["null", "string"] - - }, - "accessed_at":{ - "type": ["null", "string"] - - }, - "product_roles":{ - "type": ["null", "array"] - - }, - "enabled":{ - "type": ["null", "boolean"] - - }, - "paid_seat":{ - "type": ["null", "boolean"] - - }, - "administrator":{ - "type": ["null", "boolean"] - - }, - "administrator_roles":{ - "type": ["null", "object"] - - }, - "identity_provider":{ - "type": ["null", "object"] - - } - } - } \ No newline at end of file + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "updated_at": { + "type": ["null", "string"] + }, + "accessed_at": { + "type": ["null", "string"] + }, + "product_roles": { + "type": ["null", "array"] + }, + "enabled": { + "type": ["null", "boolean"] + }, + "paid_seat": { + "type": ["null", "boolean"] + }, + "administrator": { + "type": ["null", "boolean"] + }, + "administrator_roles": { + "type": ["null", "object"] + }, + "identity_provider": { + "type": ["null", "object"] + } + } +} From 6abf691de7723fea295dda4e54ac8313a664fbcc Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 15 Nov 2022 02:43:02 +0000 Subject: [PATCH 12/12] auto-bump connector version --- .../src/main/resources/seed/source_specs.yaml | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 36c9eb2c46af6..0ab3010507a6d 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -219,6 +219,29 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-aha:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.com/integrations/sources/aha" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Aha Spec" + type: "object" + required: + - "api_key" + - "url" + additionalProperties: true + properties: + api_key: + type: "string" + description: "API Key" + title: "API Bearer Token" + url: + type: "string" + description: "URL" + title: "Aha Url Instance" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-alpha-vantage:0.1.0" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/alpha-vantage"