From 57b279c0883876c328d5f5775388fa2e5318cac0 Mon Sep 17 00:00:00 2001 From: Rodrigo Parra Date: Thu, 16 Jun 2022 21:38:52 -0400 Subject: [PATCH 1/8] Add Glassfrog native connector --- .../connectors/source-glassfrog/.dockerignore | 6 + .../connectors/source-glassfrog/Dockerfile | 38 +++++ .../connectors/source-glassfrog/README.md | 132 ++++++++++++++++ .../acceptance-test-config.yml | 20 +++ .../acceptance-test-docker.sh | 16 ++ .../connectors/source-glassfrog/bootstrap.md | 28 ++++ .../connectors/source-glassfrog/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/catalog.json | 39 +++++ .../integration_tests/configured_catalog.json | 76 +++++++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-glassfrog/main.py | 13 ++ .../source-glassfrog/requirements.txt | 2 + .../connectors/source-glassfrog/setup.py | 29 ++++ .../source_glassfrog/__init__.py | 8 + .../source-glassfrog/source_glassfrog/auth.py | 15 ++ .../source_glassfrog/schemas/TODO.md | 25 +++ .../source_glassfrog/schemas/assignments.json | 36 +++++ .../schemas/checklist_items.json | 41 +++++ .../source_glassfrog/schemas/circles.json | 60 ++++++++ .../schemas/custom_fields.json | 29 ++++ .../source_glassfrog/schemas/metrics.json | 41 +++++ .../source_glassfrog/schemas/people.json | 53 +++++++ .../source_glassfrog/schemas/projects.json | 79 ++++++++++ .../source_glassfrog/schemas/roles.json | 82 ++++++++++ .../source_glassfrog/source.py | 145 ++++++++++++++++++ .../source_glassfrog/spec.yaml | 14 ++ .../source-glassfrog/unit_tests/__init__.py | 3 + .../unit_tests/test_source.py | 22 +++ .../unit_tests/test_streams.py | 83 ++++++++++ docs/integrations/README.md | 1 + docs/integrations/sources/glassfrog.md | 55 +++++++ 36 files changed, 1235 insertions(+) create mode 100644 airbyte-integrations/connectors/source-glassfrog/.dockerignore create mode 100644 airbyte-integrations/connectors/source-glassfrog/Dockerfile create mode 100644 airbyte-integrations/connectors/source-glassfrog/README.md create mode 100644 airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-glassfrog/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-glassfrog/build.gradle create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/main.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/requirements.txt create mode 100644 airbyte-integrations/connectors/source-glassfrog/setup.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/__init__.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml create mode 100644 airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/glassfrog.md diff --git a/airbyte-integrations/connectors/source-glassfrog/.dockerignore b/airbyte-integrations/connectors/source-glassfrog/.dockerignore new file mode 100644 index 000000000000..4f24b2e55ec1 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_glassfrog +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-glassfrog/Dockerfile b/airbyte-integrations/connectors/source-glassfrog/Dockerfile new file mode 100644 index 000000000000..80ae0eda635b --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_glassfrog ./source_glassfrog + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-glassfrog diff --git a/airbyte-integrations/connectors/source-glassfrog/README.md b/airbyte-integrations/connectors/source-glassfrog/README.md new file mode 100644 index 000000000000..b9bdfa189c10 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/README.md @@ -0,0 +1,132 @@ +# Glassfrog Source + +This is the repository for the Glassfrog source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/glassfrog). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-glassfrog:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/glassfrog) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_glassfrog/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source glassfrog test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-glassfrog:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-glassfrog:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-glassfrog:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-glassfrog:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-glassfrog:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-glassfrog:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-glassfrog:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-glassfrog:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml new file mode 100644 index 000000000000..142215f18acf --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-glassfrog:dev +tests: + spec: + - spec_path: "source_glassfrog/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-glassfrog/bootstrap.md b/airbyte-integrations/connectors/source-glassfrog/bootstrap.md new file mode 100644 index 000000000000..03a1255ab014 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/bootstrap.md @@ -0,0 +1,28 @@ +# Glassfrog + +## Overview + +Glassfrog is a team management platform that supports the implementation of [Holacracy](https://www.holacracy.org). It helps organizations gain transparency and hold meetings as required by the methodology. + +## Endpoints + +This Source is capable of syncing the following Streams: + +* [Assignments](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#db2934bd-8c07-1951-b273-51fbc2dc6422) +* [Checklist items](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#a81716d4-b492-79ff-1348-9048fd9dc527) +* [Circles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#ed696857-c3d8-fba1-a174-fbe63de07798) +* [Custom fields](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#901f8ec2-a986-0291-2fa2-281c16622107) +* [Metrics](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#00d4f5fb-d6e5-5521-a77d-bdce50a9fb84) +* [People](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#78b74b9f-72b7-63fc-a18c-18518932944b) +* [Projects](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#110bde88-a319-ae9c-077a-9752fd2f0843) +* [Roles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#d1f31f7a-1d42-8c86-be1d-a36e640bf993) + +## Additional notes + +* Authentication is handled via an API key, which is [obtained from the platform](https://app.glassfrog.com/org/27355/api_keys). +* API endpoints return all results for the resource, there is no documented pagination mechanism. +* There are no documented rate limits for the API. + +## API reference + +See the [API reference documents](https://support.glassfrog.com/support/solutions/articles/9000066846-how-do-i-get-api-keys-). diff --git a/airbyte-integrations/connectors/source-glassfrog/build.gradle b/airbyte-integrations/connectors/source-glassfrog/build.gradle new file mode 100644 index 000000000000..f6fdd76c641d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_glassfrog' +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/__init__.py b/airbyte-integrations/connectors/source-glassfrog/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..f3510ab3ff54 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/configured_catalog.json @@ -0,0 +1,76 @@ +{ + "streams": [ + { + "stream": { + "name": "assignments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "checklist_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "circles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "custom_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "people", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "roles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json new file mode 100644 index 000000000000..40061e658e4d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "xxxxxxx" +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/main.py b/airbyte-integrations/connectors/source-glassfrog/main.py new file mode 100644 index 000000000000..28550e048e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_glassfrog import SourceGlassfrog + +if __name__ == "__main__": + source = SourceGlassfrog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-glassfrog/requirements.txt b/airbyte-integrations/connectors/source-glassfrog/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-glassfrog/setup.py b/airbyte-integrations/connectors/source-glassfrog/setup.py new file mode 100644 index 000000000000..bb4bfddfa4db --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1.56", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_glassfrog", + description="Source implementation for Glassfrog.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/__init__.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/__init__.py new file mode 100644 index 000000000000..4a5e6ac55b3b --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceGlassfrog + +__all__ = ["SourceGlassfrog"] diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py new file mode 100644 index 000000000000..d83faaef9ddd --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, Mapping + +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +class GlassfrogAuthenticator(TokenAuthenticator): + def __init__(self, config: Mapping[str, Any]): + self.config = config + + def get_auth_header(self) -> Mapping[str, Any]: + return {"X-Auth-Token": self.config.get("api_key", "")} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json new file mode 100644 index 000000000000..8256445e8199 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "election": { + "type": ["null", "string"], + "default": null + }, + "exclude_from_meetings": { + "type": "boolean", + "default": false + }, + "focus": { + "type": ["null", "string"], + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "person": { + "type": ["null", "integer"], + "default": null + }, + "role": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json new file mode 100644 index 000000000000..48f23256a63d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "frequency": { + "type": "string", + "default": "" + }, + "link": { + "type": "string", + "default": "" + }, + "global": { + "type": "boolean", + "default": false + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circle": { + "type": ["null", "integer"], + "default": null + }, + "role": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json new file mode 100644 index 000000000000..4cc77fa28cd8 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json @@ -0,0 +1,60 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "name": { + "type": "string", + "default": "" + }, + "short_name": { + "type": "string", + "default": "" + }, + "strategy": { + "type": ["null", "string"], + "default": null + }, + "organization_id": { + "type": "integer", + "default": 0 + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "roles": { + "type": "array", + "default": [], + "items": { + "type": "integer" + } + }, + "policies": { + "type": "array", + "default": [], + "items": { + "type": "integer", + "default": 0 + } + }, + "domain": { + "type": "array", + "default": [], + "items": { + "type": "integer", + "default": 0 + } + }, + "supported_role": { + "type": "integer", + "default": 0 + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json new file mode 100644 index 000000000000..e27c5e884fa8 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "field_name": { + "type": "string", + "default": "" + }, + "field_value": { + "type": "string", + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json new file mode 100644 index 000000000000..a8f66cf08d9c --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "frequency": { + "type": "string", + "default": "" + }, + "link": { + "type": "string", + "default": "" + }, + "global": { + "type": "boolean", + "default": false + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null + }, + "circle": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json new file mode 100644 index 000000000000..7e3fc298f69f --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "tag_names": { + "type": "array", + "default": [], + "items": { + "type": "string", + "default": "" + } + }, + "name": { + "type": "string", + "default": "" + }, + "email": { + "type": "string", + "default": "" + }, + "external_id": { + "type": ["null", "integer"], + "default": null + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circles": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + }, + "organization_ids": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json new file mode 100644 index 000000000000..7862f80e55df --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "status": { + "type": "string", + "default": "" + }, + "waiting_on_who": { + "type": ["null", "string"], + "default": null + }, + "waiting_on_what": { + "type": ["null", "string"], + "default": null + }, + "link": { + "type": ["null", "string"], + "default": null + }, + "value": { + "type": ["null", "integer"], + "default": null + }, + "effort": { + "type": ["null", "integer"], + "default": null + }, + "roi": { + "type": ["null", "number"], + "default": null + }, + "private_to_circle": { + "type": "boolean", + "default": false + }, + "created_at": { + "type": "string", + "format": "date-time", + "default": "" + }, + "archived_at": { + "type": ["null", "string"], + "format": "date-time", + "default": null + }, + "type": { + "type": "string", + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null + }, + "person": { + "type": ["null", "integer"], + "default": null + }, + "circle": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json new file mode 100644 index 000000000000..526a0daa389e --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "tag_names": { + "type": "array", + "default": [], + "items": { + "type": "string", + "default": "" + } + }, + "name": { + "type": "string", + "default": "" + }, + "is_core": { + "type": "boolean", + "default": false + }, + "name_with_circle_for_core_roles": { + "type": "string", + "default": "" + }, + "purpose": { + "type": "string", + "default": "" + }, + "elected_until": { + "type": ["null", "string"], + "format": "date", + "default": null + }, + "organization_id": { + "type": "integer", + "default": 0 + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circle": { + "type": ["null", "integer"], + "default": null + }, + "supporting_circle": { + "type": ["null", "integer"], + "default": null + }, + "domains": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + }, + "accountabilities": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + }, + "people": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py new file mode 100644 index 000000000000..2716284d316a --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py @@ -0,0 +1,145 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream + +from .auth import GlassfrogAuthenticator + + +# Basic full refresh stream +class GlassfrogStream(HttpStream, ABC): + url_base = "https://api.glassfrog.com/api/v3/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + json_response = response.json() + records = json_response.get(self.data_field, []) if self.data_field is not None else json_response + + for record in records: + yield record + + +class Assignments(GlassfrogStream): + data_field = "assignments" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class ChecklistItems(GlassfrogStream): + data_field = "checklist_items" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class Circles(GlassfrogStream): + data_field = "circles" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class CustomFields(GlassfrogStream): + data_field = "custom_fields" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class Metrics(GlassfrogStream): + data_field = "metrics" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class People(GlassfrogStream): + data_field = "people" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class Projects(GlassfrogStream): + data_field = "projects" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class Roles(GlassfrogStream): + data_field = "roles" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +# Source +class SourceGlassfrog(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + url = "https://api.glassfrog.com/api/v3/people" + headers = {"X-Auth-Token": config["api_key"]} + + r = requests.get(url, headers=headers) + r.raise_for_status() + return True, None + except Exception as error: + return False, f"Unable to connect to Glassfrog API with the provided credentials - {repr(error)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = GlassfrogAuthenticator(config=config) + return [ + Assignments(authenticator=auth), + ChecklistItems(authenticator=auth), + Circles(authenticator=auth), + CustomFields(authenticator=auth), + Metrics(authenticator=auth), + People(authenticator=auth), + Projects(authenticator=auth), + Roles(authenticator=auth), + ] diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml new file mode 100644 index 000000000000..b5423c2f0ce5 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml @@ -0,0 +1,14 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/glassfrog +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Glassfrog Spec + type: object + required: + - api_key + additionalProperties: false + properties: + api_key: + type: string + description: API key provided by Glassfrog + airbyte_secret: true + diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py new file mode 100644 index 000000000000..9cdc3db8b407 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_glassfrog.source import SourceGlassfrog + + +def test_check_connection(mocker): + source = SourceGlassfrog() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceGlassfrog() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py new file mode 100644 index 000000000000..a781d6617bfb --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_glassfrog.source import GlassfrogStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GlassfrogStream, "path", "v0/example_endpoint") + mocker.patch.object(GlassfrogStream, "primary_key", "test_primary_key") + mocker.patch.object(GlassfrogStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = GlassfrogStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = GlassfrogStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/README.md b/docs/integrations/README.md index c8beb01ca914..1cc6f4345970 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -60,6 +60,7 @@ For more information about the grading system, see [Product Release Stages](http | [Freshservice](sources/freshservice.md) | Alpha | No | | [GitHub](sources/github.md) | Beta | Yes | | [GitLab](sources/gitlab.md) | Alpha | Yes | +| [Glassfrog](sources/glassfrog.md) | Alpha | No | | [Google Ads](sources/google-ads.md) | Generally Available | Yes | | [Google Analytics v4](sources/google-analytics-v4.md) | Beta | Yes | | [Google Directory](sources/google-directory.md) | Alpha | Yes | diff --git a/docs/integrations/sources/glassfrog.md b/docs/integrations/sources/glassfrog.md new file mode 100644 index 000000000000..dd5d1edf5555 --- /dev/null +++ b/docs/integrations/sources/glassfrog.md @@ -0,0 +1,55 @@ +# Shortio + +## Sync overview + +The Glassfrog source supports only Full Refresh syncs. This source can sync data for the [Glassfrog API](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY). + +This Source Connector is based on the [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). + +### Output schema + +This Source is capable of syncing the following Streams: + +* [Assignments](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#db2934bd-8c07-1951-b273-51fbc2dc6422) +* [Checklist items](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#a81716d4-b492-79ff-1348-9048fd9dc527) +* [Circles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#ed696857-c3d8-fba1-a174-fbe63de07798) +* [Custom fields](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#901f8ec2-a986-0291-2fa2-281c16622107) +* [Metrics](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#00d4f5fb-d6e5-5521-a77d-bdce50a9fb84) +* [People](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#78b74b9f-72b7-63fc-a18c-18518932944b) +* [Projects](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#110bde88-a319-ae9c-077a-9752fd2f0843) +* [Roles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#d1f31f7a-1d42-8c86-be1d-a36e640bf993) + + + + + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | No | | +| Namespaces | No | | + +## Getting started + +1. Sign in at `app.glassfrog.com`. +2. Go to `Profile & Settings`. +3. In the API tab, enter the label for your new API key (e.g. `Airbyte`) and clik on the button `Create new API Key`. +4. Use the created secret key to configure your source! + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2022-06-16 | [13868g](https://github.com/airbytehq/airbyte/pull/13868) | Add Native Glassfrog Source Connector | + From 97b0523a7bbda449be4174a0aa7f986ca23bf364 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Fri, 22 Jul 2022 00:29:43 +0530 Subject: [PATCH 2/8] fix: tests and formatting --- .../acceptance-test-config.yml | 2 +- .../integration_tests/catalog.json | 39 ----- .../integration_tests/sample_config.json | 3 - .../integration_tests/sample_state.json | 5 - .../source-glassfrog/source_glassfrog/auth.py | 2 +- .../source_glassfrog/schemas/assignments.json | 64 ++++---- .../schemas/checklist_items.json | 74 ++++----- .../source_glassfrog/schemas/circles.json | 102 ++++++------ .../schemas/custom_fields.json | 52 +++--- .../source_glassfrog/schemas/metrics.json | 74 ++++----- .../source_glassfrog/schemas/people.json | 94 +++++------ .../source_glassfrog/schemas/projects.json | 148 +++++++++--------- .../source_glassfrog/schemas/roles.json | 148 +++++++++--------- .../source_glassfrog/source.py | 19 ++- .../source_glassfrog/spec.yaml | 1 - .../unit_tests/test_source.py | 23 ++- .../unit_tests/test_streams.py | 17 -- 17 files changed, 404 insertions(+), 463 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json delete mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json delete mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json diff --git a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml index 142215f18acf..cfd1b79ada46 100644 --- a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: ["custom_fields", "checklist_items", "metrics", "projects"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json deleted file mode 100644 index 6799946a6851..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "streams": [ - { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json deleted file mode 100644 index ecc4913b84c7..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "fix-me": "TODO" -} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json deleted file mode 100644 index 3587e579822d..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "value" - } -} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py index d83faaef9ddd..31adcf863e7f 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py @@ -2,7 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, Dict, Mapping +from typing import Any, Mapping from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json index 8256445e8199..a2b84c67c560 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json @@ -1,36 +1,36 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "election": { + "type": ["null", "string"], + "default": null + }, + "exclude_from_meetings": { + "type": "boolean", + "default": false + }, + "focus": { + "type": ["null", "string"], + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "person": { + "type": ["null", "integer"], + "default": null }, - "election": { - "type": ["null", "string"], - "default": null - }, - "exclude_from_meetings": { - "type": "boolean", - "default": false - }, - "focus": { - "type": ["null", "string"], - "default": "" - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "person": { - "type": ["null", "integer"], - "default": null - }, - "role": { - "type": ["null", "integer"], - "default": null - } - } + "role": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json index 48f23256a63d..2ec48403a82b 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json @@ -1,41 +1,41 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "frequency": { + "type": "string", + "default": "" + }, + "link": { + "type": "string", + "default": "" + }, + "global": { + "type": "boolean", + "default": false + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circle": { + "type": ["null", "integer"], + "default": null }, - "description": { - "type": "string", - "default": "" - }, - "frequency": { - "type": "string", - "default": "" - }, - "link": { - "type": "string", - "default": "" - }, - "global": { - "type": "boolean", - "default": false - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "circle": { - "type": ["null", "integer"], - "default": null - }, - "role": { - "type": ["null", "integer"], - "default": null - } - } + "role": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json index 4cc77fa28cd8..626ba40b61f2 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json @@ -1,60 +1,60 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "name": { + "type": "string", + "default": "" + }, + "short_name": { + "type": "string", + "default": "" + }, + "strategy": { + "type": ["null", "string"], + "default": null + }, + "organization_id": { + "type": "integer", + "default": 0 + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "roles": { + "type": "array", + "default": [], + "items": { + "type": "integer" + } + }, + "policies": { + "type": "array", + "default": [], + "items": { "type": "integer", "default": 0 + } }, - "name": { - "type": "string", - "default": "" - }, - "short_name": { - "type": "string", - "default": "" - }, - "strategy": { - "type": ["null", "string"], - "default": null - }, - "organization_id": { + "domain": { + "type": "array", + "default": [], + "items": { "type": "integer", "default": 0 + } }, - "links": { - "type": "object", - "default": {}, - "properties": { - "roles": { - "type": "array", - "default": [], - "items": { - "type": "integer" - } - }, - "policies": { - "type": "array", - "default": [], - "items": { - "type": "integer", - "default": 0 - } - }, - "domain": { - "type": "array", - "default": [], - "items": { - "type": "integer", - "default": 0 - } - }, - "supported_role": { - "type": "integer", - "default": 0 - } - } + "supported_role": { + "type": "integer", + "default": 0 } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json index e27c5e884fa8..dbaaebf08779 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json @@ -1,29 +1,29 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 - }, - "field_name": { - "type": "string", - "default": "" - }, - "field_value": { - "type": "string", - "default": "" - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "role": { - "type": ["null", "integer"], - "default": null - } - } + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "field_name": { + "type": "string", + "default": "" + }, + "field_value": { + "type": "string", + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json index a8f66cf08d9c..82bee0742c63 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json @@ -1,41 +1,41 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "frequency": { + "type": "string", + "default": "" + }, + "link": { + "type": "string", + "default": "" + }, + "global": { + "type": "boolean", + "default": false + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null }, - "description": { - "type": "string", - "default": "" - }, - "frequency": { - "type": "string", - "default": "" - }, - "link": { - "type": "string", - "default": "" - }, - "global": { - "type": "boolean", - "default": false - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "role": { - "type": ["null", "integer"], - "default": null - }, - "circle": { - "type": ["null", "integer"], - "default": null - } - } + "circle": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json index 7e3fc298f69f..ea91a9c84c9e 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json @@ -1,53 +1,53 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 - }, - "tag_names": { - "type": "array", - "default": [], - "items": { - "type": "string", - "default": "" - } - }, - "name": { - "type": "string", - "default": "" - }, - "email": { - "type": "string", - "default": "" - }, - "external_id": { + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "tag_names": { + "type": "array", + "default": [], + "items": { + "type": "string", + "default": "" + } + }, + "name": { + "type": "string", + "default": "" + }, + "email": { + "type": "string", + "default": "" + }, + "external_id": { + "type": ["null", "integer"], + "default": null + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circles": { + "type": "array", + "default": [], + "items": { "type": ["null", "integer"], "default": null + } }, - "links": { - "type": "object", - "default": {}, - "properties": { - "circles": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - }, - "organization_ids": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - } - } + "organization_ids": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json index 7862f80e55df..c9e5b6466fa8 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json @@ -1,79 +1,79 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "status": { + "type": "string", + "default": "" + }, + "waiting_on_who": { + "type": ["null", "string"], + "default": null + }, + "waiting_on_what": { + "type": ["null", "string"], + "default": null + }, + "link": { + "type": ["null", "string"], + "default": null + }, + "value": { + "type": ["null", "integer"], + "default": null + }, + "effort": { + "type": ["null", "integer"], + "default": null + }, + "roi": { + "type": ["null", "number"], + "default": null + }, + "private_to_circle": { + "type": "boolean", + "default": false + }, + "created_at": { + "type": "string", + "format": "date-time", + "default": "" + }, + "archived_at": { + "type": ["null", "string"], + "format": "date-time", + "default": null + }, + "type": { + "type": "string", + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null }, - "description": { - "type": "string", - "default": "" + "person": { + "type": ["null", "integer"], + "default": null }, - "status": { - "type": "string", - "default": "" - }, - "waiting_on_who": { - "type": ["null", "string"], - "default": null - }, - "waiting_on_what": { - "type": ["null", "string"], - "default": null - }, - "link": { - "type": ["null", "string"], - "default": null - }, - "value": { - "type": ["null", "integer"], - "default": null - }, - "effort": { - "type": ["null", "integer"], - "default": null - }, - "roi": { - "type": ["null", "number"], - "default": null - }, - "private_to_circle": { - "type": "boolean", - "default": false - }, - "created_at": { - "type": "string", - "format": "date-time", - "default": "" - }, - "archived_at": { - "type": ["null", "string"], - "format": "date-time", - "default": null - }, - "type": { - "type": "string", - "default": "" - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "role": { - "type": ["null", "integer"], - "default": null - }, - "person": { - "type": ["null", "integer"], - "default": null - }, - "circle": { - "type": ["null", "integer"], - "default": null - } - } + "circle": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json index 526a0daa389e..5af77e698d70 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json @@ -1,82 +1,82 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "tag_names": { + "type": "array", + "default": [], + "items": { + "type": "string", + "default": "" + } + }, + "name": { + "type": "string", + "default": "" + }, + "is_core": { + "type": "boolean", + "default": false + }, + "name_with_circle_for_core_roles": { + "type": "string", + "default": "" + }, + "purpose": { + "type": "string", + "default": "" + }, + "elected_until": { + "type": ["null", "string"], + "format": "date", + "default": null + }, + "organization_id": { + "type": "integer", + "default": 0 + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circle": { + "type": ["null", "integer"], + "default": null }, - "tag_names": { - "type": "array", - "default": [], - "items": { - "type": "string", - "default": "" - } + "supporting_circle": { + "type": ["null", "integer"], + "default": null }, - "name": { - "type": "string", - "default": "" - }, - "is_core": { - "type": "boolean", - "default": false - }, - "name_with_circle_for_core_roles": { - "type": "string", - "default": "" - }, - "purpose": { - "type": "string", - "default": "" - }, - "elected_until": { - "type": ["null", "string"], - "format": "date", + "domains": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], "default": null + } }, - "organization_id": { - "type": "integer", - "default": 0 + "accountabilities": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } }, - "links": { - "type": "object", - "default": {}, - "properties": { - "circle": { - "type": ["null", "integer"], - "default": null - }, - "supporting_circle": { - "type": ["null", "integer"], - "default": null - }, - "domains": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - }, - "accountabilities": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - }, - "people": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - } - } + "people": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py index 2716284d316a..e533cb192136 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py @@ -4,7 +4,7 @@ from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterable, List, Mapping, Optional, Tuple import requests from airbyte_cdk.sources import AbstractSource @@ -21,15 +21,6 @@ class GlassfrogStream(HttpStream, ABC): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {} - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: json_response = response.json() records = json_response.get(self.data_field, []) if self.data_field is not None else json_response @@ -39,6 +30,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class Assignments(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#db2934bd-8c07-1951-b273-51fbc2dc6422 data_field = "assignments" primary_key = "id" @@ -49,6 +41,7 @@ def path( class ChecklistItems(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#a81716d4-b492-79ff-1348-9048fd9dc527 data_field = "checklist_items" primary_key = "id" @@ -59,6 +52,7 @@ def path( class Circles(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#ed696857-c3d8-fba1-a174-fbe63de07798 data_field = "circles" primary_key = "id" @@ -69,6 +63,7 @@ def path( class CustomFields(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#901f8ec2-a986-0291-2fa2-281c16622107 data_field = "custom_fields" primary_key = "id" @@ -79,6 +74,7 @@ def path( class Metrics(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#00d4f5fb-d6e5-5521-a77d-bdce50a9fb84 data_field = "metrics" primary_key = "id" @@ -89,6 +85,7 @@ def path( class People(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#78b74b9f-72b7-63fc-a18c-18518932944b data_field = "people" primary_key = "id" @@ -99,6 +96,7 @@ def path( class Projects(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#110bde88-a319-ae9c-077a-9752fd2f0843 data_field = "projects" primary_key = "id" @@ -109,6 +107,7 @@ def path( class Roles(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#d1f31f7a-1d42-8c86-be1d-a36e640bf993 data_field = "roles" primary_key = "id" diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml index b5423c2f0ce5..2ce188f84be4 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml @@ -11,4 +11,3 @@ connectionSpecification: type: string description: API key provided by Glassfrog airbyte_secret: true - diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py index 9cdc3db8b407..a69ca4c30861 100644 --- a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py @@ -4,19 +4,26 @@ from unittest.mock import MagicMock +from pytest import fixture from source_glassfrog.source import SourceGlassfrog -def test_check_connection(mocker): +@fixture() +def config(request): + args = {"api_key": "xxxxxxx"} + return args + + +def test_check_connection(mocker, config): source = SourceGlassfrog() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) + logger_mock = MagicMock() + (connection_status, error) = source.check_connection(logger_mock, config) + expected_status = False + assert connection_status == expected_status -def test_streams(mocker): +def test_streams(mocker, config): source = SourceGlassfrog() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 + streams = source.streams(config) + expected_streams_number = 8 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py index a781d6617bfb..050c4c830929 100644 --- a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py @@ -11,7 +11,6 @@ @pytest.fixture def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class mocker.patch.object(GlassfrogStream, "path", "v0/example_endpoint") mocker.patch.object(GlassfrogStream, "primary_key", "test_primary_key") mocker.patch.object(GlassfrogStream, "__abstractmethods__", set()) @@ -19,43 +18,27 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): stream = GlassfrogStream() - # TODO: replace this with your input parameters inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters expected_params = {} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): stream = GlassfrogStream() - # TODO: replace this with your input parameters inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token expected_token = None assert stream.next_page_token(**inputs) == expected_token -def test_parse_response(patch_base_class): - stream = GlassfrogStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - def test_request_headers(patch_base_class): stream = GlassfrogStream() - # TODO: replace this with your input parameters inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers expected_headers = {} assert stream.request_headers(**inputs) == expected_headers def test_http_method(patch_base_class): stream = GlassfrogStream() - # TODO: replace this with your expected http request method expected_method = "GET" assert stream.http_method == expected_method From f12f4eb026c5510fd59dd91bf208f739db66216e Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Fri, 22 Jul 2022 01:15:11 +0530 Subject: [PATCH 3/8] chore: added connector to definitions --- .../src/main/resources/icons/glassfrog.svg | 899 ++++++++++++++++++ .../resources/seed/source_definitions.yaml | 8 + 2 files changed, 907 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/glassfrog.svg diff --git a/airbyte-config/init/src/main/resources/icons/glassfrog.svg b/airbyte-config/init/src/main/resources/icons/glassfrog.svg new file mode 100644 index 000000000000..7f7e69cea9f7 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/glassfrog.svg @@ -0,0 +1,899 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index c98b9edc4058..a7d1d03aea3e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -308,6 +308,14 @@ icon: gitlab.svg sourceType: api releaseStage: alpha +- name: Glassfrog + sourceDefinitionId: cf8ff320-6272-4faa-89e6-4402dc17e5d5 + dockerRepository: airbyte/source-glassfrog + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/glassfrog + icon: glassfrog.svg + sourceType: api + releaseStage: alpha - name: Google Ads sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 dockerRepository: airbyte/source-google-ads From d582d742e82a48488641ecbf3f5311d8cb63183d Mon Sep 17 00:00:00 2001 From: Rodrigo Parra Date: Thu, 16 Jun 2022 21:38:52 -0400 Subject: [PATCH 4/8] Add Glassfrog native connector --- .../connectors/source-glassfrog/.dockerignore | 6 + .../connectors/source-glassfrog/Dockerfile | 38 +++++ .../connectors/source-glassfrog/README.md | 132 ++++++++++++++++ .../acceptance-test-config.yml | 20 +++ .../acceptance-test-docker.sh | 16 ++ .../connectors/source-glassfrog/bootstrap.md | 28 ++++ .../connectors/source-glassfrog/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/catalog.json | 39 +++++ .../integration_tests/configured_catalog.json | 76 +++++++++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-glassfrog/main.py | 13 ++ .../source-glassfrog/requirements.txt | 2 + .../connectors/source-glassfrog/setup.py | 29 ++++ .../source_glassfrog/__init__.py | 8 + .../source-glassfrog/source_glassfrog/auth.py | 15 ++ .../source_glassfrog/schemas/TODO.md | 25 +++ .../source_glassfrog/schemas/assignments.json | 36 +++++ .../schemas/checklist_items.json | 41 +++++ .../source_glassfrog/schemas/circles.json | 60 ++++++++ .../schemas/custom_fields.json | 29 ++++ .../source_glassfrog/schemas/metrics.json | 41 +++++ .../source_glassfrog/schemas/people.json | 53 +++++++ .../source_glassfrog/schemas/projects.json | 79 ++++++++++ .../source_glassfrog/schemas/roles.json | 82 ++++++++++ .../source_glassfrog/source.py | 145 ++++++++++++++++++ .../source_glassfrog/spec.yaml | 14 ++ .../source-glassfrog/unit_tests/__init__.py | 3 + .../unit_tests/test_source.py | 22 +++ .../unit_tests/test_streams.py | 83 ++++++++++ docs/integrations/README.md | 1 + docs/integrations/sources/glassfrog.md | 55 +++++++ 36 files changed, 1235 insertions(+) create mode 100644 airbyte-integrations/connectors/source-glassfrog/.dockerignore create mode 100644 airbyte-integrations/connectors/source-glassfrog/Dockerfile create mode 100644 airbyte-integrations/connectors/source-glassfrog/README.md create mode 100644 airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-glassfrog/bootstrap.md create mode 100644 airbyte-integrations/connectors/source-glassfrog/build.gradle create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/main.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/requirements.txt create mode 100644 airbyte-integrations/connectors/source-glassfrog/setup.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/__init__.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml create mode 100644 airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py create mode 100644 docs/integrations/sources/glassfrog.md diff --git a/airbyte-integrations/connectors/source-glassfrog/.dockerignore b/airbyte-integrations/connectors/source-glassfrog/.dockerignore new file mode 100644 index 000000000000..4f24b2e55ec1 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_glassfrog +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-glassfrog/Dockerfile b/airbyte-integrations/connectors/source-glassfrog/Dockerfile new file mode 100644 index 000000000000..80ae0eda635b --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_glassfrog ./source_glassfrog + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-glassfrog diff --git a/airbyte-integrations/connectors/source-glassfrog/README.md b/airbyte-integrations/connectors/source-glassfrog/README.md new file mode 100644 index 000000000000..b9bdfa189c10 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/README.md @@ -0,0 +1,132 @@ +# Glassfrog Source + +This is the repository for the Glassfrog source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/glassfrog). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.9.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-glassfrog:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/glassfrog) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_glassfrog/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source glassfrog test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-glassfrog:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-glassfrog:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-glassfrog:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-glassfrog:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-glassfrog:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-glassfrog:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-glassfrog:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-glassfrog:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml new file mode 100644 index 000000000000..142215f18acf --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml @@ -0,0 +1,20 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-glassfrog:dev +tests: + spec: + - spec_path: "source_glassfrog/spec.yaml" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-glassfrog/bootstrap.md b/airbyte-integrations/connectors/source-glassfrog/bootstrap.md new file mode 100644 index 000000000000..03a1255ab014 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/bootstrap.md @@ -0,0 +1,28 @@ +# Glassfrog + +## Overview + +Glassfrog is a team management platform that supports the implementation of [Holacracy](https://www.holacracy.org). It helps organizations gain transparency and hold meetings as required by the methodology. + +## Endpoints + +This Source is capable of syncing the following Streams: + +* [Assignments](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#db2934bd-8c07-1951-b273-51fbc2dc6422) +* [Checklist items](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#a81716d4-b492-79ff-1348-9048fd9dc527) +* [Circles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#ed696857-c3d8-fba1-a174-fbe63de07798) +* [Custom fields](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#901f8ec2-a986-0291-2fa2-281c16622107) +* [Metrics](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#00d4f5fb-d6e5-5521-a77d-bdce50a9fb84) +* [People](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#78b74b9f-72b7-63fc-a18c-18518932944b) +* [Projects](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#110bde88-a319-ae9c-077a-9752fd2f0843) +* [Roles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#d1f31f7a-1d42-8c86-be1d-a36e640bf993) + +## Additional notes + +* Authentication is handled via an API key, which is [obtained from the platform](https://app.glassfrog.com/org/27355/api_keys). +* API endpoints return all results for the resource, there is no documented pagination mechanism. +* There are no documented rate limits for the API. + +## API reference + +See the [API reference documents](https://support.glassfrog.com/support/solutions/articles/9000066846-how-do-i-get-api-keys-). diff --git a/airbyte-integrations/connectors/source-glassfrog/build.gradle b/airbyte-integrations/connectors/source-glassfrog/build.gradle new file mode 100644 index 000000000000..f6fdd76c641d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_glassfrog' +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/__init__.py b/airbyte-integrations/connectors/source-glassfrog/integration_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py new file mode 100644 index 000000000000..1302b2f57e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..f3510ab3ff54 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/configured_catalog.json @@ -0,0 +1,76 @@ +{ + "streams": [ + { + "stream": { + "name": "assignments", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "checklist_items", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "circles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "custom_fields", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "people", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "projects", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "roles", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + } + ] +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json new file mode 100644 index 000000000000..40061e658e4d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "api_key": "xxxxxxx" +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/main.py b/airbyte-integrations/connectors/source-glassfrog/main.py new file mode 100644 index 000000000000..28550e048e10 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_glassfrog import SourceGlassfrog + +if __name__ == "__main__": + source = SourceGlassfrog() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-glassfrog/requirements.txt b/airbyte-integrations/connectors/source-glassfrog/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-glassfrog/setup.py b/airbyte-integrations/connectors/source-glassfrog/setup.py new file mode 100644 index 000000000000..bb4bfddfa4db --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1.56", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_glassfrog", + description="Source implementation for Glassfrog.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/__init__.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/__init__.py new file mode 100644 index 000000000000..4a5e6ac55b3b --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceGlassfrog + +__all__ = ["SourceGlassfrog"] diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py new file mode 100644 index 000000000000..d83faaef9ddd --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py @@ -0,0 +1,15 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Dict, Mapping + +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +class GlassfrogAuthenticator(TokenAuthenticator): + def __init__(self, config: Mapping[str, Any]): + self.config = config + + def get_auth_header(self) -> Mapping[str, Any]: + return {"X-Auth-Token": self.config.get("api_key", "")} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json new file mode 100644 index 000000000000..8256445e8199 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json @@ -0,0 +1,36 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "election": { + "type": ["null", "string"], + "default": null + }, + "exclude_from_meetings": { + "type": "boolean", + "default": false + }, + "focus": { + "type": ["null", "string"], + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "person": { + "type": ["null", "integer"], + "default": null + }, + "role": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json new file mode 100644 index 000000000000..48f23256a63d --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "frequency": { + "type": "string", + "default": "" + }, + "link": { + "type": "string", + "default": "" + }, + "global": { + "type": "boolean", + "default": false + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circle": { + "type": ["null", "integer"], + "default": null + }, + "role": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json new file mode 100644 index 000000000000..4cc77fa28cd8 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json @@ -0,0 +1,60 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "name": { + "type": "string", + "default": "" + }, + "short_name": { + "type": "string", + "default": "" + }, + "strategy": { + "type": ["null", "string"], + "default": null + }, + "organization_id": { + "type": "integer", + "default": 0 + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "roles": { + "type": "array", + "default": [], + "items": { + "type": "integer" + } + }, + "policies": { + "type": "array", + "default": [], + "items": { + "type": "integer", + "default": 0 + } + }, + "domain": { + "type": "array", + "default": [], + "items": { + "type": "integer", + "default": 0 + } + }, + "supported_role": { + "type": "integer", + "default": 0 + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json new file mode 100644 index 000000000000..e27c5e884fa8 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json @@ -0,0 +1,29 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "field_name": { + "type": "string", + "default": "" + }, + "field_value": { + "type": "string", + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json new file mode 100644 index 000000000000..a8f66cf08d9c --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json @@ -0,0 +1,41 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "frequency": { + "type": "string", + "default": "" + }, + "link": { + "type": "string", + "default": "" + }, + "global": { + "type": "boolean", + "default": false + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null + }, + "circle": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json new file mode 100644 index 000000000000..7e3fc298f69f --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "tag_names": { + "type": "array", + "default": [], + "items": { + "type": "string", + "default": "" + } + }, + "name": { + "type": "string", + "default": "" + }, + "email": { + "type": "string", + "default": "" + }, + "external_id": { + "type": ["null", "integer"], + "default": null + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circles": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + }, + "organization_ids": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json new file mode 100644 index 000000000000..7862f80e55df --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json @@ -0,0 +1,79 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "status": { + "type": "string", + "default": "" + }, + "waiting_on_who": { + "type": ["null", "string"], + "default": null + }, + "waiting_on_what": { + "type": ["null", "string"], + "default": null + }, + "link": { + "type": ["null", "string"], + "default": null + }, + "value": { + "type": ["null", "integer"], + "default": null + }, + "effort": { + "type": ["null", "integer"], + "default": null + }, + "roi": { + "type": ["null", "number"], + "default": null + }, + "private_to_circle": { + "type": "boolean", + "default": false + }, + "created_at": { + "type": "string", + "format": "date-time", + "default": "" + }, + "archived_at": { + "type": ["null", "string"], + "format": "date-time", + "default": null + }, + "type": { + "type": "string", + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null + }, + "person": { + "type": ["null", "integer"], + "default": null + }, + "circle": { + "type": ["null", "integer"], + "default": null + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json new file mode 100644 index 000000000000..526a0daa389e --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json @@ -0,0 +1,82 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "tag_names": { + "type": "array", + "default": [], + "items": { + "type": "string", + "default": "" + } + }, + "name": { + "type": "string", + "default": "" + }, + "is_core": { + "type": "boolean", + "default": false + }, + "name_with_circle_for_core_roles": { + "type": "string", + "default": "" + }, + "purpose": { + "type": "string", + "default": "" + }, + "elected_until": { + "type": ["null", "string"], + "format": "date", + "default": null + }, + "organization_id": { + "type": "integer", + "default": 0 + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circle": { + "type": ["null", "integer"], + "default": null + }, + "supporting_circle": { + "type": ["null", "integer"], + "default": null + }, + "domains": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + }, + "accountabilities": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + }, + "people": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py new file mode 100644 index 000000000000..2716284d316a --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py @@ -0,0 +1,145 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream + +from .auth import GlassfrogAuthenticator + + +# Basic full refresh stream +class GlassfrogStream(HttpStream, ABC): + url_base = "https://api.glassfrog.com/api/v3/" + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + json_response = response.json() + records = json_response.get(self.data_field, []) if self.data_field is not None else json_response + + for record in records: + yield record + + +class Assignments(GlassfrogStream): + data_field = "assignments" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class ChecklistItems(GlassfrogStream): + data_field = "checklist_items" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class Circles(GlassfrogStream): + data_field = "circles" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class CustomFields(GlassfrogStream): + data_field = "custom_fields" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class Metrics(GlassfrogStream): + data_field = "metrics" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class People(GlassfrogStream): + data_field = "people" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class Projects(GlassfrogStream): + data_field = "projects" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +class Roles(GlassfrogStream): + data_field = "roles" + primary_key = "id" + + def path( + self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> str: + return self.data_field + + +# Source +class SourceGlassfrog(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + try: + url = "https://api.glassfrog.com/api/v3/people" + headers = {"X-Auth-Token": config["api_key"]} + + r = requests.get(url, headers=headers) + r.raise_for_status() + return True, None + except Exception as error: + return False, f"Unable to connect to Glassfrog API with the provided credentials - {repr(error)}" + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + auth = GlassfrogAuthenticator(config=config) + return [ + Assignments(authenticator=auth), + ChecklistItems(authenticator=auth), + Circles(authenticator=auth), + CustomFields(authenticator=auth), + Metrics(authenticator=auth), + People(authenticator=auth), + Projects(authenticator=auth), + Roles(authenticator=auth), + ] diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml new file mode 100644 index 000000000000..b5423c2f0ce5 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml @@ -0,0 +1,14 @@ +documentationUrl: https://docs.airbyte.io/integrations/sources/glassfrog +connectionSpecification: + $schema: http://json-schema.org/draft-07/schema# + title: Glassfrog Spec + type: object + required: + - api_key + additionalProperties: false + properties: + api_key: + type: string + description: API key provided by Glassfrog + airbyte_secret: true + diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py new file mode 100644 index 000000000000..9cdc3db8b407 --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_glassfrog.source import SourceGlassfrog + + +def test_check_connection(mocker): + source = SourceGlassfrog() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceGlassfrog() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py new file mode 100644 index 000000000000..a781d6617bfb --- /dev/null +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_glassfrog.source import GlassfrogStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(GlassfrogStream, "path", "v0/example_endpoint") + mocker.patch.object(GlassfrogStream, "primary_key", "test_primary_key") + mocker.patch.object(GlassfrogStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = GlassfrogStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = GlassfrogStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = GlassfrogStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time diff --git a/docs/integrations/README.md b/docs/integrations/README.md index 0744e20fdd09..fdb0a0aa0899 100644 --- a/docs/integrations/README.md +++ b/docs/integrations/README.md @@ -63,6 +63,7 @@ For more information about the grading system, see [Product Release Stages](http | [Freshservice](sources/freshservice.md) | Alpha | No | | [GitHub](sources/github.md) | Generally Available | Yes | | [GitLab](sources/gitlab.md) | Alpha | Yes | +| [Glassfrog](sources/glassfrog.md) | Alpha | No | | [Google Ads](sources/google-ads.md) | Generally Available | Yes | | [Google Analytics (v4)](sources/google-analytics-v4.md) | Alpha | No | | [Google Analytics (Universal Analytics)](sources/google-analytics-universal-analytics.md) | Generally Available | Yes | diff --git a/docs/integrations/sources/glassfrog.md b/docs/integrations/sources/glassfrog.md new file mode 100644 index 000000000000..dd5d1edf5555 --- /dev/null +++ b/docs/integrations/sources/glassfrog.md @@ -0,0 +1,55 @@ +# Shortio + +## Sync overview + +The Glassfrog source supports only Full Refresh syncs. This source can sync data for the [Glassfrog API](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY). + +This Source Connector is based on the [Airbyte CDK](https://docs.airbyte.io/connector-development/cdk-python). + +### Output schema + +This Source is capable of syncing the following Streams: + +* [Assignments](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#db2934bd-8c07-1951-b273-51fbc2dc6422) +* [Checklist items](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#a81716d4-b492-79ff-1348-9048fd9dc527) +* [Circles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#ed696857-c3d8-fba1-a174-fbe63de07798) +* [Custom fields](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#901f8ec2-a986-0291-2fa2-281c16622107) +* [Metrics](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#00d4f5fb-d6e5-5521-a77d-bdce50a9fb84) +* [People](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#78b74b9f-72b7-63fc-a18c-18518932944b) +* [Projects](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#110bde88-a319-ae9c-077a-9752fd2f0843) +* [Roles](https://documenter.getpostman.com/view/1014385/glassfrog-api-v3/2SJViY#d1f31f7a-1d42-8c86-be1d-a36e640bf993) + + + + + +### Data type mapping + +| Integration Type | Airbyte Type | Notes | +| :--- | :--- | :--- | +| `string` | `string` | | +| `number` | `number` | | +| `array` | `array` | | +| `object` | `object` | | + +### Features + +| Feature | Supported?\(Yes/No\) | Notes | +| :--- | :--- | :--- | +| Full Refresh Sync | Yes | | +| Incremental - Append Sync | No | | +| Namespaces | No | | + +## Getting started + +1. Sign in at `app.glassfrog.com`. +2. Go to `Profile & Settings`. +3. In the API tab, enter the label for your new API key (e.g. `Airbyte`) and clik on the button `Create new API Key`. +4. Use the created secret key to configure your source! + +## Changelog + +| Version | Date | Pull Request | Subject | +| :--- | :--- | :--- | :--- | +| 0.1.0 | 2022-06-16 | [13868g](https://github.com/airbytehq/airbyte/pull/13868) | Add Native Glassfrog Source Connector | + From 813bf8e4f913728b7dda1fc731ed630274869615 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Fri, 22 Jul 2022 00:29:43 +0530 Subject: [PATCH 5/8] fix: tests and formatting --- .../acceptance-test-config.yml | 2 +- .../integration_tests/catalog.json | 39 ----- .../integration_tests/sample_config.json | 3 - .../integration_tests/sample_state.json | 5 - .../source-glassfrog/source_glassfrog/auth.py | 2 +- .../source_glassfrog/schemas/assignments.json | 64 ++++---- .../schemas/checklist_items.json | 74 ++++----- .../source_glassfrog/schemas/circles.json | 102 ++++++------ .../schemas/custom_fields.json | 52 +++--- .../source_glassfrog/schemas/metrics.json | 74 ++++----- .../source_glassfrog/schemas/people.json | 94 +++++------ .../source_glassfrog/schemas/projects.json | 148 +++++++++--------- .../source_glassfrog/schemas/roles.json | 148 +++++++++--------- .../source_glassfrog/source.py | 19 ++- .../source_glassfrog/spec.yaml | 1 - .../unit_tests/test_source.py | 23 ++- .../unit_tests/test_streams.py | 17 -- 17 files changed, 404 insertions(+), 463 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json delete mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json delete mode 100644 airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json diff --git a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml index 142215f18acf..cfd1b79ada46 100644 --- a/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-glassfrog/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: ["custom_fields", "checklist_items", "metrics", "projects"] full_refresh: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json deleted file mode 100644 index 6799946a6851..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/catalog.json +++ /dev/null @@ -1,39 +0,0 @@ -{ - "streams": [ - { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - }, - { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": false, - "json_schema": { - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "column1": { - "type": "string" - }, - "column2": { - "type": "number" - } - } - } - } - ] -} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json deleted file mode 100644 index ecc4913b84c7..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "fix-me": "TODO" -} diff --git a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json deleted file mode 100644 index 3587e579822d..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/integration_tests/sample_state.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "todo-stream-name": { - "todo-field-name": "value" - } -} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py index d83faaef9ddd..31adcf863e7f 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/auth.py @@ -2,7 +2,7 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, Dict, Mapping +from typing import Any, Mapping from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json index 8256445e8199..a2b84c67c560 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json @@ -1,36 +1,36 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "election": { + "type": ["null", "string"], + "default": null + }, + "exclude_from_meetings": { + "type": "boolean", + "default": false + }, + "focus": { + "type": ["null", "string"], + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "person": { + "type": ["null", "integer"], + "default": null }, - "election": { - "type": ["null", "string"], - "default": null - }, - "exclude_from_meetings": { - "type": "boolean", - "default": false - }, - "focus": { - "type": ["null", "string"], - "default": "" - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "person": { - "type": ["null", "integer"], - "default": null - }, - "role": { - "type": ["null", "integer"], - "default": null - } - } + "role": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json index 48f23256a63d..2ec48403a82b 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json @@ -1,41 +1,41 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "frequency": { + "type": "string", + "default": "" + }, + "link": { + "type": "string", + "default": "" + }, + "global": { + "type": "boolean", + "default": false + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circle": { + "type": ["null", "integer"], + "default": null }, - "description": { - "type": "string", - "default": "" - }, - "frequency": { - "type": "string", - "default": "" - }, - "link": { - "type": "string", - "default": "" - }, - "global": { - "type": "boolean", - "default": false - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "circle": { - "type": ["null", "integer"], - "default": null - }, - "role": { - "type": ["null", "integer"], - "default": null - } - } + "role": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json index 4cc77fa28cd8..626ba40b61f2 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json @@ -1,60 +1,60 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "name": { + "type": "string", + "default": "" + }, + "short_name": { + "type": "string", + "default": "" + }, + "strategy": { + "type": ["null", "string"], + "default": null + }, + "organization_id": { + "type": "integer", + "default": 0 + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "roles": { + "type": "array", + "default": [], + "items": { + "type": "integer" + } + }, + "policies": { + "type": "array", + "default": [], + "items": { "type": "integer", "default": 0 + } }, - "name": { - "type": "string", - "default": "" - }, - "short_name": { - "type": "string", - "default": "" - }, - "strategy": { - "type": ["null", "string"], - "default": null - }, - "organization_id": { + "domain": { + "type": "array", + "default": [], + "items": { "type": "integer", "default": 0 + } }, - "links": { - "type": "object", - "default": {}, - "properties": { - "roles": { - "type": "array", - "default": [], - "items": { - "type": "integer" - } - }, - "policies": { - "type": "array", - "default": [], - "items": { - "type": "integer", - "default": 0 - } - }, - "domain": { - "type": "array", - "default": [], - "items": { - "type": "integer", - "default": 0 - } - }, - "supported_role": { - "type": "integer", - "default": 0 - } - } + "supported_role": { + "type": "integer", + "default": 0 } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json index e27c5e884fa8..dbaaebf08779 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json @@ -1,29 +1,29 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 - }, - "field_name": { - "type": "string", - "default": "" - }, - "field_value": { - "type": "string", - "default": "" - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "role": { - "type": ["null", "integer"], - "default": null - } - } + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "field_name": { + "type": "string", + "default": "" + }, + "field_value": { + "type": "string", + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json index a8f66cf08d9c..82bee0742c63 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json @@ -1,41 +1,41 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "frequency": { + "type": "string", + "default": "" + }, + "link": { + "type": "string", + "default": "" + }, + "global": { + "type": "boolean", + "default": false + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null }, - "description": { - "type": "string", - "default": "" - }, - "frequency": { - "type": "string", - "default": "" - }, - "link": { - "type": "string", - "default": "" - }, - "global": { - "type": "boolean", - "default": false - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "role": { - "type": ["null", "integer"], - "default": null - }, - "circle": { - "type": ["null", "integer"], - "default": null - } - } + "circle": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json index 7e3fc298f69f..ea91a9c84c9e 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json @@ -1,53 +1,53 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 - }, - "tag_names": { - "type": "array", - "default": [], - "items": { - "type": "string", - "default": "" - } - }, - "name": { - "type": "string", - "default": "" - }, - "email": { - "type": "string", - "default": "" - }, - "external_id": { + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "tag_names": { + "type": "array", + "default": [], + "items": { + "type": "string", + "default": "" + } + }, + "name": { + "type": "string", + "default": "" + }, + "email": { + "type": "string", + "default": "" + }, + "external_id": { + "type": ["null", "integer"], + "default": null + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circles": { + "type": "array", + "default": [], + "items": { "type": ["null", "integer"], "default": null + } }, - "links": { - "type": "object", - "default": {}, - "properties": { - "circles": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - }, - "organization_ids": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - } - } + "organization_ids": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json index 7862f80e55df..c9e5b6466fa8 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json @@ -1,79 +1,79 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "description": { + "type": "string", + "default": "" + }, + "status": { + "type": "string", + "default": "" + }, + "waiting_on_who": { + "type": ["null", "string"], + "default": null + }, + "waiting_on_what": { + "type": ["null", "string"], + "default": null + }, + "link": { + "type": ["null", "string"], + "default": null + }, + "value": { + "type": ["null", "integer"], + "default": null + }, + "effort": { + "type": ["null", "integer"], + "default": null + }, + "roi": { + "type": ["null", "number"], + "default": null + }, + "private_to_circle": { + "type": "boolean", + "default": false + }, + "created_at": { + "type": "string", + "format": "date-time", + "default": "" + }, + "archived_at": { + "type": ["null", "string"], + "format": "date-time", + "default": null + }, + "type": { + "type": "string", + "default": "" + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "role": { + "type": ["null", "integer"], + "default": null }, - "description": { - "type": "string", - "default": "" + "person": { + "type": ["null", "integer"], + "default": null }, - "status": { - "type": "string", - "default": "" - }, - "waiting_on_who": { - "type": ["null", "string"], - "default": null - }, - "waiting_on_what": { - "type": ["null", "string"], - "default": null - }, - "link": { - "type": ["null", "string"], - "default": null - }, - "value": { - "type": ["null", "integer"], - "default": null - }, - "effort": { - "type": ["null", "integer"], - "default": null - }, - "roi": { - "type": ["null", "number"], - "default": null - }, - "private_to_circle": { - "type": "boolean", - "default": false - }, - "created_at": { - "type": "string", - "format": "date-time", - "default": "" - }, - "archived_at": { - "type": ["null", "string"], - "format": "date-time", - "default": null - }, - "type": { - "type": "string", - "default": "" - }, - "links": { - "type": "object", - "default": {}, - "properties": { - "role": { - "type": ["null", "integer"], - "default": null - }, - "person": { - "type": ["null", "integer"], - "default": null - }, - "circle": { - "type": ["null", "integer"], - "default": null - } - } + "circle": { + "type": ["null", "integer"], + "default": null } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json index 526a0daa389e..5af77e698d70 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json @@ -1,82 +1,82 @@ { - "$schema": "https://json-schema.org/draft-07/schema", - "type": "object", - "default": {}, - "properties": { - "id": { - "type": "integer", - "default": 0 + "$schema": "https://json-schema.org/draft-07/schema", + "type": "object", + "default": {}, + "properties": { + "id": { + "type": "integer", + "default": 0 + }, + "tag_names": { + "type": "array", + "default": [], + "items": { + "type": "string", + "default": "" + } + }, + "name": { + "type": "string", + "default": "" + }, + "is_core": { + "type": "boolean", + "default": false + }, + "name_with_circle_for_core_roles": { + "type": "string", + "default": "" + }, + "purpose": { + "type": "string", + "default": "" + }, + "elected_until": { + "type": ["null", "string"], + "format": "date", + "default": null + }, + "organization_id": { + "type": "integer", + "default": 0 + }, + "links": { + "type": "object", + "default": {}, + "properties": { + "circle": { + "type": ["null", "integer"], + "default": null }, - "tag_names": { - "type": "array", - "default": [], - "items": { - "type": "string", - "default": "" - } + "supporting_circle": { + "type": ["null", "integer"], + "default": null }, - "name": { - "type": "string", - "default": "" - }, - "is_core": { - "type": "boolean", - "default": false - }, - "name_with_circle_for_core_roles": { - "type": "string", - "default": "" - }, - "purpose": { - "type": "string", - "default": "" - }, - "elected_until": { - "type": ["null", "string"], - "format": "date", + "domains": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], "default": null + } }, - "organization_id": { - "type": "integer", - "default": 0 + "accountabilities": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } }, - "links": { - "type": "object", - "default": {}, - "properties": { - "circle": { - "type": ["null", "integer"], - "default": null - }, - "supporting_circle": { - "type": ["null", "integer"], - "default": null - }, - "domains": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - }, - "accountabilities": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - }, - "people": { - "type": "array", - "default": [], - "items": { - "type": ["null", "integer"], - "default": null - } - } - } + "people": { + "type": "array", + "default": [], + "items": { + "type": ["null", "integer"], + "default": null + } } + } } -} \ No newline at end of file + } +} diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py index 2716284d316a..e533cb192136 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/source.py @@ -4,7 +4,7 @@ from abc import ABC -from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +from typing import Any, Iterable, List, Mapping, Optional, Tuple import requests from airbyte_cdk.sources import AbstractSource @@ -21,15 +21,6 @@ class GlassfrogStream(HttpStream, ABC): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: return None - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. - """ - return {} - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: json_response = response.json() records = json_response.get(self.data_field, []) if self.data_field is not None else json_response @@ -39,6 +30,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp class Assignments(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#db2934bd-8c07-1951-b273-51fbc2dc6422 data_field = "assignments" primary_key = "id" @@ -49,6 +41,7 @@ def path( class ChecklistItems(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#a81716d4-b492-79ff-1348-9048fd9dc527 data_field = "checklist_items" primary_key = "id" @@ -59,6 +52,7 @@ def path( class Circles(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#ed696857-c3d8-fba1-a174-fbe63de07798 data_field = "circles" primary_key = "id" @@ -69,6 +63,7 @@ def path( class CustomFields(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#901f8ec2-a986-0291-2fa2-281c16622107 data_field = "custom_fields" primary_key = "id" @@ -79,6 +74,7 @@ def path( class Metrics(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#00d4f5fb-d6e5-5521-a77d-bdce50a9fb84 data_field = "metrics" primary_key = "id" @@ -89,6 +85,7 @@ def path( class People(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#78b74b9f-72b7-63fc-a18c-18518932944b data_field = "people" primary_key = "id" @@ -99,6 +96,7 @@ def path( class Projects(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#110bde88-a319-ae9c-077a-9752fd2f0843 data_field = "projects" primary_key = "id" @@ -109,6 +107,7 @@ def path( class Roles(GlassfrogStream): + # https://documenter.getpostman.com/view/1014385/2SJViY?version=latest#d1f31f7a-1d42-8c86-be1d-a36e640bf993 data_field = "roles" primary_key = "id" diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml index b5423c2f0ce5..2ce188f84be4 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml @@ -11,4 +11,3 @@ connectionSpecification: type: string description: API key provided by Glassfrog airbyte_secret: true - diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py index 9cdc3db8b407..a69ca4c30861 100644 --- a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_source.py @@ -4,19 +4,26 @@ from unittest.mock import MagicMock +from pytest import fixture from source_glassfrog.source import SourceGlassfrog -def test_check_connection(mocker): +@fixture() +def config(request): + args = {"api_key": "xxxxxxx"} + return args + + +def test_check_connection(mocker, config): source = SourceGlassfrog() - logger_mock, config_mock = MagicMock(), MagicMock() - assert source.check_connection(logger_mock, config_mock) == (True, None) + logger_mock = MagicMock() + (connection_status, error) = source.check_connection(logger_mock, config) + expected_status = False + assert connection_status == expected_status -def test_streams(mocker): +def test_streams(mocker, config): source = SourceGlassfrog() - config_mock = MagicMock() - streams = source.streams(config_mock) - # TODO: replace this with your streams number - expected_streams_number = 2 + streams = source.streams(config) + expected_streams_number = 8 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py index a781d6617bfb..050c4c830929 100644 --- a/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-glassfrog/unit_tests/test_streams.py @@ -11,7 +11,6 @@ @pytest.fixture def patch_base_class(mocker): - # Mock abstract methods to enable instantiating abstract class mocker.patch.object(GlassfrogStream, "path", "v0/example_endpoint") mocker.patch.object(GlassfrogStream, "primary_key", "test_primary_key") mocker.patch.object(GlassfrogStream, "__abstractmethods__", set()) @@ -19,43 +18,27 @@ def patch_base_class(mocker): def test_request_params(patch_base_class): stream = GlassfrogStream() - # TODO: replace this with your input parameters inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request parameters expected_params = {} assert stream.request_params(**inputs) == expected_params def test_next_page_token(patch_base_class): stream = GlassfrogStream() - # TODO: replace this with your input parameters inputs = {"response": MagicMock()} - # TODO: replace this with your expected next page token expected_token = None assert stream.next_page_token(**inputs) == expected_token -def test_parse_response(patch_base_class): - stream = GlassfrogStream() - # TODO: replace this with your input parameters - inputs = {"response": MagicMock()} - # TODO: replace this with your expected parced object - expected_parsed_object = {} - assert next(stream.parse_response(**inputs)) == expected_parsed_object - - def test_request_headers(patch_base_class): stream = GlassfrogStream() - # TODO: replace this with your input parameters inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} - # TODO: replace this with your expected request headers expected_headers = {} assert stream.request_headers(**inputs) == expected_headers def test_http_method(patch_base_class): stream = GlassfrogStream() - # TODO: replace this with your expected http request method expected_method = "GET" assert stream.http_method == expected_method From cb6d0303635ef15bdac5ee74687f7d41507eb260 Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Fri, 22 Jul 2022 01:15:11 +0530 Subject: [PATCH 6/8] chore: added connector to definitions --- .../src/main/resources/icons/glassfrog.svg | 899 ++++++++++++++++++ .../resources/seed/source_definitions.yaml | 8 + 2 files changed, 907 insertions(+) create mode 100644 airbyte-config/init/src/main/resources/icons/glassfrog.svg diff --git a/airbyte-config/init/src/main/resources/icons/glassfrog.svg b/airbyte-config/init/src/main/resources/icons/glassfrog.svg new file mode 100644 index 000000000000..7f7e69cea9f7 --- /dev/null +++ b/airbyte-config/init/src/main/resources/icons/glassfrog.svg @@ -0,0 +1,899 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 8956e55f097f..ebb5e73624e4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -316,6 +316,14 @@ icon: gitlab.svg sourceType: api releaseStage: alpha +- name: Glassfrog + sourceDefinitionId: cf8ff320-6272-4faa-89e6-4402dc17e5d5 + dockerRepository: airbyte/source-glassfrog + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/glassfrog + icon: glassfrog.svg + sourceType: api + releaseStage: alpha - name: Google Ads sourceDefinitionId: 253487c0-2246-43ba-a21f-5116b20a2c50 dockerRepository: airbyte/source-google-ads From acff87eb0d78bb2dcac6142750a3d3b694bd84fb Mon Sep 17 00:00:00 2001 From: Harshith Mullapudi Date: Tue, 26 Jul 2022 12:43:20 +0530 Subject: [PATCH 7/8] fix: tests and formatting --- .../source_glassfrog/schemas/TODO.md | 25 ------------------- .../source_glassfrog/schemas/assignments.json | 1 + .../schemas/checklist_items.json | 2 +- .../source_glassfrog/schemas/circles.json | 2 +- .../schemas/custom_fields.json | 2 +- .../source_glassfrog/schemas/metrics.json | 2 +- .../source_glassfrog/schemas/people.json | 2 +- .../source_glassfrog/schemas/projects.json | 2 +- .../source_glassfrog/schemas/roles.json | 2 +- .../source_glassfrog/spec.yaml | 2 +- 10 files changed, 9 insertions(+), 33 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c..000000000000 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json index a2b84c67c560..4d23450daa56 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/assignments.json @@ -1,6 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema", "type": "object", + "additionalProperties": true, "properties": { "id": { "type": "integer", diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json index 2ec48403a82b..a34d736f12b9 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/checklist_items.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema", "type": "object", - "default": {}, + "additionalProperties": true, "properties": { "id": { "type": "integer", diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json index 626ba40b61f2..67c718b8fb95 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/circles.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema", "type": "object", - "default": {}, + "additionalProperties": true, "properties": { "id": { "type": "integer", diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json index dbaaebf08779..4165620483d3 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/custom_fields.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema", "type": "object", - "default": {}, + "additionalProperties": true, "properties": { "id": { "type": "integer", diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json index 82bee0742c63..8f61822d2ad0 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/metrics.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema", "type": "object", - "default": {}, + "additionalProperties": true, "properties": { "id": { "type": "integer", diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json index ea91a9c84c9e..63b4024f04b2 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/people.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema", "type": "object", - "default": {}, + "additionalProperties": true, "properties": { "id": { "type": "integer", diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json index c9e5b6466fa8..1504ac121b22 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/projects.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema", "type": "object", - "default": {}, + "additionalProperties": true, "properties": { "id": { "type": "integer", diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json index 5af77e698d70..adbca758a4e4 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/schemas/roles.json @@ -1,7 +1,7 @@ { "$schema": "https://json-schema.org/draft-07/schema", "type": "object", - "default": {}, + "additionalProperties": true, "properties": { "id": { "type": "integer", diff --git a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml index 2ce188f84be4..46be13d613ae 100644 --- a/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml +++ b/airbyte-integrations/connectors/source-glassfrog/source_glassfrog/spec.yaml @@ -5,7 +5,7 @@ connectionSpecification: type: object required: - api_key - additionalProperties: false + additionalProperties: true properties: api_key: type: string From 2e522296eddbe721708bb56143827725e1c3b6e2 Mon Sep 17 00:00:00 2001 From: Octavia Squidington III Date: Tue, 26 Jul 2022 08:03:31 +0000 Subject: [PATCH 8/8] auto-bump connector version [ci skip] --- .../src/main/resources/seed/source_specs.yaml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 41d4a7d4f948..09806756e50c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2781,6 +2781,24 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-glassfrog:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/glassfrog" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Glassfrog Spec" + type: "object" + required: + - "api_key" + additionalProperties: true + properties: + api_key: + type: "string" + description: "API key provided by Glassfrog" + airbyte_secret: true + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-google-ads:0.1.43" spec: documentationUrl: "https://docs.airbyte.com/integrations/sources/google-ads"