From caa128cda0e28b057cfd666656affc4c4e97dd78 Mon Sep 17 00:00:00 2001 From: Ghilman Date: Fri, 22 Apr 2022 12:38:17 +0500 Subject: [PATCH 1/8] WIP create junip source --- .../source-junip-reviews/.dockerignore | 6 + .../source-junip-reviews/Dockerfile | 38 +++++ .../connectors/source-junip-reviews/README.md | 132 ++++++++++++++++ .../acceptance-test-config.yml | 30 ++++ .../acceptance-test-docker.sh | 16 ++ .../source-junip-reviews/build.gradle | 9 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 ++ .../integration_tests/catalog.json | 39 +++++ .../integration_tests/configured_catalog.json | 22 +++ .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../connectors/source-junip-reviews/main.py | 13 ++ .../source-junip-reviews/requirements.txt | 2 + .../sample_files/config.json | 3 + .../sample_files/configured_catalog.json | 65 ++++++++ .../connectors/source-junip-reviews/setup.py | 29 ++++ .../source_junip_reviews/__init__.py | 8 + .../source_junip_reviews/schemas/TODO.md | 25 +++ .../schemas/customers.json | 16 ++ .../schemas/employees.json | 19 +++ .../schemas/product_list_stream.json | 45 ++++++ .../source_junip_reviews/source.py | 148 ++++++++++++++++++ .../source_junip_reviews/spec.json | 16 ++ .../source_junip_reviews/streams.py | 68 ++++++++ .../unit_tests/__init__.py | 3 + .../unit_tests/test_incremental_streams.py | 59 +++++++ .../unit_tests/test_source.py | 22 +++ .../unit_tests/test_streams.py | 83 ++++++++++ 31 files changed, 951 insertions(+) create mode 100644 airbyte-integrations/connectors/source-junip-reviews/.dockerignore create mode 100644 airbyte-integrations/connectors/source-junip-reviews/Dockerfile create mode 100644 airbyte-integrations/connectors/source-junip-reviews/README.md create mode 100644 airbyte-integrations/connectors/source-junip-reviews/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-junip-reviews/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-junip-reviews/build.gradle create mode 100644 airbyte-integrations/connectors/source-junip-reviews/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/main.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/requirements.txt create mode 100644 airbyte-integrations/connectors/source-junip-reviews/sample_files/config.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/setup.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/__init__.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_list_stream.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/spec.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/unit_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_source.py create mode 100644 airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_streams.py diff --git a/airbyte-integrations/connectors/source-junip-reviews/.dockerignore b/airbyte-integrations/connectors/source-junip-reviews/.dockerignore new file mode 100644 index 000000000000..243f65234f7a --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_junip_reviews +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-junip-reviews/Dockerfile b/airbyte-integrations/connectors/source-junip-reviews/Dockerfile new file mode 100644 index 000000000000..46c60cb6c393 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/Dockerfile @@ -0,0 +1,38 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_junip_reviews ./source_junip_reviews + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-junip-reviews diff --git a/airbyte-integrations/connectors/source-junip-reviews/README.md b/airbyte-integrations/connectors/source-junip-reviews/README.md new file mode 100644 index 000000000000..7f409bb4350c --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/README.md @@ -0,0 +1,132 @@ +# Junip Reviews Source + +This is the repository for the Junip Reviews source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/junip-reviews). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-junip-reviews:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/junip-reviews) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_junip_reviews/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source junip-reviews test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-junip-reviews:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-junip-reviews:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-junip-reviews:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-junip-reviews:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-junip-reviews:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-junip-reviews:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +python -m pytest integration_tests -p integration_tests.acceptance +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-junip-reviews:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-junip-reviews:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-config.yml b/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-config.yml new file mode 100644 index 000000000000..2ed049b10348 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-config.yml @@ -0,0 +1,30 @@ +# See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-junip-reviews:dev +tests: + spec: + - spec_path: "source_junip_reviews/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.txt" +# extra_fields: no +# exact_order: no +# extra_records: yes + incremental: # TODO if your connector does not implement incremental sync, remove this block + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-docker.sh new file mode 100644 index 000000000000..c51577d10690 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2-) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-junip-reviews/build.gradle b/airbyte-integrations/connectors/source-junip-reviews/build.gradle new file mode 100644 index 000000000000..9fe15a2a7580 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_junip_reviews' +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/__init__.py b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/acceptance.py new file mode 100644 index 000000000000..056971f95450 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json new file mode 100644 index 000000000000..6799946a6851 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json @@ -0,0 +1,39 @@ +{ + "streams": [ + { + "name": "TODO fix this file", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": "column1", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + }, + { + "name": "table1", + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "column1": { + "type": "string" + }, + "column2": { + "type": "number" + } + } + } + } + ] +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..36f0468db0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/main.py b/airbyte-integrations/connectors/source-junip-reviews/main.py new file mode 100644 index 000000000000..d274d0445ddc --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_junip_reviews import SourceJunipReviews + +if __name__ == "__main__": + source = SourceJunipReviews() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-junip-reviews/requirements.txt b/airbyte-integrations/connectors/source-junip-reviews/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-junip-reviews/sample_files/config.json b/airbyte-integrations/connectors/source-junip-reviews/sample_files/config.json new file mode 100644 index 000000000000..d37911f66d06 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/sample_files/config.json @@ -0,0 +1,3 @@ +{ + "junip_store_key": "aGErmrHSAe3Hf9zjRaYRbHmu" +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json new file mode 100644 index 000000000000..d14082967d68 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json @@ -0,0 +1,65 @@ +{ + "streams": [ + { + "stream": { + "name": "product_list_stream", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "deleted_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "integer" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "remote_handle": { + "type": "string" + }, + "remote_id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "unreviewable": { + "type": "boolean" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + } + ] +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-junip-reviews/setup.py b/airbyte-integrations/connectors/source-junip-reviews/setup.py new file mode 100644 index 000000000000..5055c51e8a33 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/setup.py @@ -0,0 +1,29 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "source-acceptance-test", +] + +setup( + name="source_junip_reviews", + description="Source implementation for Junip Reviews.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/__init__.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/__init__.py new file mode 100644 index 000000000000..b756c7dbb17d --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceJunipReviews + +__all__ = ["SourceJunipReviews"] diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/TODO.md b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/TODO.md new file mode 100644 index 000000000000..cf1efadb3c9c --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/TODO.md @@ -0,0 +1,25 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. + +The schema of a stream is the return value of `Stream.get_json_schema`. + +## Static schemas +By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. + +Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. + +## Dynamic schemas +If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). + +## Dynamically modifying static schemas +Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: +``` +def get_json_schema(self): + schema = super().get_json_schema() + schema['dynamically_determined_property'] = "property" + return schema +``` + +Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/customers.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/customers.json new file mode 100644 index 000000000000..9a4b13485836 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/customers.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/employees.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/employees.json new file mode 100644 index 000000000000..2fa01a0fa1ff --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/employees.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_list_stream.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_list_stream.json new file mode 100644 index 000000000000..e94b34a44bd2 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_list_stream.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:24:48.702Z"] + }, + "deleted_at": { + "type": "string", + "examples": ["2021-06-18T18:24:48.702Z"] + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "integer" + }, + "rating_distribution": { + "type": "array" + }, + "recommended_count": { + "type": "integer" + }, + "remote_handle": { + "type": "string" + }, + "remote_id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "unreviewable": { + "type": "boolean" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:24:48.702Z"] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py new file mode 100644 index 000000000000..7e2572d7f893 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py @@ -0,0 +1,148 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +from .streams import ProductListStream + +import requests +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + +""" +TODO: Most comments in this class are instructive and should be deleted after the source is implemented. + +This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an +incremental syncs from an HTTP API. + +The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental +stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. + +The approach here is not authoritative, and devs are free to use their own judgement. + +There are additional required TODOs in the files within the integration_tests folder and the spec.json file. +""" + + +# Basic full refresh stream + +# class Customers(JunipReviewsStream): +# """ +# TODO: Change class name to match the table/data source this stream corresponds to. +# """ +# +# # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. +# primary_key = "customer_id" +# +# def path( +# self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None +# ) -> str: +# """ +# TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this +# should return "customers". Required. +# """ +# return "customers" + + +# Basic incremental stream +# class IncrementalJunipReviewsStream(JunipReviewsStream, ABC): +# """ +# TODO fill in details of this class to implement functionality related to incremental syncs for your connector. +# if you do not need to implement incremental sync for any streams, remove this class. +# """ +# +# # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. +# state_checkpoint_interval = None +# +# @property +# def cursor_field(self) -> str: +# """ +# TODO +# Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is +# usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. +# +# :return str: The name of the cursor field. +# """ +# return [] +# +# def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: +# """ +# Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and +# the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. +# """ +# return {} +# +# +# class Employees(IncrementalJunipReviewsStream): +# """ +# TODO: Change class name to match the table/data source this stream corresponds to. +# """ +# +# # TODO: Fill in the cursor_field. Required. +# cursor_field = "start_date" +# +# # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. +# primary_key = "" +# +# def path(self, **kwargs) -> str: +# """ +# TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should +# return "single". Required. +# """ +# return "employees" +# +# def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: +# """ +# TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. +# +# Slices control when state is saved. Specifically, state is saved after a slice has been fully read. +# This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" +# section of the docs for more information. +# +# The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the +# necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. +# This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. +# +# An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help +# craft that specific request. +# +# For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement +# this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date +# till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into +# the date query param. +# """ +# raise NotImplementedError("Implement stream slices or delete this method!") + + +# Source +class SourceJunipReviews(AbstractSource): + def check_connection(self, logger, config) -> Tuple[bool, any]: + """ + TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API + + See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 + for an example. + + :param config: the user-input config object conforming to the connector's spec.json + :param logger: logger object + :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. + """ + url = "https://api.juniphq.com/v1/product_overviews" + headers = { + 'Junip-Store-Key': config["junip_store_key"] + } + + try: + response = requests.request("GET", url, headers=headers) + except Exception as e: + return False, e + return True, None + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + + return [ProductListStream(junip_store_key=config.get("junip_store_key"))] diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/spec.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/spec.json new file mode 100644 index 000000000000..22fd441e21b0 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/spec.json @@ -0,0 +1,16 @@ +{ + "documentationUrl": "https://docsurl.com", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Junip Reviews Spec", + "type": "object", + "required": ["junip_store_key"], + "additionalProperties": false, + "properties": { + "junip_store_key": { + "type": "string", + "description": "junip store key" + } + } + } +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py new file mode 100644 index 000000000000..e2a82804afe5 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py @@ -0,0 +1,68 @@ + +from abc import ABC +from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple + +import requests +from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator + + +class JunipReviewsStream(HttpStream, ABC): + url_base = "https://api.juniphq.com/v1/" + primary_key = None + + def __init__(self, junip_store_key,**kwargs): + super().__init__(**kwargs) + self.junip_store_key = junip_store_key + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. + + This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed + to most other methods in this class to help you form headers, request bodies, query params, etc.. + + For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a + 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. + The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + + :param response: the most recent response from the API + :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. + If there are no more pages in the result, return None. + """ + return None + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + """ + TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. + Usually contains common params e.g. pagination size etc. + """ + return {} + + def request_headers( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + + return {"Junip_Store_key": self.junip_store_key} + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + TODO: Override this method to define how a response is parsed. + :return an iterable containing each record in the response + """ + return [response.json()] + + +class ProductListStream(JunipReviewsStream): + + def path( + self, + *, + stream_state: Mapping[str, Any] = None, + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> str: + + return "products" diff --git a/airbyte-integrations/connectors/source-junip-reviews/unit_tests/__init__.py b/airbyte-integrations/connectors/source-junip-reviews/unit_tests/__init__.py new file mode 100644 index 000000000000..46b7376756ec --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/unit_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_incremental_streams.py new file mode 100644 index 000000000000..4fd707618c72 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_incremental_streams.py @@ -0,0 +1,59 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + + +from airbyte_cdk.models import SyncMode +from pytest import fixture +from source_junip_reviews.source import IncrementalJunipReviewsStream + + +@fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(IncrementalJunipReviewsStream, "path", "v0/example_endpoint") + mocker.patch.object(IncrementalJunipReviewsStream, "primary_key", "test_primary_key") + mocker.patch.object(IncrementalJunipReviewsStream, "__abstractmethods__", set()) + + +def test_cursor_field(patch_incremental_base_class): + stream = IncrementalJunipReviewsStream() + # TODO: replace this with your expected cursor field + expected_cursor_field = [] + assert stream.cursor_field == expected_cursor_field + + +def test_get_updated_state(patch_incremental_base_class): + stream = IncrementalJunipReviewsStream() + # TODO: replace this with your input parameters + inputs = {"current_stream_state": None, "latest_record": None} + # TODO: replace this with your expected updated stream state + expected_state = {} + assert stream.get_updated_state(**inputs) == expected_state + + +def test_stream_slices(patch_incremental_base_class): + stream = IncrementalJunipReviewsStream() + # TODO: replace this with your input parameters + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": {}} + # TODO: replace this with your expected stream slices list + expected_stream_slice = [None] + assert stream.stream_slices(**inputs) == expected_stream_slice + + +def test_supports_incremental(patch_incremental_base_class, mocker): + mocker.patch.object(IncrementalJunipReviewsStream, "cursor_field", "dummy_field") + stream = IncrementalJunipReviewsStream() + assert stream.supports_incremental + + +def test_source_defined_cursor(patch_incremental_base_class): + stream = IncrementalJunipReviewsStream() + assert stream.source_defined_cursor + + +def test_stream_checkpoint_interval(patch_incremental_base_class): + stream = IncrementalJunipReviewsStream() + # TODO: replace this with your expected checkpoint interval + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval diff --git a/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_source.py b/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_source.py new file mode 100644 index 000000000000..7e056bc48746 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_source.py @@ -0,0 +1,22 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +from source_junip_reviews.source import SourceJunipReviews + + +def test_check_connection(mocker): + source = SourceJunipReviews() + logger_mock, config_mock = MagicMock(), MagicMock() + assert source.check_connection(logger_mock, config_mock) == (True, None) + + +def test_streams(mocker): + source = SourceJunipReviews() + config_mock = MagicMock() + streams = source.streams(config_mock) + # TODO: replace this with your streams number + expected_streams_number = 2 + assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_streams.py new file mode 100644 index 000000000000..0af040ff63f7 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/unit_tests/test_streams.py @@ -0,0 +1,83 @@ +# +# Copyright (c) 2021 Airbyte, Inc., all rights reserved. +# + +from http import HTTPStatus +from unittest.mock import MagicMock + +import pytest +from source_junip_reviews.source import JunipReviewsStream + + +@pytest.fixture +def patch_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(JunipReviewsStream, "path", "v0/example_endpoint") + mocker.patch.object(JunipReviewsStream, "primary_key", "test_primary_key") + mocker.patch.object(JunipReviewsStream, "__abstractmethods__", set()) + + +def test_request_params(patch_base_class): + stream = JunipReviewsStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request parameters + expected_params = {} + assert stream.request_params(**inputs) == expected_params + + +def test_next_page_token(patch_base_class): + stream = JunipReviewsStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected next page token + expected_token = None + assert stream.next_page_token(**inputs) == expected_token + + +def test_parse_response(patch_base_class): + stream = JunipReviewsStream() + # TODO: replace this with your input parameters + inputs = {"response": MagicMock()} + # TODO: replace this with your expected parced object + expected_parsed_object = {} + assert next(stream.parse_response(**inputs)) == expected_parsed_object + + +def test_request_headers(patch_base_class): + stream = JunipReviewsStream() + # TODO: replace this with your input parameters + inputs = {"stream_slice": None, "stream_state": None, "next_page_token": None} + # TODO: replace this with your expected request headers + expected_headers = {} + assert stream.request_headers(**inputs) == expected_headers + + +def test_http_method(patch_base_class): + stream = JunipReviewsStream() + # TODO: replace this with your expected http request method + expected_method = "GET" + assert stream.http_method == expected_method + + +@pytest.mark.parametrize( + ("http_status", "should_retry"), + [ + (HTTPStatus.OK, False), + (HTTPStatus.BAD_REQUEST, False), + (HTTPStatus.TOO_MANY_REQUESTS, True), + (HTTPStatus.INTERNAL_SERVER_ERROR, True), + ], +) +def test_should_retry(patch_base_class, http_status, should_retry): + response_mock = MagicMock() + response_mock.status_code = http_status + stream = JunipReviewsStream() + assert stream.should_retry(response_mock) == should_retry + + +def test_backoff_time(patch_base_class): + response_mock = MagicMock() + stream = JunipReviewsStream() + expected_backoff_time = None + assert stream.backoff_time(response_mock) == expected_backoff_time From fc22319cd2954a39f809244f6f3c827470d524ab Mon Sep 17 00:00:00 2001 From: Ghilman Date: Fri, 22 Apr 2022 15:18:07 +0500 Subject: [PATCH 2/8] :boom: add junip reviews source --- .../sample_files/configured_catalog.json | 247 +++++++++++++++++- .../schemas/product_overviews.json | 33 +++ .../schemas/product_reviews.json | 59 +++++ ...product_list_stream.json => products.json} | 0 .../schemas/store_reviews.json | 56 ++++ .../source_junip_reviews/schemas/stores.json | 41 +++ .../source_junip_reviews/source.py | 13 +- .../source_junip_reviews/streams.py | 60 +++-- 8 files changed, 488 insertions(+), 21 deletions(-) create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_overviews.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_reviews.json rename airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/{product_list_stream.json => products.json} (100%) create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/store_reviews.json create mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/stores.json diff --git a/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json index d14082967d68..1347a53c1fb3 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json @@ -2,7 +2,7 @@ "streams": [ { "stream": { - "name": "product_list_stream", + "name": "products", "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", @@ -60,6 +60,251 @@ }, "sync_mode": "full_refresh", "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "product_overviews", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "deleted_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "integer" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "product_reviews", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "body": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:07.033Z" + ] + }, + "customer_id": { + "type": "integer" + }, + "down_vote_count": { + "type": "integer" + }, + "featured": { + "type": "boolean" + }, + "product_id": { + "type": "integer" + }, + "rating": { + "type": "integer" + }, + "response": { + "type": [ + "null", + "object" + ] + }, + "target_title": { + "type": "string" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "up_vote_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:07.033Z" + ] + }, + "verified_buyer": { + "type": "boolean" + }, + "would_recommend": { + "type": [ + "null", + "string" + ] + }, + "photo_urls": { + "type": "array" + }, + "video_urls": { + "type": "array" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "stores", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:25:17.951Z"] + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "number" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "slug": { + "type": "string" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:25:17.951Z"] + }, + "url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "store_reviews", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "body": { + "type": ["null", "object"] + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:25:16.161Z"] + }, + "customer_id": { + "type": "integer" + }, + "down_vote_count": { + "type": "number" + }, + "featured": { + "type": "boolean" + }, + "rating": { + "type": "number" + }, + "response": { + "type": ["null", "object"] + }, + "target_title": { + "type": "string" + }, + "title": { + "type": ["null", "string"] + }, + "up_vote_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:25:16.161Z"] + }, + "verified_buyer": { + "type": "boolean" + }, + "would_recommend": { + "type": ["null", "string"] + }, + "photo_urls": { + "type": "array" + }, + "video_urls": { + "type": "array" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" } ] } \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_overviews.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_overviews.json new file mode 100644 index 000000000000..10bdac39571e --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_overviews.json @@ -0,0 +1,33 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:24:48.702Z"] + }, + "deleted_at": { + "type": "string", + "examples": ["2021-06-18T18:24:48.702Z"] + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "integer" + }, + "rating_distribution": { + "type": "array" + }, + "recommended_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:24:48.702Z"] + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_reviews.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_reviews.json new file mode 100644 index 000000000000..24c69cbd23c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_reviews.json @@ -0,0 +1,59 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "body": { + "type": ["null", "string"] + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:25:07.033Z"] + }, + "customer_id": { + "type": "integer" + }, + "down_vote_count": { + "type": "integer" + }, + "featured": { + "type": "boolean" + }, + "product_id": { + "type": "integer" + }, + "rating": { + "type": "integer" + }, + "response": { + "type": ["null", "object"] + }, + "target_title": { + "type": "string" + }, + "title": { + "type": ["string", "null"] + }, + "up_vote_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:25:07.033Z"] + }, + "verified_buyer": { + "type": "boolean" + }, + "would_recommend": { + "type": ["null", "string"] + }, + "photo_urls": { + "type": "array" + }, + "video_urls": { + "type": "array" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_list_stream.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/products.json similarity index 100% rename from airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_list_stream.json rename to airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/products.json diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/store_reviews.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/store_reviews.json new file mode 100644 index 000000000000..5624d9c36f56 --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/store_reviews.json @@ -0,0 +1,56 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "body": { + "type": ["null", "object"] + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:25:16.161Z"] + }, + "customer_id": { + "type": "integer" + }, + "down_vote_count": { + "type": "number" + }, + "featured": { + "type": "boolean" + }, + "rating": { + "type": "number" + }, + "response": { + "type": ["null", "object"] + }, + "target_title": { + "type": "string" + }, + "title": { + "type": ["null", "string"] + }, + "up_vote_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:25:16.161Z"] + }, + "verified_buyer": { + "type": "boolean" + }, + "would_recommend": { + "type": ["null", "string"] + }, + "photo_urls": { + "type": "array" + }, + "video_urls": { + "type": "array" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/stores.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/stores.json new file mode 100644 index 000000000000..8297e5bd530f --- /dev/null +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/stores.json @@ -0,0 +1,41 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:25:17.951Z"] + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "number" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "slug": { + "type": "string" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:25:17.951Z"] + }, + "url": { + "type": "string" + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py index 7e2572d7f893..c399e9c70942 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py @@ -6,7 +6,7 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from .streams import ProductListStream +from .streams import Products, ProductOverviews, ProductReviews, Stores, StoreReviews import requests from airbyte_cdk.sources import AbstractSource @@ -144,5 +144,14 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: return True, None def streams(self, config: Mapping[str, Any]) -> List[Stream]: + args = { + "junip_store_key": config.get("junip_store_key") + } - return [ProductListStream(junip_store_key=config.get("junip_store_key"))] + return [ + Products(**args), + ProductOverviews(**args), + ProductReviews(**args), + Stores(**args), + StoreReviews(**args) + ] diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py index e2a82804afe5..9744b072bdad 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py @@ -16,20 +16,12 @@ def __init__(self, junip_store_key,**kwargs): self.junip_store_key = junip_store_key def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: - """ - TODO: Override this method to define a pagination strategy. If you will not be using pagination, no action is required - just return None. - - This method should return a Mapping (e.g: dict) containing whatever information required to make paginated requests. This dict is passed - to most other methods in this class to help you form headers, request bodies, query params, etc.. - - For example, if the API accepts a 'page' parameter to determine which page of the result to return, and a response from the API contains a - 'page' number, then this method should probably return a dict {'page': response.json()['page'] + 1} to increment the page count by 1. - The request_params method should then read the input next_page_token and set the 'page' param to next_page_token['page']. + decoded_response = response.json() - :param response: the most recent response from the API - :return If there is another page in the result, a mapping (e.g: dict) containing information needed to query the next page in the response. - If there are no more pages in the result, return None. - """ + if decoded_response.get("after") is not None: + return { + "page[after]": decoded_response.get("after") + } return None def request_params( @@ -39,7 +31,12 @@ def request_params( TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. Usually contains common params e.g. pagination size etc. """ - return {} + params = {} + + if next_page_token: + params.update(next_page_token) + + return params def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None @@ -54,9 +51,6 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp """ return [response.json()] - -class ProductListStream(JunipReviewsStream): - def path( self, *, @@ -65,4 +59,34 @@ def path( next_page_token: Mapping[str, Any] = None, ) -> str: - return "products" + return self.name + + +class Products(JunipReviewsStream): + """ + url: "self.base_url/self.name" + """ + + +class ProductOverviews(JunipReviewsStream): + """ + url: "self.base_url/self.name" + """ + + +class ProductReviews(JunipReviewsStream): + """ + url: "self.base_url/self.name" + """ + + +class Stores(JunipReviewsStream): + """ + url: "self.base_url/self.name" + """ + + +class StoreReviews(JunipReviewsStream): + """ + url: "self.base_url/self.name" + """ From d9e191f89a20fccf45825832c67acf4052f0b329 Mon Sep 17 00:00:00 2001 From: Ghilman Date: Sat, 23 Apr 2022 00:52:21 +0500 Subject: [PATCH 3/8] :memo: minor change in README.md --- airbyte-integrations/connectors/source-junip-reviews/README.md | 2 +- .../connectors/source-junip-reviews/sample_files/config.json | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-junip-reviews/sample_files/config.json diff --git a/airbyte-integrations/connectors/source-junip-reviews/README.md b/airbyte-integrations/connectors/source-junip-reviews/README.md index 7f409bb4350c..04c5d4ba6dcf 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/README.md +++ b/airbyte-integrations/connectors/source-junip-reviews/README.md @@ -60,7 +60,7 @@ python main.py read --config secrets/config.json --catalog integration_tests/con #### Build First, make sure you build the latest Docker image: ``` -docker build . -t airbyte/source-junip-reviews:dev +docker build . -t ghilman17/source-junip-reviews:0.1 ``` You can also build the connector image via Gradle: diff --git a/airbyte-integrations/connectors/source-junip-reviews/sample_files/config.json b/airbyte-integrations/connectors/source-junip-reviews/sample_files/config.json deleted file mode 100644 index d37911f66d06..000000000000 --- a/airbyte-integrations/connectors/source-junip-reviews/sample_files/config.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "junip_store_key": "aGErmrHSAe3Hf9zjRaYRbHmu" -} \ No newline at end of file From 98a6be54df4e6eba249ffaf43fb68d5f3c909a5d Mon Sep 17 00:00:00 2001 From: Ghilman Date: Mon, 25 Apr 2022 15:51:41 +0500 Subject: [PATCH 4/8] :zap: requested issues fixed --- .../integration_tests/abnormal_state.json | 6 +- .../integration_tests/acceptance.py | 3 +- .../integration_tests/catalog.json | 305 +++++++++++++++++- .../integration_tests/configured_catalog.json | 304 ++++++++++++++++- .../integration_tests/invalid_config.json | 2 +- .../integration_tests/sample_config.json | 2 +- .../integration_tests/sample_state.json | 6 +- .../sample_files/configured_catalog.json | 2 +- .../source_junip_reviews/source.py | 129 ++------ .../source_junip_reviews/streams.py | 7 +- 10 files changed, 621 insertions(+), 145 deletions(-) diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/abnormal_state.json index 52b0f2c2118f..8b137891791f 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/abnormal_state.json @@ -1,5 +1 @@ -{ - "todo-stream-name": { - "todo-field-name": "todo-abnormal-value" - } -} + diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/acceptance.py index 056971f95450..c1f0803ff9a6 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/acceptance.py +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/acceptance.py @@ -11,6 +11,5 @@ @pytest.fixture(scope="session", autouse=True) def connector_setup(): """This fixture is a placeholder for external resources that acceptance test might require.""" - # TODO: setup test dependencies if needed. otherwise remove the TODO comments yield - # TODO: clean up test dependencies + diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json index 6799946a6851..336b3f6920c7 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json @@ -1,39 +1,320 @@ { "streams": [ { - "name": "TODO fix this file", - "supported_sync_modes": ["full_refresh", "incremental"], - "source_defined_cursor": true, - "default_cursor_field": "column1", + "name": "products", + "source_defined_cursor": false, "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "column1": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "deleted_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "integer" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "remote_handle": { + "type": "string" + }, + "remote_id": { + "type": "string" + }, + "title": { "type": "string" }, - "column2": { + "unreviewable": { + "type": "boolean" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + { + "name": "product_overviews", + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "deleted_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "rating_average": { "type": "number" + }, + "rating_count": { + "type": "integer" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] } } - } + }, + "supported_sync_modes": [ + "full_refresh" + ] }, { - "name": "table1", - "supported_sync_modes": ["full_refresh", "incremental"], + "name": "product_reviews", "source_defined_cursor": false, "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": { - "column1": { + "id": { + "type": "integer" + }, + "body": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:07.033Z" + ] + }, + "customer_id": { + "type": "integer" + }, + "down_vote_count": { + "type": "integer" + }, + "featured": { + "type": "boolean" + }, + "product_id": { + "type": "integer" + }, + "rating": { + "type": "integer" + }, + "response": { + "type": [ + "null", + "object" + ] + }, + "target_title": { "type": "string" }, - "column2": { + "title": { + "type": [ + "string", + "null" + ] + }, + "up_vote_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:07.033Z" + ] + }, + "verified_buyer": { + "type": "boolean" + }, + "would_recommend": { + "type": [ + "null", + "string" + ] + }, + "photo_urls": { + "type": "array" + }, + "video_urls": { + "type": "array" + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + { + "name": "stores", + "source_defined_cursor": false, + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:17.951Z" + ] + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "rating_average": { "type": "number" + }, + "rating_count": { + "type": "number" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "slug": { + "type": "string" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:17.951Z" + ] + }, + "url": { + "type": "string" + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + { + "name": "store_reviews", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "body": { + "type": [ + "null", + "object" + ] + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:16.161Z" + ] + }, + "customer_id": { + "type": "integer" + }, + "down_vote_count": { + "type": "number" + }, + "featured": { + "type": "boolean" + }, + "rating": { + "type": "number" + }, + "response": { + "type": [ + "null", + "object" + ] + }, + "target_title": { + "type": "string" + }, + "title": { + "type": [ + "null", + "string" + ] + }, + "up_vote_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:16.161Z" + ] + }, + "verified_buyer": { + "type": "boolean" + }, + "would_recommend": { + "type": [ + "null", + "string" + ] + }, + "photo_urls": { + "type": "array" + }, + "video_urls": { + "type": "array" } } - } + }, + "supported_sync_modes": [ + "full_refresh" + ] } ] } diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json index 36f0468db0d8..c83aafd1cb2e 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json @@ -2,21 +2,309 @@ "streams": [ { "stream": { - "name": "customers", - "json_schema": {}, + "name": "products", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "deleted_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "integer" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "remote_handle": { + "type": "string" + }, + "remote_id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "unreviewable": { + "type": "boolean" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "product_overviews", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "deleted_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "integer" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:24:48.702Z" + ] + } + } + }, + "supported_sync_modes": [ + "full_refresh" + ] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "product_reviews", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "body": { + "type": [ + "null", + "string" + ] + }, + "created_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:07.033Z" + ] + }, + "customer_id": { + "type": "integer" + }, + "down_vote_count": { + "type": "integer" + }, + "featured": { + "type": "boolean" + }, + "product_id": { + "type": "integer" + }, + "rating": { + "type": "integer" + }, + "response": { + "type": [ + "null", + "object" + ] + }, + "target_title": { + "type": "string" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "up_vote_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": [ + "2021-06-18T18:25:07.033Z" + ] + }, + "verified_buyer": { + "type": "boolean" + }, + "would_recommend": { + "type": [ + "null", + "string" + ] + }, + "photo_urls": { + "type": "array" + }, + "video_urls": { + "type": "array" + } + } + }, "supported_sync_modes": ["full_refresh"] }, "sync_mode": "full_refresh", - "destination_sync_mode": "overwrite" + "destination_sync_mode": "append_dedup" }, { "stream": { - "name": "employees", - "json_schema": {}, - "supported_sync_modes": ["full_refresh", "incremental"] + "name": "stores", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:25:17.951Z"] + }, + "key": { + "type": "string" + }, + "name": { + "type": "string" + }, + "rating_average": { + "type": "number" + }, + "rating_count": { + "type": "number" + }, + "rating_distribution": { + "type": "object" + }, + "recommended_count": { + "type": "integer" + }, + "slug": { + "type": "string" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:25:17.951Z"] + }, + "url": { + "type": "string" + } + } + }, + "supported_sync_modes": ["full_refresh"] }, - "sync_mode": "incremental", - "destination_sync_mode": "append" + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" + }, + { + "stream": { + "name": "store_reviews", + "json_schema": { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": "integer" + }, + "body": { + "type": ["null", "object"] + }, + "created_at": { + "type": "string", + "examples": ["2021-06-18T18:25:16.161Z"] + }, + "customer_id": { + "type": "integer" + }, + "down_vote_count": { + "type": "number" + }, + "featured": { + "type": "boolean" + }, + "rating": { + "type": "number" + }, + "response": { + "type": ["null", "object"] + }, + "target_title": { + "type": "string" + }, + "title": { + "type": ["null", "string"] + }, + "up_vote_count": { + "type": "integer" + }, + "updated_at": { + "type": "string", + "examples": ["2021-06-18T18:25:16.161Z"] + }, + "verified_buyer": { + "type": "boolean" + }, + "would_recommend": { + "type": ["null", "string"] + }, + "photo_urls": { + "type": "array" + }, + "video_urls": { + "type": "array" + } + } + }, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "append_dedup" } ] } diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json index f3732995784f..a0737190a5b3 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "todo-wrong-field": "this should be an incomplete config file, used in standard tests" + "junip_store_key": "", } diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_config.json index ecc4913b84c7..040b2986df22 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_config.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_config.json @@ -1,3 +1,3 @@ { - "fix-me": "TODO" + "junip_store_key": "" } diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_state.json index 3587e579822d..8b137891791f 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/sample_state.json @@ -1,5 +1 @@ -{ - "todo-stream-name": { - "todo-field-name": "value" - } -} + diff --git a/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json index 1347a53c1fb3..c83aafd1cb2e 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json @@ -307,4 +307,4 @@ "destination_sync_mode": "append_dedup" } ] -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py index c399e9c70942..0680c10fcf37 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py @@ -6,125 +6,38 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple -from .streams import Products, ProductOverviews, ProductReviews, Stores, StoreReviews +from .streams import Products, ProductOverviews, ProductReviews, Stores, StoreReviews, JunipReviewsStream import requests from airbyte_cdk.sources import AbstractSource from airbyte_cdk.sources.streams import Stream -from airbyte_cdk.sources.streams.http import HttpStream -from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator -""" -TODO: Most comments in this class are instructive and should be deleted after the source is implemented. -This file provides a stubbed example of how to use the Airbyte CDK to develop both a source connector which supports full refresh or and an -incremental syncs from an HTTP API. - -The various TODOs are both implementation hints and steps - fulfilling all the TODOs should be sufficient to implement one basic and one incremental -stream from a source. This pattern is the same one used by Airbyte internally to implement connectors. - -The approach here is not authoritative, and devs are free to use their own judgement. - -There are additional required TODOs in the files within the integration_tests folder and the spec.json file. -""" +class IncrementalJunipReviewsStream(JunipReviewsStream, ABC): + state_checkpoint_interval = None + @property + def cursor_field(self) -> str: + """ + TODO + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. -# Basic full refresh stream + :return str: The name of the cursor field. + """ + return [] -# class Customers(JunipReviewsStream): -# """ -# TODO: Change class name to match the table/data source this stream corresponds to. -# """ -# -# # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. -# primary_key = "customer_id" -# -# def path( -# self, stream_state: Mapping[str, Any] = None, stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None -# ) -> str: -# """ -# TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/customers then this -# should return "customers". Required. -# """ -# return "customers" - - -# Basic incremental stream -# class IncrementalJunipReviewsStream(JunipReviewsStream, ABC): -# """ -# TODO fill in details of this class to implement functionality related to incremental syncs for your connector. -# if you do not need to implement incremental sync for any streams, remove this class. -# """ -# -# # TODO: Fill in to checkpoint stream reads after N records. This prevents re-reading of data if the stream fails for any reason. -# state_checkpoint_interval = None -# -# @property -# def cursor_field(self) -> str: -# """ -# TODO -# Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is -# usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. -# -# :return str: The name of the cursor field. -# """ -# return [] -# -# def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: -# """ -# Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and -# the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. -# """ -# return {} -# -# -# class Employees(IncrementalJunipReviewsStream): -# """ -# TODO: Change class name to match the table/data source this stream corresponds to. -# """ -# -# # TODO: Fill in the cursor_field. Required. -# cursor_field = "start_date" -# -# # TODO: Fill in the primary key. Required. This is usually a unique field in the stream, like an ID or a timestamp. -# primary_key = "" -# -# def path(self, **kwargs) -> str: -# """ -# TODO: Override this method to define the path this stream corresponds to. E.g. if the url is https://example-api.com/v1/employees then this should -# return "single". Required. -# """ -# return "employees" -# -# def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: -# """ -# TODO: Optionally override this method to define this stream's slices. If slicing is not needed, delete this method. -# -# Slices control when state is saved. Specifically, state is saved after a slice has been fully read. -# This is useful if the API offers reads by groups or filters, and can be paired with the state object to make reads efficient. See the "concepts" -# section of the docs for more information. -# -# The function is called before reading any records in a stream. It returns an Iterable of dicts, each containing the -# necessary data to craft a request for a slice. The stream state is usually referenced to determine what slices need to be created. -# This means that data in a slice is usually closely related to a stream's cursor_field and stream_state. -# -# An HTTP request is made for each returned slice. The same slice can be accessed in the path, request_params and request_header functions to help -# craft that specific request. -# -# For example, if https://example-api.com/v1/employees offers a date query params that returns data for that particular day, one way to implement -# this would be to consult the stream state object for the last synced date, then return a slice containing each date from the last synced date -# till now. The request_params function would then grab the date from the stream_slice and make it part of the request by injecting it into -# the date query param. -# """ -# raise NotImplementedError("Implement stream slices or delete this method!") + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + return {} -# Source class SourceJunipReviews(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: """ - TODO: Implement a connection check to validate that the user-provided config can be used to connect to the underlying API - See https://github.com/airbytehq/airbyte/blob/master/airbyte-integrations/connectors/source-stripe/source_stripe/source.py#L232 for an example. @@ -139,9 +52,11 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: try: response = requests.request("GET", url, headers=headers) + connection = True, None except Exception as e: - return False, e - return True, None + connection = False, e + + return connection def streams(self, config: Mapping[str, Any]) -> List[Stream]: args = { diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py index 9744b072bdad..61d11bbdd306 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py @@ -28,8 +28,7 @@ def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: """ - TODO: Override this method to define any query parameters to be set. Remove this method if you don't need to define request params. - Usually contains common params e.g. pagination size etc. + This method is for add query params in request URL """ params = {} @@ -41,7 +40,9 @@ def request_params( def request_headers( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> Mapping[str, Any]: - + """ + Override to return any non-auth headers. Authentication headers will overwrite any overlapping headers returned from this method. + """ return {"Junip_Store_key": self.junip_store_key} def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: From 5ff45726bdcd7eb28e4d79e6549c199301ca671f Mon Sep 17 00:00:00 2001 From: Ghilman Date: Mon, 25 Apr 2022 15:58:55 +0500 Subject: [PATCH 5/8] refactor code and remove extra code --- .../acceptance-test-config.yml | 8 +----- .../connectors/source-junip-reviews/setup.py | 4 +-- .../source_junip_reviews/schemas/TODO.md | 25 ------------------- .../schemas/customers.json | 16 ------------ .../schemas/employees.json | 19 -------------- .../schemas/product_overviews.json | 2 +- .../schemas/product_reviews.json | 2 +- .../schemas/products.json | 2 +- .../schemas/store_reviews.json | 2 +- .../source_junip_reviews/schemas/stores.json | 2 +- .../source_junip_reviews/source.py | 22 ---------------- 11 files changed, 8 insertions(+), 96 deletions(-) delete mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/TODO.md delete mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/customers.json delete mode 100644 airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/employees.json diff --git a/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-config.yml b/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-config.yml index 2ed049b10348..4c4ff991cde9 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-junip-reviews/acceptance-test-config.yml @@ -15,13 +15,7 @@ tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] -# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file -# expect_records: -# path: "integration_tests/expected_records.txt" -# extra_fields: no -# exact_order: no -# extra_records: yes - incremental: # TODO if your connector does not implement incremental sync, remove this block + incremental: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" future_state_path: "integration_tests/abnormal_state.json" diff --git a/airbyte-integrations/connectors/source-junip-reviews/setup.py b/airbyte-integrations/connectors/source-junip-reviews/setup.py index 5055c51e8a33..f9d34e7df542 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/setup.py +++ b/airbyte-integrations/connectors/source-junip-reviews/setup.py @@ -18,8 +18,8 @@ setup( name="source_junip_reviews", description="Source implementation for Junip Reviews.", - author="Airbyte", - author_email="contact@airbyte.io", + author="Ghilman Randhawa", + author_email="ghilman.randhawa@cogentlabs.co", packages=find_packages(), install_requires=MAIN_REQUIREMENTS, package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/TODO.md b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/TODO.md deleted file mode 100644 index cf1efadb3c9c..000000000000 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/TODO.md +++ /dev/null @@ -1,25 +0,0 @@ -# TODO: Define your stream schemas -Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). - -The simplest way to do this is to describe the schema of your streams using one `.json` file per stream. You can also dynamically generate the schema of your stream in code, or you can combine both approaches: start with a `.json` file and dynamically add properties to it. - -The schema of a stream is the return value of `Stream.get_json_schema`. - -## Static schemas -By default, `Stream.get_json_schema` reads a `.json` file in the `schemas/` directory whose name is equal to the value of the `Stream.name` property. In turn `Stream.name` by default returns the name of the class in snake case. Therefore, if you have a class `class EmployeeBenefits(HttpStream)` the default behavior will look for a file called `schemas/employee_benefits.json`. You can override any of these behaviors as you need. - -Important note: any objects referenced via `$ref` should be placed in the `shared/` directory in their own `.json` files. - -## Dynamic schemas -If you'd rather define your schema in code, override `Stream.get_json_schema` in your stream class to return a `dict` describing the schema using [JSONSchema](https://json-schema.org). - -## Dynamically modifying static schemas -Override `Stream.get_json_schema` to run the default behavior, edit the returned value, then return the edited value: -``` -def get_json_schema(self): - schema = super().get_json_schema() - schema['dynamically_determined_property'] = "property" - return schema -``` - -Delete this file once you're done. Or don't. Up to you :) diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/customers.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/customers.json deleted file mode 100644 index 9a4b13485836..000000000000 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/customers.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "signup_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/employees.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/employees.json deleted file mode 100644 index 2fa01a0fa1ff..000000000000 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/employees.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "type": "object", - "properties": { - "id": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "years_of_service": { - "type": ["null", "integer"] - }, - "start_date": { - "type": ["null", "string"], - "format": "date-time" - } - } -} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_overviews.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_overviews.json index 10bdac39571e..c6bf8f480f8e 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_overviews.json +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_overviews.json @@ -30,4 +30,4 @@ "examples": ["2021-06-18T18:24:48.702Z"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_reviews.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_reviews.json index 24c69cbd23c5..30b658e643a1 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_reviews.json +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/product_reviews.json @@ -56,4 +56,4 @@ "type": "array" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/products.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/products.json index e94b34a44bd2..82b330400eee 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/products.json +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/products.json @@ -42,4 +42,4 @@ "examples": ["2021-06-18T18:24:48.702Z"] } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/store_reviews.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/store_reviews.json index 5624d9c36f56..d00ab7ed73c0 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/store_reviews.json +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/store_reviews.json @@ -53,4 +53,4 @@ "type": "array" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/stores.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/stores.json index 8297e5bd530f..cabc2b044577 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/stores.json +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/schemas/stores.json @@ -38,4 +38,4 @@ "type": "string" } } -} \ No newline at end of file +} diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py index 0680c10fcf37..f5609a3a7ab6 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py @@ -13,28 +13,6 @@ from airbyte_cdk.sources.streams import Stream -class IncrementalJunipReviewsStream(JunipReviewsStream, ABC): - state_checkpoint_interval = None - - @property - def cursor_field(self) -> str: - """ - TODO - Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is - usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. - - :return str: The name of the cursor field. - """ - return [] - - def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: - """ - Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and - the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. - """ - return {} - - class SourceJunipReviews(AbstractSource): def check_connection(self, logger, config) -> Tuple[bool, any]: """ From 9247a1a2873fa77e232105045a68dca633a6c970 Mon Sep 17 00:00:00 2001 From: Ghilman Date: Mon, 25 Apr 2022 16:11:04 +0500 Subject: [PATCH 6/8] :momo: refactor doc --- .../connectors/source-junip-reviews/README.md | 2 +- .../source_junip_reviews/spec.json | 3 ++- .../source_junip_reviews/streams.py | 15 ++++++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/airbyte-integrations/connectors/source-junip-reviews/README.md b/airbyte-integrations/connectors/source-junip-reviews/README.md index 04c5d4ba6dcf..f677d2582e2b 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/README.md +++ b/airbyte-integrations/connectors/source-junip-reviews/README.md @@ -1,7 +1,7 @@ # Junip Reviews Source This is the repository for the Junip Reviews source connector, written in Python. -For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/junip-reviews). +For information about how to use this connector within Airbyte, see [the documentation](https://junip.co/docs/api/). ## Local development diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/spec.json b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/spec.json index 22fd441e21b0..5196346eb598 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/spec.json +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/spec.json @@ -9,7 +9,8 @@ "properties": { "junip_store_key": { "type": "string", - "description": "junip store key" + "description": "junip store key", + "airbyte_secret": true } } } diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py index 61d11bbdd306..b494aa3ac2ad 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py @@ -11,18 +11,24 @@ class JunipReviewsStream(HttpStream, ABC): url_base = "https://api.juniphq.com/v1/" primary_key = None - def __init__(self, junip_store_key,**kwargs): + def __init__(self, junip_store_key, **kwargs): super().__init__(**kwargs) self.junip_store_key = junip_store_key def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + """ + This method used to do pagination strategy + + Returns: Next page token + """ decoded_response = response.json() + page_token = None - if decoded_response.get("after") is not None: - return { + if decoded_response.get("after"): + page_token = { "page[after]": decoded_response.get("after") } - return None + return page_token def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None @@ -47,7 +53,6 @@ def request_headers( def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: """ - TODO: Override this method to define how a response is parsed. :return an iterable containing each record in the response """ return [response.json()] From 252f2830439241887622a769646d9255d64e0dc9 Mon Sep 17 00:00:00 2001 From: Ghilman Date: Wed, 27 Apr 2022 17:32:46 +0500 Subject: [PATCH 7/8] requested changes fixed --- .../integration_tests/catalog.json | 28 +++-- .../integration_tests/configured_catalog.json | 11 +- .../integration_tests/invalid_config.json | 2 +- .../sample_files/configured_catalog.json | 12 +- .../source_junip_reviews/source.py | 7 +- .../source_junip_reviews/streams.py | 118 ++++++++++++++++-- 6 files changed, 145 insertions(+), 33 deletions(-) diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json index 336b3f6920c7..2bc1957c363c 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/catalog.json @@ -2,7 +2,7 @@ "streams": [ { "name": "products", - "source_defined_cursor": false, + "source_defined_cursor": true, "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", @@ -56,11 +56,13 @@ }, "supported_sync_modes": [ "full_refresh" - ] + ], + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] }, { "name": "product_overviews", - "source_defined_cursor": false, + "source_defined_cursor": true, "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", @@ -102,11 +104,13 @@ }, "supported_sync_modes": [ "full_refresh" - ] + ], + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] }, { "name": "product_reviews", - "source_defined_cursor": false, + "source_defined_cursor": true, "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", @@ -184,11 +188,13 @@ }, "supported_sync_modes": [ "full_refresh" - ] + ], + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] }, { "name": "stores", - "source_defined_cursor": false, + "source_defined_cursor": true, "json_schema": { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", @@ -236,7 +242,9 @@ }, "supported_sync_modes": [ "full_refresh" - ] + ], + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] }, { "name": "store_reviews", @@ -314,7 +322,9 @@ }, "supported_sync_modes": [ "full_refresh" - ] + ], + "default_cursor_field": ["created_at"], + "source_defined_primary_key": [["id"]] } ] } diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json index c83aafd1cb2e..8f790ac53285 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/configured_catalog.json @@ -104,7 +104,8 @@ } }, "supported_sync_modes": [ - "full_refresh" + "full_refresh", + "incremental" ] }, "sync_mode": "full_refresh", @@ -188,7 +189,7 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append_dedup" @@ -237,9 +238,9 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, - "sync_mode": "full_refresh", + "sync_mode": "incremental", "destination_sync_mode": "append_dedup" }, { @@ -301,7 +302,7 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append_dedup" diff --git a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json index a0737190a5b3..64a18d6a5f50 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json +++ b/airbyte-integrations/connectors/source-junip-reviews/integration_tests/invalid_config.json @@ -1,3 +1,3 @@ { - "junip_store_key": "", + "junip_store_key": "234567854454", } diff --git a/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json index c83aafd1cb2e..4ec5a4458a38 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json +++ b/airbyte-integrations/connectors/source-junip-reviews/sample_files/configured_catalog.json @@ -55,7 +55,8 @@ } }, "supported_sync_modes": [ - "full_refresh" + "full_refresh", + "incremental" ] }, "sync_mode": "full_refresh", @@ -104,7 +105,8 @@ } }, "supported_sync_modes": [ - "full_refresh" + "full_refresh", + "incremental" ] }, "sync_mode": "full_refresh", @@ -188,7 +190,7 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append_dedup" @@ -237,7 +239,7 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append_dedup" @@ -301,7 +303,7 @@ } } }, - "supported_sync_modes": ["full_refresh"] + "supported_sync_modes": ["full_refresh", "incremental"] }, "sync_mode": "full_refresh", "destination_sync_mode": "append_dedup" diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py index f5609a3a7ab6..066657b52169 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/source.py @@ -23,13 +23,10 @@ def check_connection(self, logger, config) -> Tuple[bool, any]: :param logger: logger object :return Tuple[bool, any]: (True, None) if the input config can be used to connect to the API successfully, (False, error) otherwise. """ - url = "https://api.juniphq.com/v1/product_overviews" - headers = { - 'Junip-Store-Key': config["junip_store_key"] - } + junip_store_key = config["junip_store_key"] try: - response = requests.request("GET", url, headers=headers) + Products(junip_store_key=junip_store_key) connection = True, None except Exception as e: connection = False, e diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py index b494aa3ac2ad..fb69d3ac3b31 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py @@ -2,6 +2,7 @@ from abc import ABC from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Tuple +import datetime import requests from airbyte_cdk.sources.streams.http import HttpStream from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator @@ -55,7 +56,7 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp """ :return an iterable containing each record in the response """ - return [response.json()] + return {} def path( self, @@ -68,31 +69,132 @@ def path( return self.name -class Products(JunipReviewsStream): +class IncrementalJunipReviewsStream(JunipReviewsStream, ABC): """ + Baseclass for all incremental streams of Bold source. Override cursor field property in order to use + incremental stream. + """ + state_checkpoint_interval = None + + @property + def cursor_field(self) -> str: + """ + Override to return the cursor field used by this stream e.g: an API entity might always use created_at as the cursor field. This is + usually id or date based. This field's presence tells the framework this in an incremental stream. Required for incremental. + :return str: The name of the cursor field. + """ + pass + + def _convert_date_to_timestamp(self, date: datetime): + return datetime.datetime.strptime(date, "%Y-%m-%dT%H:%M:%S.%fZ") + + def get_updated_state(self, current_stream_state: MutableMapping[str, Any], latest_record: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Override to determine the latest state after reading the latest record. This typically compared the cursor_field from the latest record and + the current state and picks the 'most' recent cursor. This is how a stream's state is determined. Required for incremental. + """ + base_date = ( + datetime.datetime.combine( + datetime.date.fromtimestamp(0), + datetime.datetime.min.time() + ).strftime("%Y-%m-%dT%H:%M:%S.%fZ") + ) + state_dt = self._convert_date_to_timestamp(current_stream_state.get(self.cursor_field, base_date)) + latest_record = self._convert_date_to_timestamp(latest_record.get(self.cursor_field, base_date)) + + return {self.cursor_field: max(latest_record, state_dt)} + + +class Products(IncrementalJunipReviewsStream): + """ + Ref: https://junip.co/docs/api/ + url: "self.base_url/self.name" """ + cursor_field = "created_at" + primary_key = "id" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + Ref: https://junip.co/docs/api/ + """ + json_response = response.json() + for product in json_response.get("products"): + yield product -class ProductOverviews(JunipReviewsStream): + +class ProductOverviews(IncrementalJunipReviewsStream): """ + Ref: https://junip.co/docs/api/ + url: "self.base_url/self.name" """ + cursor_field = "created_at" + primary_key = "id" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + Ref: https://junip.co/docs/api/ + """ + + json_response = response.json() + for product_overview in json_response.get("product_overviews"): + yield product_overview -class ProductReviews(JunipReviewsStream): +class ProductReviews(IncrementalJunipReviewsStream): """ - url: "self.base_url/self.name" - """ + Ref: https://junip.co/docs/api/ + url: "self.base_url/self.name" + """ + cursor_field = "created_at" + primary_key = "id" -class Stores(JunipReviewsStream): + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + Ref: https://junip.co/docs/api/ + """ + + json_response = response.json() + for product_reviews in json_response.get("product_reviews"): + yield product_reviews + + +class Stores(IncrementalJunipReviewsStream): """ + Ref: https://junip.co/docs/api/ + url: "self.base_url/self.name" """ + cursor_field = "created_at" + primary_key = "id" + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + Ref: https://junip.co/docs/api/ + """ -class StoreReviews(JunipReviewsStream): + json_response = response.json() + for store in json_response.get("stores"): + yield store + + +class StoreReviews(IncrementalJunipReviewsStream): """ + Ref: https://junip.co/docs/api/ + url: "self.base_url/self.name" """ + cursor_field = "created_at" + primary_key = "id" + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + """ + Ref: https://junip.co/docs/api/ + """ + + json_response = response.json() + for store_reviews in json_response.get("store_reviews"): + yield store_reviews From 237f8a7a039a23db40720ec3ae89e547468b5603 Mon Sep 17 00:00:00 2001 From: Ghilman Date: Thu, 28 Apr 2022 01:41:44 +0500 Subject: [PATCH 8/8] minor changes in streams.py --- .../source-junip-reviews/source_junip_reviews/streams.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py index fb69d3ac3b31..3f5cce5f15bf 100644 --- a/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py +++ b/airbyte-integrations/connectors/source-junip-reviews/source_junip_reviews/streams.py @@ -18,7 +18,7 @@ def __init__(self, junip_store_key, **kwargs): def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: """ - This method used to do pagination strategy + Implements the pagination approach for stream. Returns: Next page token """ @@ -35,7 +35,7 @@ def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: """ - This method is for add query params in request URL + Adds query params in requested URL. """ params = {}