-
Notifications
You must be signed in to change notification settings - Fork 4.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
🎉 New Source: Retently #6966
🎉 New Source: Retently #6966
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
* | ||
!Dockerfile | ||
!Dockerfile.test | ||
!main.py | ||
!source_retently | ||
!setup.py | ||
!secrets |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
.venv/ | ||
__pycache__ | ||
*.egg-info |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
FROM python:3.7-slim as base | ||
|
||
# build and load all requirements | ||
FROM base as builder | ||
WORKDIR /airbyte/integration_code | ||
|
||
# upgrade pip to the latest version | ||
# RUN apk --no-cache upgrade \ | ||
# && pip install --upgrade pip \ | ||
# && apk --no-cache add tzdata | ||
|
||
|
||
COPY setup.py ./ | ||
# install necessary packages to a temporary folder | ||
RUN pip install --prefix=/install . | ||
|
||
# build a clean environment | ||
FROM base | ||
WORKDIR /airbyte/integration_code | ||
|
||
# copy all loaded and built libraries to a pure basic image | ||
COPY --from=builder /install /usr/local | ||
# add default timezone settings | ||
COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime | ||
RUN echo "Etc/UTC" > /etc/timezone | ||
|
||
# bash is installed for more convenient debugging. | ||
# RUN apk --no-cache add bash | ||
|
||
# copy payload code only | ||
COPY main.py ./ | ||
COPY source_retently ./source_retently | ||
|
||
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" | ||
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] | ||
|
||
LABEL io.airbyte.version=0.1.0 | ||
LABEL io.airbyte.name=airbyte/source_retently | ||
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
## [Quickrun](https://docs.airbyte.io/connector-development/tutorials/cdk-speedrun) | ||
|
||
1. `python -m venv .venv` | ||
2. `source .venv/bin/activate` | ||
3. `pip install -r requirements.txt` (Alter path in requirements.txt) | ||
4. `python main.py check --config sample_files/config.json` should report SUCCESS | ||
5. `python main.py check --config sample_files/invalid_config.json` should report FAILED | ||
6. `python main.py discover --config sample_files/config.json` presents schema | ||
7. `python main.py read --config sample_files/config.json --catalog sample_files/configured_catalog.json` runs source | ||
8. `docker build . -t <repo>/source-retently:<n>` builds docker image | ||
10. `docker push <repo>/source-retently:<n>` .. and push | ||
11. Add connector from http://localhost:8000/settings/source | ||
12. `pip install .[tests]` | ||
13. `python -m pytest unit_tests` | ||
14. `python -m pytest integration_tests` | ||
15. `python -m pytest integration_tests -p integration_tests.acceptance` | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Please keep the README as same from other connectors. |
||
|
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
@@ -0,0 +1,15 @@ | ||||||
connector_image: airbyte/source-retently | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||
tests: | ||||||
spec: | ||||||
- spec_path: "source_retently/spec.json" | ||||||
connection: | ||||||
- config_path: "secrets/config.json" | ||||||
status: "succeed" | ||||||
- config_path: "integration_tests/invalid_config.json" | ||||||
status: "exception" | ||||||
discovery: | ||||||
- config_path: "secrets/config.json" | ||||||
basic_read: | ||||||
- config_path: "secrets/config.json" | ||||||
configured_catalog_path: "integration_tests/configured_catalog.json" | ||||||
empty_streams: ["reports"] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,16 @@ | ||
#!/usr/bin/env sh | ||
|
||
# Build latest connector image | ||
docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) | ||
|
||
# Pull latest acctest image | ||
docker pull airbyte/source-acceptance-test:latest | ||
|
||
# Run | ||
docker run --rm -it \ | ||
-v /var/run/docker.sock:/var/run/docker.sock \ | ||
-v /tmp:/tmp \ | ||
-v $(pwd):/test_input \ | ||
airbyte/source-acceptance-test \ | ||
--acceptance-test-config /test_input | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
plugins { | ||
id 'airbyte-python' | ||
id 'airbyte-docker' | ||
id 'airbyte-source-acceptance-test' | ||
} | ||
|
||
airbytePython { | ||
moduleDirectory 'source_retently' | ||
} | ||
|
||
dependencies { | ||
implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) | ||
implementation files(project(':airbyte-integrations:bases:base-python').airbyteDocker.outputs) | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{ | ||
"todo-stream-name": { | ||
"todo-field-name": "todo-abnormal-value" | ||
} | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,36 @@ | ||
# | ||
# MIT License | ||
# | ||
# Copyright (c) 2020 Airbyte | ||
# | ||
# Permission is hereby granted, free of charge, to any person obtaining a copy | ||
# of this software and associated documentation files (the "Software"), to deal | ||
# in the Software without restriction, including without limitation the rights | ||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
# copies of the Software, and to permit persons to whom the Software is | ||
# furnished to do so, subject to the following conditions: | ||
# | ||
# The above copyright notice and this permission notice shall be included in all | ||
# copies or substantial portions of the Software. | ||
# | ||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
# SOFTWARE. | ||
# | ||
|
||
|
||
import pytest | ||
|
||
pytest_plugins = ("source_acceptance_test.plugin",) | ||
|
||
|
||
@pytest.fixture(scope="session", autouse=True) | ||
def connector_setup(): | ||
""" This fixture is a placeholder for external resources that acceptance test might require.""" | ||
# TODO: setup test dependencies if needed. otherwise remove the TODO comments | ||
yield | ||
# TODO: clean up test dependencies |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
{ | ||
"streams": [ | ||
{ | ||
"stream": { | ||
"name": "customers", | ||
"json_schema": { | ||
"$schema": "http://json-schema.org/draft-04/schema#", | ||
"type": "object" | ||
}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite" | ||
}, | ||
{ | ||
"stream": { | ||
"name": "companies", | ||
"json_schema": { | ||
"$schema": "http://json-schema.org/draft-04/schema#", | ||
"type": "object" | ||
}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite" | ||
}, | ||
{ | ||
"stream": { | ||
"name": "reports", | ||
"json_schema": { | ||
"$schema": "http://json-schema.org/draft-04/schema#", | ||
"type": "object" | ||
}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite" | ||
} | ||
] | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,4 @@ | ||
{ | ||
"username": "foo", | ||
"password": "bar" | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"api_key": "b89fe9a7-d33c-43ab-a215-373c27593a69" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. you dont need to let your api key here, change to only an example. |
||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,33 @@ | ||
# | ||
# MIT License | ||
# | ||
# Copyright (c) 2020 Airbyte | ||
# | ||
# Permission is hereby granted, free of charge, to any person obtaining a copy | ||
# of this software and associated documentation files (the "Software"), to deal | ||
# in the Software without restriction, including without limitation the rights | ||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
# copies of the Software, and to permit persons to whom the Software is | ||
# furnished to do so, subject to the following conditions: | ||
# | ||
# The above copyright notice and this permission notice shall be included in all | ||
# copies or substantial portions of the Software. | ||
# | ||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
# SOFTWARE. | ||
# | ||
|
||
|
||
import sys | ||
|
||
from airbyte_cdk.entrypoint import launch | ||
from source_retently import SourceRetently | ||
|
||
if __name__ == "__main__": | ||
source = SourceRetently() | ||
launch(source, sys.argv[1:]) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,2 @@ | ||
-e ../../bases/source-acceptance-test | ||
-e . |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
{ | ||
"api_key": "b89fe9a7-d33c-43ab-a215-373c27593a69" | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
{ | ||
"streams": [ | ||
{ | ||
"stream": { | ||
"name": "customers", | ||
"json_schema": { | ||
"$schema": "http://json-schema.org/draft-04/schema#", | ||
"type": "object" | ||
}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite" | ||
}, | ||
{ | ||
"stream": { | ||
"name": "companies", | ||
"json_schema": { | ||
"$schema": "http://json-schema.org/draft-04/schema#", | ||
"type": "object" | ||
}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite" | ||
}, | ||
{ | ||
"stream": { | ||
"name": "reports", | ||
"json_schema": { | ||
"$schema": "http://json-schema.org/draft-04/schema#", | ||
"type": "object" | ||
}, | ||
"supported_sync_modes": ["full_refresh"] | ||
}, | ||
"sync_mode": "full_refresh", | ||
"destination_sync_mode": "overwrite" | ||
} | ||
] | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,5 @@ | ||
{ | ||
"username": "foo", | ||
"password": "bar" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. the config only use |
||
} | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
# | ||
# MIT License | ||
# | ||
# Copyright (c) 2020 Airbyte | ||
# | ||
# Permission is hereby granted, free of charge, to any person obtaining a copy | ||
# of this software and associated documentation files (the "Software"), to deal | ||
# in the Software without restriction, including without limitation the rights | ||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
# copies of the Software, and to permit persons to whom the Software is | ||
# furnished to do so, subject to the following conditions: | ||
# | ||
# The above copyright notice and this permission notice shall be included in all | ||
# copies or substantial portions of the Software. | ||
# | ||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
# SOFTWARE. | ||
# | ||
|
||
|
||
from setuptools import find_packages, setup | ||
|
||
MAIN_REQUIREMENTS = [ | ||
"airbyte-cdk", | ||
] | ||
|
||
TEST_REQUIREMENTS = [ | ||
"pytest~=6.1", | ||
"pytest-mock~=3.6.1", | ||
"source-acceptance-test", | ||
] | ||
|
||
setup( | ||
name="source_retently", | ||
description="Source implementation for Retently", | ||
author="Airbyte", | ||
author_email="contact@airbyte.io", | ||
packages=find_packages(), | ||
install_requires=MAIN_REQUIREMENTS, | ||
package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, | ||
extras_require={ | ||
"tests": TEST_REQUIREMENTS, | ||
}, | ||
) |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,28 @@ | ||
# | ||
# MIT License | ||
# | ||
# Copyright (c) 2020 Airbyte | ||
# | ||
# Permission is hereby granted, free of charge, to any person obtaining a copy | ||
# of this software and associated documentation files (the "Software"), to deal | ||
# in the Software without restriction, including without limitation the rights | ||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell | ||
# copies of the Software, and to permit persons to whom the Software is | ||
# furnished to do so, subject to the following conditions: | ||
# | ||
# The above copyright notice and this permission notice shall be included in all | ||
# copies or substantial portions of the Software. | ||
# | ||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR | ||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, | ||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE | ||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER | ||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, | ||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | ||
# SOFTWARE. | ||
# | ||
|
||
|
||
from .source import SourceRetently | ||
|
||
__all__ = ["SourceRetently"] |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,14 @@ | ||
{ | ||
"type": "object", | ||
"properties": { | ||
|
||
"id": { "type": "string" }, | ||
"createdDate": { "type": "string" }, | ||
"tags": { "type": "array", "items": { "type": "string" } }, | ||
"cxMetrics": { "type": "object" }, | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. cxMetrics is not possible to create the object or is dynamic? |
||
"industryName": { "type": "string" }, | ||
"domain": { "type": "string" }, | ||
"companyName": { "type": "string" } | ||
|
||
} | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.