-
Notifications
You must be signed in to change notification settings - Fork 4.1k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into grubberr/13925-source-github
- Loading branch information
Showing
233 changed files
with
4,018 additions
and
1,547 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,126 @@ | ||
name: Publish OSS Artifacts for Cloud | ||
concurrency: | ||
group: ${{ github.workflow }}-${{ github.ref }} | ||
|
||
on: | ||
workflow_dispatch: | ||
inputs: | ||
oss_ref: | ||
description: "Publish artifacts for the following git ref (if unspecified, uses the latest commit for the current branch):" | ||
required: false | ||
jobs: | ||
find_valid_pat: | ||
name: "Find a PAT with room for actions" | ||
timeout-minutes: 10 | ||
runs-on: ubuntu-latest | ||
outputs: | ||
pat: ${{ steps.variables.outputs.pat }} | ||
steps: | ||
- name: Checkout Airbyte | ||
uses: actions/checkout@v2 | ||
- name: Check PAT rate limits | ||
id: variables | ||
run: | | ||
./tools/bin/find_non_rate_limited_PAT \ | ||
${{ secrets.AIRBYTEIO_PAT }} \ | ||
${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ | ||
${{ secrets.SUPERTOPHER_PAT }} \ | ||
${{ secrets.DAVINCHIA_PAT }} | ||
start-runner: | ||
name: "Start Runner on AWS" | ||
needs: find_valid_pat | ||
timeout-minutes: 10 | ||
runs-on: ubuntu-latest | ||
outputs: | ||
label: ${{ steps.start-ec2-runner.outputs.label }} | ||
ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} | ||
steps: | ||
- name: Checkout Airbyte | ||
uses: actions/checkout@v2 | ||
- name: Start AWS Runner | ||
id: start-ec2-runner | ||
uses: ./.github/actions/start-aws-runner | ||
with: | ||
aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | ||
aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | ||
github-token: ${{ needs.find_valid_pat.outputs.pat }} | ||
|
||
generate-tags: | ||
name: "Generate Tags" | ||
runs-on: ubuntu-latest | ||
outputs: | ||
dev_tag: ${{ steps.set-outputs.outputs.dev_tag }} | ||
master_tag: ${{ steps.set-outputs.outputs.master_tag }} | ||
steps: | ||
- name: Checkout Airbyte | ||
uses: actions/checkout@v2 | ||
with: | ||
ref: ${{ github.event.inputs.oss_ref || github.ref }} | ||
- name: Generate Outputs | ||
id: set-outputs | ||
shell: bash | ||
run: |- | ||
set -x | ||
commit_sha=$(git rev-parse --short HEAD) | ||
# set dev_tag | ||
# AirbyteVersion.java allows versions that have a prefix of 'dev' | ||
echo "::set-output name=dev_tag::dev-${commit_sha}" | ||
# If this commit is on the master branch, also set master_tag | ||
if test 0 -eq $(git merge-base --is-ancestor "${commit_sha}" master); then | ||
echo "::set-output name=master_tag::${commit_sha}" | ||
fi | ||
oss-branch-build: | ||
name: "Build and Push Images from Branch" | ||
needs: | ||
- start-runner | ||
- generate-tags | ||
runs-on: ${{ needs.start-runner.outputs.label }} | ||
steps: | ||
- name: Checkout Airbyte | ||
uses: actions/checkout@v2 | ||
with: | ||
ref: ${{ github.event.inputs.oss_ref || github.ref }} | ||
|
||
- name: Build Branch | ||
uses: ./.github/actions/build-branch | ||
with: | ||
branch_version_tag: ${{ needs.generate-tags.outputs.dev_tag }} | ||
|
||
- name: Login to Docker (on Master) | ||
uses: docker/login-action@v1 | ||
with: | ||
username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
|
||
- name: Push Dev Docker Images | ||
run: | | ||
GIT_REVISION=$(git rev-parse HEAD) | ||
[ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1 | ||
docker buildx create --name oss-buildx --driver docker-container --use | ||
VERSION=${{ needs.generate-tags.outputs.dev_tag }} | ||
VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push | ||
docker buildx rm oss-buildx | ||
shell: bash | ||
|
||
- name: Push Master Docker Images | ||
if: needs.generate-tags.outputs.master_tag != "" | ||
run: | | ||
GIT_REVISION=$(git rev-parse HEAD) | ||
[ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1 | ||
docker buildx create --name oss-buildx --driver docker-container --use | ||
VERSION=${{ needs.generate-tags.outputs.master_tag }} | ||
VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push | ||
docker buildx rm oss-buildx | ||
shell: bash | ||
|
||
- name: Publish Dev Jars | ||
shell: bash | ||
run: VERSION=${{ needs.generate-tags.outputs.dev_tag }} SUB_BUILD=PLATFORM ./gradlew publish | ||
- name: Publish Master Jars | ||
if: needs.generate-tags.outputs.master_tag != "" | ||
shell: bash | ||
run: VERSION=${{ needs.generate-tags.outputs.master_tag }} SUB_BUILD=PLATFORM ./gradlew publish |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
3 changes: 3 additions & 0 deletions
3
airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
# | ||
# Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
# |
50 changes: 50 additions & 0 deletions
50
...k/sources/declarative/requesters/request_options/interpolated_request_options_provider.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,50 @@ | ||
# | ||
# Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
from typing import Any, Mapping, MutableMapping, Optional, Union | ||
|
||
from airbyte_cdk.sources.declarative.requesters.interpolated_request_input_provider import InterpolatedRequestInputProvider | ||
from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider | ||
|
||
|
||
class InterpolatedRequestOptionsProvider(RequestOptionsProvider): | ||
def __init__(self, *, config, request_parameters=None, request_body_data=None, request_body_json=None): | ||
if request_parameters is None: | ||
request_parameters = {} | ||
if request_body_data is None: | ||
request_body_data = "" | ||
if request_body_json is None: | ||
request_body_json = {} | ||
|
||
if request_body_json and request_body_data: | ||
raise ValueError("RequestOptionsProvider should only contain either 'request_body_data' or 'request_body_json' not both") | ||
|
||
self._parameter_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_parameters) | ||
self._body_data_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_body_data) | ||
self._body_json_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_body_json) | ||
|
||
def request_params( | ||
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None | ||
) -> MutableMapping[str, Any]: | ||
interpolated_value = self._parameter_interpolator.request_inputs(stream_state, stream_slice, next_page_token) | ||
if isinstance(interpolated_value, dict): | ||
return interpolated_value | ||
return {} | ||
|
||
def request_body_data( | ||
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None | ||
) -> Optional[Union[Mapping, str]]: | ||
return self._body_data_interpolator.request_inputs(stream_state, stream_slice, next_page_token) | ||
|
||
def request_body_json( | ||
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None | ||
) -> Optional[Mapping]: | ||
return self._body_json_interpolator.request_inputs(stream_state, stream_slice, next_page_token) | ||
|
||
def request_kwargs( | ||
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None | ||
) -> Mapping[str, Any]: | ||
# todo: there are a few integrations that override the request_kwargs() method, but the use case for why kwargs over existing | ||
# constructs is a little unclear. We may revisit this, but for now lets leave it out of the DSL | ||
return {} |
32 changes: 32 additions & 0 deletions
32
...on/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
# | ||
# Copyright (c) 2022 Airbyte, Inc., all rights reserved. | ||
# | ||
|
||
from abc import ABC, abstractmethod | ||
from typing import Any, Mapping, MutableMapping, Optional, Union | ||
|
||
|
||
class RequestOptionsProvider(ABC): | ||
@abstractmethod | ||
def request_params( | ||
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None | ||
) -> MutableMapping[str, Any]: | ||
pass | ||
|
||
@abstractmethod | ||
def request_body_data( | ||
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None | ||
) -> Optional[Union[Mapping, str]]: | ||
pass | ||
|
||
@abstractmethod | ||
def request_body_json( | ||
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None | ||
) -> Optional[Mapping]: | ||
pass | ||
|
||
@abstractmethod | ||
def request_kwargs( | ||
self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None | ||
) -> Mapping[str, Any]: | ||
pass |
3 changes: 0 additions & 3 deletions
3
airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/__init__.py
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.