Merge branch 'master' into grubberr/13925-source-github

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.39.21-alpha
+current_version = 0.39.23-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.39.21-alpha
+VERSION=0.39.23-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data

.github/actions/start-aws-runner/action.yml

@@ -41,14 +41,17 @@ runs:
         aws-region: us-east-2
     - name: Start EC2 runner
       id: start-ec2-runner
-      uses: supertopher/ec2-github-runner@base64v1.0.10
+      uses: airbytehq/ec2-github-runner@base64v1.1.0
       with:
         mode: start
         github-token: ${{ inputs.github-token }}
         ec2-image-id: ${{ inputs.ec2-image-id }}
         ec2-instance-type: ${{ inputs.ec2-instance-type }}
         subnet-id: ${{ inputs.subnet-id }}
         security-group-id: ${{ inputs.security-group-id }}
+        # this adds a label to group any EC2 runners spun up within the same action run
+        # this enables creating a pool of runners to run multiple/matrix jobs on in parallel
+        label: runner-pool-${{ github.run_id }}
         aws-resource-tags: >
           [
             {"Key": "BuildType", "Value": "oss"},

.github/workflows/publish-oss-for-cloud.yml

@@ -0,0 +1,126 @@
+name: Publish OSS Artifacts for Cloud
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      oss_ref:
+        description: "Publish artifacts for the following git ref (if unspecified, uses the latest commit for the current branch):"
+        required: false
+jobs:
+  find_valid_pat:
+    name: "Find a PAT with room for actions"
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      pat: ${{ steps.variables.outputs.pat }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Check PAT rate limits
+        id: variables
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.AIRBYTEIO_PAT }} \
+            ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
+            ${{ secrets.SUPERTOPHER_PAT }} \
+            ${{ secrets.DAVINCHIA_PAT }}
+  start-runner:
+    name: "Start Runner on AWS"
+    needs: find_valid_pat
+    timeout-minutes: 10
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Start AWS Runner
+        id: start-ec2-runner
+        uses: ./.github/actions/start-aws-runner
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+
+  generate-tags:
+    name: "Generate Tags"
+    runs-on: ubuntu-latest
+    outputs:
+      dev_tag: ${{ steps.set-outputs.outputs.dev_tag }}
+      master_tag: ${{ steps.set-outputs.outputs.master_tag }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.inputs.oss_ref || github.ref }}
+      - name: Generate Outputs
+        id: set-outputs
+        shell: bash
+        run: |-
+          set -x
+          
+          commit_sha=$(git rev-parse --short HEAD)
+          
+          # set dev_tag
+          # AirbyteVersion.java allows versions that have a prefix of 'dev'
+          echo "::set-output name=dev_tag::dev-${commit_sha}"
+          
+          # If this commit is on the master branch, also set master_tag
+          if test 0 -eq $(git merge-base --is-ancestor "${commit_sha}" master); then
+            echo "::set-output name=master_tag::${commit_sha}"
+          fi
+
+  oss-branch-build:
+    name: "Build and Push Images from Branch"
+    needs:
+      - start-runner
+      - generate-tags
+    runs-on: ${{ needs.start-runner.outputs.label }}
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+        with:
+          ref: ${{ github.event.inputs.oss_ref || github.ref }}
+
+      - name: Build Branch
+        uses: ./.github/actions/build-branch
+        with:
+          branch_version_tag: ${{ needs.generate-tags.outputs.dev_tag }}
+
+      - name: Login to Docker (on Master)
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+
+      - name: Push Dev Docker Images
+        run: |
+          GIT_REVISION=$(git rev-parse HEAD)
+          [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
+          docker buildx create --name oss-buildx --driver docker-container --use
+          VERSION=${{ needs.generate-tags.outputs.dev_tag }}
+          VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push
+          docker buildx rm oss-buildx
+        shell: bash
+
+      - name: Push Master Docker Images
+        if: needs.generate-tags.outputs.master_tag != ""
+        run: |
+          GIT_REVISION=$(git rev-parse HEAD)
+          [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1
+          docker buildx create --name oss-buildx --driver docker-container --use
+          VERSION=${{ needs.generate-tags.outputs.master_tag }}
+          VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push
+          docker buildx rm oss-buildx
+        shell: bash
+
+      - name: Publish Dev Jars
+        shell: bash
+        run: VERSION=${{ needs.generate-tags.outputs.dev_tag }} SUB_BUILD=PLATFORM ./gradlew publish
+      - name: Publish Master Jars
+        if: needs.generate-tags.outputs.master_tag != ""
+        shell: bash
+        run: VERSION=${{ needs.generate-tags.outputs.master_tag }} SUB_BUILD=PLATFORM ./gradlew publish

.github/workflows/terminate-zombie-build-instances.yml

@@ -34,9 +34,12 @@ jobs:
 
           # See https://docs.aws.amazon.com/cli/latest/reference/ec2/terminate-instances.html for terminate command.
           echo $to_terminate |  jq '.[]  | .InstanceId' | xargs --no-run-if-empty --max-args=1 aws ec2 terminate-instances --instance-ids
-
+  terminate-github-instances:
+    runs-on: ubuntu-latest
     steps:
-      - shell: List and Terminate GH actions in status 'offline'
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: List and Terminate GH actions in status 'offline'
         env:
           GITHUB_PAT: ${{ secrets.OCTAVIA_PAT }}
         run: ./tools/bin/gh_action_zombie_killer

airbyte-bootloader/Dockerfile

@@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.1
 ARG JDK_IMAGE=openjdk:${JDK_VERSION}-slim
 FROM ${JDK_IMAGE}
 
-ARG VERSION=0.39.21-alpha
+ARG VERSION=0.39.23-alpha
 
 ENV APPLICATION airbyte-bootloader
 ENV VERSION ${VERSION}

airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py

@@ -10,10 +10,10 @@
     InterpolatedRequestHeaderProvider,
 )
 from airbyte_cdk.sources.declarative.requesters.request_headers.request_header_provider import RequestHeaderProvider
-from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import (
-    InterpolatedRequestParameterProvider,
+from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import (
+    InterpolatedRequestOptionsProvider,
 )
-from airbyte_cdk.sources.declarative.requesters.request_params.request_parameters_provider import RequestParameterProvider
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider
 from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester
 from airbyte_cdk.sources.declarative.requesters.retriers.retrier import Retrier
 from airbyte_cdk.sources.declarative.types import Config
@@ -28,14 +28,16 @@ def __init__(
         url_base: [str, InterpolatedString],
         path: [str, InterpolatedString],
         http_method: Union[str, HttpMethod],
-        request_parameters_provider: RequestParameterProvider = None,
+        request_options_provider: RequestOptionsProvider = None,
         request_headers_provider: RequestHeaderProvider = None,
         authenticator: HttpAuthenticator,
         retrier: Retrier,
         config: Config,
     ):
-        if request_parameters_provider is None:
-            request_parameters_provider = InterpolatedRequestParameterProvider(config=config, request_headers={})
+        if request_options_provider is None:
+            request_options_provider = InterpolatedRequestOptionsProvider(
+                config=config, request_parameters={}, request_body_data="", request_body_json={}
+            )
         if request_headers_provider is None:
             request_headers_provider = InterpolatedRequestHeaderProvider(config=config, request_headers={})
         self._name = name
@@ -49,15 +51,15 @@ def __init__(
         if type(http_method) == str:
             http_method = HttpMethod[http_method]
         self._method = http_method
-        self._request_parameters_provider = request_parameters_provider
+        self._request_options_provider = request_options_provider
         self._request_headers_provider = request_headers_provider
         self._retrier = retrier
         self._config = config
 
     def request_params(
         self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
     ) -> MutableMapping[str, Any]:
-        return self._request_parameters_provider.request_params(stream_state, stream_slice, next_page_token)
+        return self._request_options_provider.request_params(stream_state, stream_slice, next_page_token)
 
     def get_authenticator(self):
         return self._authenticator
@@ -100,20 +102,17 @@ def request_headers(
     def request_body_data(
         self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
     ) -> Optional[Union[Mapping, str]]:
-        # FIXME: this should be declarative
-        return dict()
+        return self._request_options_provider.request_body_data(stream_state, stream_slice, next_page_token)
 
     def request_body_json(
         self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
     ) -> Optional[Mapping]:
-        # FIXME: this should be declarative
-        return dict()
+        return self._request_options_provider.request_body_json(stream_state, stream_slice, next_page_token)
 
     def request_kwargs(
         self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
     ) -> Mapping[str, Any]:
-        # FIXME: this should be declarative
-        return dict()
+        return self._request_options_provider.request_kwargs(stream_state, stream_slice, next_page_token)
 
     @property
     def cache_filename(self) -> str:

airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/interpolated_request_input_provider.py

@@ -2,27 +2,35 @@
 # Copyright (c) 2022 Airbyte, Inc., all rights reserved.
 #
 
-from typing import Any, Mapping, MutableMapping
+from typing import Any, Mapping, Union
 
 from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping
+from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString
 from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation
 
 
 class InterpolatedRequestInputProvider:
     """
-    Helper class that generically performs string interpolation on the provided dictionary input
+    Helper class that generically performs string interpolation on the provided dictionary or string input
     """
 
     def __init__(self, *, config, request_inputs=None):
+        self._config = config
+
         if request_inputs is None:
             request_inputs = {}
-        self._interpolator = InterpolatedMapping(request_inputs, JinjaInterpolation())
-        self._config = config
+        if isinstance(request_inputs, str):
+            self._interpolator = InterpolatedString(request_inputs, "")
+        else:
+            self._interpolator = InterpolatedMapping(request_inputs, JinjaInterpolation())
 
     def request_inputs(
         self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
-    ) -> MutableMapping[str, Any]:
+    ) -> Union[Mapping, str]:
         kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token}
-        interpolated_values = self._interpolator.eval(self._config, **kwargs)  # dig into this function a little more
-        non_null_tokens = {k: v for k, v in interpolated_values.items() if v}
-        return non_null_tokens
+        interpolated_value = self._interpolator.eval(self._config, **kwargs)
+
+        if isinstance(interpolated_value, dict):
+            non_null_tokens = {k: v for k, v in interpolated_value.items() if v}
+            return non_null_tokens
+        return interpolated_value

airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py

@@ -0,0 +1,3 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#

airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py

@@ -0,0 +1,50 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from typing import Any, Mapping, MutableMapping, Optional, Union
+
+from airbyte_cdk.sources.declarative.requesters.interpolated_request_input_provider import InterpolatedRequestInputProvider
+from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider
+
+
+class InterpolatedRequestOptionsProvider(RequestOptionsProvider):
+    def __init__(self, *, config, request_parameters=None, request_body_data=None, request_body_json=None):
+        if request_parameters is None:
+            request_parameters = {}
+        if request_body_data is None:
+            request_body_data = ""
+        if request_body_json is None:
+            request_body_json = {}
+
+        if request_body_json and request_body_data:
+            raise ValueError("RequestOptionsProvider should only contain either 'request_body_data' or 'request_body_json' not both")
+
+        self._parameter_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_parameters)
+        self._body_data_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_body_data)
+        self._body_json_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_body_json)
+
+    def request_params(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> MutableMapping[str, Any]:
+        interpolated_value = self._parameter_interpolator.request_inputs(stream_state, stream_slice, next_page_token)
+        if isinstance(interpolated_value, dict):
+            return interpolated_value
+        return {}
+
+    def request_body_data(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Optional[Union[Mapping, str]]:
+        return self._body_data_interpolator.request_inputs(stream_state, stream_slice, next_page_token)
+
+    def request_body_json(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Optional[Mapping]:
+        return self._body_json_interpolator.request_inputs(stream_state, stream_slice, next_page_token)
+
+    def request_kwargs(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Mapping[str, Any]:
+        # todo: there are a few integrations that override the request_kwargs() method, but the use case for why kwargs over existing
+        #  constructs is a little unclear. We may revisit this, but for now lets leave it out of the DSL
+        return {}

airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py

@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2022 Airbyte, Inc., all rights reserved.
+#
+
+from abc import ABC, abstractmethod
+from typing import Any, Mapping, MutableMapping, Optional, Union
+
+
+class RequestOptionsProvider(ABC):
+    @abstractmethod
+    def request_params(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> MutableMapping[str, Any]:
+        pass
+
+    @abstractmethod
+    def request_body_data(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Optional[Union[Mapping, str]]:
+        pass
+
+    @abstractmethod
+    def request_body_json(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Optional[Mapping]:
+        pass
+
+    @abstractmethod
+    def request_kwargs(
+        self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None
+    ) -> Mapping[str, Any]:
+        pass