diff --git a/.bumpversion.cfg b/.bumpversion.cfg index d6672d438e2b..b96c35229234 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.39.21-alpha +current_version = 0.39.23-alpha commit = False tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-[a-z]+)? diff --git a/.env b/.env index 10840d35c43e..48f3379d6ea4 100644 --- a/.env +++ b/.env @@ -10,7 +10,7 @@ ### SHARED ### -VERSION=0.39.21-alpha +VERSION=0.39.23-alpha # When using the airbyte-db via default docker image CONFIG_ROOT=/data diff --git a/.github/actions/start-aws-runner/action.yml b/.github/actions/start-aws-runner/action.yml index 7f268783fc60..c3b94df610b6 100644 --- a/.github/actions/start-aws-runner/action.yml +++ b/.github/actions/start-aws-runner/action.yml @@ -41,7 +41,7 @@ runs: aws-region: us-east-2 - name: Start EC2 runner id: start-ec2-runner - uses: supertopher/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: start github-token: ${{ inputs.github-token }} @@ -49,6 +49,9 @@ runs: ec2-instance-type: ${{ inputs.ec2-instance-type }} subnet-id: ${{ inputs.subnet-id }} security-group-id: ${{ inputs.security-group-id }} + # this adds a label to group any EC2 runners spun up within the same action run + # this enables creating a pool of runners to run multiple/matrix jobs on in parallel + label: runner-pool-${{ github.run_id }} aws-resource-tags: > [ {"Key": "BuildType", "Value": "oss"}, diff --git a/.github/workflows/publish-command.yml b/.github/workflows/publish-command.yml index 60bf4453586a..a0814d20cef5 100644 --- a/.github/workflows/publish-command.yml +++ b/.github/workflows/publish-command.yml @@ -24,6 +24,10 @@ on: description: "after publishing, the workflow will automatically bump the connector version in definitions and generate seed spec" required: true default: "true" + parallel: + description: "Switching this to true will spin up 5 build agents instead of 1 and allow multi connector publishes to run in parallel" + required: true + default: "false" jobs: find_valid_pat: @@ -45,8 +49,8 @@ jobs: ${{ secrets.DAVINCHIA_PAT }} ## Gradle Build # In case of self-hosted EC2 errors, remove this block. - start-publish-image-runner: - name: Start Build EC2 Runner + start-publish-image-runner-0: + name: Start Build EC2 Runner 0 runs-on: ubuntu-latest needs: find_valid_pat outputs: @@ -65,19 +69,154 @@ jobs: aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} github-token: ${{ needs.find_valid_pat.outputs.pat }} - publish-image: - timeout-minutes: 240 - needs: start-publish-image-runner - runs-on: ${{ needs.start-publish-image-runner.outputs.label }} - environment: more-secrets + label: ${{ github.run_id }}-publisher + start-publish-image-runner-1: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 1 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher + start-publish-image-runner-2: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 2 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} steps: - - name: Link comment to workflow run - if: github.event.inputs.comment-id + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher + start-publish-image-runner-3: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 3 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher + start-publish-image-runner-4: + if: github.event.inputs.parallel == 'true' && success() + name: Start Build EC2 Runner 4 + runs-on: ubuntu-latest + needs: find_valid_pat + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + repository: ${{ github.event.inputs.repo }} + ref: ${{ github.event.inputs.gitref }} + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ github.run_id }}-publisher + preprocess-matrix: + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + outputs: + connectorjson: ${{ steps.preprocess.outputs.connectorjson }} + steps: + # given a string input of a single connector or comma separated list of connectors e.g. connector1, connector2 + # this step builds an array, by removing whitespace, add in quotation marks around connectors and braces [ ] at the start and end + # finally, it sets it as output from this job so we can use this array of connectors as our matrix strategy for publishing + - id: preprocess + run: | + start="[\"" + replace="\",\"" + end="\"]" + stripped_connector="$(echo "${{ github.event.inputs.connector }}" | tr -d ' ')" + middle=${stripped_connector//,/$replace} + full="$start$middle$end" + echo "::set-output name=connectorjson::$full" + write-initial-output-to-comment: + name: Set up git comment + if: github.event.inputs.comment-id + needs: start-publish-image-runner-0 + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - name: Print start message + if: github.event.inputs.comment-id && success() + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + > :clock2: Publishing the following connectors:
${{ github.event.inputs.connector }}
Running tests before publishing: **${{ github.event.inputs.run-tests }}**
https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} + - name: Create table header uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :clock2: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +
+ + | Connector | Published | Definitions generated | + - name: Create table separator + uses: peter-evans/create-or-update-comment@v1 + with: + comment-id: ${{ github.event.inputs.comment-id }} + body: | + | --- | --- | --- | + publish-image: + timeout-minutes: 240 + needs: + - start-publish-image-runner-0 + - preprocess-matrix + - write-initial-output-to-comment + strategy: + max-parallel: 5 + fail-fast: false + matrix: + connector: ${{ fromJSON(needs.preprocess-matrix.outputs.connectorjson) }} + runs-on: runner-pool-${{ github.run_id }} + environment: more-secrets + steps: - name: Set up Cloud SDK uses: google-github-actions/setup-gcloud@v0 with: @@ -89,9 +228,9 @@ jobs: with: regex_pattern: "^(connectors|bases)/[a-zA-Z0-9-_]+$" regex_flags: "i" # required to be set for this plugin - search_string: ${{ github.event.inputs.connector }} + search_string: ${{ matrix.connector }} - name: Validate input workflow format - if: steps.regex.outputs.first_match != github.event.inputs.connector + if: steps.regex.outputs.first_match != matrix.connector run: echo "The connector provided has an invalid format!" && exit 1 - name: Checkout Airbyte uses: actions/checkout@v2 @@ -110,6 +249,7 @@ jobs: - name: Install Pyenv and Tox run: | python3 -m pip install --quiet virtualenv==16.7.9 --user + rm -r venv || echo "no pre-existing venv" python3 -m virtualenv venv source venv/bin/activate pip install --quiet tox==3.24.4 @@ -126,35 +266,35 @@ jobs: source venv/bin/activate tox -r -c ./tools/tox_ci.ini pip install --quiet -e ./tools/ci_* - - name: Write Integration Test Credentials for ${{ github.event.inputs.connector }} + - name: Write Integration Test Credentials for ${{ matrix.connector }} run: | source venv/bin/activate - ci_credentials ${{ github.event.inputs.connector }} + ci_credentials ${{ matrix.connector }} env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - name: Set Name and Version Environment Vars - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | source tools/lib/lib.sh - DOCKERFILE=airbyte-integrations/${{ github.event.inputs.connector }}/Dockerfile - echo "IMAGE_NAME=$(echo ${{ github.event.inputs.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV + DOCKERFILE=airbyte-integrations/${{ matrix.connector }}/Dockerfile + echo "IMAGE_NAME=$(echo ${{ matrix.connector }} | cut -d"/" -f2)" >> $GITHUB_ENV echo "IMAGE_VERSION=$(_get_docker_image_version ${DOCKERFILE})" >> $GITHUB_ENV - name: Prepare Sentry - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | - curl -sL https://sentry.io/get-cli/ | bash + curl -sL https://sentry.io/get-cli/ | bash || echo "sentry cli already installed" - name: Create Sentry Release - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | sentry-cli releases set-commits "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" --auto --ignore-missing env: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} SENTRY_ORG: airbyte-5j SENTRY_PROJECT: airbyte-connectors - - name: Publish ${{ github.event.inputs.connector }} + - name: Publish ${{ matrix.connector }} run: | echo "$SPEC_CACHE_SERVICE_ACCOUNT_KEY" > spec_cache_key_file.json && docker login -u ${DOCKER_HUB_USERNAME} -p ${DOCKER_HUB_PASSWORD} - ./tools/integrations/manage.sh publish airbyte-integrations/${{ github.event.inputs.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache + ./tools/integrations/manage.sh publish airbyte-integrations/${{ matrix.connector }} ${{ github.event.inputs.run-tests }} --publish_spec_to_cache id: publish env: DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} @@ -162,27 +302,13 @@ jobs: # Oracle expects this variable to be set. Although usually present, this is not set by default on Github virtual runners. TZ: UTC - name: Finalize Sentry release - if: startsWith(github.event.inputs.connector, 'connectors') + if: startsWith(matrix.connector, 'connectors') run: | sentry-cli releases finalize "${{ env.IMAGE_NAME }}@${{ env.IMAGE_VERSION }}" env: SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_CONNECTOR_RELEASE_AUTH_TOKEN }} SENTRY_ORG: airbyte-5j SENTRY_PROJECT: airbyte-connectors - - name: Add Published Success Comment - if: github.event.inputs.comment-id && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Successfully published ${{github.event.inputs.connector}} - - name: Add Published Failure Comment - if: github.event.inputs.comment-id && !success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :x: Failed to publish ${{github.event.inputs.connector}} - name: Check if connector in definitions yaml if: github.event.inputs.auto-bump-version == 'true' && success() run: | @@ -220,36 +346,103 @@ jobs: git commit -m "auto-bump connector version" git pull origin ${{ github.event.inputs.gitref }} git push origin ${{ github.event.inputs.gitref }} - - name: Add Version Bump Success Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && success() - uses: peter-evans/create-or-update-comment@v1 - with: - comment-id: ${{ github.event.inputs.comment-id }} - body: | - > :rocket: Auto-bumped version for ${{github.event.inputs.connector}} - - name: Add Version Bump Failure Comment - if: github.event.inputs.comment-id && github.event.inputs.auto-bump-version == 'true' && !success() + id: auto-bump + - name: Process outcomes into emojis + if: ${{ always() && github.event.inputs.comment-id }} + run: | + if [[ ${{ steps.publish.outcome }} = "success" ]]; then + echo "PUBLISH_OUTCOME=:white_check_mark:" >> $GITHUB_ENV + else + echo "PUBLISH_OUTCOME=:x:" >> $GITHUB_ENV + fi + if [[ ${{ steps.auto-bump.outcome }} = "success" ]]; then + echo "AUTO_BUMP_OUTCOME=:white_check_mark:" >> $GITHUB_ENV + else + echo "AUTO_BUMP_OUTCOME=:x:" >> $GITHUB_ENV + fi + - name: Add connector outcome line to table + if: ${{ always() && github.event.inputs.comment-id }} uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :x: Couldn't auto-bump version for ${{github.event.inputs.connector}} - - name: Add Final Success Comment - if: github.event.inputs.comment-id && success() + | ${{ matrix.connector }} | ${{ env.PUBLISH_OUTCOME }} | ${{ env.AUTO_BUMP_OUTCOME }} | + add-helpful-info-to-git-comment: + if: ${{ always() && github.event.inputs.comment-id }} + name: Add extra info to git comment + needs: + - start-publish-image-runner-0 # required to get output from the start-runner job + - publish-image # required to wait when the main job is done + runs-on: ${{ needs.start-publish-image-runner-0.outputs.label }} + steps: + - name: Add hint for manual seed definition update uses: peter-evans/create-or-update-comment@v1 with: comment-id: ${{ github.event.inputs.comment-id }} body: | - > :white_check_mark: ${{github.event.inputs.connector}} https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} +
+ + if you have connectors that successfully published but failed definition generation, follow [step 4 here ▶️](https://docs.airbyte.com/connector-development/#publishing-a-connector) # In case of self-hosted EC2 errors, remove this block. - stop-publish-image-runner: + stop-publish-image-runner-0: + if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs name: Stop Build EC2 Runner needs: - - start-publish-image-runner # required to get output from the start-runner job + - start-publish-image-runner-0 # required to get output from the start-runner job + - preprocess-matrix - publish-image # required to wait when the main job is done - find_valid_pat + - add-helpful-info-to-git-comment + runs-on: ubuntu-latest + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + - name: Stop EC2 runner + uses: airbytehq/ec2-github-runner@base64v1.1.0 + with: + mode: stop + github-token: ${{ needs.find_valid_pat.outputs.pat }} + label: ${{ needs.start-publish-image-runner-0.outputs.label }} + ec2-instance-id: ${{ needs.start-publish-image-runner-0.outputs.ec2-instance-id }} + stop-publish-image-runner-multi: + if: ${{ always() && github.event.inputs.parallel == 'true' }} + name: Stop Build EC2 Runner + needs: + - start-publish-image-runner-0 + - start-publish-image-runner-1 + - start-publish-image-runner-2 + - start-publish-image-runner-3 + - start-publish-image-runner-4 + - preprocess-matrix + - publish-image # required to wait when the main job is done + - find_valid_pat + strategy: + fail-fast: false + matrix: + ec2-instance: + [ + { + "label": "${{ needs.start-publish-image-runner-1.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-1.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-2.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-2.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-3.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-3.outputs.ec2-instance-id }}", + }, + { + "label": "${{ needs.start-publish-image-runner-4.outputs.label }}", + "id": "${{ needs.start-publish-image-runner-4.outputs.ec2-instance-id }}", + }, + ] runs-on: ubuntu-latest - if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs steps: - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v1 @@ -258,9 +451,9 @@ jobs: aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} aws-region: us-east-2 - name: Stop EC2 runner - uses: supertopher/ec2-github-runner@base64v1.0.10 + uses: airbytehq/ec2-github-runner@base64v1.1.0 with: mode: stop github-token: ${{ needs.find_valid_pat.outputs.pat }} - label: ${{ needs.start-publish-image-runner.outputs.label }} - ec2-instance-id: ${{ needs.start-publish-image-runner.outputs.ec2-instance-id }} + label: ${{ matrix.ec2-instance.label }} + ec2-instance-id: ${{ matrix.ec2-instance.id }} diff --git a/.github/workflows/publish-oss-for-cloud.yml b/.github/workflows/publish-oss-for-cloud.yml new file mode 100644 index 000000000000..847ed5c8a501 --- /dev/null +++ b/.github/workflows/publish-oss-for-cloud.yml @@ -0,0 +1,126 @@ +name: Publish OSS Artifacts for Cloud +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + +on: + workflow_dispatch: + inputs: + oss_ref: + description: "Publish artifacts for the following git ref (if unspecified, uses the latest commit for the current branch):" + required: false +jobs: + find_valid_pat: + name: "Find a PAT with room for actions" + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + pat: ${{ steps.variables.outputs.pat }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Check PAT rate limits + id: variables + run: | + ./tools/bin/find_non_rate_limited_PAT \ + ${{ secrets.AIRBYTEIO_PAT }} \ + ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \ + ${{ secrets.SUPERTOPHER_PAT }} \ + ${{ secrets.DAVINCHIA_PAT }} + start-runner: + name: "Start Runner on AWS" + needs: find_valid_pat + timeout-minutes: 10 + runs-on: ubuntu-latest + outputs: + label: ${{ steps.start-ec2-runner.outputs.label }} + ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: Start AWS Runner + id: start-ec2-runner + uses: ./.github/actions/start-aws-runner + with: + aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} + github-token: ${{ needs.find_valid_pat.outputs.pat }} + + generate-tags: + name: "Generate Tags" + runs-on: ubuntu-latest + outputs: + dev_tag: ${{ steps.set-outputs.outputs.dev_tag }} + master_tag: ${{ steps.set-outputs.outputs.master_tag }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.oss_ref || github.ref }} + - name: Generate Outputs + id: set-outputs + shell: bash + run: |- + set -x + + commit_sha=$(git rev-parse --short HEAD) + + # set dev_tag + # AirbyteVersion.java allows versions that have a prefix of 'dev' + echo "::set-output name=dev_tag::dev-${commit_sha}" + + # If this commit is on the master branch, also set master_tag + if test 0 -eq $(git merge-base --is-ancestor "${commit_sha}" master); then + echo "::set-output name=master_tag::${commit_sha}" + fi + + oss-branch-build: + name: "Build and Push Images from Branch" + needs: + - start-runner + - generate-tags + runs-on: ${{ needs.start-runner.outputs.label }} + steps: + - name: Checkout Airbyte + uses: actions/checkout@v2 + with: + ref: ${{ github.event.inputs.oss_ref || github.ref }} + + - name: Build Branch + uses: ./.github/actions/build-branch + with: + branch_version_tag: ${{ needs.generate-tags.outputs.dev_tag }} + + - name: Login to Docker (on Master) + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: Push Dev Docker Images + run: | + GIT_REVISION=$(git rev-parse HEAD) + [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1 + docker buildx create --name oss-buildx --driver docker-container --use + VERSION=${{ needs.generate-tags.outputs.dev_tag }} + VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push + docker buildx rm oss-buildx + shell: bash + + - name: Push Master Docker Images + if: needs.generate-tags.outputs.master_tag != "" + run: | + GIT_REVISION=$(git rev-parse HEAD) + [ [ -z "$GIT_REVISION" ] ] && echo "Couldn't get the git revision..." && exit 1 + docker buildx create --name oss-buildx --driver docker-container --use + VERSION=${{ needs.generate-tags.outputs.master_tag }} + VERSION=$VERSION GIT_REVISION=$GIT_REVISION docker buildx bake --platform=linux/amd64,linux/arm64 -f docker-compose-cloud.build.yaml --push + docker buildx rm oss-buildx + shell: bash + + - name: Publish Dev Jars + shell: bash + run: VERSION=${{ needs.generate-tags.outputs.dev_tag }} SUB_BUILD=PLATFORM ./gradlew publish + - name: Publish Master Jars + if: needs.generate-tags.outputs.master_tag != "" + shell: bash + run: VERSION=${{ needs.generate-tags.outputs.master_tag }} SUB_BUILD=PLATFORM ./gradlew publish diff --git a/.github/workflows/terminate-zombie-build-instances.yml b/.github/workflows/terminate-zombie-build-instances.yml index 2fcdc4e5120f..42901385695c 100644 --- a/.github/workflows/terminate-zombie-build-instances.yml +++ b/.github/workflows/terminate-zombie-build-instances.yml @@ -34,9 +34,12 @@ jobs: # See https://docs.aws.amazon.com/cli/latest/reference/ec2/terminate-instances.html for terminate command. echo $to_terminate | jq '.[] | .InstanceId' | xargs --no-run-if-empty --max-args=1 aws ec2 terminate-instances --instance-ids - + terminate-github-instances: + runs-on: ubuntu-latest steps: - - shell: List and Terminate GH actions in status 'offline' + - name: Checkout Airbyte + uses: actions/checkout@v2 + - name: List and Terminate GH actions in status 'offline' env: GITHUB_PAT: ${{ secrets.OCTAVIA_PAT }} run: ./tools/bin/gh_action_zombie_killer diff --git a/airbyte-bootloader/Dockerfile b/airbyte-bootloader/Dockerfile index 591a5470cf2b..f820b9134dc6 100644 --- a/airbyte-bootloader/Dockerfile +++ b/airbyte-bootloader/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.1 ARG JDK_IMAGE=openjdk:${JDK_VERSION}-slim FROM ${JDK_IMAGE} -ARG VERSION=0.39.21-alpha +ARG VERSION=0.39.23-alpha ENV APPLICATION airbyte-bootloader ENV VERSION ${VERSION} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py index 92e29eec4307..81eaf56fc16c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/http_requester.py @@ -10,10 +10,10 @@ InterpolatedRequestHeaderProvider, ) from airbyte_cdk.sources.declarative.requesters.request_headers.request_header_provider import RequestHeaderProvider -from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import ( - InterpolatedRequestParameterProvider, +from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import ( + InterpolatedRequestOptionsProvider, ) -from airbyte_cdk.sources.declarative.requesters.request_params.request_parameters_provider import RequestParameterProvider +from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod, Requester from airbyte_cdk.sources.declarative.requesters.retriers.retrier import Retrier from airbyte_cdk.sources.declarative.types import Config @@ -28,14 +28,16 @@ def __init__( url_base: [str, InterpolatedString], path: [str, InterpolatedString], http_method: Union[str, HttpMethod], - request_parameters_provider: RequestParameterProvider = None, + request_options_provider: RequestOptionsProvider = None, request_headers_provider: RequestHeaderProvider = None, authenticator: HttpAuthenticator, retrier: Retrier, config: Config, ): - if request_parameters_provider is None: - request_parameters_provider = InterpolatedRequestParameterProvider(config=config, request_headers={}) + if request_options_provider is None: + request_options_provider = InterpolatedRequestOptionsProvider( + config=config, request_parameters={}, request_body_data="", request_body_json={} + ) if request_headers_provider is None: request_headers_provider = InterpolatedRequestHeaderProvider(config=config, request_headers={}) self._name = name @@ -49,7 +51,7 @@ def __init__( if type(http_method) == str: http_method = HttpMethod[http_method] self._method = http_method - self._request_parameters_provider = request_parameters_provider + self._request_options_provider = request_options_provider self._request_headers_provider = request_headers_provider self._retrier = retrier self._config = config @@ -57,7 +59,7 @@ def __init__( def request_params( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: - return self._request_parameters_provider.request_params(stream_state, stream_slice, next_page_token) + return self._request_options_provider.request_params(stream_state, stream_slice, next_page_token) def get_authenticator(self): return self._authenticator @@ -100,20 +102,17 @@ def request_headers( def request_body_data( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> Optional[Union[Mapping, str]]: - # FIXME: this should be declarative - return dict() + return self._request_options_provider.request_body_data(stream_state, stream_slice, next_page_token) def request_body_json( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> Optional[Mapping]: - # FIXME: this should be declarative - return dict() + return self._request_options_provider.request_body_json(stream_state, stream_slice, next_page_token) def request_kwargs( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None ) -> Mapping[str, Any]: - # FIXME: this should be declarative - return dict() + return self._request_options_provider.request_kwargs(stream_state, stream_slice, next_page_token) @property def cache_filename(self) -> str: diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/interpolated_request_input_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/interpolated_request_input_provider.py index 43dbbc8aeda0..cf8063fba5c4 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/interpolated_request_input_provider.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/interpolated_request_input_provider.py @@ -2,27 +2,35 @@ # Copyright (c) 2022 Airbyte, Inc., all rights reserved. # -from typing import Any, Mapping, MutableMapping +from typing import Any, Mapping, Union from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString from airbyte_cdk.sources.declarative.interpolation.jinja import JinjaInterpolation class InterpolatedRequestInputProvider: """ - Helper class that generically performs string interpolation on the provided dictionary input + Helper class that generically performs string interpolation on the provided dictionary or string input """ def __init__(self, *, config, request_inputs=None): + self._config = config + if request_inputs is None: request_inputs = {} - self._interpolator = InterpolatedMapping(request_inputs, JinjaInterpolation()) - self._config = config + if isinstance(request_inputs, str): + self._interpolator = InterpolatedString(request_inputs, "") + else: + self._interpolator = InterpolatedMapping(request_inputs, JinjaInterpolation()) def request_inputs( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: + ) -> Union[Mapping, str]: kwargs = {"stream_state": stream_state, "stream_slice": stream_slice, "next_page_token": next_page_token} - interpolated_values = self._interpolator.eval(self._config, **kwargs) # dig into this function a little more - non_null_tokens = {k: v for k, v in interpolated_values.items() if v} - return non_null_tokens + interpolated_value = self._interpolator.eval(self._config, **kwargs) + + if isinstance(interpolated_value, dict): + non_null_tokens = {k: v for k, v in interpolated_value.items() if v} + return non_null_tokens + return interpolated_value diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py new file mode 100644 index 000000000000..60b3c444378c --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/interpolated_request_options_provider.py @@ -0,0 +1,50 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from typing import Any, Mapping, MutableMapping, Optional, Union + +from airbyte_cdk.sources.declarative.requesters.interpolated_request_input_provider import InterpolatedRequestInputProvider +from airbyte_cdk.sources.declarative.requesters.request_options.request_options_provider import RequestOptionsProvider + + +class InterpolatedRequestOptionsProvider(RequestOptionsProvider): + def __init__(self, *, config, request_parameters=None, request_body_data=None, request_body_json=None): + if request_parameters is None: + request_parameters = {} + if request_body_data is None: + request_body_data = "" + if request_body_json is None: + request_body_json = {} + + if request_body_json and request_body_data: + raise ValueError("RequestOptionsProvider should only contain either 'request_body_data' or 'request_body_json' not both") + + self._parameter_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_parameters) + self._body_data_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_body_data) + self._body_json_interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_body_json) + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + interpolated_value = self._parameter_interpolator.request_inputs(stream_state, stream_slice, next_page_token) + if isinstance(interpolated_value, dict): + return interpolated_value + return {} + + def request_body_data( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Optional[Union[Mapping, str]]: + return self._body_data_interpolator.request_inputs(stream_state, stream_slice, next_page_token) + + def request_body_json( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Optional[Mapping]: + return self._body_json_interpolator.request_inputs(stream_state, stream_slice, next_page_token) + + def request_kwargs( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + # todo: there are a few integrations that override the request_kwargs() method, but the use case for why kwargs over existing + # constructs is a little unclear. We may revisit this, but for now lets leave it out of the DSL + return {} diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py new file mode 100644 index 000000000000..e7a8571e1c55 --- /dev/null +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_options/request_options_provider.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Any, Mapping, MutableMapping, Optional, Union + + +class RequestOptionsProvider(ABC): + @abstractmethod + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + pass + + @abstractmethod + def request_body_data( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Optional[Union[Mapping, str]]: + pass + + @abstractmethod + def request_body_json( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Optional[Mapping]: + pass + + @abstractmethod + def request_kwargs( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> Mapping[str, Any]: + pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/__init__.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/interpolated_request_parameter_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/interpolated_request_parameter_provider.py deleted file mode 100644 index 17afe7d9feca..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/interpolated_request_parameter_provider.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from typing import Any, Mapping, MutableMapping - -from airbyte_cdk.sources.declarative.requesters.interpolated_request_input_provider import InterpolatedRequestInputProvider -from airbyte_cdk.sources.declarative.requesters.request_params.request_parameters_provider import RequestParameterProvider - - -class InterpolatedRequestParameterProvider(RequestParameterProvider): - def __init__(self, *, config, request_parameters=None): - if request_parameters is None: - request_parameters = {} - self._interpolator = InterpolatedRequestInputProvider(config=config, request_inputs=request_parameters) - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return self._interpolator.request_inputs(stream_state, stream_slice, next_page_token) diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/request_parameters_provider.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/request_parameters_provider.py deleted file mode 100644 index 30f1431695eb..000000000000 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/request_params/request_parameters_provider.py +++ /dev/null @@ -1,14 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from abc import ABC, abstractmethod -from typing import Any, Mapping, MutableMapping - - -class RequestParameterProvider(ABC): - @abstractmethod - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - pass diff --git a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py index 2dba6311415a..7ce5f3aeeb81 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/declarative/requesters/requester.py @@ -12,6 +12,7 @@ class HttpMethod(Enum): GET = "GET" + POST = "POST" class Requester(ABC): diff --git a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py index 528711ea6bd1..03e977ebc58c 100644 --- a/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py +++ b/airbyte-cdk/python/airbyte_cdk/sources/streams/http/http.py @@ -22,7 +22,7 @@ from .rate_limiting import default_backoff_handler, user_defined_backoff_handler # list of all possible HTTP methods which can be used for sending of request bodies -BODY_REQUEST_METHODS = ("POST", "PUT", "PATCH") +BODY_REQUEST_METHODS = ("GET", "POST", "PUT", "PATCH") logging.getLogger("vcr").setLevel(logging.ERROR) @@ -248,7 +248,12 @@ def backoff_time(self, response: requests.Response) -> Optional[float]: return None def _create_prepared_request( - self, path: str, headers: Mapping = None, params: Mapping = None, json: Any = None, data: Any = None + self, + path: str, + headers: Mapping = None, + params: Mapping = None, + json: Any = None, + data: Any = None, ) -> requests.PreparedRequest: args = {"method": self.http_method, "url": urljoin(self.url_base, path), "headers": headers, "params": params} if self.http_method.upper() in BODY_REQUEST_METHODS: diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_request_parameter_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_request_parameter_provider.py deleted file mode 100644 index 1699a62a9497..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/interpolation/test_interpolated_request_parameter_provider.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import ( - InterpolatedRequestParameterProvider, -) - -state = {"date": "2021-01-01"} -stream_slice = {"start_date": "2020-01-01"} -next_page_token = {"offset": "12345"} -config = {"option": "OPTION"} - - -def test(): - request_parameters = {"a_static_request_param": "a_static_value"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_parameters == request_params - - -def test_value_depends_on_state(): - request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == state["date"] - - -def test_value_depends_on_stream_slice(): - request_parameters = {"a_static_request_param": "{{ stream_slice['start_date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == stream_slice["start_date"] - - -def test_value_depends_on_next_page_token(): - request_parameters = {"a_static_request_param": "{{ next_page_token['offset'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == next_page_token["offset"] - - -def test_value_depends_on_config(): - request_parameters = {"a_static_request_param": "{{ config['option'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == config["option"] - - -def test_parameter_is_interpolated(): - request_parameters = { - "{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC" - } - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params[f"{state['date']} - {stream_slice['start_date']} - {next_page_token['offset']} - {config['option']}"] == "ABC" - - -def test_none_value(): - request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params({}, stream_slice, next_page_token) - - assert len(request_params) == 0 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_interpolated_request_parameter_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_interpolated_request_parameter_provider.py deleted file mode 100644 index eff1dd651d4f..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/iterators/test_interpolated_request_parameter_provider.py +++ /dev/null @@ -1,77 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - -from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import ( - InterpolatedRequestParameterProvider, -) - -state = {"date": "2021-01-01"} -stream_slice = {"start_date": "2020-01-01"} -next_page_token = {"offset": "12345"} -config = {"option": "OPTION"} - - -def test(): - request_parameters = {"a_static_request_param": "a_static_value"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_parameters == request_params - - -def test_value_depends_on_state(): - request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == state["date"] - - -def test_value_depends_on_stream_slice(): - request_parameters = {"a_static_request_param": "{{ stream_slice['start_date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == stream_slice["start_date"] - - -def test_value_depends_on_next_page_token(): - request_parameters = {"a_static_request_param": "{{ next_page_token['offset'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == next_page_token["offset"] - - -def test_value_depends_on_config(): - request_parameters = {"a_static_request_param": "{{ config['option'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == config["option"] - - -def test_parameter_is_interpolated(): - request_parameters = { - "{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC" - } - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params[f"{state['date']} - {stream_slice['start_date']} - {next_page_token['offset']} - {config['option']}"] == "ABC" - - -def test_none_value(): - request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params({}, stream_slice, next_page_token) - - assert len(request_params) == 0 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/__init__.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/__init__.py new file mode 100644 index 000000000000..1100c1c58cf5 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py new file mode 100644 index 000000000000..0dc242b076a4 --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_options/test_interpolated_request_options_provider.py @@ -0,0 +1,96 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest +from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import ( + InterpolatedRequestOptionsProvider, +) + +state = {"date": "2021-01-01"} +stream_slice = {"start_date": "2020-01-01"} +next_page_token = {"offset": "12345", "page": "27"} +config = {"option": "OPTION"} + + +@pytest.mark.parametrize( + "test_name, input_request_params, expected_request_params", + [ + ("test_static_param", {"a_static_request_param": "a_static_value"}, {"a_static_request_param": "a_static_value"}), + ("test_value_depends_on_state", {"read_from_state": "{{ stream_state['date'] }}"}, {"read_from_state": "2021-01-01"}), + ("test_value_depends_on_stream_slice", {"read_from_slice": "{{ stream_slice['start_date'] }}"}, {"read_from_slice": "2020-01-01"}), + ("test_value_depends_on_next_page_token", {"read_from_token": "{{ next_page_token['offset'] }}"}, {"read_from_token": "12345"}), + ("test_value_depends_on_config", {"read_from_config": "{{ config['option'] }}"}, {"read_from_config": "OPTION"}), + ("test_none_value", {"missing_param": "{{ fake_path['date'] }}"}, {}), + ("test_return_empty_dict_for_string_templates", "Should return empty dict {{ stream_state['date'] }}", {}), + ( + "test_parameter_is_interpolated", + {"{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC"}, + {"2021-01-01 - 2020-01-01 - 12345 - OPTION": "ABC"}, + ), + ], +) +def test_interpolated_request_params(test_name, input_request_params, expected_request_params): + provider = InterpolatedRequestOptionsProvider(config=config, request_parameters=input_request_params) + + actual_request_params = provider.request_params(state, stream_slice, next_page_token) + + assert actual_request_params == expected_request_params + + +@pytest.mark.parametrize( + "test_name, input_request_json, expected_request_json", + [ + ("test_static_json", {"a_static_request_param": "a_static_value"}, {"a_static_request_param": "a_static_value"}), + ("test_value_depends_on_state", {"read_from_state": "{{ stream_state['date'] }}"}, {"read_from_state": "2021-01-01"}), + ("test_value_depends_on_stream_slice", {"read_from_slice": "{{ stream_slice['start_date'] }}"}, {"read_from_slice": "2020-01-01"}), + ("test_value_depends_on_next_page_token", {"read_from_token": "{{ next_page_token['offset'] }}"}, {"read_from_token": "12345"}), + ("test_value_depends_on_config", {"read_from_config": "{{ config['option'] }}"}, {"read_from_config": "OPTION"}), + ("test_none_value", {"missing_json": "{{ fake_path['date'] }}"}, {}), + ( + "test_interpolated_keys", + {"{{ stream_state['date'] }}": 123, "{{ config['option'] }}": "ABC"}, + {"2021-01-01": 123, "OPTION": "ABC"}, + ), + ], +) +def test_interpolated_request_json(test_name, input_request_json, expected_request_json): + provider = InterpolatedRequestOptionsProvider(config=config, request_body_json=input_request_json) + + actual_request_json = provider.request_body_json(state, stream_slice, next_page_token) + + assert actual_request_json == expected_request_json + + +@pytest.mark.parametrize( + "test_name, input_request_data, expected_request_data", + [ + ("test_static_map_data", {"a_static_request_param": "a_static_value"}, {"a_static_request_param": "a_static_value"}), + ("test_static_string_data", "a_static_value", "a_static_value"), + ("test_string_depends_on_state", "key={{ stream_state['date'] }}", "key=2021-01-01"), + ("test_map_depends_on_stream_slice", {"read_from_slice": "{{ stream_slice['start_date'] }}"}, {"read_from_slice": "2020-01-01"}), + ("test_string_depends_on_next_page_token", "{{ next_page_token['page'] }} and {{ next_page_token['offset'] }}", "27 and 12345"), + ("test_map_depends_on_config", {"read_from_config": "{{ config['option'] }}"}, {"read_from_config": "OPTION"}), + ("test_defaults_to_empty_string", None, ""), + ("test_interpolated_keys", {"{{ stream_state['date'] }} - {{ next_page_token['offset'] }}": "ABC"}, {"2021-01-01 - 12345": "ABC"}), + ], +) +def test_interpolated_request_data(test_name, input_request_data, expected_request_data): + provider = InterpolatedRequestOptionsProvider(config=config, request_body_data=input_request_data) + + actual_request_data = provider.request_body_data(state, stream_slice, next_page_token) + + assert actual_request_data == expected_request_data + + +def test_error_on_create_for_both_request_json_and_data(): + request_json = {"body_key": "{{ stream_slice['start_date'] }}"} + request_data = "interpolate_me=5&invalid={{ config['option'] }}" + with pytest.raises(ValueError): + InterpolatedRequestOptionsProvider(config=config, request_body_json=request_json, request_body_data=request_data) + + +def test_interpolated_request_kwargs_is_empty(): + provider = InterpolatedRequestOptionsProvider(config=config) + actual_request_kwargs = provider.request_kwargs(state, stream_slice, next_page_token) + assert {} == actual_request_kwargs diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_params/__init__.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_params/__init__.py deleted file mode 100644 index 46b7376756ec..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_params/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -# -# Copyright (c) 2021 Airbyte, Inc., all rights reserved. -# diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_params/test_interpolated_request_parameter_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_params/test_interpolated_request_parameter_provider.py deleted file mode 100644 index 1699a62a9497..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/request_params/test_interpolated_request_parameter_provider.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import ( - InterpolatedRequestParameterProvider, -) - -state = {"date": "2021-01-01"} -stream_slice = {"start_date": "2020-01-01"} -next_page_token = {"offset": "12345"} -config = {"option": "OPTION"} - - -def test(): - request_parameters = {"a_static_request_param": "a_static_value"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_parameters == request_params - - -def test_value_depends_on_state(): - request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == state["date"] - - -def test_value_depends_on_stream_slice(): - request_parameters = {"a_static_request_param": "{{ stream_slice['start_date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == stream_slice["start_date"] - - -def test_value_depends_on_next_page_token(): - request_parameters = {"a_static_request_param": "{{ next_page_token['offset'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == next_page_token["offset"] - - -def test_value_depends_on_config(): - request_parameters = {"a_static_request_param": "{{ config['option'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == config["option"] - - -def test_parameter_is_interpolated(): - request_parameters = { - "{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC" - } - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params[f"{state['date']} - {stream_slice['start_date']} - {next_page_token['offset']} - {config['option']}"] == "ABC" - - -def test_none_value(): - request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params({}, stream_slice, next_page_token) - - assert len(request_params) == 0 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py index a9891445c8a7..3df9cbf781e4 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_http_requester.py @@ -11,9 +11,13 @@ def test(): http_method = "GET" - request_parameters_provider = MagicMock() + request_options_provider = MagicMock() request_params = {"param": "value"} - request_parameters_provider.request_params.return_value = request_params + request_body_data = "body_key_1=value_1&body_key_2=value2" + request_body_json = {"body_field": "body_value"} + request_options_provider.request_params.return_value = request_params + request_options_provider.request_body_data.return_value = request_body_data + request_options_provider.request_body_json.return_value = request_body_json request_headers_provider = MagicMock() request_headers = {"header": "value"} @@ -39,7 +43,7 @@ def test(): url_base="{{ config['url'] }}", path="v1/{{ stream_slice['id'] }}", http_method=http_method, - request_parameters_provider=request_parameters_provider, + request_options_provider=request_options_provider, request_headers_provider=request_headers_provider, authenticator=authenticator, retrier=retrier, @@ -47,10 +51,12 @@ def test(): ) assert requester.get_url_base() == "https://airbyte.io" - assert requester.get_path(stream_state=None, stream_slice=stream_slice, next_page_token=None) == "v1/1234" + assert requester.get_path(stream_state={}, stream_slice=stream_slice, next_page_token={}) == "v1/1234" assert requester.get_authenticator() == authenticator assert requester.get_method() == HttpMethod.GET - assert requester.request_params(stream_state=None, stream_slice=None, next_page_token=None) == request_params + assert requester.request_params(stream_state={}, stream_slice=None, next_page_token=None) == request_params + assert requester.request_body_data(stream_state={}, stream_slice=None, next_page_token=None) == request_body_data + assert requester.request_body_json(stream_state={}, stream_slice=None, next_page_token=None) == request_body_json assert requester.max_retries == max_retries assert requester.should_retry(requests.Response()) == should_retry assert requester.backoff_time(requests.Response()) == backoff_time diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py new file mode 100644 index 000000000000..625f9c05cd4a --- /dev/null +++ b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_input_provider.py @@ -0,0 +1,48 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pytest as pytest +from airbyte_cdk.sources.declarative.interpolation.interpolated_mapping import InterpolatedMapping +from airbyte_cdk.sources.declarative.interpolation.interpolated_string import InterpolatedString +from airbyte_cdk.sources.declarative.requesters.interpolated_request_input_provider import InterpolatedRequestInputProvider + + +@pytest.mark.parametrize( + "test_name, input_request_data, expected_request_data", + [ + ("test_static_string_data", "a_static_value", "a_static_value"), + ("test_string_depends_on_state", "key={{ stream_state['state_key'] }}", "key=state_value"), + ("test_string_depends_on_next_page_token", "{{ next_page_token['token_key'] }} + ultra", "token_value + ultra"), + ], +) +def test_interpolated_string_request_input_provider(test_name, input_request_data, expected_request_data): + config = {"config_key": "value_of_config"} + stream_state = {"state_key": "state_value"} + next_page_token = {"token_key": "token_value"} + + provider = InterpolatedRequestInputProvider(config=config, request_inputs=input_request_data) + actual_request_data = provider.request_inputs(stream_state=stream_state, next_page_token=next_page_token) + + assert isinstance(provider._interpolator, InterpolatedString) + assert actual_request_data == expected_request_data + + +@pytest.mark.parametrize( + "test_name, input_request_data, expected_request_data", + [ + ("test_static_map_data", {"a_static_request_param": "a_static_value"}, {"a_static_request_param": "a_static_value"}), + ("test_map_depends_on_stream_slice", {"read_from_slice": "{{ stream_slice['slice_key'] }}"}, {"read_from_slice": "slice_value"}), + ("test_map_depends_on_config", {"read_from_config": "{{ config['config_key'] }}"}, {"read_from_config": "value_of_config"}), + ("test_defaults_to_empty_dictionary", None, {}), + ], +) +def test_initialize_interpolated_mapping_request_input_provider(test_name, input_request_data, expected_request_data): + config = {"config_key": "value_of_config"} + stream_slice = {"slice_key": "slice_value"} + + provider = InterpolatedRequestInputProvider(config=config, request_inputs=input_request_data) + actual_request_data = provider.request_inputs(stream_state={}, stream_slice=stream_slice) + + assert isinstance(provider._interpolator, InterpolatedMapping) + assert actual_request_data == expected_request_data diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_parameter_provider.py b/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_parameter_provider.py deleted file mode 100644 index 1699a62a9497..000000000000 --- a/airbyte-cdk/python/unit_tests/sources/declarative/requesters/test_interpolated_request_parameter_provider.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) 2022 Airbyte, Inc., all rights reserved. -# - - -from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import ( - InterpolatedRequestParameterProvider, -) - -state = {"date": "2021-01-01"} -stream_slice = {"start_date": "2020-01-01"} -next_page_token = {"offset": "12345"} -config = {"option": "OPTION"} - - -def test(): - request_parameters = {"a_static_request_param": "a_static_value"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_parameters == request_params - - -def test_value_depends_on_state(): - request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == state["date"] - - -def test_value_depends_on_stream_slice(): - request_parameters = {"a_static_request_param": "{{ stream_slice['start_date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == stream_slice["start_date"] - - -def test_value_depends_on_next_page_token(): - request_parameters = {"a_static_request_param": "{{ next_page_token['offset'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == next_page_token["offset"] - - -def test_value_depends_on_config(): - request_parameters = {"a_static_request_param": "{{ config['option'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params["a_static_request_param"] == config["option"] - - -def test_parameter_is_interpolated(): - request_parameters = { - "{{ stream_state['date'] }} - {{stream_slice['start_date']}} - {{next_page_token['offset']}} - {{config['option']}}": "ABC" - } - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params(state, stream_slice, next_page_token) - - assert request_params[f"{state['date']} - {stream_slice['start_date']} - {next_page_token['offset']} - {config['option']}"] == "ABC" - - -def test_none_value(): - request_parameters = {"a_static_request_param": "{{ stream_state['date'] }}"} - provider = InterpolatedRequestParameterProvider(request_parameters=request_parameters, config=config) - - request_params = provider.request_params({}, stream_slice, next_page_token) - - assert len(request_params) == 0 diff --git a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py index ea2d055b8a75..7335f20029fd 100644 --- a/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py +++ b/airbyte-cdk/python/unit_tests/sources/declarative/test_factory.py @@ -6,8 +6,8 @@ from airbyte_cdk.sources.declarative.decoders.json_decoder import JsonDecoder from airbyte_cdk.sources.declarative.parsers.factory import DeclarativeComponentFactory from airbyte_cdk.sources.declarative.parsers.yaml_parser import YamlParser -from airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider import ( - InterpolatedRequestParameterProvider, +from airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider import ( + InterpolatedRequestOptionsProvider, ) from airbyte_cdk.sources.declarative.requesters.requester import HttpMethod from airbyte_cdk.sources.declarative.retrievers.simple_retriever import SimpleRetriever @@ -26,15 +26,19 @@ def test_factory(): offset_request_parameters: offset: "{{ next_page_token['offset'] }}" limit: "*ref(limit)" - offset_pagination_request_parameters: - class_name: airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider.InterpolatedRequestParameterProvider + request_options: + class_name: airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider.InterpolatedRequestOptionsProvider request_parameters: "*ref(offset_request_parameters)" + request_body_json: + body_offset: "{{ next_page_token['offset'] }}" """ config = parser.parse(content) - offset_pagination_request_parameters = factory.create_component(config["offset_pagination_request_parameters"], input_config)() - assert type(offset_pagination_request_parameters) == InterpolatedRequestParameterProvider - assert offset_pagination_request_parameters._interpolator._config == input_config - assert offset_pagination_request_parameters._interpolator._interpolator._mapping["offset"] == "{{ next_page_token['offset'] }}" + request_options_provider = factory.create_component(config["request_options"], input_config)() + assert type(request_options_provider) == InterpolatedRequestOptionsProvider + assert request_options_provider._parameter_interpolator._config == input_config + assert request_options_provider._parameter_interpolator._interpolator._mapping["offset"] == "{{ next_page_token['offset'] }}" + assert request_options_provider._body_json_interpolator._config == input_config + assert request_options_provider._body_json_interpolator._interpolator._mapping["body_offset"] == "{{ next_page_token['offset'] }}" def test_interpolate_config(): @@ -89,8 +93,8 @@ def test_full_config(): next_page_url_from_token_partial: class_name: "airbyte_cdk.sources.declarative.interpolation.interpolated_string.InterpolatedString" string: "{{ next_page_token['next_page_url'] }}" -request_parameters_provider: - class_name: airbyte_cdk.sources.declarative.requesters.request_params.interpolated_request_parameter_provider.InterpolatedRequestParameterProvider +request_options_provider: + class_name: airbyte_cdk.sources.declarative.requesters.request_options.interpolated_request_options_provider.InterpolatedRequestOptionsProvider requester: class_name: airbyte_cdk.sources.declarative.requesters.http_requester.HttpRequester name: "{{ options['name'] }}" @@ -99,7 +103,7 @@ def test_full_config(): authenticator: class_name: airbyte_cdk.sources.streams.http.requests_native_auth.token.TokenAuthenticator token: "{{ config['apikey'] }}" - request_parameters_provider: "*ref(request_parameters_provider)" + request_parameters_provider: "*ref(request_options_provider)" retrier: class_name: airbyte_cdk.sources.declarative.requesters.retriers.default_retrier.DefaultRetrier retriever: diff --git a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py index 5d9f11bcb2d6..fe87508e51a5 100644 --- a/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py +++ b/airbyte-cdk/python/unit_tests/sources/streams/http/test_http.py @@ -328,13 +328,13 @@ def test_text_json_body(self, mocker, requests_mock): list(stream.read_records(sync_mode=SyncMode.full_refresh)) def test_body_for_all_methods(self, mocker, requests_mock): - """Stream must send a body for POST/PATCH/PUT methods only""" + """Stream must send a body for GET/POST/PATCH/PUT methods only""" stream = PostHttpStream() methods = { "POST": True, "PUT": True, "PATCH": True, - "GET": False, + "GET": True, "DELETE": False, "OPTIONS": False, } diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java index 8381f15262da..740fb94bcb73 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/Configs.java @@ -171,7 +171,36 @@ public interface Configs { */ boolean runDatabaseMigrationOnStartup(); + // Temporal Cloud - Internal-Use Only + + /** + * Define if Temporal Cloud should be used. Internal-use only. + */ + boolean temporalCloudEnabled(); + + /** + * Temporal Cloud target endpoint, usually with form ${namespace}.tmprl.cloud:7233. Internal-use + * only. + */ + String getTemporalCloudHost(); + + /** + * Temporal Cloud namespace. Internal-use only. + */ + String getTemporalCloudNamespace(); + + /** + * Temporal Cloud client cert for SSL. Internal-use only. + */ + String getTemporalCloudClientCert(); + + /** + * Temporal Cloud client key for SSL. Internal-use only. + */ + String getTemporalCloudClientKey(); + // Airbyte Services + /** * Define the url where Temporal is hosted at. Please include the port. Airbyte services use this * information. diff --git a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java index 486575bc242e..9bf3ef300cec 100644 --- a/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java +++ b/airbyte-config/config-models/src/main/java/io/airbyte/config/EnvConfigs.java @@ -109,6 +109,12 @@ public class EnvConfigs implements Configs { public static final String STATE_STORAGE_GCS_BUCKET_NAME = "STATE_STORAGE_GCS_BUCKET_NAME"; public static final String STATE_STORAGE_GCS_APPLICATION_CREDENTIALS = "STATE_STORAGE_GCS_APPLICATION_CREDENTIALS"; + private static final String TEMPORAL_CLOUD_ENABLED = "TEMPORAL_CLOUD_ENABLED"; + private static final String TEMPORAL_CLOUD_HOST = "TEMPORAL_CLOUD_HOST"; + private static final String TEMPORAL_CLOUD_NAMESPACE = "TEMPORAL_CLOUD_NAMESPACE"; + private static final String TEMPORAL_CLOUD_CLIENT_CERT = "TEMPORAL_CLOUD_CLIENT_CERT"; + private static final String TEMPORAL_CLOUD_CLIENT_KEY = "TEMPORAL_CLOUD_CLIENT_KEY"; + public static final String ACTIVITY_MAX_TIMEOUT_SECOND = "ACTIVITY_MAX_TIMEOUT_SECOND"; public static final String ACTIVITY_MAX_ATTEMPT = "ACTIVITY_MAX_ATTEMPT"; public static final String ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS = "ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS"; @@ -390,6 +396,32 @@ public boolean runDatabaseMigrationOnStartup() { return getEnvOrDefault(RUN_DATABASE_MIGRATION_ON_STARTUP, true); } + // Temporal Cloud + @Override + public boolean temporalCloudEnabled() { + return getEnvOrDefault(TEMPORAL_CLOUD_ENABLED, false); + } + + @Override + public String getTemporalCloudHost() { + return getEnvOrDefault(TEMPORAL_CLOUD_HOST, ""); + } + + @Override + public String getTemporalCloudNamespace() { + return getEnvOrDefault(TEMPORAL_CLOUD_NAMESPACE, ""); + } + + @Override + public String getTemporalCloudClientCert() { + return getEnvOrDefault(TEMPORAL_CLOUD_CLIENT_CERT, ""); + } + + @Override + public String getTemporalCloudClientKey() { + return getEnvOrDefault(TEMPORAL_CLOUD_CLIENT_KEY, ""); + } + // Airbyte Services @Override public String getTemporalHost() { diff --git a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml index eb1a0cb131f8..6b0047f07c18 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_definitions.yaml @@ -183,7 +183,7 @@ - name: MySQL destinationDefinitionId: ca81ee7c-3163-4246-af40-094cc31e5e42 dockerRepository: airbyte/destination-mysql - dockerImageTag: 0.1.18 + dockerImageTag: 0.1.20 documentationUrl: https://docs.airbyte.io/integrations/destinations/mysql icon: mysql.svg releaseStage: alpha @@ -238,7 +238,7 @@ - name: Rockset destinationDefinitionId: 2c9d93a7-9a17-4789-9de9-f46f0097eb70 dockerRepository: airbyte/destination-rockset - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.3 documentationUrl: https://docs.airbyte.io/integrations/destinations/rockset releaseStage: alpha - name: S3 diff --git a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml index fda39c9d11e4..dcd20018abfd 100644 --- a/airbyte-config/init/src/main/resources/seed/destination_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/destination_specs.yaml @@ -2807,7 +2807,7 @@ supported_destination_sync_modes: - "overwrite" - "append" -- dockerImage: "airbyte/destination-mysql:0.1.18" +- dockerImage: "airbyte/destination-mysql:0.1.20" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/mysql" connectionSpecification: @@ -3780,7 +3780,7 @@ - "overwrite" - "append" - "append_dedup" -- dockerImage: "airbyte/destination-rockset:0.1.2" +- dockerImage: "airbyte/destination-rockset:0.1.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/destinations/rockset" connectionSpecification: diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index acbda234019f..26db7e596f58 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -525,7 +525,7 @@ - name: Marketo sourceDefinitionId: 9e0556f4-69df-4522-a3fb-03264d36b348 dockerRepository: airbyte/source-marketo - dockerImageTag: 0.1.3 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/marketo icon: marketo.svg sourceType: api @@ -613,7 +613,7 @@ - name: OpenWeather sourceDefinitionId: d8540a80-6120-485d-b7d6-272bca477d9b dockerRepository: airbyte/source-openweather - dockerImageTag: 0.1.4 + dockerImageTag: 0.1.5 documentationUrl: https://docs.airbyte.io/integrations/sources/openweather sourceType: api releaseStage: alpha @@ -929,7 +929,7 @@ - name: TiDB sourceDefinitionId: 0dad1a35-ccf8-4d03-b73e-6788c00b13ae dockerRepository: airbyte/source-tidb - dockerImageTag: 0.1.1 + dockerImageTag: 0.1.2 documentationUrl: https://docs.airbyte.io/integrations/sources/tidb icon: tidb.svg sourceType: database diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 488f7dd1a518..d5b9183689b4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -4799,7 +4799,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-marketo:0.1.3" +- dockerImage: "airbyte/source-marketo:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/marketo" connectionSpecification: @@ -5910,7 +5910,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-openweather:0.1.4" +- dockerImage: "airbyte/source-openweather:0.1.5" spec: documentationUrl: "https://docsurl.com" connectionSpecification: @@ -8884,7 +8884,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-tidb:0.1.1" +- dockerImage: "airbyte/source-tidb:0.1.2" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/tidb" connectionSpecification: diff --git a/airbyte-container-orchestrator/Dockerfile b/airbyte-container-orchestrator/Dockerfile index 5c2875c5a1cc..bc619d0cadef 100644 --- a/airbyte-container-orchestrator/Dockerfile +++ b/airbyte-container-orchestrator/Dockerfile @@ -28,7 +28,7 @@ RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] htt RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y kubectl # Don't change this manually. Bump version expects to make moves based on this string -ARG VERSION=0.39.21-alpha +ARG VERSION=0.39.23-alpha ENV APPLICATION airbyte-container-orchestrator ENV VERSION=${VERSION} diff --git a/airbyte-integrations/bases/debezium-v1-4-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java b/airbyte-integrations/bases/debezium-v1-4-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java index a1049f0b7450..04cd2bfc20b8 100644 --- a/airbyte-integrations/bases/debezium-v1-4-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java +++ b/airbyte-integrations/bases/debezium-v1-4-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java @@ -316,7 +316,7 @@ void testDelete() throws Exception { .format("DELETE FROM %s.%s WHERE %s = %s", MODELS_SCHEMA, MODELS_STREAM_NAME, COL_ID, 11)); - final JsonNode state = stateMessages1.get(0).getData(); + final JsonNode state = Jsons.jsonNode(stateMessages1); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); @@ -347,7 +347,7 @@ void testUpdate() throws Exception { .format("UPDATE %s.%s SET %s = '%s' WHERE %s = %s", MODELS_SCHEMA, MODELS_STREAM_NAME, COL_MODEL, updatedModel, COL_ID, 11)); - final JsonNode state = stateMessages1.get(0).getData(); + final JsonNode state = Jsons.jsonNode(stateMessages1); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); @@ -403,7 +403,7 @@ void testRecordsProducedDuringAndAfterSync() throws Exception { recordsCreated[0]++; } - final JsonNode state = stateAfterFirstBatch.get(0).getData(); + final JsonNode state = Jsons.jsonNode(stateAfterFirstBatch); final AutoCloseableIterator secondBatchIterator = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List dataFromSecondBatch = AutoCloseableIterators @@ -492,7 +492,7 @@ void testCdcAndFullRefreshInSameSync() throws Exception { .jsonNode(ImmutableMap.of(COL_ID, 100, COL_MAKE_ID, 3, COL_MODEL, "Punto")); writeModelRecord(puntoRecord); - final JsonNode state = extractStateMessages(actualRecords1).get(0).getData(); + final JsonNode state = Jsons.jsonNode(extractStateMessages(actualRecords1)); final AutoCloseableIterator read2 = getSource() .read(getConfig(), configuredCatalog, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); @@ -535,7 +535,7 @@ void testNoDataOnSecondSync() throws Exception { final AutoCloseableIterator read1 = getSource() .read(getConfig(), CONFIGURED_CATALOG, null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); - final JsonNode state = extractStateMessages(actualRecords1).get(0).getData(); + final JsonNode state = Jsons.jsonNode(extractStateMessages(actualRecords1)); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); diff --git a/airbyte-integrations/bases/debezium-v1-9-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java b/airbyte-integrations/bases/debezium-v1-9-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java index 79d6dbbd5b31..441de6ff481e 100644 --- a/airbyte-integrations/bases/debezium-v1-9-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java +++ b/airbyte-integrations/bases/debezium-v1-9-2/src/testFixtures/java/io/airbyte/integrations/debezium/CdcSourceTest.java @@ -316,7 +316,7 @@ void testDelete() throws Exception { .format("DELETE FROM %s.%s WHERE %s = %s", MODELS_SCHEMA, MODELS_STREAM_NAME, COL_ID, 11)); - final JsonNode state = stateMessages1.get(0).getData(); + final JsonNode state = Jsons.jsonNode(stateMessages1); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); @@ -347,7 +347,7 @@ void testUpdate() throws Exception { .format("UPDATE %s.%s SET %s = '%s' WHERE %s = %s", MODELS_SCHEMA, MODELS_STREAM_NAME, COL_MODEL, updatedModel, COL_ID, 11)); - final JsonNode state = stateMessages1.get(0).getData(); + final JsonNode state = Jsons.jsonNode(stateMessages1); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); @@ -399,7 +399,7 @@ protected void testRecordsProducedDuringAndAfterSync() throws Exception { writeModelRecord(record); } - final JsonNode state = stateAfterFirstBatch.get(0).getData(); + final JsonNode state = Jsons.jsonNode(stateAfterFirstBatch); final AutoCloseableIterator secondBatchIterator = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List dataFromSecondBatch = AutoCloseableIterators @@ -488,7 +488,7 @@ void testCdcAndFullRefreshInSameSync() throws Exception { .jsonNode(ImmutableMap.of(COL_ID, 100, COL_MAKE_ID, 3, COL_MODEL, "Punto")); writeModelRecord(puntoRecord); - final JsonNode state = extractStateMessages(actualRecords1).get(0).getData(); + final JsonNode state = Jsons.jsonNode(extractStateMessages(actualRecords1)); final AutoCloseableIterator read2 = getSource() .read(getConfig(), configuredCatalog, state); final List actualRecords2 = AutoCloseableIterators.toListAndClose(read2); @@ -531,7 +531,7 @@ void testNoDataOnSecondSync() throws Exception { final AutoCloseableIterator read1 = getSource() .read(getConfig(), CONFIGURED_CATALOG, null); final List actualRecords1 = AutoCloseableIterators.toListAndClose(read1); - final JsonNode state = extractStateMessages(actualRecords1).get(0).getData(); + final JsonNode state = Jsons.jsonNode(extractStateMessages(actualRecords1)); final AutoCloseableIterator read2 = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); diff --git a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java index 186d0b3c14ad..a6e2d50c85aa 100644 --- a/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java +++ b/airbyte-integrations/bases/standard-source-test/src/main/java/io/airbyte/integrations/standardtest/source/SourceAcceptanceTest.java @@ -13,6 +13,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.google.common.collect.Iterables; import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; import io.airbyte.config.StandardCheckConnectionOutput.Status; @@ -106,6 +107,18 @@ public abstract class SourceAcceptanceTest extends AbstractSourceConnectorTest { */ protected abstract JsonNode getState() throws Exception; + /** + * Tests whether the connector under test supports the per-stream state format or should use the + * legacy format for data generated by this test. + * + * @return {@code true} if the connector supports the per-stream state format or {@code false} if it + * does not support the per-stream state format (e.g. legacy format supported). Default + * value is {@code false}. + */ + protected boolean supportsPerStream() { + return false; + } + /** * Verify that a spec operation issued to the connector returns a valid spec. */ @@ -236,7 +249,7 @@ public void testIncrementalSyncWithState() throws Exception { // when we run incremental sync again there should be no new records. Run a sync with the latest // state message and assert no records were emitted. - final JsonNode latestState = stateMessages.get(stateMessages.size() - 1).getData(); + final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); final List secondSyncRecords = filterRecords(runRead(configuredCatalog, latestState)); assertTrue( secondSyncRecords.isEmpty(), diff --git a/airbyte-integrations/connectors/destination-jdbc/Dockerfile b/airbyte-integrations/connectors/destination-jdbc/Dockerfile index aa9e1177a2b1..a35e7fb7b3f2 100644 --- a/airbyte-integrations/connectors/destination-jdbc/Dockerfile +++ b/airbyte-integrations/connectors/destination-jdbc/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-jdbc COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.3.13 +LABEL io.airbyte.version=0.3.14 LABEL io.airbyte.name=airbyte/destination-jdbc diff --git a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java index 2115990996e8..49dcaadfa742 100644 --- a/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java +++ b/airbyte-integrations/connectors/destination-mongodb/src/test-integration/java/io/airbyte/integrations/destination/mongodb/MongodbDestinationAcceptanceTest.java @@ -111,26 +111,6 @@ protected void tearDown(final TestDestinationEnv testEnv) { container.close(); } - @Override - protected TestDataComparator getTestDataComparator() { - return new AdvancedTestDataComparator(); - } - - @Override - protected boolean supportBasicDataTypeTest() { - return true; - } - - @Override - protected boolean supportArrayDataTypeTest() { - return true; - } - - @Override - protected boolean supportObjectDataTypeTest() { - return true; - } - /* Helpers */ private JsonNode getAuthTypeConfig() { diff --git a/airbyte-integrations/connectors/destination-mysql/Dockerfile b/airbyte-integrations/connectors/destination-mysql/Dockerfile index bc324b2bff11..29fa71d00ceb 100644 --- a/airbyte-integrations/connectors/destination-mysql/Dockerfile +++ b/airbyte-integrations/connectors/destination-mysql/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-mysql COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.18 +LABEL io.airbyte.version=0.1.20 LABEL io.airbyte.name=airbyte/destination-mysql diff --git a/airbyte-integrations/connectors/destination-rockset/Dockerfile b/airbyte-integrations/connectors/destination-rockset/Dockerfile index 73477dc97bb6..136dbcd02b48 100644 --- a/airbyte-integrations/connectors/destination-rockset/Dockerfile +++ b/airbyte-integrations/connectors/destination-rockset/Dockerfile @@ -16,5 +16,5 @@ ENV APPLICATION destination-rockset COPY --from=build /airbyte /airbyte -LABEL io.airbyte.version=0.1.2 +LABEL io.airbyte.version=0.1.3 LABEL io.airbyte.name=airbyte/destination-rockset diff --git a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java index 909194580404..01e1837b7992 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/test/java/io/airbyte/integrations/source/jdbc/AbstractJdbcSourceAcceptanceTest.java @@ -15,8 +15,14 @@ import io.airbyte.integrations.base.IntegrationRunner; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.protocol.models.AirbyteGlobalState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.AirbyteStreamState; import io.airbyte.test.utils.PostgreSQLContainerHelper; import java.sql.JDBCType; +import java.util.List; import java.util.Set; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -82,6 +88,11 @@ public String getDriverClass() { return PostgresTestSource.DRIVER_CLASS; } + @Override + protected boolean supportsPerStream() { + return true; + } + @AfterAll static void cleanUp() { PSQL_DB.close(); @@ -118,6 +129,27 @@ public Set getExcludedInternalNameSpaces() { return Set.of("information_schema", "pg_catalog", "pg_internal", "catalog_history"); } + // TODO This is a temporary override so that the Postgres source can take advantage of per-stream + // state + @Override + protected List generateEmptyInitialState(final JsonNode config) { + if (getSupportedStateType(config) == AirbyteStateType.GLOBAL) { + final AirbyteGlobalState globalState = new AirbyteGlobalState() + .withSharedState(Jsons.jsonNode(new CdcState())) + .withStreamStates(List.of()); + return List.of(new AirbyteStateMessage().withStateType(AirbyteStateType.GLOBAL).withGlobal(globalState)); + } else { + return List.of(new AirbyteStateMessage() + .withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState())); + } + } + + @Override + protected AirbyteStateType getSupportedStateType(final JsonNode config) { + return AirbyteStateType.STREAM; + } + public static void main(final String[] args) throws Exception { final Source source = new PostgresTestSource(); LOGGER.info("starting source: {}", PostgresTestSource.class); diff --git a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java index 802d8ac79bc7..74d8d7add0af 100644 --- a/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-jdbc/src/testFixtures/java/io/airbyte/integrations/source/jdbc/test/JdbcSourceAcceptanceTest.java @@ -13,10 +13,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Lists; import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; import io.airbyte.commons.json.Jsons; import io.airbyte.commons.resources.MoreResources; @@ -39,7 +35,9 @@ import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.AirbyteStreamState; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.ConfiguredAirbyteStream; @@ -47,6 +45,7 @@ import io.airbyte.protocol.models.DestinationSyncMode; import io.airbyte.protocol.models.Field; import io.airbyte.protocol.models.JsonSchemaType; +import io.airbyte.protocol.models.StreamDescriptor; import io.airbyte.protocol.models.SyncMode; import java.math.BigDecimal; import java.sql.SQLException; @@ -54,6 +53,7 @@ import java.util.Collections; import java.util.Comparator; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.function.Function; @@ -82,7 +82,7 @@ public abstract class JdbcSourceAcceptanceTest { // otherwise parallel runs can interfere with each other public static String SCHEMA_NAME = Strings.addRandomSuffix("jdbc_integration_test1", "_", 5).toLowerCase(); public static String SCHEMA_NAME2 = Strings.addRandomSuffix("jdbc_integration_test2", "_", 5).toLowerCase(); - public static Set TEST_SCHEMAS = ImmutableSet.of(SCHEMA_NAME, SCHEMA_NAME2); + public static Set TEST_SCHEMAS = Set.of(SCHEMA_NAME, SCHEMA_NAME2); public static String TABLE_NAME = "id_and_name"; public static String TABLE_NAME_WITH_SPACES = "id and name"; @@ -255,7 +255,7 @@ public void setup() throws Exception { connection.createStatement().execute( createTableQuery(getFullyQualifiedTableName(TABLE_NAME_COMPOSITE_PK), COLUMN_CLAUSE_WITH_COMPOSITE_PK, - primaryKeyClause(ImmutableList.of("first_name", "last_name")))); + primaryKeyClause(List.of("first_name", "last_name")))); connection.createStatement().execute( String.format( "INSERT INTO %s(first_name, last_name, updated_at) VALUES ('first' ,'picard', '2004-10-19')", @@ -354,12 +354,15 @@ void testDiscoverWithMultipleSchemas() throws Exception { final AirbyteCatalog actual = source.discover(config); final AirbyteCatalog expected = getCatalog(getDefaultNamespace()); - expected.getStreams().add(CatalogHelpers + final List catalogStreams = new ArrayList<>(); + catalogStreams.addAll(expected.getStreams()); + catalogStreams.add(CatalogHelpers .createAirbyteStream(TABLE_NAME, SCHEMA_NAME2, Field.of(COL_ID, JsonSchemaType.STRING), Field.of(COL_NAME, JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))); + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL))); + expected.setStreams(catalogStreams); // sort streams by name so that we are comparing lists with the same order. final Comparator schemaTableCompare = Comparator.comparing(stream -> stream.getNamespace() + "." + stream.getName()); expected.getStreams().sort(schemaTableCompare); @@ -389,9 +392,8 @@ void testReadOneColumn() throws Exception { setEmittedAtToNull(actualMessages); final List expectedMessages = getAirbyteMessagesReadOneColumn(); - assertTrue(expectedMessages.size() == actualMessages.size()); - assertTrue(expectedMessages.containsAll(actualMessages)); - assertTrue(actualMessages.containsAll(expectedMessages)); + assertEquals(expectedMessages.size(), actualMessages.size()); + assertEquals(expectedMessages, actualMessages); } protected List getAirbyteMessagesReadOneColumn() { @@ -437,8 +439,7 @@ void testReadMultipleTables() throws Exception { Field.of(COL_ID, JsonSchemaType.NUMBER), Field.of(COL_NAME, JsonSchemaType.STRING))); - final List secondStreamExpectedMessages = getAirbyteMessagesSecondSync(streamName2); - expectedMessages.addAll(secondStreamExpectedMessages); + expectedMessages.addAll(getAirbyteMessagesSecondSync(streamName2)); } final List actualMessages = MoreIterators @@ -446,12 +447,11 @@ void testReadMultipleTables() throws Exception { setEmittedAtToNull(actualMessages); - assertTrue(expectedMessages.size() == actualMessages.size()); - assertTrue(expectedMessages.containsAll(actualMessages)); - assertTrue(actualMessages.containsAll(expectedMessages)); + assertEquals(expectedMessages.size(), actualMessages.size()); + assertEquals(expectedMessages, actualMessages); } - protected List getAirbyteMessagesSecondSync(String streamName2) { + protected List getAirbyteMessagesSecondSync(final String streamName2) { return getTestMessages() .stream() .map(Jsons::clone) @@ -471,7 +471,7 @@ void testTablesWithQuoting() throws Exception { final ConfiguredAirbyteStream streamForTableWithSpaces = createTableWithSpaces(); final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() - .withStreams(Lists.newArrayList( + .withStreams(List.of( getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0), streamForTableWithSpaces)); final List actualMessages = MoreIterators @@ -479,16 +479,14 @@ void testTablesWithQuoting() throws Exception { setEmittedAtToNull(actualMessages); - final List secondStreamExpectedMessages = getAirbyteMessagesForTablesWithQuoting(streamForTableWithSpaces); final List expectedMessages = new ArrayList<>(getTestMessages()); - expectedMessages.addAll(secondStreamExpectedMessages); + expectedMessages.addAll(getAirbyteMessagesForTablesWithQuoting(streamForTableWithSpaces)); - assertTrue(expectedMessages.size() == actualMessages.size()); - assertTrue(expectedMessages.containsAll(actualMessages)); - assertTrue(actualMessages.containsAll(expectedMessages)); + assertEquals(expectedMessages.size(), actualMessages.size()); + assertEquals(expectedMessages, actualMessages); } - protected List getAirbyteMessagesForTablesWithQuoting(ConfiguredAirbyteStream streamForTableWithSpaces) { + protected List getAirbyteMessagesForTablesWithQuoting(final ConfiguredAirbyteStream streamForTableWithSpaces) { return getTestMessages() .stream() .map(Jsons::clone) @@ -509,7 +507,7 @@ void testReadFailure() { final ConfiguredAirbyteStream spiedAbStream = spy( getConfiguredCatalogWithOneStream(getDefaultNamespace()).getStreams().get(0)); final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() - .withStreams(Lists.newArrayList(spiedAbStream)); + .withStreams(List.of(spiedAbStream)); doCallRealMethod().doThrow(new RuntimeException()).when(spiedAbStream).getStream(); assertThrows(RuntimeException.class, () -> source.read(config, catalog, null)); @@ -521,7 +519,7 @@ void testIncrementalNoPreviousState() throws Exception { COL_ID, null, "3", - Lists.newArrayList(getTestMessages())); + getTestMessages()); } @Test @@ -530,7 +528,7 @@ void testIncrementalIntCheckCursor() throws Exception { COL_ID, "2", "3", - Lists.newArrayList(getTestMessages().get(2))); + List.of(getTestMessages().get(2))); } @Test @@ -539,14 +537,14 @@ void testIncrementalStringCheckCursor() throws Exception { COL_NAME, "patent", "vash", - Lists.newArrayList(getTestMessages().get(0), getTestMessages().get(2))); + List.of(getTestMessages().get(0), getTestMessages().get(2))); } @Test void testIncrementalStringCheckCursorSpaceInColumnName() throws Exception { final ConfiguredAirbyteStream streamWithSpaces = createTableWithSpaces(); - final ArrayList expectedRecordMessages = getAirbyteMessagesCheckCursorSpaceInColumnName(streamWithSpaces); + final List expectedRecordMessages = getAirbyteMessagesCheckCursorSpaceInColumnName(streamWithSpaces); incrementalCursorCheck( COL_LAST_NAME_WITH_SPACE, COL_LAST_NAME_WITH_SPACE, @@ -556,7 +554,7 @@ void testIncrementalStringCheckCursorSpaceInColumnName() throws Exception { streamWithSpaces); } - protected ArrayList getAirbyteMessagesCheckCursorSpaceInColumnName(ConfiguredAirbyteStream streamWithSpaces) { + protected List getAirbyteMessagesCheckCursorSpaceInColumnName(final ConfiguredAirbyteStream streamWithSpaces) { final AirbyteMessage firstMessage = getTestMessages().get(0); firstMessage.getRecord().setStream(streamWithSpaces.getStream().getName()); ((ObjectNode) firstMessage.getRecord().getData()).remove(COL_UPDATED_AT); @@ -569,9 +567,7 @@ protected ArrayList getAirbyteMessagesCheckCursorSpaceInColumnNa ((ObjectNode) secondMessage.getRecord().getData()).set(COL_LAST_NAME_WITH_SPACE, ((ObjectNode) secondMessage.getRecord().getData()).remove(COL_NAME)); - Lists.newArrayList(getTestMessages().get(0), getTestMessages().get(2)); - - return Lists.newArrayList(firstMessage, secondMessage); + return List.of(firstMessage, secondMessage); } @Test @@ -584,7 +580,7 @@ protected void incrementalDateCheck() throws Exception { COL_UPDATED_AT, "2005-10-18T00:00:00Z", "2006-10-19T00:00:00Z", - Lists.newArrayList(getTestMessages().get(1), getTestMessages().get(2))); + List.of(getTestMessages().get(1), getTestMessages().get(2))); } @Test @@ -597,7 +593,7 @@ void testIncrementalCursorChanges() throws Exception { // records to (incorrectly) be filtered out. "data", "vash", - Lists.newArrayList(getTestMessages())); + getTestMessages()); } @Test @@ -606,14 +602,12 @@ void testReadOneTableIncrementallyTwice() throws Exception { final ConfiguredAirbyteCatalog configuredCatalog = getConfiguredCatalogWithOneStream(namespace); configuredCatalog.getStreams().forEach(airbyteStream -> { airbyteStream.setSyncMode(SyncMode.INCREMENTAL); - airbyteStream.setCursorField(Lists.newArrayList(COL_ID)); + airbyteStream.setCursorField(List.of(COL_ID)); airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); }); - final DbState state = new DbState() - .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source.read(config, configuredCatalog, createEmptyState(streamName, namespace))); final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() .filter(r -> r.getType() == Type.STATE).findFirst(); @@ -622,8 +616,7 @@ void testReadOneTableIncrementallyTwice() throws Exception { executeStatementReadIncrementallyTwice(); final List actualMessagesSecondSync = MoreIterators - .toList(source.read(config, configuredCatalog, - stateAfterFirstSyncOptional.get().getState().getData())); + .toList(source.read(config, configuredCatalog, extractState(stateAfterFirstSyncOptional.get()))); assertEquals(2, (int) actualMessagesSecondSync.stream().filter(r -> r.getType() == Type.RECORD).count()); @@ -631,9 +624,8 @@ void testReadOneTableIncrementallyTwice() throws Exception { setEmittedAtToNull(actualMessagesSecondSync); - assertTrue(expectedMessages.size() == actualMessagesSecondSync.size()); - assertTrue(expectedMessages.containsAll(actualMessagesSecondSync)); - assertTrue(actualMessagesSecondSync.containsAll(expectedMessages)); + assertEquals(expectedMessages.size(), actualMessagesSecondSync.size()); + assertEquals(expectedMessages, actualMessagesSecondSync); } protected void executeStatementReadIncrementallyTwice() throws SQLException { @@ -647,30 +639,26 @@ protected void executeStatementReadIncrementallyTwice() throws SQLException { }); } - protected List getExpectedAirbyteMessagesSecondSync(String namespace) { + protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_4, COL_NAME, "riker", COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); expectedMessages.add(new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_5, COL_NAME, "data", COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); - expectedMessages.add(new AirbyteMessage() - .withType(Type.STATE) - .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(new DbState() - .withCdc(false) - .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(ImmutableList.of(COL_ID)) - .withCursor("5"))))))); + final DbStreamState state = new DbStreamState() + .withStreamName(streamName) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("5"); + expectedMessages.addAll(createExpectedTestMessages(List.of(state))); return expectedMessages; } @@ -702,14 +690,12 @@ void testReadMultipleTablesIncrementally() throws Exception { Field.of(COL_NAME, JsonSchemaType.STRING))); configuredCatalog.getStreams().forEach(airbyteStream -> { airbyteStream.setSyncMode(SyncMode.INCREMENTAL); - airbyteStream.setCursorField(Lists.newArrayList(COL_ID)); + airbyteStream.setCursorField(List.of(COL_ID)); airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); }); - final DbState state = new DbState() - .withStreams(Lists.newArrayList(new DbStreamState().withStreamName(streamName).withStreamNamespace(namespace))); final List actualMessagesFirstSync = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source.read(config, configuredCatalog, createEmptyState(streamName, namespace))); // get last state message. final Optional stateAfterFirstSyncOptional = actualMessagesFirstSync.stream() @@ -720,49 +706,44 @@ void testReadMultipleTablesIncrementally() throws Exception { // we know the second streams messages are the same as the first minus the updated at column. so we // cheat and generate the expected messages off of the first expected messages. final List secondStreamExpectedMessages = getAirbyteMessagesSecondStreamWithNamespace(streamName2); - final List expectedMessagesFirstSync = new ArrayList<>(getTestMessages()); - expectedMessagesFirstSync.add(new AirbyteMessage() - .withType(Type.STATE) - .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(new DbState() - .withCdc(false) - .withStreams(Lists.newArrayList( - new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(ImmutableList.of(COL_ID)) - .withCursor("3"), - new DbStreamState() - .withStreamName(streamName2) - .withStreamNamespace(namespace) - .withCursorField(ImmutableList.of(COL_ID)))))))); + // Represents the state after the first stream has been updated + final List expectedStateStreams1 = List.of( + new DbStreamState() + .withStreamName(streamName) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("3"), + new DbStreamState() + .withStreamName(streamName2) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID))); + + // Represents the state after both streams have been updated + final List expectedStateStreams2 = List.of( + new DbStreamState() + .withStreamName(streamName) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("3"), + new DbStreamState() + .withStreamName(streamName2) + .withStreamNamespace(namespace) + .withCursorField(List.of(COL_ID)) + .withCursor("3")); + + final List expectedMessagesFirstSync = new ArrayList<>(getTestMessages()); + expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams1.get(0), expectedStateStreams1)); expectedMessagesFirstSync.addAll(secondStreamExpectedMessages); - expectedMessagesFirstSync.add(new AirbyteMessage() - .withType(Type.STATE) - .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(new DbState() - .withCdc(false) - .withStreams(Lists.newArrayList( - new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(ImmutableList.of(COL_ID)) - .withCursor("3"), - new DbStreamState() - .withStreamName(streamName2) - .withStreamNamespace(namespace) - .withCursorField(ImmutableList.of(COL_ID)) - .withCursor("3"))))))); + expectedMessagesFirstSync.add(createStateMessage(expectedStateStreams2.get(1), expectedStateStreams2)); setEmittedAtToNull(actualMessagesFirstSync); - assertTrue(expectedMessagesFirstSync.size() == actualMessagesFirstSync.size()); - assertTrue(expectedMessagesFirstSync.containsAll(actualMessagesFirstSync)); - assertTrue(actualMessagesFirstSync.containsAll(expectedMessagesFirstSync)); + assertEquals(expectedMessagesFirstSync.size(), actualMessagesFirstSync.size()); + assertEquals(expectedMessagesFirstSync, actualMessagesFirstSync); } - protected List getAirbyteMessagesSecondStreamWithNamespace(String streamName2) { + protected List getAirbyteMessagesSecondStreamWithNamespace(final String streamName2) { return getTestMessages() .stream() .map(Jsons::clone) @@ -807,39 +788,34 @@ private void incrementalCursorCheck( final ConfiguredAirbyteStream airbyteStream) throws Exception { airbyteStream.setSyncMode(SyncMode.INCREMENTAL); - airbyteStream.setCursorField(Lists.newArrayList(cursorField)); + airbyteStream.setCursorField(List.of(cursorField)); airbyteStream.setDestinationSyncMode(DestinationSyncMode.APPEND); - final DbState state = new DbState() - .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(airbyteStream.getStream().getName()) - .withStreamNamespace(airbyteStream.getStream().getNamespace()) - .withCursorField(ImmutableList.of(initialCursorField)) - .withCursor(initialCursorValue))); - final ConfiguredAirbyteCatalog configuredCatalog = new ConfiguredAirbyteCatalog() - .withStreams(ImmutableList.of(airbyteStream)); + .withStreams(List.of(airbyteStream)); + + final DbStreamState dbStreamState = new DbStreamState() + .withStreamName(airbyteStream.getStream().getName()) + .withStreamNamespace(airbyteStream.getStream().getNamespace()) + .withCursorField(List.of(initialCursorField)) + .withCursor(initialCursorValue); final List actualMessages = MoreIterators - .toList(source.read(config, configuredCatalog, Jsons.jsonNode(state))); + .toList(source.read(config, configuredCatalog, Jsons.jsonNode(createState(List.of(dbStreamState))))); setEmittedAtToNull(actualMessages); + final List expectedStreams = List.of( + new DbStreamState() + .withStreamName(airbyteStream.getStream().getName()) + .withStreamNamespace(airbyteStream.getStream().getNamespace()) + .withCursorField(List.of(cursorField)) + .withCursor(endCursorValue)); final List expectedMessages = new ArrayList<>(expectedRecordMessages); - expectedMessages.add(new AirbyteMessage() - .withType(Type.STATE) - .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(new DbState() - .withCdc(false) - .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(airbyteStream.getStream().getName()) - .withStreamNamespace(airbyteStream.getStream().getNamespace()) - .withCursorField(ImmutableList.of(cursorField)) - .withCursor(endCursorValue))))))); - - assertTrue(expectedMessages.size() == actualMessages.size()); - assertTrue(expectedMessages.containsAll(actualMessages)); - assertTrue(actualMessages.containsAll(expectedMessages)); + expectedMessages.addAll(createExpectedTestMessages(expectedStreams)); + + assertEquals(expectedMessages.size(), actualMessages.size()); + assertEquals(expectedMessages, actualMessages); } // get catalog and perform a defensive copy. @@ -853,14 +829,14 @@ protected ConfiguredAirbyteCatalog getConfiguredCatalogWithOneStream(final Strin } protected AirbyteCatalog getCatalog(final String defaultNamespace) { - return new AirbyteCatalog().withStreams(Lists.newArrayList( + return new AirbyteCatalog().withStreams(List.of( CatalogHelpers.createAirbyteStream( TABLE_NAME, defaultNamespace, Field.of(COL_ID, JsonSchemaType.NUMBER), Field.of(COL_NAME, JsonSchemaType.STRING), Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(List.of(List.of(COL_ID))), CatalogHelpers.createAirbyteStream( TABLE_NAME_WITHOUT_PK, @@ -868,7 +844,7 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { Field.of(COL_ID, JsonSchemaType.NUMBER), Field.of(COL_NAME, JsonSchemaType.STRING), Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey(Collections.emptyList()), CatalogHelpers.createAirbyteStream( TABLE_NAME_COMPOSITE_PK, @@ -876,34 +852,62 @@ protected AirbyteCatalog getCatalog(final String defaultNamespace) { Field.of(COL_FIRST_NAME, JsonSchemaType.STRING), Field.of(COL_LAST_NAME, JsonSchemaType.STRING), Field.of(COL_UPDATED_AT, JsonSchemaType.STRING)) - .withSupportedSyncModes(Lists.newArrayList(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) + .withSupportedSyncModes(List.of(SyncMode.FULL_REFRESH, SyncMode.INCREMENTAL)) .withSourceDefinedPrimaryKey( List.of(List.of(COL_FIRST_NAME), List.of(COL_LAST_NAME))))); } protected List getTestMessages() { - return Lists.newArrayList( + return List.of( new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_1, COL_NAME, "picard", COL_UPDATED_AT, "2004-10-19T00:00:00Z")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_2, COL_NAME, "crusher", COL_UPDATED_AT, "2005-10-19T00:00:00Z")))), new AirbyteMessage().withType(Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(getDefaultNamespace()) - .withData(Jsons.jsonNode(ImmutableMap + .withData(Jsons.jsonNode(Map .of(COL_ID, ID_VALUE_3, COL_NAME, "vash", COL_UPDATED_AT, "2006-10-19T00:00:00Z"))))); } + protected List createExpectedTestMessages(final List states) { + return supportsPerStream() + ? states.stream() + .map(s -> new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s))) + .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(states))))) + .collect( + Collectors.toList()) + : List.of(new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withStateType(AirbyteStateType.LEGACY) + .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(states))))); + } + + protected List createState(final List states) { + return supportsPerStream() + ? states.stream() + .map(s -> new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s)))) + .collect( + Collectors.toList()) + : List.of(new AirbyteStateMessage().withStateType(AirbyteStateType.LEGACY).withData(Jsons.jsonNode(new DbState().withStreams(states)))); + } + protected ConfiguredAirbyteStream createTableWithSpaces() throws SQLException { final String tableNameWithSpaces = TABLE_NAME_WITH_SPACES + "2"; final String streamName2 = tableNameWithSpaces; @@ -994,4 +998,67 @@ protected static void setEmittedAtToNull(final Iterable messages } } + /** + * Tests whether the connector under test supports the per-stream state format or should use the + * legacy format for data generated by this test. + * + * @return {@code true} if the connector supports the per-stream state format or {@code false} if it + * does not support the per-stream state format (e.g. legacy format supported). Default + * value is {@code false}. + */ + protected boolean supportsPerStream() { + return false; + } + + /** + * Creates empty state with the provided stream name and namespace. + * + * @param streamName The stream name. + * @param streamNamespace The stream namespace. + * @return {@link JsonNode} representation of the generated empty state. + */ + protected JsonNode createEmptyState(final String streamName, final String streamNamespace) { + if (supportsPerStream()) { + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() + .withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(streamName).withNamespace(streamNamespace))); + return Jsons.jsonNode(List.of(airbyteStateMessage)); + } else { + final DbState dbState = new DbState() + .withStreams(List.of(new DbStreamState().withStreamName(streamName).withStreamNamespace(streamNamespace))); + return Jsons.jsonNode(dbState); + } + } + + /** + * Extracts the state component from the provided {@link AirbyteMessage} based on the value returned + * by {@link #supportsPerStream()}. + * + * @param airbyteMessage An {@link AirbyteMessage} that contains state. + * @return A {@link JsonNode} representation of the state contained in the {@link AirbyteMessage}. + */ + protected JsonNode extractState(final AirbyteMessage airbyteMessage) { + if (supportsPerStream()) { + return Jsons.jsonNode(List.of(airbyteMessage.getState())); + } else { + return airbyteMessage.getState().getData(); + } + } + + protected AirbyteMessage createStateMessage(final DbStreamState dbStreamState, final List legacyStates) { + if (supportsPerStream()) { + return new AirbyteMessage().withType(Type.STATE) + .withState( + new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(dbStreamState.getStreamNamespace()) + .withName(dbStreamState.getStreamName())) + .withStreamState(Jsons.jsonNode(dbStreamState))) + .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(legacyStates)))); + } else { + return new AirbyteMessage().withType(Type.STATE).withState(new AirbyteStateMessage().withStateType(AirbyteStateType.LEGACY) + .withData(Jsons.jsonNode(new DbState().withCdc(false).withStreams(legacyStates)))); + } + } + } diff --git a/airbyte-integrations/connectors/source-marketo/Dockerfile b/airbyte-integrations/connectors/source-marketo/Dockerfile index 83d335c1d40b..b84ad46b586f 100644 --- a/airbyte-integrations/connectors/source-marketo/Dockerfile +++ b/airbyte-integrations/connectors/source-marketo/Dockerfile @@ -34,5 +34,5 @@ COPY source_marketo ./source_marketo ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-marketo diff --git a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml index 4fe9da86cfff..583c5ba56660 100644 --- a/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-marketo/acceptance-test-config.yml @@ -14,7 +14,7 @@ tests: basic_read: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" - empty_streams: [] + empty_streams: ["activities_visit_webpage"] timeout_seconds: 3600 expect_records: path: "integration_tests/expected_records.txt" diff --git a/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.txt b/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.txt index 90bdb0ecdb91..81f2152caff2 100644 --- a/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.txt +++ b/airbyte-integrations/connectors/source-marketo/integration_tests/expected_records.txt @@ -1,11 +1,11 @@ -{"stream": "programs", "data": {"id": 1016, "name": "123", "description": "", "createdAt": "2021-09-01T16:02:30Z", "updatedAt": "2021-09-01T16:06:57Z", "url": "https://app-sj32.marketo.com/#EBP1016A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "unlocked", "workspace": "Default"}, "emitted_at": 1638527519000} -{"stream": "programs", "data": {"id": 1017, "name": "air", "description": "", "createdAt": "2021-09-01T16:09:23Z", "updatedAt": "2021-09-01T16:09:23Z", "url": "https://app-sj32.marketo.com/#EBP1017A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "unlocked", "workspace": "Default"}, "emitted_at": 1638527519000} -{"stream": "programs", "data": {"id": 1003, "name": "API Test Program", "description": "Sample API Program", "createdAt": "2021-01-18T13:55:44Z", "updatedAt": "2021-09-01T16:19:32Z", "url": "https://app-sj32.marketo.com/#PG1003A1", "type": "Default", "channel": "Online Advertising", "folder": {"type": "Folder", "value": 45, "folderName": "Active Marketing Programs"}, "status": "", "workspace": "Default"}, "emitted_at": 1638527519000} -{"stream": "programs", "data": {"id": 1018, "name": "Jean Lafleur", "description": "", "createdAt": "2021-09-08T12:49:49Z", "updatedAt": "2021-09-08T12:49:49Z", "url": "https://app-sj32.marketo.com/#PG1018A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default"}, "emitted_at": 1638527519000} -{"stream": "programs", "data": {"id": 1019, "name": "Test", "description": "", "createdAt": "2021-09-08T12:59:25Z", "updatedAt": "2021-09-08T12:59:25Z", "url": "https://app-sj32.marketo.com/#PG1019A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default"}, "emitted_at": 1638527519000} -{"stream": "programs", "data": {"id": 1020, "name": "TEST1", "description": "", "createdAt": "2021-09-08T13:21:41Z", "updatedAt": "2021-09-08T13:21:41Z", "url": "https://app-sj32.marketo.com/#PG1020A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default"}, "emitted_at": 1638527519000} -{"stream": "programs", "data": {"id": 1021, "name": "TEST_23", "description": "This is for Test", "createdAt": "2021-09-09T09:00:21Z", "updatedAt": "2021-09-09T09:00:22Z", "url": "https://app-sj32.marketo.com/#PG1021A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default"}, "emitted_at": 1638527519000} -{"stream": "programs", "data": {"id": 1022, "name": "Test_Demo1", "description": "bla bla", "createdAt": "2021-09-09T14:40:14Z", "updatedAt": "2021-09-09T14:40:14Z", "url": "https://app-sj32.marketo.com/#PG1022A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default"}, "emitted_at": 1638527519000} +{"stream": "programs", "data": {"id": 1016, "name": "123", "description": "", "createdAt": "2021-09-01T16:02:30Z", "updatedAt": "2022-06-21T06:50:32Z", "url": "https://app-sj32.marketo.com/#EBP1016A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "locked", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476224} +{"stream": "programs", "data": {"id": 1017, "name": "air", "description": "", "createdAt": "2021-09-01T16:09:23Z", "updatedAt": "2022-06-21T06:51:01Z", "url": "https://app-sj32.marketo.com/#EBP1017A1", "type": "Email", "channel": "Email Send", "folder": {"type": "Program", "value": 1003, "folderName": "API Test Program"}, "status": "locked", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476226} +{"stream": "programs", "data": {"id": 1003, "name": "API Test Program", "description": "Sample API Program", "createdAt": "2021-01-18T13:55:44Z", "updatedAt": "2022-06-21T06:54:59Z", "url": "https://app-sj32.marketo.com/#PG1003A1", "type": "Default", "channel": "Email Blast", "folder": {"type": "Folder", "value": 45, "folderName": "Active Marketing Programs"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476226} +{"stream": "programs", "data": {"id": 1018, "name": "Jean Lafleur", "description": "", "createdAt": "2021-09-08T12:49:49Z", "updatedAt": "2022-06-21T06:53:28Z", "url": "https://app-sj32.marketo.com/#PG1018A1", "type": "Default", "channel": "Online Advertising", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} +{"stream": "programs", "data": {"id": 1019, "name": "Test", "description": "", "createdAt": "2021-09-08T12:59:25Z", "updatedAt": "2022-06-21T06:53:45Z", "url": "https://app-sj32.marketo.com/#PG1019A1", "type": "Default", "channel": "List Import", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} +{"stream": "programs", "data": {"id": 1020, "name": "TEST1", "description": "", "createdAt": "2021-09-08T13:21:41Z", "updatedAt": "2022-06-21T06:54:03Z", "url": "https://app-sj32.marketo.com/#PG1020A1", "type": "Default", "channel": "Operational", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} +{"stream": "programs", "data": {"id": 1021, "name": "TEST_23", "description": "This is for Test", "createdAt": "2021-09-09T09:00:21Z", "updatedAt": "2022-06-21T06:54:16Z", "url": "https://app-sj32.marketo.com/#PG1021A1", "type": "Default", "channel": "Web Content", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} +{"stream": "programs", "data": {"id": 1022, "name": "Test_Demo1", "description": "bla bla", "createdAt": "2021-09-09T14:40:14Z", "updatedAt": "2022-06-21T06:54:29Z", "url": "https://app-sj32.marketo.com/#PG1022A1", "type": "Default", "channel": "Web Request", "folder": {"type": "Folder", "value": 52, "folderName": "Web Forms"}, "status": "", "workspace": "Default", "headStart": false}, "emitted_at": 1655800476227} {"stream": "campaigns", "data": {"id": 1019, "name": "Form Smart Campaign", "type": "trigger", "programName": "Form Program", "programId": 1002, "workspaceName": "Default", "createdAt": "2020-04-09T20:18:24Z", "updatedAt": "2020-10-22T09:03:44Z", "active": false}, "emitted_at": 1638527708000} {"stream": "campaigns", "data": {"id": 1020, "name": "Smart Campaign number 02", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:37:24Z", "updatedAt": "2021-01-19T22:50:17Z", "active": false}, "emitted_at": 1638527708000} {"stream": "campaigns", "data": {"id": 1021, "name": "Smart Campaign 03", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:38:53Z", "updatedAt": "2021-01-18T13:38:53Z", "active": false}, "emitted_at": 1638527708000} @@ -19,7 +19,7 @@ {"stream": "campaigns", "data": {"id": 1029, "name": "Smart Campaign Number 8", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:48Z", "updatedAt": "2021-01-18T13:48:48Z", "active": false}, "emitted_at": 1638527708000} {"stream": "campaigns", "data": {"id": 1030, "name": "Smart Campaign Number 9", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:49Z", "updatedAt": "2021-01-18T13:48:49Z", "active": false}, "emitted_at": 1638527708000} {"stream": "campaigns", "data": {"id": 1031, "name": "Smart Campaign Number 10", "description": "This is a smart campaign creation test.", "type": "batch", "workspaceName": "Default", "createdAt": "2021-01-18T13:48:50Z", "updatedAt": "2021-01-18T13:48:50Z", "active": false}, "emitted_at": 1638527708000} -{"stream": "lists", "data": {"id": 1001, "name": "Test list", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:27:23Z", "updatedAt": "2021-01-19T20:27:24Z"}, "emitted_at": 1638527852000} +{"stream": "lists", "data": {"id": 1001, "name": "Test list", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:27:23Z", "updatedAt": "2022-06-21T06:58:01Z"}, "emitted_at": 1638527852000} {"stream": "lists", "data": {"id": 1002, "name": "Test list number 1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:00Z", "updatedAt": "2021-01-19T21:55:54Z"}, "emitted_at": 1638527852000} {"stream": "lists", "data": {"id": 1003, "name": "Test list number 2", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:07Z", "updatedAt": "2021-01-19T20:28:09Z"}, "emitted_at": 1638527852000} {"stream": "lists", "data": {"id": 1004, "name": "Test list number 3", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-01-19T20:28:13Z", "updatedAt": "2021-01-19T20:28:15Z"}, "emitted_at": 1638527852000} @@ -34,10 +34,8 @@ {"stream": "lists", "data": {"id": 1012, "name": "airbyte", "programName": "EM - Auteur - v1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-09-02T09:30:58Z", "updatedAt": "2021-09-02T09:30:59Z"}, "emitted_at": 1638527852000} {"stream": "lists", "data": {"id": 1012, "name": "airbyte", "programName": "EM - Auteur - v1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-09-02T09:30:58Z", "updatedAt": "2021-09-02T09:30:59Z"}, "emitted_at": 1638527852000} {"stream": "lists", "data": {"id": 1012, "name": "airbyte", "programName": "EM - Auteur - v1", "workspaceId": 1, "workspaceName": "Default", "createdAt": "2021-09-02T09:30:58Z", "updatedAt": "2021-09-02T09:30:59Z"}, "emitted_at": 1638527853000} -{"stream": "leads", "data": {"company": null, "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 863, "mktoName": "Test-1", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Test-1", "middleName": null, "lastName": null, "email": "test-1@test.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "77", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": "00000", "personTimeZone": null, "originalSourceType": "Web service API", "originalSourceInfo": "Web service API", "registrationSourceType": "Web service API", "registrationSourceInfo": "Web service API", "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2021-08-23T12:35:27Z", "updatedAt": "2021-08-23T12:35:27Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "863", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "863", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1638529087000} -{"stream": "leads", "data": {"company": "Airbyte", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 864, "mktoName": "yuriiyurii", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "yuriiyurii", "middleName": null, "lastName": null, "email": "integration-test@airbyte.io", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "78", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": "http://mkto-sj320154.com/u/NjAyLUVVTy01OTgAAAF_QLVQN_CmMgjmeDlv2KOH8SvdmQFkcr5E7bB6_u9nyy4qyi8TLSRagKEl2yDz4A8JdOXvOps=", "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": true, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2021-09-01T14:09:58Z", "updatedAt": "2021-09-01T14:47:26Z", "cookies": "_mch-marketo.com-1630506111294-76141,_mch-marketo.com-1630507625996-85446,_mch-marketo.com-1630509534684-98098,_mch-marketo.com-1630509805945-33648,_mch-marketo.com-1630514099902-54557", "externalSalesPersonId": null, "leadPerson": "864", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "864", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1638529087000} -{"stream": "leads", "data": {"company": "Airbyte", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": "airbyte.io", "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 866, "mktoName": "yurii yurii", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "yurii", "middleName": null, "lastName": "yurii", "email": "integration-test@airbyte.io", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "79", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": "http://na-sj32.marketo.com/lp/datalineaedev/UnsubscribePage.html?mkt_unsubscribe=1&mkt_tok=NjAyLUVVTy01OTgAAAF_QLVRDCgLykiaUiUq2HHzdAieIK6v1qqh8ssBkS0UG5PAMCUj-e56dwddm82ciLtx9jCsvAndW4xV5GaiveYVSKEql_F4eao37V3Za92pqCFJOV9sXpl69DnXdozZk1WLLGBcUtTujEgBGL87", "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": true, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": "93.177.75.198", "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2021-09-01T14:38:02Z", "updatedAt": "2021-09-01T14:47:37Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "866", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "866", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1638529087000} -{"stream": "leads", "data": {"company": "Airbyte", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 867, "mktoName": "Yurii Yurii", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Yurii", "middleName": null, "lastName": "Yurii", "email": "yurii.cherniaiev@globallogic.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "80", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2021-09-01T15:21:44Z", "updatedAt": "2021-09-01T15:21:44Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "867", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "867", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1638529087000} -{"stream": "leads", "data": {"company": "Airbyte", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 868, "mktoName": "Yurii Yurii", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Yurii", "middleName": null, "lastName": "Yurii", "email": "yurii.cherniaiev@globallogic.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "81", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2021-09-01T15:22:28Z", "updatedAt": "2021-09-01T15:22:28Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "868", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "868", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1638529087000} -{"stream": "leads", "data": {"company": "Airbyte", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 869, "mktoName": "Yurii Yurii", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Yurii", "middleName": null, "lastName": "Yurii", "email": "yurii.chenriaiev@globallogic.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "82", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2021-09-01T15:23:07Z", "updatedAt": "2021-09-01T15:23:07Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "869", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "869", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1638529087000} -{"stream": "leads", "data": {"company": null, "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 875, "mktoName": "TEST-1-1", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "TEST-1-1", "middleName": null, "lastName": null, "email": "test-test-test@test.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "83", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": "1111", "personTimeZone": null, "originalSourceType": "Web service API", "originalSourceInfo": "Web service API", "registrationSourceType": "Web service API", "registrationSourceInfo": "Web service API", "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2021-11-08T22:03:32Z", "updatedAt": "2021-11-08T22:03:32Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "875", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "875", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1638529399000} +{"stream": "leads", "data": {"company": "Airbyte", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 876, "mktoName": "Expecto Patronum", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Expecto", "middleName": null, "lastName": "Patronum", "email": "expecto@patronum.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "84", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T07:49:25Z", "updatedAt": "2022-06-21T07:50:05Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "876", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "876", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613397} +{"stream": "leads", "data": {"company": "FedEx", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 877, "mktoName": "Frodo Baggins", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Frodo", "middleName": null, "lastName": "Baggins", "email": "frodo@baggins.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "85", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:30:55Z", "updatedAt": "2022-06-21T08:30:55Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "877", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "877", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613399} +{"stream": "leads", "data": {"company": "PizzaHouse", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 878, "mktoName": "Peter Petegrew", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Peter", "middleName": null, "lastName": "Petegrew", "email": "peter@petegrew.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "86", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:31:42Z", "updatedAt": "2022-06-21T08:31:42Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "878", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "878", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613400} +{"stream": "leads", "data": {"company": "SportLife", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 879, "mktoName": "Dudley Dursley", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Dudley", "middleName": null, "lastName": "Dursley", "email": "dudley@dursley.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "87", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:32:37Z", "updatedAt": "2022-06-21T08:32:37Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "879", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "879", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613400} +{"stream": "leads", "data": {"company": "KeenEye", "site": null, "billingStreet": null, "billingCity": null, "billingState": null, "billingCountry": null, "billingPostalCode": null, "website": null, "mainPhone": null, "annualRevenue": null, "numberOfEmployees": null, "industry": null, "sicCode": null, "mktoCompanyNotes": null, "externalCompanyId": null, "id": 880, "mktoName": "Alastor Moody", "personType": "contact", "mktoIsPartner": false, "isLead": true, "mktoIsCustomer": false, "isAnonymous": false, "salutation": null, "firstName": "Alastor", "middleName": null, "lastName": "Moody", "email": "alastor@moody.com", "phone": null, "mobilePhone": null, "fax": null, "title": null, "contactCompany": "88", "dateOfBirth": null, "address": null, "city": null, "state": null, "country": null, "postalCode": null, "personTimeZone": null, "originalSourceType": "New lead", "originalSourceInfo": null, "registrationSourceType": "New lead", "registrationSourceInfo": null, "originalSearchEngine": null, "originalSearchPhrase": null, "originalReferrer": null, "emailInvalid": false, "emailInvalidCause": null, "unsubscribed": false, "unsubscribedReason": null, "doNotCall": false, "mktoDoNotCallCause": null, "doNotCallReason": null, "marketingSuspended": false, "marketingSuspendedCause": null, "blackListed": false, "blackListedCause": null, "mktoPersonNotes": null, "anonymousIP": null, "inferredCompany": null, "inferredCountry": null, "inferredCity": null, "inferredStateRegion": null, "inferredPostalCode": null, "inferredMetropolitanArea": null, "inferredPhoneAreaCode": null, "emailSuspended": null, "emailSuspendedCause": null, "emailSuspendedAt": null, "department": null, "createdAt": "2022-06-21T08:34:25Z", "updatedAt": "2022-06-21T08:34:25Z", "cookies": null, "externalSalesPersonId": null, "leadPerson": "880", "leadRole": null, "leadSource": null, "leadStatus": null, "leadScore": null, "urgency": null, "priority": null, "relativeScore": null, "relativeUrgency": null, "rating": null, "personPrimaryLeadInterest": "880", "leadPartitionId": "1", "leadRevenueCycleModelId": null, "leadRevenueStageId": null, "acquisitionProgramId": null, "mktoAcquisitionDate": null}, "emitted_at": 1655800613401} diff --git a/airbyte-integrations/connectors/source-marketo/setup.py b/airbyte-integrations/connectors/source-marketo/setup.py index 9f645e47ae16..054f5677afb3 100644 --- a/airbyte-integrations/connectors/source-marketo/setup.py +++ b/airbyte-integrations/connectors/source-marketo/setup.py @@ -12,6 +12,7 @@ TEST_REQUIREMENTS = [ "pytest~=6.1", "pytest-mock~=3.6.1", + "requests-mock", "source-acceptance-test", ] diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/schemas/programs.json b/airbyte-integrations/connectors/source-marketo/source_marketo/schemas/programs.json index 58071b52dcb4..54aa592f2497 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/schemas/programs.json +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/schemas/programs.json @@ -34,6 +34,9 @@ "workspace": { "type": ["null", "string"] }, + "headStart": { + "type": ["null", "boolean"] + }, "folder": { "type": ["object", "null"], "properties": { diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py index ce119e1635af..59295de6b781 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/source.py +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/source.py @@ -99,7 +99,7 @@ def get_updated_state(self, current_stream_state: MutableMapping[str, Any], late ) } - def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[MutableMapping[str, any]]]: """ Override default stream_slices CDK method to provide date_slices as page chunks for data fetch. Returns list of dict, example: [{ @@ -172,7 +172,9 @@ def get_export_status(self, stream_slice): def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: return f"bulk/v1/{self.stream_name}/export/{stream_slice['id']}/file.json" - def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + def stream_slices( + self, sync_mode, stream_state: MutableMapping[str, Any] = None, **kwargs + ) -> Iterable[Optional[MutableMapping[str, any]]]: date_slices = super().stream_slices(sync_mode, stream_state, **kwargs) for date_slice in date_slices: @@ -182,8 +184,12 @@ def stream_slices(self, sync_mode, stream_state: Mapping[str, Any] = None, **kwa export = self.create_export(param) - date_slice["id"] = export["exportId"] - return date_slices + status, export_id = export.get("status", "").lower(), export.get("exportId") + if status != "created" or not export_id: + self.logger.warning(f"Failed to create export job for data slice {date_slice}!") + continue + date_slice["id"] = export_id + yield date_slice def sleep_till_export_completed(self, stream_slice: Mapping[str, Any]) -> bool: while True: diff --git a/airbyte-integrations/connectors/source-marketo/source_marketo/spec.json b/airbyte-integrations/connectors/source-marketo/source_marketo/spec.json index 5e5d57747c42..9af488bf4cdc 100644 --- a/airbyte-integrations/connectors/source-marketo/source_marketo/spec.json +++ b/airbyte-integrations/connectors/source-marketo/source_marketo/spec.json @@ -18,7 +18,6 @@ "client_id": { "title": "Client ID", "type": "string", - "title": "Client ID", "description": "The Client ID of your Marketo developer application. See the docs for info on how to obtain this.", "order": 0, "airbyte_secret": true @@ -26,7 +25,6 @@ "client_secret": { "title": "Client Secret", "type": "string", - "title": "Client Secret", "description": "The Client Secret of your Marketo developer application. See the docs for info on how to obtain this.", "order": 1, "airbyte_secret": true @@ -35,7 +33,6 @@ "title": "Start Date", "type": "string", "order": 2, - "title": "Start Date", "description": "UTC date and time in the format 2017-01-25T00:00:00Z. Any data before this date will not be replicated.", "examples": ["2020-09-25T00:00:00Z"], "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py new file mode 100644 index 000000000000..03a9195f4799 --- /dev/null +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/conftest.py @@ -0,0 +1,58 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pendulum +import pytest +from source_marketo.source import Activities, MarketoAuthenticator + + +@pytest.fixture(autouse=True) +def mock_requests(requests_mock): + requests_mock.register_uri( + "GET", "https://602-euo-598.mktorest.com/identity/oauth/token", json={"access_token": "token", "expires_in": 3600} + ) + requests_mock.register_uri( + "POST", + "https://602-euo-598.mktorest.com/bulk/v1/activities/export/create.json", + [ + {"json": {"result": [{"exportId": "2c09ce6d", "format": "CSV", "status": "Created", "createdAt": "2022-06-20T08:44:08Z"}]}}, + {"json": {"result": [{"exportId": "cd465f55", "format": "CSV", "status": "Created", "createdAt": "2022-06-20T08:45:08Z"}]}}, + {"json": {"result": [{"exportId": "null", "format": "CSV", "status": "Failed", "createdAt": "2022-06-20T08:46:08Z"}]}}, + {"json": {"result": [{"exportId": "232aafb4", "format": "CSV", "status": "Created", "createdAt": "2022-06-20T08:47:08Z"}]}}, + ], + ) + + +@pytest.fixture +def config(): + start_date = pendulum.now().subtract(days=100).strftime("%Y-%m-%dT%H:%M:%SZ") + config = { + "client_id": "client-id", + "client_secret": "********", + "domain_url": "https://602-EUO-598.mktorest.com", + "start_date": start_date, + "window_in_days": 30, + } + config["authenticator"] = MarketoAuthenticator(config) + return config + + +@pytest.fixture +def send_email_stream(config): + activity = { + "id": 6, + "name": "send_email", + "description": "Send Marketo Email to a person", + "primaryAttribute": {"name": "Mailing ID", "dataType": "integer"}, + "attributes": [ + {"name": "Campaign Run ID", "dataType": "integer"}, + {"name": "Choice Number", "dataType": "integer"}, + {"name": "Has Predictive", "dataType": "boolean"}, + {"name": "Step ID", "dataType": "integer"}, + {"name": "Test Variant", "dataType": "integer"}, + ], + } + stream_name = f"activities_{activity['name']}" + cls = type(stream_name, (Activities,), {"activity": activity}) + return cls(config) diff --git a/airbyte-integrations/connectors/source-marketo/unit_tests/test_stream_slices.py b/airbyte-integrations/connectors/source-marketo/unit_tests/test_stream_slices.py new file mode 100644 index 000000000000..6d1b6aac923e --- /dev/null +++ b/airbyte-integrations/connectors/source-marketo/unit_tests/test_stream_slices.py @@ -0,0 +1,19 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from unittest.mock import ANY + +from airbyte_cdk.models.airbyte_protocol import SyncMode + + +def test_create_export_job(send_email_stream, caplog): + caplog.set_level(logging.WARNING) + slices = list(send_email_stream.stream_slices(sync_mode=SyncMode.incremental)) + assert slices == [ + {"endAt": ANY, "id": "2c09ce6d", "startAt": ANY}, + {"endAt": ANY, "id": "cd465f55", "startAt": ANY}, + {"endAt": ANY, "id": "232aafb4", "startAt": ANY}, + ] + assert "Failed to create export job for data slice " in caplog.records[-1].message diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java index 63f92f7977c4..ad275bda45c2 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlCdcStateHandler.java @@ -10,13 +10,14 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.debezium.CdcStateHandler; -import io.airbyte.integrations.source.relationaldb.StateManager; import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteStateMessage; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,7 +42,11 @@ public AirbyteMessage saveState(final Map offset, final String d final CdcState cdcState = new CdcState().withState(asJson); stateManager.getCdcStateManager().setCdcState(cdcState); - final AirbyteStateMessage stateMessage = stateManager.emit(); + /* + * Namespace pair is ignored by global state manager, but is needed for satisfy the API contract. + * Therefore, provide an empty optional. + */ + final AirbyteStateMessage stateMessage = stateManager.emit(Optional.empty()); return new AirbyteMessage().withType(Type.STATE).withState(stateMessage); } diff --git a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java index 2a770d8e1ddd..1eea401030f1 100644 --- a/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java +++ b/airbyte-integrations/connectors/source-mssql/src/main/java/io/airbyte/integrations/source/mssql/MssqlSource.java @@ -25,8 +25,8 @@ import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.mssql.MssqlCdcHelper.SnapshotIsolation; -import io.airbyte.integrations.source.relationaldb.StateManager; import io.airbyte.integrations.source.relationaldb.TableInfo; +import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java index d6171c06ff82..e896f3082ce7 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlCdcStateHandler.java @@ -10,13 +10,14 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.debezium.CdcStateHandler; -import io.airbyte.integrations.source.relationaldb.StateManager; import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteStateMessage; import java.util.HashMap; import java.util.Map; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +43,11 @@ public AirbyteMessage saveState(final Map offset, final String d final CdcState cdcState = new CdcState().withState(asJson); stateManager.getCdcStateManager().setCdcState(cdcState); - final AirbyteStateMessage stateMessage = stateManager.emit(); + /* + * Namespace pair is ignored by global state manager, but is needed for satisfy the API contract. + * Therefore, provide an empty optional. + */ + final AirbyteStateMessage stateMessage = stateManager.emit(Optional.empty()); return new AirbyteMessage().withType(Type.STATE).withState(stateMessage); } diff --git a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java index ea435043efc9..5c2ef9b99a01 100644 --- a/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java +++ b/airbyte-integrations/connectors/source-mysql/src/main/java/io/airbyte/integrations/source/mysql/MySqlSource.java @@ -25,9 +25,9 @@ import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.mysql.helpers.CdcConfigurationHelper; -import io.airbyte.integrations.source.relationaldb.StateManager; import io.airbyte.integrations.source.relationaldb.TableInfo; import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteStream; diff --git a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java index f1008f08b40c..b23b8953fc82 100644 --- a/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-mysql/src/test-integration/java/io/airbyte/integrations/source/mysql/CdcMySqlSourceAcceptanceTest.java @@ -10,6 +10,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Lists; import io.airbyte.commons.json.Jsons; import io.airbyte.db.Database; @@ -174,7 +175,7 @@ public void testIncrementalSyncFailedIfBinlogIsDeleted() throws Exception { // when we run incremental sync again there should be no new records. Run a sync with the latest // state message and assert no records were emitted. - final JsonNode latestState = stateMessages.get(stateMessages.size() - 1).getData(); + final JsonNode latestState = Jsons.jsonNode(supportsPerStream() ? stateMessages : List.of(Iterables.getLast(stateMessages))); // RESET MASTER removes all binary log files that are listed in the index file, // leaving only a single, empty binary log file with a numeric suffix of .000001 executeQuery("RESET MASTER;"); diff --git a/airbyte-integrations/connectors/source-openweather/Dockerfile b/airbyte-integrations/connectors/source-openweather/Dockerfile index b344b066bd47..264f36fd53b7 100644 --- a/airbyte-integrations/connectors/source-openweather/Dockerfile +++ b/airbyte-integrations/connectors/source-openweather/Dockerfile @@ -34,5 +34,5 @@ COPY source_openweather ./source_openweather ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.4 +LABEL io.airbyte.version=0.1.5 LABEL io.airbyte.name=airbyte/source-openweather diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcStateHandler.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcStateHandler.java index 50c93d0405ce..6175f81c904f 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcStateHandler.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresCdcStateHandler.java @@ -7,12 +7,13 @@ import com.fasterxml.jackson.databind.JsonNode; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.debezium.CdcStateHandler; -import io.airbyte.integrations.source.relationaldb.StateManager; import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteStateMessage; import java.util.Map; +import java.util.Optional; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,7 +32,11 @@ public AirbyteMessage saveState(final Map offset, final String d LOGGER.info("debezium state: {}", asJson); final CdcState cdcState = new CdcState().withState(asJson); stateManager.getCdcStateManager().setCdcState(cdcState); - final AirbyteStateMessage stateMessage = stateManager.emit(); + /* + * Namespace pair is ignored by global state manager, but is needed for satisfy the API contract. + * Therefore, provide an empty optional. + */ + final AirbyteStateMessage stateMessage = stateManager.emit(Optional.empty()); return new AirbyteMessage().withType(Type.STATE).withState(stateMessage); } diff --git a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java index cb83f7324c69..76aaa2c88d11 100644 --- a/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java +++ b/airbyte-integrations/connectors/source-postgres/src/main/java/io/airbyte/integrations/source/postgres/PostgresSource.java @@ -26,12 +26,17 @@ import io.airbyte.integrations.debezium.AirbyteDebeziumHandler; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.dto.JdbcPrivilegeDto; -import io.airbyte.integrations.source.relationaldb.StateManager; import io.airbyte.integrations.source.relationaldb.TableInfo; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteConnectionStatus; +import io.airbyte.protocol.models.AirbyteGlobalState; import io.airbyte.protocol.models.AirbyteMessage; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.AirbyteStreamState; import io.airbyte.protocol.models.CommonField; import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; import io.airbyte.protocol.models.SyncMode; @@ -404,6 +409,27 @@ private static AirbyteStream addCdcMetadataColumns(final AirbyteStream stream) { return stream; } + // TODO This is a temporary override so that the Postgres source can take advantage of per-stream + // state + @Override + protected List generateEmptyInitialState(final JsonNode config) { + if (getSupportedStateType(config) == AirbyteStateType.GLOBAL) { + final AirbyteGlobalState globalState = new AirbyteGlobalState() + .withSharedState(Jsons.jsonNode(new CdcState())) + .withStreamStates(List.of()); + return List.of(new AirbyteStateMessage().withStateType(AirbyteStateType.GLOBAL).withGlobal(globalState)); + } else { + return List.of(new AirbyteStateMessage() + .withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState())); + } + } + + @Override + protected AirbyteStateType getSupportedStateType(final JsonNode config) { + return isCdc(config) ? AirbyteStateType.GLOBAL : AirbyteStateType.STREAM; + } + public static void main(final String[] args) throws Exception { final Source source = PostgresSource.sshWrappedSource(); LOGGER.info("starting source: {}", PostgresSource.class); diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java index 633e9715f59c..911a24f02f21 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/AbstractSshPostgresSourceAcceptanceTest.java @@ -135,4 +135,9 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } + @Override + protected boolean supportsPerStream() { + return true; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java index acd1da14241f..623d2ef11e80 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceAcceptanceTest.java @@ -134,4 +134,9 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } + @Override + protected boolean supportsPerStream() { + return true; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java index 569d84d6e6cb..6752036e504e 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test-integration/java/io/airbyte/integrations/io/airbyte/integration_tests/sources/PostgresSourceStrictEncryptAcceptanceTest.java @@ -130,4 +130,9 @@ protected JsonNode getState() { return Jsons.jsonNode(new HashMap<>()); } + @Override + protected boolean supportsPerStream() { + return true; + } + } diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java index 6d2caa067420..2aa5e03ebfda 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/CdcPostgresSourceTest.java @@ -287,7 +287,7 @@ public void testRecordsProducedDuringAndAfterSync() throws Exception { writeModelRecord(record); } - final JsonNode state = stateAfterFirstBatch.get(0).getData(); + final JsonNode state = Jsons.jsonNode(stateAfterFirstBatch); final AutoCloseableIterator secondBatchIterator = getSource() .read(getConfig(), CONFIGURED_CATALOG, state); final List dataFromSecondBatch = AutoCloseableIterators diff --git a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java index 459a44fa86e3..1695d4ed8543 100644 --- a/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java +++ b/airbyte-integrations/connectors/source-postgres/src/test/java/io/airbyte/integrations/source/postgres/PostgresJdbcSourceAcceptanceTest.java @@ -22,12 +22,10 @@ import io.airbyte.db.jdbc.streaming.AdaptiveStreamingQueryConfig; import io.airbyte.integrations.source.jdbc.AbstractJdbcSource; import io.airbyte.integrations.source.jdbc.test.JdbcSourceAcceptanceTest; -import io.airbyte.integrations.source.relationaldb.models.DbState; import io.airbyte.integrations.source.relationaldb.models.DbStreamState; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteRecordMessage; -import io.airbyte.protocol.models.AirbyteStateMessage; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.ConfiguredAirbyteStream; import io.airbyte.protocol.models.ConnectorSpecification; @@ -175,7 +173,7 @@ protected List getAirbyteMessagesReadOneColumn() { } @Override - protected ArrayList getAirbyteMessagesCheckCursorSpaceInColumnName(ConfiguredAirbyteStream streamWithSpaces) { + protected ArrayList getAirbyteMessagesCheckCursorSpaceInColumnName(final ConfiguredAirbyteStream streamWithSpaces) { final AirbyteMessage firstMessage = getTestMessages().get(0); firstMessage.getRecord().setStream(streamWithSpaces.getStream().getName()); ((ObjectNode) firstMessage.getRecord().getData()).remove(COL_UPDATED_AT); @@ -200,7 +198,7 @@ protected ArrayList getAirbyteMessagesCheckCursorSpaceInColumnNa } @Override - protected List getAirbyteMessagesSecondSync(String streamName2) { + protected List getAirbyteMessagesSecondSync(final String streamName2) { return getTestMessages() .stream() .map(Jsons::clone) @@ -217,7 +215,7 @@ protected List getAirbyteMessagesSecondSync(String streamName2) .collect(Collectors.toList()); } - protected List getAirbyteMessagesSecondStreamWithNamespace(String streamName2) { + protected List getAirbyteMessagesSecondStreamWithNamespace(final String streamName2) { return getTestMessages() .stream() .map(Jsons::clone) @@ -233,7 +231,7 @@ protected List getAirbyteMessagesSecondStreamWithNamespace(Strin .collect(Collectors.toList()); } - protected List getAirbyteMessagesForTablesWithQuoting(ConfiguredAirbyteStream streamForTableWithSpaces) { + protected List getAirbyteMessagesForTablesWithQuoting(final ConfiguredAirbyteStream streamForTableWithSpaces) { return getTestMessages() .stream() .map(Jsons::clone) @@ -410,7 +408,7 @@ protected JdbcSourceOperations getSourceOperations() { } @Override - protected List getExpectedAirbyteMessagesSecondSync(String namespace) { + protected List getExpectedAirbyteMessagesSecondSync(final String namespace) { final List expectedMessages = new ArrayList<>(); expectedMessages.add(new AirbyteMessage().withType(AirbyteMessage.Type.RECORD) .withRecord(new AirbyteRecordMessage().withStream(streamName).withNamespace(namespace) @@ -430,17 +428,18 @@ protected List getExpectedAirbyteMessagesSecondSync(String names COL_WAKEUP_AT, "12:12:12.123456-05:00", COL_LAST_VISITED_AT, "2006-10-19T17:23:54.123456Z", COL_LAST_COMMENT_AT, "2006-01-01T17:23:54.123456"))))); - expectedMessages.add(new AirbyteMessage() - .withType(AirbyteMessage.Type.STATE) - .withState(new AirbyteStateMessage() - .withData(Jsons.jsonNode(new DbState() - .withCdc(false) - .withStreams(Lists.newArrayList(new DbStreamState() - .withStreamName(streamName) - .withStreamNamespace(namespace) - .withCursorField(ImmutableList.of(COL_ID)) - .withCursor("5"))))))); + final DbStreamState state = new DbStreamState() + .withStreamName(streamName) + .withStreamNamespace(namespace) + .withCursorField(ImmutableList.of(COL_ID)) + .withCursor("5"); + expectedMessages.addAll(createExpectedTestMessages(List.of(state))); return expectedMessages; } + @Override + protected boolean supportsPerStream() { + return true; + } + } diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java index 6ebdc7aa751e..389d7e555432 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/AbstractDbSource.java @@ -20,12 +20,17 @@ import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; import io.airbyte.integrations.base.Source; import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.state.AirbyteStateMessageListTypeReference; +import io.airbyte.integrations.source.relationaldb.state.StateManager; +import io.airbyte.integrations.source.relationaldb.state.StateManagerFactory; import io.airbyte.protocol.models.AirbyteCatalog; import io.airbyte.protocol.models.AirbyteConnectionStatus; import io.airbyte.protocol.models.AirbyteConnectionStatus.Status; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; import io.airbyte.protocol.models.AirbyteStream; import io.airbyte.protocol.models.CatalogHelpers; import io.airbyte.protocol.models.CommonField; @@ -103,9 +108,8 @@ public AutoCloseableIterator read(final JsonNode config, final ConfiguredAirbyteCatalog catalog, final JsonNode state) throws Exception { - final StateManager stateManager = new StateManager( - state == null ? StateManager.emptyState() : Jsons.object(state, DbState.class), - catalog); + final StateManager stateManager = + StateManagerFactory.createStateManager(getSupportedStateType(config), deserializeInitialState(state, config), catalog); final Instant emittedAt = Instant.now(); final Database database = createDatabaseInternal(config); @@ -509,4 +513,45 @@ private Database createDatabaseInternal(final JsonNode sourceConfig) throws Exce return database; } + /** + * Deserializes the state represented as JSON into an object representation. + * + * @param initialStateJson The state as JSON. + * @param config The connector configuration. + * @return The deserialized object representation of the state. + */ + protected List deserializeInitialState(final JsonNode initialStateJson, final JsonNode config) { + if (initialStateJson == null) { + return generateEmptyInitialState(config); + } else { + try { + return Jsons.object(initialStateJson, new AirbyteStateMessageListTypeReference()); + } catch (final IllegalArgumentException e) { + LOGGER.warn("Defaulting to legacy state object..."); + return List.of(new AirbyteStateMessage().withStateType(AirbyteStateType.LEGACY).withData(initialStateJson)); + } + } + } + + /** + * Generates an empty, initial state for use by the connector. + * + * @param config The connector configuration. + * @return The empty, initial state. + */ + protected List generateEmptyInitialState(final JsonNode config) { + // For backwards compatibility with existing connectors + return List.of(new AirbyteStateMessage().withStateType(AirbyteStateType.LEGACY).withData(Jsons.jsonNode(new DbState()))); + } + + /** + * Returns the {@link AirbyteStateType} supported by this connector. + * + * @param config The connector configuration. + * @return A {@link AirbyteStateType} representing the state supported by this connector. + */ + protected AirbyteStateType getSupportedStateType(final JsonNode config) { + return AirbyteStateType.LEGACY; + } + } diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java index db33dfd6167b..7b855e6c9770 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/CdcStateManager.java @@ -4,7 +4,6 @@ package io.airbyte.integrations.source.relationaldb; -import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.json.Jsons; import io.airbyte.integrations.source.relationaldb.models.CdcState; import org.slf4j.Logger; @@ -12,14 +11,13 @@ public class CdcStateManager { - private static final Logger LOGGER = LoggerFactory.getLogger(StateManager.class); + private static final Logger LOGGER = LoggerFactory.getLogger(CdcStateManager.class); private final CdcState initialState; private CdcState currentState; - @VisibleForTesting - CdcStateManager(final CdcState serialized) { + public CdcStateManager(final CdcState serialized) { this.initialState = serialized; this.currentState = serialized; diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java index 122d62ddbb65..7eabaad9eb31 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIterator.java @@ -7,6 +7,7 @@ import com.google.common.collect.AbstractIterator; import io.airbyte.db.IncrementalUtils; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteStateMessage; @@ -40,7 +41,6 @@ public StateDecoratingIterator(final Iterator messageIterator, this.cursorField = cursorField; this.cursorType = cursorType; this.maxCursor = initialCursor; - stateManager.setIsCdc(false); } private String getCursorCandidate(final AirbyteMessage message) { diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateManager.java deleted file mode 100644 index 3e509e2869d9..000000000000 --- a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/StateManager.java +++ /dev/null @@ -1,197 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.relationaldb; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; -import io.airbyte.protocol.models.AirbyteStateMessage; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Set; -import java.util.stream.Collectors; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Handles the state machine for the state of source implementations. - */ -public class StateManager { - - private static final Logger LOGGER = LoggerFactory.getLogger(StateManager.class); - - private final Map pairToCursorInfo; - private Boolean isCdc; - private final CdcStateManager cdcStateManager; - - public static DbState emptyState() { - return new DbState(); - } - - public StateManager(final DbState serialized, final ConfiguredAirbyteCatalog catalog) { - this.cdcStateManager = new CdcStateManager(serialized.getCdcState()); - this.isCdc = serialized.getCdc(); - if (serialized.getCdc() == null) { - this.isCdc = false; - } - - pairToCursorInfo = - new ImmutableMap.Builder().putAll(createCursorInfoMap(serialized, catalog)).build(); - } - - private static Map createCursorInfoMap(final DbState serialized, - final ConfiguredAirbyteCatalog catalog) { - final Set allStreamNames = catalog.getStreams() - .stream() - .map(ConfiguredAirbyteStream::getStream) - .map(AirbyteStreamNameNamespacePair::fromAirbyteSteam) - .collect(Collectors.toSet()); - allStreamNames.addAll(serialized.getStreams().stream().map(StateManager::toAirbyteStreamNameNamespacePair).collect(Collectors.toSet())); - - final Map localMap = new HashMap<>(); - final Map pairToState = serialized.getStreams() - .stream() - .collect(Collectors.toMap(StateManager::toAirbyteStreamNameNamespacePair, a -> a)); - final Map pairToConfiguredAirbyteStream = catalog.getStreams().stream() - .collect(Collectors.toMap(AirbyteStreamNameNamespacePair::fromConfiguredAirbyteSteam, s -> s)); - - for (final AirbyteStreamNameNamespacePair pair : allStreamNames) { - final Optional stateOptional = Optional.ofNullable(pairToState.get(pair)); - final Optional streamOptional = Optional.ofNullable(pairToConfiguredAirbyteStream.get(pair)); - localMap.put(pair, createCursorInfoForStream(pair, stateOptional, streamOptional)); - } - - return localMap; - } - - private static AirbyteStreamNameNamespacePair toAirbyteStreamNameNamespacePair(final DbStreamState state) { - return new AirbyteStreamNameNamespacePair(state.getStreamName(), state.getStreamNamespace()); - } - - @VisibleForTesting - @SuppressWarnings("OptionalUsedAsFieldOrParameterType") - static CursorInfo createCursorInfoForStream(final AirbyteStreamNameNamespacePair pair, - final Optional stateOptional, - final Optional streamOptional) { - final String originalCursorField = stateOptional - .map(DbStreamState::getCursorField) - .flatMap(f -> f.size() > 0 ? Optional.of(f.get(0)) : Optional.empty()) - .orElse(null); - final String originalCursor = stateOptional.map(DbStreamState::getCursor).orElse(null); - - final String cursor; - final String cursorField; - - // if cursor field is set in catalog. - if (streamOptional.map(ConfiguredAirbyteStream::getCursorField).isPresent()) { - cursorField = streamOptional - .map(ConfiguredAirbyteStream::getCursorField) - .flatMap(f -> f.size() > 0 ? Optional.of(f.get(0)) : Optional.empty()) - .orElse(null); - // if cursor field is set in state. - if (stateOptional.map(DbStreamState::getCursorField).isPresent()) { - // if cursor field in catalog and state are the same. - if (stateOptional.map(DbStreamState::getCursorField).equals(streamOptional.map(ConfiguredAirbyteStream::getCursorField))) { - cursor = stateOptional.map(DbStreamState::getCursor).orElse(null); - LOGGER.info("Found matching cursor in state. Stream: {}. Cursor Field: {} Value: {}", pair, cursorField, cursor); - // if cursor field in catalog and state are different. - } else { - cursor = null; - LOGGER.info( - "Found cursor field. Does not match previous cursor field. Stream: {}. Original Cursor Field: {}. New Cursor Field: {}. Resetting cursor value.", - pair, originalCursorField, cursorField); - } - // if cursor field is not set in state but is set in catalog. - } else { - LOGGER.info("No cursor field set in catalog but not present in state. Stream: {}, New Cursor Field: {}. Resetting cursor value", pair, - cursorField); - cursor = null; - } - // if cursor field is not set in catalog. - } else { - LOGGER.info( - "Cursor field set in state but not present in catalog. Stream: {}. Original Cursor Field: {}. Original value: {}. Resetting cursor.", - pair, originalCursorField, originalCursor); - cursorField = null; - cursor = null; - } - - return new CursorInfo(originalCursorField, originalCursor, cursorField, cursor); - } - - private Optional getCursorInfo(final AirbyteStreamNameNamespacePair pair) { - return Optional.ofNullable(pairToCursorInfo.get(pair)); - } - - public Optional getOriginalCursorField(final AirbyteStreamNameNamespacePair pair) { - return getCursorInfo(pair).map(CursorInfo::getOriginalCursorField); - } - - public Optional getOriginalCursor(final AirbyteStreamNameNamespacePair pair) { - return getCursorInfo(pair).map(CursorInfo::getOriginalCursor); - } - - public Optional getCursorField(final AirbyteStreamNameNamespacePair pair) { - return getCursorInfo(pair).map(CursorInfo::getCursorField); - } - - public Optional getCursor(final AirbyteStreamNameNamespacePair pair) { - return getCursorInfo(pair).map(CursorInfo::getCursor); - } - - synchronized public AirbyteStateMessage updateAndEmit(final AirbyteStreamNameNamespacePair pair, final String cursor) { - // cdc file gets updated by debezium so the "update" part is a no op. - if (!isCdc) { - final Optional cursorInfo = getCursorInfo(pair); - Preconditions.checkState(cursorInfo.isPresent(), "Could not find cursor information for stream: " + pair); - cursorInfo.get().setCursor(cursor); - } - - return toState(); - } - - public void setIsCdc(final boolean isCdc) { - if (this.isCdc == null) { - this.isCdc = isCdc; - } else { - Preconditions.checkState(this.isCdc == isCdc, "attempt to set cdc to {}, but is already set to {}.", isCdc, this.isCdc); - } - } - - public CdcStateManager getCdcStateManager() { - return cdcStateManager; - } - - public AirbyteStateMessage emit() { - return toState(); - } - - private AirbyteStateMessage toState() { - final DbState DbState = new DbState() - .withCdc(isCdc) - .withStreams(pairToCursorInfo.entrySet().stream() - .sorted(Entry.comparingByKey()) // sort by stream name then namespace for sanity. - .map(e -> new DbStreamState() - .withStreamName(e.getKey().getName()) - .withStreamNamespace(e.getKey().getNamespace()) - .withCursorField(e.getValue().getCursorField() == null ? Collections.emptyList() : Lists.newArrayList(e.getValue().getCursorField())) - .withCursor(e.getValue().getCursor())) - .collect(Collectors.toList())) - .withCdcState(cdcStateManager.getCdcState()); - - return new AirbyteStateMessage().withData(Jsons.jsonNode(DbState)); - } - -} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java new file mode 100644 index 000000000000..dec78ec39fac --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AbstractStateManager.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.function.Function; +import java.util.function.Supplier; + +/** + * Abstract implementation of the {@link StateManager} interface that provides common functionality + * for state manager implementations. + * + * @param The type associated with the state object managed by this manager. + * @param The type associated with the state object stored in the state managed by this manager. + */ +public abstract class AbstractStateManager implements StateManager { + + /** + * The {@link CursorManager} responsible for keeping track of the current cursor value for each + * stream managed by this state manager. + */ + private final CursorManager cursorManager; + + /** + * Constructs a new state manager for the given configured connector. + * + * @param catalog The connector's configured catalog. + * @param streamSupplier A {@link Supplier} that provides the cursor manager with the collection of + * streams tracked by the connector's state. + * @param cursorFunction A {@link Function} that extracts the current cursor from a stream stored in + * the connector's state. + * @param cursorFieldFunction A {@link Function} that extracts the cursor field name from a stream + * stored in the connector's state. + * @param namespacePairFunction A {@link Function} that generates a + * {@link AirbyteStreamNameNamespacePair} that identifies each stream in the connector's + * state. + */ + public AbstractStateManager(final ConfiguredAirbyteCatalog catalog, + final Supplier> streamSupplier, + final Function cursorFunction, + final Function> cursorFieldFunction, + final Function namespacePairFunction) { + cursorManager = new CursorManager(catalog, streamSupplier, cursorFunction, cursorFieldFunction, namespacePairFunction); + } + + @Override + public Map getPairToCursorInfoMap() { + return cursorManager.getPairToCursorInfo(); + } + + @Override + public abstract AirbyteStateMessage toState(final Optional pair); + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AirbyteStateMessageListTypeReference.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AirbyteStateMessageListTypeReference.java new file mode 100644 index 000000000000..c7e153e6d79a --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/AirbyteStateMessageListTypeReference.java @@ -0,0 +1,13 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import com.fasterxml.jackson.core.type.TypeReference; +import io.airbyte.protocol.models.AirbyteStateMessage; +import java.util.List; + +public class AirbyteStateMessageListTypeReference extends TypeReference> { + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java new file mode 100644 index 000000000000..207b51ad5bad --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/CursorManager.java @@ -0,0 +1,222 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import com.google.common.annotations.VisibleForTesting; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.function.Function; +import java.util.function.Supplier; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Manages the map of streams to current cursor values for state management. + * + * @param The type that represents the stream object which holds the current cursor information + * in the state. + */ +public class CursorManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(CursorManager.class); + + /** + * Map of streams (name/namespace tuple) to the current cursor information stored in the state. + */ + private final Map pairToCursorInfo; + + /** + * Constructs a new {@link CursorManager} based on the configured connector and current state + * information. + * + * @param catalog The connector's configured catalog. + * @param streamSupplier A {@link Supplier} that provides the cursor manager with the collection of + * streams tracked by the connector's state. + * @param cursorFunction A {@link Function} that extracts the current cursor from a stream stored in + * the connector's state. + * @param cursorFieldFunction A {@link Function} that extracts the cursor field name from a stream + * stored in the connector's state. + * @param namespacePairFunction A {@link Function} that generates a + * {@link AirbyteStreamNameNamespacePair} that identifies each stream in the connector's + * state. + */ + public CursorManager(final ConfiguredAirbyteCatalog catalog, + final Supplier> streamSupplier, + final Function cursorFunction, + final Function> cursorFieldFunction, + final Function namespacePairFunction) { + pairToCursorInfo = createCursorInfoMap(catalog, streamSupplier, cursorFunction, cursorFieldFunction, namespacePairFunction); + } + + /** + * Creates the cursor information map that associates stream name/namespace tuples with the current + * cursor information for that stream as stored in the connector's state. + * + * @param catalog The connector's configured catalog. + * @param streamSupplier A {@link Supplier} that provides the cursor manager with the collection of + * streams tracked by the connector's state. + * @param cursorFunction A {@link Function} that extracts the current cursor from a stream stored in + * the connector's state. + * @param cursorFieldFunction A {@link Function} that extracts the cursor field name from a stream + * stored in the connector's state. + * @param namespacePairFunction A {@link Function} that generates a + * {@link AirbyteStreamNameNamespacePair} that identifies each stream in the connector's + * state. + * @return A map of streams to current cursor information for the stream. + */ + @VisibleForTesting + protected Map createCursorInfoMap( + final ConfiguredAirbyteCatalog catalog, + final Supplier> streamSupplier, + final Function cursorFunction, + final Function> cursorFieldFunction, + final Function namespacePairFunction) { + final Set allStreamNames = catalog.getStreams() + .stream() + .map(ConfiguredAirbyteStream::getStream) + .map(AirbyteStreamNameNamespacePair::fromAirbyteSteam) + .collect(Collectors.toSet()); + allStreamNames.addAll(streamSupplier.get().stream().map(namespacePairFunction).filter(n -> n != null).collect(Collectors.toSet())); + + final Map localMap = new HashMap<>(); + final Map pairToState = streamSupplier.get() + .stream() + .collect(Collectors.toMap(namespacePairFunction,Function.identity())); + final Map pairToConfiguredAirbyteStream = catalog.getStreams().stream() + .collect(Collectors.toMap(AirbyteStreamNameNamespacePair::fromConfiguredAirbyteSteam, Function.identity())); + + for (final AirbyteStreamNameNamespacePair pair : allStreamNames) { + final Optional stateOptional = Optional.ofNullable(pairToState.get(pair)); + final Optional streamOptional = Optional.ofNullable(pairToConfiguredAirbyteStream.get(pair)); + localMap.put(pair, createCursorInfoForStream(pair, stateOptional, streamOptional, cursorFunction, cursorFieldFunction)); + } + + return localMap; + } + + /** + * Generates a {@link CursorInfo} object based on the data currently stored in the connector's state + * for the given stream. + * + * @param pair A {@link AirbyteStreamNameNamespacePair} that identifies a specific stream managed by + * the connector. + * @param stateOptional {@link Optional} containing the current state associated with the stream. + * @param streamOptional {@link Optional} containing the {@link ConfiguredAirbyteStream} associated + * with the stream. + * @param cursorFunction A {@link Function} that provides the current cursor from the state + * associated with the stream. + * @param cursorFieldFunction A {@link Function} that provides the cursor field name for the cursor + * stored in the state associated with the stream. + * @return A {@link CursorInfo} object based on the data currently stored in the connector's state + * for the given stream. + */ + @SuppressWarnings("OptionalUsedAsFieldOrParameterType") + @VisibleForTesting + protected CursorInfo createCursorInfoForStream(final AirbyteStreamNameNamespacePair pair, + final Optional stateOptional, + final Optional streamOptional, + final Function cursorFunction, + final Function> cursorFieldFunction) { + final String originalCursorField = stateOptional + .map(cursorFieldFunction) + .flatMap(f -> f.size() > 0 ? Optional.of(f.get(0)) : Optional.empty()) + .orElse(null); + final String originalCursor = stateOptional.map(cursorFunction).orElse(null); + + final String cursor; + final String cursorField; + + // if cursor field is set in catalog. + if (streamOptional.map(ConfiguredAirbyteStream::getCursorField).isPresent()) { + cursorField = streamOptional + .map(ConfiguredAirbyteStream::getCursorField) + .flatMap(f -> f.size() > 0 ? Optional.of(f.get(0)) : Optional.empty()) + .orElse(null); + // if cursor field is set in state. + if (stateOptional.map(cursorFieldFunction).isPresent()) { + // if cursor field in catalog and state are the same. + if (stateOptional.map(cursorFieldFunction).equals(streamOptional.map(ConfiguredAirbyteStream::getCursorField))) { + cursor = stateOptional.map(cursorFunction).orElse(null); + LOGGER.info("Found matching cursor in state. Stream: {}. Cursor Field: {} Value: {}", pair, cursorField, cursor); + // if cursor field in catalog and state are different. + } else { + cursor = null; + LOGGER.info( + "Found cursor field. Does not match previous cursor field. Stream: {}. Original Cursor Field: {}. New Cursor Field: {}. Resetting cursor value.", + pair, originalCursorField, cursorField); + } + // if cursor field is not set in state but is set in catalog. + } else { + LOGGER.info("No cursor field set in catalog but not present in state. Stream: {}, New Cursor Field: {}. Resetting cursor value", pair, + cursorField); + cursor = null; + } + // if cursor field is not set in catalog. + } else { + LOGGER.info( + "Cursor field set in state but not present in catalog. Stream: {}. Original Cursor Field: {}. Original value: {}. Resetting cursor.", + pair, originalCursorField, originalCursor); + cursorField = null; + cursor = null; + } + + return new CursorInfo(originalCursorField, originalCursor, cursorField, cursor); + } + + /** + * Retrieves a copy of the stream name/namespace tuple to current cursor information map. + * + * @return A copy of the stream name/namespace tuple to current cursor information map. + */ + public Map getPairToCursorInfo() { + return Map.copyOf(pairToCursorInfo); + } + + /** + * Retrieves an {@link Optional} possibly containing the current {@link CursorInfo} associated with + * the provided stream name/namespace tuple. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} which identifies a stream. + * @return An {@link Optional} possibly containing the current {@link CursorInfo} associated with + * the provided stream name/namespace tuple. + */ + public Optional getCursorInfo(final AirbyteStreamNameNamespacePair pair) { + return Optional.ofNullable(pairToCursorInfo.get(pair)); + } + + /** + * Retrieves an {@link Optional} possibly containing the cursor field name associated with the + * cursor tracked in the state associated with the provided stream name/namespace tuple. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} which identifies a stream. + * @return An {@link Optional} possibly containing the cursor field name associated with the cursor + * tracked in the state associated with the provided stream name/namespace tuple. + */ + public Optional getCursorField(final AirbyteStreamNameNamespacePair pair) { + return getCursorInfo(pair).map(CursorInfo::getCursorField); + } + + /** + * Retrieves an {@link Optional} possibly containing the cursor value tracked in the state + * associated with the provided stream name/namespace tuple. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} which identifies a stream. + * @return An {@link Optional} possibly containing the cursor value tracked in the state associated + * with the provided stream name/namespace tuple. + */ + public Optional getCursor(final AirbyteStreamNameNamespacePair pair) { + return getCursorInfo(pair).map(CursorInfo::getCursor); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java new file mode 100644 index 000000000000..ca8b516c7cb3 --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManager.java @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FIELD_FUNCTION; +import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FUNCTION; +import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.NAME_NAMESPACE_PAIR_FUNCTION; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.protocol.models.AirbyteGlobalState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.AirbyteStreamState; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.StreamDescriptor; +import java.util.Collection; +import java.util.List; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Global implementation of the {@link StateManager} interface. + * + * This implementation generates a single, global state object for the state tracked by this + * manager. + */ +public class GlobalStateManager extends AbstractStateManager { + + /** + * Legacy {@link CdcStateManager} used to manage state for connectors that support Change Data + * Capture (CDC). + */ + private final CdcStateManager cdcStateManager; + + /** + * Constructs a new {@link GlobalStateManager} that is seeded with the provided + * {@link AirbyteStateMessage}. + * + * @param airbyteStateMessage The initial state represented as an {@link AirbyteStateMessage}. + * @param catalog The {@link ConfiguredAirbyteCatalog} for the connector associated with this state + * manager. + */ + public GlobalStateManager(final AirbyteStateMessage airbyteStateMessage, final ConfiguredAirbyteCatalog catalog) { + super(catalog, + getStreamsSupplier(airbyteStateMessage), + CURSOR_FUNCTION, + CURSOR_FIELD_FUNCTION, + NAME_NAMESPACE_PAIR_FUNCTION); + + this.cdcStateManager = new CdcStateManager(extractCdcState(airbyteStateMessage)); + } + + @Override + public CdcStateManager getCdcStateManager() { + return cdcStateManager; + } + + @Override + public AirbyteStateMessage toState(final Optional pair) { + // Populate global state + final AirbyteGlobalState globalState = new AirbyteGlobalState(); + globalState.setSharedState(Jsons.jsonNode(getCdcStateManager().getCdcState())); + globalState.setStreamStates(StateGeneratorUtils.generateStreamStateList(getPairToCursorInfoMap())); + + // Generate the legacy state for backwards compatibility + final DbState dbState = StateGeneratorUtils.generateDbState(getPairToCursorInfoMap()) + .withCdc(true) + .withCdcState(getCdcStateManager().getCdcState()); + + return new AirbyteStateMessage() + .withStateType(AirbyteStateType.GLOBAL) + // Temporarily include legacy state for backwards compatibility with the platform + .withData(Jsons.jsonNode(dbState)) + .withGlobal(globalState); + } + + /** + * Extracts the Change Data Capture (CDC) state stored in the initial state provided to this state + * manager. + * + * @param airbyteStateMessage The {@link AirbyteStateMessage} that contains the initial state + * provided to the state manager. + * @return The {@link CdcState} stored in the state, if any. Note that this will not be {@code null} + * but may be empty. + */ + private CdcState extractCdcState(final AirbyteStateMessage airbyteStateMessage) { + if (airbyteStateMessage.getStateType() == AirbyteStateType.GLOBAL) { + return Jsons.object(airbyteStateMessage.getGlobal().getSharedState(), CdcState.class); + } else { + return Jsons.object(airbyteStateMessage.getData(), DbState.class).getCdcState(); + } + } + + /** + * Generates the {@link Supplier} that will be used to extract the streams from the incoming + * {@link AirbyteStateMessage}. + * + * @param airbyteStateMessage The {@link AirbyteStateMessage} supplied to this state manager with + * the initial state. + * @return A {@link Supplier} that will be used to fetch the streams present in the initial state. + */ + private static Supplier> getStreamsSupplier(final AirbyteStateMessage airbyteStateMessage) { + /* + * If the incoming message has the state type set to GLOBAL, it is using the new format. Therefore, + * we can look for streams in the "global" field of the message. Otherwise, the message is still + * storing state in the legacy "data" field. + */ + return () -> { + if (airbyteStateMessage.getStateType() == AirbyteStateType.GLOBAL) { + return airbyteStateMessage.getGlobal().getStreamStates(); + } else if (airbyteStateMessage.getData() != null) { + return Jsons.object(airbyteStateMessage.getData(), DbState.class).getStreams().stream() + .map(s -> new AirbyteStreamState().withStreamState(Jsons.jsonNode(s)) + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName()))) + .collect( + Collectors.toList()); + } else { + return List.of(); + } + }; + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java new file mode 100644 index 000000000000..64dabe9e07e2 --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManager.java @@ -0,0 +1,112 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import com.google.common.base.Preconditions; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.List; +import java.util.Optional; +import java.util.function.Function; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Legacy implementation (pre-per-stream state support) of the {@link StateManager} interface. + * + * This implementation assumes that the state matches the {@link DbState} object and effectively + * tracks state as global across the streams managed by a connector. + * + * @deprecated This manager may be removed in the future if/once all connectors support per-stream + * state management. + */ +@Deprecated(forRemoval = true) +public class LegacyStateManager extends AbstractStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(LegacyStateManager.class); + + /** + * {@link Function} that extracts the cursor from the stream state. + */ + private static final Function CURSOR_FUNCTION = DbStreamState::getCursor; + + /** + * {@link Function} that extracts the cursor field(s) from the stream state. + */ + private static final Function> CURSOR_FIELD_FUNCTION = DbStreamState::getCursorField; + + /** + * {@link Function} that creates an {@link AirbyteStreamNameNamespacePair} from the stream state. + */ + private static final Function NAME_NAMESPACE_PAIR_FUNCTION = + s -> new AirbyteStreamNameNamespacePair(s.getStreamName(), s.getStreamNamespace()); + + /** + * Tracks whether the connector associated with this state manager supports CDC. + */ + private Boolean isCdc; + + /** + * {@link CdcStateManager} used to manage state for connectors that support CDC. + */ + private final CdcStateManager cdcStateManager; + + /** + * Constructs a new {@link LegacyStateManager} that is seeded with the provided {@link DbState} + * instance. + * + * @param dbState The initial state represented as an {@link DbState} instance. + * @param catalog The {@link ConfiguredAirbyteCatalog} for the connector associated with this state + * manager. + */ + public LegacyStateManager(final DbState dbState, final ConfiguredAirbyteCatalog catalog) { + super(catalog, + () -> dbState.getStreams(), + CURSOR_FUNCTION, + CURSOR_FIELD_FUNCTION, + NAME_NAMESPACE_PAIR_FUNCTION); + + this.cdcStateManager = new CdcStateManager(dbState.getCdcState()); + this.isCdc = dbState.getCdc(); + if (dbState.getCdc() == null) { + this.isCdc = false; + } + } + + @Override + public CdcStateManager getCdcStateManager() { + return cdcStateManager; + } + + @Override + public AirbyteStateMessage toState(final Optional pair) { + final DbState dbState = StateGeneratorUtils.generateDbState(getPairToCursorInfoMap()) + .withCdc(isCdc) + .withCdcState(getCdcStateManager().getCdcState()); + + LOGGER.info("Generated legacy state for {} streams", dbState.getStreams().size()); + return new AirbyteStateMessage().withStateType(AirbyteStateType.LEGACY).withData(Jsons.jsonNode(dbState)); + } + + @Override + public AirbyteStateMessage updateAndEmit(final AirbyteStreamNameNamespacePair pair, final String cursor) { + // cdc file gets updated by debezium so the "update" part is a no op. + if (!isCdc) { + final Optional cursorInfo = getCursorInfo(pair); + Preconditions.checkState(cursorInfo.isPresent(), "Could not find cursor information for stream: " + pair); + cursorInfo.get().setCursor(cursor); + } + + return toState(Optional.ofNullable(pair)); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtils.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtils.java new file mode 100644 index 000000000000..493defb95e9f --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtils.java @@ -0,0 +1,216 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import com.google.common.collect.Lists; +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteGlobalState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.AirbyteStreamState; +import io.airbyte.protocol.models.StreamDescriptor; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.function.Function; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Collection of utilities that facilitate the generation of state objects. + */ +public class StateGeneratorUtils { + + private static final Logger LOGGER = LoggerFactory.getLogger(StateGeneratorUtils.class); + + /** + * {@link Function} that extracts the cursor from the stream state. + */ + public static final Function CURSOR_FUNCTION = stream -> { + final Optional dbStreamState = StateGeneratorUtils.extractState(stream); + return dbStreamState.map(DbStreamState::getCursor).orElse(null); + }; + + /** + * {@link Function} that extracts the cursor field(s) from the stream state. + */ + public static final Function> CURSOR_FIELD_FUNCTION = stream -> { + final Optional dbStreamState = StateGeneratorUtils.extractState(stream); + if (dbStreamState.isPresent()) { + return dbStreamState.get().getCursorField(); + } else { + return List.of(); + } + }; + + /** + * {@link Function} that creates an {@link AirbyteStreamNameNamespacePair} from the stream state. + */ + public static final Function NAME_NAMESPACE_PAIR_FUNCTION = + s -> isValidStreamDescriptor(s.getStreamDescriptor()) + ? new AirbyteStreamNameNamespacePair(s.getStreamDescriptor().getName(), s.getStreamDescriptor().getNamespace()) + : null; + + private StateGeneratorUtils() {} + + /** + * Generates the stream state for the given stream and cursor information. + * + * @param airbyteStreamNameNamespacePair The stream. + * @param cursorInfo The current cursor. + * @return The {@link AirbyteStreamState} representing the current state of the stream. + */ + public static AirbyteStreamState generateStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final CursorInfo cursorInfo) { + return new AirbyteStreamState() + .withStreamDescriptor( + new StreamDescriptor().withName(airbyteStreamNameNamespacePair.getName()).withNamespace(airbyteStreamNameNamespacePair.getNamespace())) + .withStreamState(Jsons.jsonNode(generateDbStreamState(airbyteStreamNameNamespacePair, cursorInfo))); + } + + /** + * Generates a list of valid stream states from the provided stream and cursor information. A stream + * state is considered to be valid if the stream has a valid descriptor (see + * {@link #isValidStreamDescriptor(StreamDescriptor)} for more details). + * + * @param pairToCursorInfoMap The map of stream name/namespace tuple to the current cursor + * information for that stream + * @return The list of stream states derived from the state information extracted from the provided + * map. + */ + public static List generateStreamStateList(final Map pairToCursorInfoMap) { + return pairToCursorInfoMap.entrySet().stream() + .sorted(Entry.comparingByKey()) + .map(e -> generateStreamState(e.getKey(), e.getValue())) + .filter(s -> isValidStreamDescriptor(s.getStreamDescriptor())) + .collect(Collectors.toList()); + } + + /** + * Generates the legacy global state for backwards compatibility. + * + * @param pairToCursorInfoMap The map of stream name/namespace tuple to the current cursor + * information for that stream + * @return The legacy {@link DbState}. + */ + public static DbState generateDbState(final Map pairToCursorInfoMap) { + return new DbState() + .withCdc(false) + .withStreams(pairToCursorInfoMap.entrySet().stream() + .sorted(Entry.comparingByKey()) // sort by stream name then namespace for sanity. + .map(e -> generateDbStreamState(e.getKey(), e.getValue())) + .collect(Collectors.toList())); + } + + /** + * Generates the {@link DbStreamState} for the given stream and cursor. + * + * @param airbyteStreamNameNamespacePair The stream. + * @param cursorInfo The current cursor. + * @return The {@link DbStreamState}. + */ + public static DbStreamState generateDbStreamState(final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair, + final CursorInfo cursorInfo) { + return new DbStreamState() + .withStreamName(airbyteStreamNameNamespacePair.getName()) + .withStreamNamespace(airbyteStreamNameNamespacePair.getNamespace()) + .withCursorField(cursorInfo.getCursorField() == null ? Collections.emptyList() : Lists.newArrayList(cursorInfo.getCursorField())) + .withCursor(cursorInfo.getCursor()); + } + + /** + * Extracts the actual state from the {@link AirbyteStreamState} object. + * + * @param state The {@link AirbyteStreamState} that contains the actual stream state as JSON. + * @return An {@link Optional} possibly containing the deserialized representation of the stream + * state or an empty {@link Optional} if the state is not present or could not be + * deserialized. + */ + public static Optional extractState(final AirbyteStreamState state) { + try { + return Optional.ofNullable(Jsons.object(state.getStreamState(), DbStreamState.class)); + } catch (final IllegalArgumentException e) { + LOGGER.error("Unable to extract state.", e); + return Optional.empty(); + } + } + + /** + * Tests whether the provided {@link StreamDescriptor} is valid. A valid descriptor is defined as + * one that has a non-{@code null} name. + * + * See https://github.com/airbytehq/airbyte/blob/e63458fabb067978beb5eaa74d2bc130919b419f/docs/understanding-airbyte/airbyte-protocol.md + * for more details + * + * @param streamDescriptor A {@link StreamDescriptor} to be validated. + * @return {@code true} if the provided {@link StreamDescriptor} is valid or {@code false} if it is + * invalid. + */ + public static boolean isValidStreamDescriptor(final StreamDescriptor streamDescriptor) { + if (streamDescriptor != null) { + return streamDescriptor.getName() != null; + } else { + return false; + } + } + + /** + * Converts a {@link AirbyteStateType#LEGACY} state message into a {@link AirbyteStateType#GLOBAL} + * message. + * + * @param airbyteStateMessage A {@link AirbyteStateType#LEGACY} state message. + * @return A {@link AirbyteStateType#GLOBAL} state message. + */ + public static AirbyteStateMessage convertLegacyStateToGlobalState(final AirbyteStateMessage airbyteStateMessage) { + final DbState dbState = Jsons.object(airbyteStateMessage.getData(), DbState.class); + final AirbyteGlobalState globalState = new AirbyteGlobalState() + .withSharedState(Jsons.jsonNode(dbState.getCdcState())) + .withStreamStates(dbState.getStreams().stream() + .map(s -> new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(s.getStreamName()).withNamespace(s.getStreamNamespace())) + .withStreamState(Jsons.jsonNode(s))) + .collect( + Collectors.toList())); + return new AirbyteStateMessage().withStateType(AirbyteStateType.GLOBAL).withGlobal(globalState); + } + + /** + * Converts a {@link AirbyteStateType#GLOBAL} state message into a list of + * {@link AirbyteStateType#STREAM} messages. + * + * @param airbyteStateMessage A {@link AirbyteStateType#GLOBAL} state message. + * @return A list {@link AirbyteStateType#STREAM} state messages. + */ + public static List convertGlobalStateToStreamState(final AirbyteStateMessage airbyteStateMessage) { + return airbyteStateMessage.getGlobal().getStreamStates().stream() + .map(s -> new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(s.getStreamDescriptor()).withStreamState(s.getStreamState()))) + .collect(Collectors.toList()); + } + + /** + * Converts a {@link AirbyteStateType#LEGACY} state message into a list of + * {@link AirbyteStateType#STREAM} messages. + * + * @param airbyteStateMessage A {@link AirbyteStateType#LEGACY} state message. + * @return A list {@link AirbyteStateType#STREAM} state messages. + */ + public static List convertLegacyStateToStreamState(final AirbyteStateMessage airbyteStateMessage) { + return Jsons.object(airbyteStateMessage.getData(), DbState.class).getStreams().stream() + .map(s -> new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withNamespace(s.getStreamNamespace()).withName(s.getStreamName())) + .withStreamState(Jsons.jsonNode(s)))) + .collect(Collectors.toList()); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManager.java new file mode 100644 index 000000000000..a4234454b06f --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManager.java @@ -0,0 +1,150 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import com.google.common.base.Preconditions; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.protocol.models.AirbyteStateMessage; +import java.util.Map; +import java.util.Optional; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Defines a manager that manages connector state. Connector state is used to keep track of the data + * synced by the connector. + * + * @param The type of the state maintained by the manager. + * @param The type of the stream(s) stored within the state maintained by the manager. + */ +public interface StateManager { + + Logger LOGGER = LoggerFactory.getLogger(StateManager.class); + + /** + * Retrieves the {@link CdcStateManager} associated with the state manager. + * + * @return The {@link CdcStateManager} + * @throws UnsupportedOperationException if the state manager does not support tracking change data + * capture (CDC) state. + */ + CdcStateManager getCdcStateManager(); + + /** + * Retrieves the map of stream name/namespace tuple to the current cursor information for that + * stream. + * + * @return The map of stream name/namespace tuple to the current cursor information for that stream + * as maintained by this state manager. + */ + Map getPairToCursorInfoMap(); + + /** + * Generates an {@link AirbyteStateMessage} that represents the current state contained in the state + * manager. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} that represents a stream managed by the + * state manager. + * @return The {@link AirbyteStateMessage} that represents the current state contained in the state + * manager. + */ + AirbyteStateMessage toState(final Optional pair); + + /** + * Retrieves an {@link Optional} possibly containing the cursor value tracked in the state + * associated with the provided stream name/namespace tuple. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} which identifies a stream. + * @return An {@link Optional} possibly containing the cursor value tracked in the state associated + * with the provided stream name/namespace tuple. + */ + default Optional getCursor(final AirbyteStreamNameNamespacePair pair) { + return getCursorInfo(pair).map(CursorInfo::getCursor); + } + + /** + * Retrieves an {@link Optional} possibly containing the cursor field name associated with the + * cursor tracked in the state associated with the provided stream name/namespace tuple. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} which identifies a stream. + * @return An {@link Optional} possibly containing the cursor field name associated with the cursor + * tracked in the state associated with the provided stream name/namespace tuple. + */ + default Optional getCursorField(final AirbyteStreamNameNamespacePair pair) { + return getCursorInfo(pair).map(CursorInfo::getCursorField); + } + + /** + * Retrieves an {@link Optional} possibly containing the original cursor value tracked in the state + * associated with the provided stream name/namespace tuple. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} which identifies a stream. + * @return An {@link Optional} possibly containing the original cursor value tracked in the state + * associated with the provided stream name/namespace tuple. + */ + default Optional getOriginalCursor(final AirbyteStreamNameNamespacePair pair) { + return getCursorInfo(pair).map(CursorInfo::getOriginalCursor); + } + + /** + * Retrieves an {@link Optional} possibly containing the original cursor field name associated with + * the cursor tracked in the state associated with the provided stream name/namespace tuple. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} which identifies a stream. + * @return An {@link Optional} possibly containing the original cursor field name associated with + * the cursor tracked in the state associated with the provided stream name/namespace tuple. + */ + default Optional getOriginalCursorField(final AirbyteStreamNameNamespacePair pair) { + return getCursorInfo(pair).map(CursorInfo::getOriginalCursorField); + } + + /** + * Retrieves the current cursor information stored in the state manager for the steam name/namespace + * tuple. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} that represents a stream managed by the + * state manager. + * @return {@link Optional} that potentially contains the current cursor information for the given + * stream name/namespace tuple. + */ + default Optional getCursorInfo(final AirbyteStreamNameNamespacePair pair) { + return Optional.ofNullable(getPairToCursorInfoMap().get(pair)); + } + + /** + * Emits the current state maintained by the manager as an {@link AirbyteStateMessage}. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} that represents a stream managed by the + * state manager. + * @return An {@link AirbyteStateMessage} that represents the current state maintained by the state + * manager. + */ + default AirbyteStateMessage emit(final Optional pair) { + return toState(pair); + } + + /** + * Updates the cursor associated with the provided stream name/namespace pair and emits the current + * state maintained by the state manager. + * + * @param pair The {@link AirbyteStreamNameNamespacePair} that represents a stream managed by the + * state manager. + * @param cursor The new value for the cursor associated with the + * {@link AirbyteStreamNameNamespacePair} that represents a stream managed by the state + * manager. + * @return An {@link AirbyteStateMessage} that represents the current state maintained by the state + * manager. + */ + default AirbyteStateMessage updateAndEmit(final AirbyteStreamNameNamespacePair pair, final String cursor) { + final Optional cursorInfo = getCursorInfo(pair); + Preconditions.checkState(cursorInfo.isPresent(), "Could not find cursor information for stream: " + pair); + LOGGER.debug("Updating cursor value for {} to {}...", pair, cursor); + cursorInfo.get().setCursor(cursor); + return emit(Optional.ofNullable(pair)); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactory.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactory.java new file mode 100644 index 000000000000..a5dddedc9ebe --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactory.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.ArrayList; +import java.util.List; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Factory class that creates {@link StateManager} instances based on the provided state. + */ +public class StateManagerFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(StateManagerFactory.class); + + /** + * Private constructor to prevent direct instantiation. + */ + private StateManagerFactory() {} + + /** + * Creates a {@link StateManager} based on the provided state object and catalog. This method will + * handle the conversion of the provided state to match the requested state manager based on the + * provided {@link AirbyteStateType}. + * + * @param supportedStateType The type of state supported by the connector. + * @param initialState The deserialized initial state that will be provided to the selected + * {@link StateManager}. + * @param catalog The {@link ConfiguredAirbyteCatalog} for the connector that will utilize the state + * manager. + * @return A newly created {@link StateManager} implementation based on the provided state. + */ + public static StateManager createStateManager(final AirbyteStateType supportedStateType, + final List initialState, + final ConfiguredAirbyteCatalog catalog) { + if (initialState != null && !initialState.isEmpty()) { + final AirbyteStateMessage airbyteStateMessage = initialState.get(0); + switch (supportedStateType) { + case LEGACY: + LOGGER.info("Legacy state manager selected to manage state object with type {}.", airbyteStateMessage.getStateType()); + return new LegacyStateManager(Jsons.object(airbyteStateMessage.getData(), DbState.class), catalog); + case GLOBAL: + LOGGER.info("Global state manager selected to manage state object with type {}.", airbyteStateMessage.getStateType()); + return new GlobalStateManager(generateGlobalState(airbyteStateMessage), catalog); + case STREAM: + default: + LOGGER.info("Stream state manager selected to manage state object with type {}.", airbyteStateMessage.getStateType()); + return new StreamStateManager(generateStreamState(initialState), catalog); + } + } else { + throw new IllegalArgumentException("Failed to create state manager due to empty state list."); + } + } + + /** + * Handles the conversion between a different state type and the global state. This method handles + * the following transitions: + *
    + *
  • Stream -> Global (not supported, results in {@link IllegalArgumentException}
  • + *
  • Legacy -> Global (supported)
  • + *
  • Global -> Global (supported/no conversion required)
  • + *
+ * + * @param airbyteStateMessage The current state that is to be converted to global state. + * @return The converted state message. + * @throws IllegalArgumentException if unable to convert between the given state type and global. + */ + private static AirbyteStateMessage generateGlobalState(final AirbyteStateMessage airbyteStateMessage) { + AirbyteStateMessage globalStateMessage = airbyteStateMessage; + + switch (airbyteStateMessage.getStateType()) { + case STREAM: + throw new IllegalArgumentException("Unable to convert connector state from stream to global. Please reset the connection to continue."); + case LEGACY: + globalStateMessage = StateGeneratorUtils.convertLegacyStateToGlobalState(airbyteStateMessage); + LOGGER.info("Legacy state converted to global state.", airbyteStateMessage.getStateType()); + break; + case GLOBAL: + default: + break; + } + + return globalStateMessage; + } + + /** + * Handles the conversion between a different state type and the stream state. This method handles + * the following transitions: + *
    + *
  • Global -> Stream (not supported, results in {@link IllegalArgumentException}
  • + *
  • Legacy -> Stream (supported)
  • + *
  • Stream -> Stream (supported/no conversion required)
  • + *
+ * + * @param states The list of current states. + * @return The converted state messages. + * @throws IllegalArgumentException if unable to convert between the given state type and stream. + */ + private static List generateStreamState(final List states) { + final AirbyteStateMessage airbyteStateMessage = states.get(0); + final List streamStates = new ArrayList<>(); + switch (airbyteStateMessage.getStateType()) { + case GLOBAL: + throw new IllegalArgumentException("Unable to convert connector state from global to stream. Please reset the connection to continue."); + case LEGACY: + streamStates.addAll(StateGeneratorUtils.convertLegacyStateToStreamState(airbyteStateMessage)); + break; + case STREAM: + default: + streamStates.addAll(states); + break; + } + + return streamStates; + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManager.java b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManager.java new file mode 100644 index 000000000000..9fee0a39ab6c --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/main/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManager.java @@ -0,0 +1,81 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FIELD_FUNCTION; +import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.CURSOR_FUNCTION; +import static io.airbyte.integrations.source.relationaldb.state.StateGeneratorUtils.NAME_NAMESPACE_PAIR_FUNCTION; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.CdcStateManager; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.AirbyteStreamState; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Per-stream implementation of the {@link StateManager} interface. + * + * This implementation generates a state object for each stream detected in catalog/map of known + * streams to cursor information stored in this manager. + */ +public class StreamStateManager extends AbstractStateManager { + + private static final Logger LOGGER = LoggerFactory.getLogger(StreamStateManager.class); + + /** + * Constructs a new {@link StreamStateManager} that is seeded with the provided + * {@link AirbyteStateMessage}. + * + * @param airbyteStateMessages The initial state represented as a list of + * {@link AirbyteStateMessage}s. + * @param catalog The {@link ConfiguredAirbyteCatalog} for the connector associated with this state + * manager. + */ + public StreamStateManager(final List airbyteStateMessages, final ConfiguredAirbyteCatalog catalog) { + super(catalog, + () -> airbyteStateMessages.stream().map(a -> a.getStream()).collect(Collectors.toList()), + CURSOR_FUNCTION, + CURSOR_FIELD_FUNCTION, + NAME_NAMESPACE_PAIR_FUNCTION); + } + + @Override + public CdcStateManager getCdcStateManager() { + throw new UnsupportedOperationException("CDC state management not supported by stream state manager."); + } + + @Override + public AirbyteStateMessage toState(final Optional pair) { + if (pair.isPresent()) { + final Map pairToCursorInfoMap = getPairToCursorInfoMap(); + final Optional cursorInfo = Optional.ofNullable(pairToCursorInfoMap.get(pair.get())); + + if (cursorInfo.isPresent()) { + LOGGER.debug("Generating state message for {}...", pair); + return new AirbyteStateMessage() + .withStateType(AirbyteStateType.STREAM) + // Temporarily include legacy state for backwards compatibility with the platform + .withData(Jsons.jsonNode(StateGeneratorUtils.generateDbState(pairToCursorInfoMap))) + .withStream(StateGeneratorUtils.generateStreamState(pair.get(), cursorInfo.get())); + } else { + LOGGER.warn("Cursor information could not be located in state for stream {}. Returning a new, empty state message...", pair); + return new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState()); + } + } else { + LOGGER.warn("Stream not provided. Returning a new, empty state message..."); + return new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState()); + } + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java index 7fb6964d2654..e464a95e40fa 100644 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateDecoratingIteratorTest.java @@ -14,6 +14,7 @@ import io.airbyte.commons.json.Jsons; import io.airbyte.commons.util.MoreIterators; import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.state.StateManager; import io.airbyte.protocol.models.AirbyteMessage; import io.airbyte.protocol.models.AirbyteMessage.Type; import io.airbyte.protocol.models.AirbyteRecordMessage; diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateManagerTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateManagerTest.java deleted file mode 100644 index 9e64edb55b7e..000000000000 --- a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/StateManagerTest.java +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (c) 2022 Airbyte, Inc., all rights reserved. - */ - -package io.airbyte.integrations.source.relationaldb; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -import io.airbyte.commons.json.Jsons; -import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; -import io.airbyte.integrations.source.relationaldb.models.DbState; -import io.airbyte.integrations.source.relationaldb.models.DbStreamState; -import io.airbyte.protocol.models.AirbyteStateMessage; -import io.airbyte.protocol.models.AirbyteStream; -import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; -import io.airbyte.protocol.models.ConfiguredAirbyteStream; -import java.util.Collections; -import java.util.Comparator; -import java.util.Optional; -import java.util.stream.Collectors; -import org.junit.jupiter.api.Test; -import org.testcontainers.shaded.com.google.common.collect.Lists; - -class StateManagerTest { - - private static final String NAMESPACE = "public"; - private static final String STREAM_NAME1 = "cars"; - private static final AirbyteStreamNameNamespacePair NAME_NAMESPACE_PAIR1 = new AirbyteStreamNameNamespacePair(STREAM_NAME1, NAMESPACE); - private static final String STREAM_NAME2 = "bicycles"; - private static final AirbyteStreamNameNamespacePair NAME_NAMESPACE_PAIR2 = new AirbyteStreamNameNamespacePair(STREAM_NAME2, NAMESPACE); - private static final String STREAM_NAME3 = "stationary_bicycles"; - private static final String CURSOR_FIELD1 = "year"; - private static final String CURSOR_FIELD2 = "generation"; - private static final String CURSOR = "2000"; - - @Test - void testCreateCursorInfoCatalogAndStateSameCursorField() { - final CursorInfo actual = - StateManager.createCursorInfoForStream(NAME_NAMESPACE_PAIR1, getState(CURSOR_FIELD1, CURSOR), getCatalog(CURSOR_FIELD1)); - assertEquals(new CursorInfo(CURSOR_FIELD1, CURSOR, CURSOR_FIELD1, CURSOR), actual); - } - - @Test - void testCreateCursorInfoCatalogAndStateSameCursorFieldButNoCursor() { - final CursorInfo actual = - StateManager.createCursorInfoForStream(NAME_NAMESPACE_PAIR1, getState(CURSOR_FIELD1, null), getCatalog(CURSOR_FIELD1)); - assertEquals(new CursorInfo(CURSOR_FIELD1, null, CURSOR_FIELD1, null), actual); - } - - @Test - void testCreateCursorInfoCatalogAndStateChangeInCursorFieldName() { - final CursorInfo actual = - StateManager.createCursorInfoForStream(NAME_NAMESPACE_PAIR1, getState(CURSOR_FIELD1, CURSOR), getCatalog(CURSOR_FIELD2)); - assertEquals(new CursorInfo(CURSOR_FIELD1, CURSOR, CURSOR_FIELD2, null), actual); - } - - @Test - void testCreateCursorInfoCatalogAndNoState() { - final CursorInfo actual = StateManager - .createCursorInfoForStream(NAME_NAMESPACE_PAIR1, Optional.empty(), getCatalog(CURSOR_FIELD1)); - assertEquals(new CursorInfo(null, null, CURSOR_FIELD1, null), actual); - } - - @Test - void testCreateCursorInfoStateAndNoCatalog() { - final CursorInfo actual = StateManager - .createCursorInfoForStream(NAME_NAMESPACE_PAIR1, getState(CURSOR_FIELD1, CURSOR), Optional.empty()); - assertEquals(new CursorInfo(CURSOR_FIELD1, CURSOR, null, null), actual); - } - - // this is what full refresh looks like. - @Test - void testCreateCursorInfoNoCatalogAndNoState() { - final CursorInfo actual = StateManager - .createCursorInfoForStream(NAME_NAMESPACE_PAIR1, Optional.empty(), Optional.empty()); - assertEquals(new CursorInfo(null, null, null, null), actual); - } - - @Test - void testCreateCursorInfoStateAndCatalogButNoCursorField() { - final CursorInfo actual = StateManager - .createCursorInfoForStream(NAME_NAMESPACE_PAIR1, getState(CURSOR_FIELD1, CURSOR), getCatalog(null)); - assertEquals(new CursorInfo(CURSOR_FIELD1, CURSOR, null, null), actual); - } - - @SuppressWarnings("SameParameterValue") - private static Optional getState(final String cursorField, final String cursor) { - return Optional.of(new DbStreamState() - .withStreamName(STREAM_NAME1) - .withCursorField(Lists.newArrayList(cursorField)) - .withCursor(cursor)); - } - - private static Optional getCatalog(final String cursorField) { - return Optional.of(new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1)) - .withCursorField(cursorField == null ? Collections.emptyList() : Lists.newArrayList(cursorField))); - } - - @Test - void testGetters() { - final DbState state = new DbState().withStreams(Lists.newArrayList( - new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(Lists.newArrayList(CURSOR_FIELD1)) - .withCursor(CURSOR), - new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE))); - - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() - .withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) - .withCursorField(Lists.newArrayList(CURSOR_FIELD1)), - new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); - - final StateManager stateManager = new StateManager(state, catalog); - - assertEquals(Optional.of(CURSOR_FIELD1), stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR1)); - assertEquals(Optional.of(CURSOR), stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR1)); - assertEquals(Optional.of(CURSOR_FIELD1), stateManager.getCursorField(NAME_NAMESPACE_PAIR1)); - assertEquals(Optional.of(CURSOR), stateManager.getCursor(NAME_NAMESPACE_PAIR1)); - - assertEquals(Optional.empty(), stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR2)); - assertEquals(Optional.empty(), stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR2)); - assertEquals(Optional.empty(), stateManager.getCursorField(NAME_NAMESPACE_PAIR2)); - assertEquals(Optional.empty(), stateManager.getCursor(NAME_NAMESPACE_PAIR2)); - } - - @Test - void testToState() { - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() - .withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) - .withCursorField(Lists.newArrayList(CURSOR_FIELD1)), - new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) - .withCursorField(Lists.newArrayList(CURSOR_FIELD2)), - new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); - - final StateManager stateManager = new StateManager(new DbState(), catalog); - - final AirbyteStateMessage expectedFirstEmission = new AirbyteStateMessage() - .withData(Jsons.jsonNode(new DbState().withStreams(Lists - .newArrayList( - new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(Lists.newArrayList(CURSOR_FIELD1)) - .withCursor("a"), - new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE).withCursorField(Lists.newArrayList(CURSOR_FIELD2)), - new DbStreamState().withStreamName(STREAM_NAME3).withStreamNamespace(NAMESPACE)) - .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())) - .withCdc(false))); - final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); - assertEquals(expectedFirstEmission, actualFirstEmission); - final AirbyteStateMessage expectedSecondEmission = new AirbyteStateMessage() - .withData(Jsons.jsonNode(new DbState().withStreams(Lists - .newArrayList( - new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(Lists.newArrayList(CURSOR_FIELD1)) - .withCursor("a"), - new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE).withCursorField(Lists.newArrayList(CURSOR_FIELD2)) - .withCursor("b"), - new DbStreamState().withStreamName(STREAM_NAME3).withStreamNamespace(NAMESPACE)) - .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())) - .withCdc(false))); - final AirbyteStateMessage actualSecondEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR2, "b"); - assertEquals(expectedSecondEmission, actualSecondEmission); - } - - @Test - void testToStateNullCursorField() { - final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() - .withStreams(Lists.newArrayList( - new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) - .withCursorField(Lists.newArrayList(CURSOR_FIELD1)), - new ConfiguredAirbyteStream() - .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); - final StateManager stateManager = new StateManager(new DbState(), catalog); - - final AirbyteStateMessage expectedFirstEmission = new AirbyteStateMessage() - .withData(Jsons.jsonNode(new DbState().withStreams(Lists - .newArrayList( - new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(Lists.newArrayList(CURSOR_FIELD1)) - .withCursor("a"), - new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE)) - .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())) - .withCdc(false))); - - final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); - assertEquals(expectedFirstEmission, actualFirstEmission); - } - -} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java new file mode 100644 index 000000000000..67b7fddc23f5 --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/CursorManagerTest.java @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.getCatalog; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.getState; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.getStream; +import static org.junit.jupiter.api.Assertions.assertEquals; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.CursorInfo; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import java.util.Collections; +import java.util.Optional; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link CursorManager} class. + */ +public class CursorManagerTest { + + @Test + void testCreateCursorInfoCatalogAndStateSameCursorField() { + final CursorManager cursorManager = createCursorManager(CURSOR_FIELD1, CURSOR, NAME_NAMESPACE_PAIR1); + final CursorInfo actual = cursorManager.createCursorInfoForStream( + NAME_NAMESPACE_PAIR1, + getState(CURSOR_FIELD1, CURSOR), + getStream(CURSOR_FIELD1), + DbStreamState::getCursor, + DbStreamState::getCursorField); + assertEquals(new CursorInfo(CURSOR_FIELD1, CURSOR, CURSOR_FIELD1, CURSOR), actual); + } + + @Test + void testCreateCursorInfoCatalogAndStateSameCursorFieldButNoCursor() { + final CursorManager cursorManager = createCursorManager(CURSOR_FIELD1, null, NAME_NAMESPACE_PAIR1); + final CursorInfo actual = cursorManager.createCursorInfoForStream( + NAME_NAMESPACE_PAIR1, + getState(CURSOR_FIELD1, null), + getStream(CURSOR_FIELD1), + DbStreamState::getCursor, + DbStreamState::getCursorField); + assertEquals(new CursorInfo(CURSOR_FIELD1, null, CURSOR_FIELD1, null), actual); + } + + @Test + void testCreateCursorInfoCatalogAndStateChangeInCursorFieldName() { + final CursorManager cursorManager = createCursorManager(CURSOR_FIELD1, CURSOR, NAME_NAMESPACE_PAIR1); + final CursorInfo actual = cursorManager.createCursorInfoForStream( + NAME_NAMESPACE_PAIR1, + getState(CURSOR_FIELD1, CURSOR), + getStream(CURSOR_FIELD2), + DbStreamState::getCursor, + DbStreamState::getCursorField); + assertEquals(new CursorInfo(CURSOR_FIELD1, CURSOR, CURSOR_FIELD2, null), actual); + } + + @Test + void testCreateCursorInfoCatalogAndNoState() { + final CursorManager cursorManager = createCursorManager(CURSOR_FIELD1, CURSOR, NAME_NAMESPACE_PAIR1); + final CursorInfo actual = cursorManager.createCursorInfoForStream( + NAME_NAMESPACE_PAIR1, + Optional.empty(), + getStream(CURSOR_FIELD1), + DbStreamState::getCursor, + DbStreamState::getCursorField); + assertEquals(new CursorInfo(null, null, CURSOR_FIELD1, null), actual); + } + + @Test + void testCreateCursorInfoStateAndNoCatalog() { + final CursorManager cursorManager = createCursorManager(CURSOR_FIELD1, CURSOR, NAME_NAMESPACE_PAIR1); + final CursorInfo actual = cursorManager.createCursorInfoForStream( + NAME_NAMESPACE_PAIR1, + getState(CURSOR_FIELD1, CURSOR), + Optional.empty(), + DbStreamState::getCursor, + DbStreamState::getCursorField); + assertEquals(new CursorInfo(CURSOR_FIELD1, CURSOR, null, null), actual); + } + + // this is what full refresh looks like. + @Test + void testCreateCursorInfoNoCatalogAndNoState() { + final CursorManager cursorManager = createCursorManager(CURSOR_FIELD1, CURSOR, NAME_NAMESPACE_PAIR1); + final CursorInfo actual = cursorManager.createCursorInfoForStream( + NAME_NAMESPACE_PAIR1, + Optional.empty(), + Optional.empty(), + DbStreamState::getCursor, + DbStreamState::getCursorField); + assertEquals(new CursorInfo(null, null, null, null), actual); + } + + @Test + void testCreateCursorInfoStateAndCatalogButNoCursorField() { + final CursorManager cursorManager = createCursorManager(CURSOR_FIELD1, CURSOR, NAME_NAMESPACE_PAIR1); + final CursorInfo actual = cursorManager.createCursorInfoForStream( + NAME_NAMESPACE_PAIR1, + getState(CURSOR_FIELD1, CURSOR), + getStream(null), + DbStreamState::getCursor, + DbStreamState::getCursorField); + assertEquals(new CursorInfo(CURSOR_FIELD1, CURSOR, null, null), actual); + } + + @Test + void testGetters() { + final CursorManager cursorManager = createCursorManager(CURSOR_FIELD1, CURSOR, NAME_NAMESPACE_PAIR1); + final CursorInfo actualCursorInfo = new CursorInfo(CURSOR_FIELD1, CURSOR, null, null); + + assertEquals(Optional.of(actualCursorInfo), cursorManager.getCursorInfo(NAME_NAMESPACE_PAIR1)); + assertEquals(Optional.empty(), cursorManager.getCursorField(NAME_NAMESPACE_PAIR1)); + assertEquals(Optional.empty(), cursorManager.getCursor(NAME_NAMESPACE_PAIR1)); + + assertEquals(Optional.empty(), cursorManager.getCursorInfo(NAME_NAMESPACE_PAIR2)); + assertEquals(Optional.empty(), cursorManager.getCursorField(NAME_NAMESPACE_PAIR2)); + assertEquals(Optional.empty(), cursorManager.getCursor(NAME_NAMESPACE_PAIR2)); + } + + private CursorManager createCursorManager(final String cursorField, + final String cursor, + final AirbyteStreamNameNamespacePair nameNamespacePair) { + final DbStreamState dbStreamState = getState(cursorField, cursor).get(); + return new CursorManager<>( + getCatalog(cursorField).orElse(null), + () -> Collections.singleton(dbStreamState), + DbStreamState::getCursor, + DbStreamState::getCursorField, + s -> nameNamespacePair); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManagerTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManagerTest.java new file mode 100644 index 000000000000..c39ca83c16d2 --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/GlobalStateManagerTest.java @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteGlobalState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.AirbyteStreamState; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.StreamDescriptor; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link GlobalStateManager} class. + */ +public class GlobalStateManagerTest { + + @Test + void testCdcStateManager() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final CdcState cdcState = new CdcState().withState(Jsons.jsonNode(Map.of("foo", "bar", "baz", 5))); + final AirbyteGlobalState globalState = new AirbyteGlobalState().withSharedState(Jsons.jsonNode(cdcState)) + .withStreamStates(List.of(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withNamespace("namespace").withName("name")) + .withStreamState(Jsons.jsonNode(new DbStreamState())))); + final StateManager stateManager = + new GlobalStateManager(new AirbyteStateMessage().withStateType(AirbyteStateType.GLOBAL).withGlobal(globalState), catalog); + assertNotNull(stateManager.getCdcStateManager()); + assertEquals(cdcState, stateManager.getCdcStateManager().getCdcState()); + } + + @Test + void testToStateFromLegacyState() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD2)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + + final CdcState cdcState = new CdcState().withState(Jsons.jsonNode(Map.of("foo", "bar", "baz", 5))); + final DbState dbState = new DbState() + .withCdc(true) + .withCdcState(cdcState) + .withStreams(List.of( + new DbStreamState() + .withStreamName(STREAM_NAME1) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState() + .withStreamName(STREAM_NAME2) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD2)), + new DbStreamState() + .withStreamName(STREAM_NAME3) + .withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())); + final StateManager stateManager = new GlobalStateManager(new AirbyteStateMessage().withData(Jsons.jsonNode(dbState)), catalog); + + final DbState expectedDbState = new DbState() + .withCdc(true) + .withCdcState(cdcState) + .withStreams(List.of( + new DbStreamState() + .withStreamName(STREAM_NAME1) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState() + .withStreamName(STREAM_NAME2) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD2)), + new DbStreamState() + .withStreamName(STREAM_NAME3) + .withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())); + + final AirbyteGlobalState expectedGlobalState = new AirbyteGlobalState() + .withSharedState(Jsons.jsonNode(cdcState)) + .withStreamStates(List.of( + new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(STREAM_NAME1) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"))), + new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(STREAM_NAME2) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD2)))), + new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME3).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(STREAM_NAME3) + .withStreamNamespace(NAMESPACE)))) + .stream().sorted(Comparator.comparing(o -> o.getStreamDescriptor().getName())).collect(Collectors.toList())); + final AirbyteStateMessage expected = new AirbyteStateMessage() + .withData(Jsons.jsonNode(expectedDbState)) + .withGlobal(expectedGlobalState) + .withStateType(AirbyteStateType.GLOBAL); + + final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); + assertEquals(expected, actualFirstEmission); + } + + @Test + void testToState() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD2)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + + final CdcState cdcState = new CdcState().withState(Jsons.jsonNode(Map.of("foo", "bar", "baz", 5))); + final AirbyteGlobalState globalState = new AirbyteGlobalState().withSharedState(Jsons.jsonNode(new DbState())).withStreamStates( + List.of(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor()).withStreamState(Jsons.jsonNode(new DbStreamState())))); + final StateManager stateManager = + new GlobalStateManager(new AirbyteStateMessage().withStateType(AirbyteStateType.GLOBAL).withGlobal(globalState), catalog); + stateManager.getCdcStateManager().setCdcState(cdcState); + + final DbState expectedDbState = new DbState() + .withCdc(true) + .withCdcState(cdcState) + .withStreams(List.of( + new DbStreamState() + .withStreamName(STREAM_NAME1) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState() + .withStreamName(STREAM_NAME2) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD2)), + new DbStreamState() + .withStreamName(STREAM_NAME3) + .withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())); + + final AirbyteGlobalState expectedGlobalState = new AirbyteGlobalState() + .withSharedState(Jsons.jsonNode(cdcState)) + .withStreamStates(List.of( + new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(STREAM_NAME1) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"))), + new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(STREAM_NAME2) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD2)))), + new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME3).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode(new DbStreamState() + .withStreamName(STREAM_NAME3) + .withStreamNamespace(NAMESPACE)))) + .stream().sorted(Comparator.comparing(o -> o.getStreamDescriptor().getName())).collect(Collectors.toList())); + final AirbyteStateMessage expected = new AirbyteStateMessage() + .withData(Jsons.jsonNode(expectedDbState)) + .withGlobal(expectedGlobalState) + .withStateType(AirbyteStateType.GLOBAL); + + final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); + assertEquals(expected, actualFirstEmission); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManagerTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManagerTest.java new file mode 100644 index 000000000000..cbf41a7415e4 --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/LegacyStateManagerTest.java @@ -0,0 +1,181 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.mockito.Mockito.mock; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link LegacyStateManager} class. + */ +public class LegacyStateManagerTest { + + @Test + void testGetters() { + final DbState state = new DbState().withStreams(List.of( + new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(List.of(CURSOR_FIELD1)) + .withCursor(CURSOR), + new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE))); + + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); + + final StateManager stateManager = new LegacyStateManager(state, catalog); + + assertEquals(Optional.of(CURSOR_FIELD1), stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR1)); + assertEquals(Optional.of(CURSOR), stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR1)); + assertEquals(Optional.of(CURSOR_FIELD1), stateManager.getCursorField(NAME_NAMESPACE_PAIR1)); + assertEquals(Optional.of(CURSOR), stateManager.getCursor(NAME_NAMESPACE_PAIR1)); + + assertEquals(Optional.empty(), stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR2)); + assertEquals(Optional.empty(), stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR2)); + assertEquals(Optional.empty(), stateManager.getCursorField(NAME_NAMESPACE_PAIR2)); + assertEquals(Optional.empty(), stateManager.getCursor(NAME_NAMESPACE_PAIR2)); + } + + @Test + void testToState() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD2)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + + final StateManager stateManager = new LegacyStateManager(new DbState(), catalog); + + final AirbyteStateMessage expectedFirstEmission = new AirbyteStateMessage() + .withStateType(AirbyteStateType.LEGACY) + .withData(Jsons.jsonNode(new DbState().withStreams(List.of( + new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE).withCursorField(List.of(CURSOR_FIELD2)), + new DbStreamState().withStreamName(STREAM_NAME3).withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())) + .withCdc(false))); + final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); + assertEquals(expectedFirstEmission, actualFirstEmission); + final AirbyteStateMessage expectedSecondEmission = new AirbyteStateMessage() + .withStateType(AirbyteStateType.LEGACY) + .withData(Jsons.jsonNode(new DbState().withStreams(List.of( + new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE).withCursorField(List.of(CURSOR_FIELD2)) + .withCursor("b"), + new DbStreamState().withStreamName(STREAM_NAME3).withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())) + .withCdc(false))); + final AirbyteStateMessage actualSecondEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR2, "b"); + assertEquals(expectedSecondEmission, actualSecondEmission); + } + + @Test + void testToStateNullCursorField() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); + final StateManager stateManager = new LegacyStateManager(new DbState(), catalog); + + final AirbyteStateMessage expectedFirstEmission = new AirbyteStateMessage() + .withStateType(AirbyteStateType.LEGACY) + .withData(Jsons.jsonNode(new DbState().withStreams(List.of( + new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())) + .withCdc(false))); + + final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); + assertEquals(expectedFirstEmission, actualFirstEmission); + } + + @Test + void testCursorNotUpdatedForCdc() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); + + final DbState state = new DbState(); + state.setCdc(true); + final StateManager stateManager = new LegacyStateManager(state, catalog); + + final AirbyteStateMessage expectedFirstEmission = new AirbyteStateMessage() + .withStateType(AirbyteStateType.LEGACY) + .withData(Jsons.jsonNode(new DbState().withStreams(List.of( + new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(List.of(CURSOR_FIELD1)) + .withCursor(null), + new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE).withCursorField(List.of())) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())) + .withCdc(true))); + final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); + assertEquals(expectedFirstEmission, actualFirstEmission); + final AirbyteStateMessage expectedSecondEmission = new AirbyteStateMessage() + .withStateType(AirbyteStateType.LEGACY) + .withData(Jsons.jsonNode(new DbState().withStreams(List.of( + new DbStreamState().withStreamName(STREAM_NAME1).withStreamNamespace(NAMESPACE).withCursorField(List.of(CURSOR_FIELD1)) + .withCursor(null), + new DbStreamState().withStreamName(STREAM_NAME2).withStreamNamespace(NAMESPACE).withCursorField(List.of()) + .withCursor(null)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())) + .withCdc(true))); + final AirbyteStateMessage actualSecondEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR2, "b"); + assertEquals(expectedSecondEmission, actualSecondEmission); + } + + @Test + void testCdcStateManager() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final CdcState cdcState = new CdcState().withState(Jsons.jsonNode(Map.of("foo", "bar", "baz", 5))); + final DbState dbState = new DbState().withCdcState(cdcState).withStreams(List.of( + new DbStreamState().withStreamNamespace(NAMESPACE).withStreamName(STREAM_NAME1))); + final StateManager stateManager = new LegacyStateManager(dbState, catalog); + assertNotNull(stateManager.getCdcStateManager()); + assertEquals(cdcState, stateManager.getCdcStateManager().getCdcState()); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java new file mode 100644 index 000000000000..9ac94775c928 --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateGeneratorUtilsTest.java @@ -0,0 +1,39 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import io.airbyte.protocol.models.StreamDescriptor; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link StateGeneratorUtils} class. + */ +public class StateGeneratorUtilsTest { + + @Test + void testValidStreamDescriptor() { + final StreamDescriptor streamDescriptor1 = null; + final StreamDescriptor streamDescriptor2 = new StreamDescriptor(); + final StreamDescriptor streamDescriptor3 = new StreamDescriptor().withName("name"); + final StreamDescriptor streamDescriptor4 = new StreamDescriptor().withNamespace("namespace"); + final StreamDescriptor streamDescriptor5 = new StreamDescriptor().withName("name").withNamespace("namespace"); + final StreamDescriptor streamDescriptor6 = new StreamDescriptor().withName("name").withNamespace(""); + final StreamDescriptor streamDescriptor7 = new StreamDescriptor().withName("").withNamespace("namespace"); + final StreamDescriptor streamDescriptor8 = new StreamDescriptor().withName("").withNamespace(""); + + assertFalse(StateGeneratorUtils.isValidStreamDescriptor(streamDescriptor1)); + assertFalse(StateGeneratorUtils.isValidStreamDescriptor(streamDescriptor2)); + assertTrue(StateGeneratorUtils.isValidStreamDescriptor(streamDescriptor3)); + assertFalse(StateGeneratorUtils.isValidStreamDescriptor(streamDescriptor4)); + assertTrue(StateGeneratorUtils.isValidStreamDescriptor(streamDescriptor5)); + assertTrue(StateGeneratorUtils.isValidStreamDescriptor(streamDescriptor6)); + assertTrue(StateGeneratorUtils.isValidStreamDescriptor(streamDescriptor7)); + assertTrue(StateGeneratorUtils.isValidStreamDescriptor(streamDescriptor8)); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactoryTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactoryTest.java new file mode 100644 index 000000000000..0127b068915a --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateManagerFactoryTest.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.source.relationaldb.models.CdcState; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteGlobalState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.AirbyteStreamState; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.StreamDescriptor; +import java.util.List; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link StateManagerFactory} class. + */ +public class StateManagerFactoryTest { + + private static final String NAMESPACE = "namespace"; + private static final String NAME = "name"; + + @Test + void testNullOrEmptyState() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + + Assertions.assertThrows(IllegalArgumentException.class, () -> { + StateManagerFactory.createStateManager(AirbyteStateType.GLOBAL, null, catalog); + }); + + Assertions.assertThrows(IllegalArgumentException.class, () -> { + StateManagerFactory.createStateManager(AirbyteStateType.GLOBAL, List.of(), catalog); + }); + + Assertions.assertThrows(IllegalArgumentException.class, () -> { + StateManagerFactory.createStateManager(AirbyteStateType.LEGACY, null, catalog); + }); + + Assertions.assertThrows(IllegalArgumentException.class, () -> { + StateManagerFactory.createStateManager(AirbyteStateType.LEGACY, List.of(), catalog); + }); + + Assertions.assertThrows(IllegalArgumentException.class, () -> { + StateManagerFactory.createStateManager(AirbyteStateType.STREAM, null, catalog); + }); + + Assertions.assertThrows(IllegalArgumentException.class, () -> { + StateManagerFactory.createStateManager(AirbyteStateType.STREAM, List.of(), catalog); + }); + } + + @Test + void testLegacyStateManagerCreationFromAirbyteStateMessage() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final AirbyteStateMessage airbyteStateMessage = mock(AirbyteStateMessage.class); + when(airbyteStateMessage.getData()).thenReturn(Jsons.jsonNode(new DbState())); + + final StateManager stateManager = StateManagerFactory.createStateManager(AirbyteStateType.LEGACY, List.of(airbyteStateMessage), catalog); + + Assertions.assertNotNull(stateManager); + Assertions.assertEquals(LegacyStateManager.class, stateManager.getClass()); + } + + @Test + void testGlobalStateManagerCreation() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final AirbyteGlobalState globalState = + new AirbyteGlobalState().withSharedState(Jsons.jsonNode(new DbState().withCdcState(new CdcState().withState(Jsons.jsonNode(new DbState()))))) + .withStreamStates(List.of(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withNamespace(NAMESPACE).withName(NAME)) + .withStreamState(Jsons.jsonNode(new DbStreamState())))); + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage().withStateType(AirbyteStateType.GLOBAL).withGlobal(globalState); + + final StateManager stateManager = StateManagerFactory.createStateManager(AirbyteStateType.GLOBAL, List.of(airbyteStateMessage), catalog); + + Assertions.assertNotNull(stateManager); + Assertions.assertEquals(GlobalStateManager.class, stateManager.getClass()); + } + + @Test + void testGlobalStateManagerCreationFromLegacyState() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final CdcState cdcState = new CdcState(); + final DbState dbState = new DbState() + .withCdcState(cdcState) + .withStreams(List.of(new DbStreamState().withStreamName(NAME).withStreamNamespace(NAMESPACE))); + final AirbyteStateMessage airbyteStateMessage = + new AirbyteStateMessage().withStateType(AirbyteStateType.LEGACY).withData(Jsons.jsonNode(dbState)); + + final StateManager stateManager = StateManagerFactory.createStateManager(AirbyteStateType.GLOBAL, List.of(airbyteStateMessage), catalog); + + Assertions.assertNotNull(stateManager); + Assertions.assertEquals(GlobalStateManager.class, stateManager.getClass()); + } + + @Test + void testGlobalStateManagerCreationFromStreamState() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(NAME).withNamespace( + NAMESPACE)).withStreamState(Jsons.jsonNode(new DbStreamState()))); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> StateManagerFactory.createStateManager(AirbyteStateType.GLOBAL, List.of(airbyteStateMessage), catalog)); + } + + @Test + void testGlobalStateManagerCreationWithLegacyDataPresent() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final AirbyteGlobalState globalState = + new AirbyteGlobalState().withSharedState(Jsons.jsonNode(new DbState().withCdcState(new CdcState().withState(Jsons.jsonNode(new DbState()))))) + .withStreamStates(List.of(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withNamespace(NAMESPACE).withName(NAME)) + .withStreamState(Jsons.jsonNode(new DbStreamState())))); + final AirbyteStateMessage airbyteStateMessage = + new AirbyteStateMessage().withStateType(AirbyteStateType.GLOBAL).withGlobal(globalState).withData(Jsons.jsonNode(new DbState())); + + final StateManager stateManager = StateManagerFactory.createStateManager(AirbyteStateType.GLOBAL, List.of(airbyteStateMessage), catalog); + + Assertions.assertNotNull(stateManager); + Assertions.assertEquals(GlobalStateManager.class, stateManager.getClass()); + } + + @Test + void testStreamStateManagerCreation() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(NAME).withNamespace( + NAMESPACE)).withStreamState(Jsons.jsonNode(new DbStreamState()))); + + final StateManager stateManager = StateManagerFactory.createStateManager(AirbyteStateType.STREAM, List.of(airbyteStateMessage), catalog); + + Assertions.assertNotNull(stateManager); + Assertions.assertEquals(StreamStateManager.class, stateManager.getClass()); + } + + @Test + void testStreamStateManagerCreationFromLegacy() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final CdcState cdcState = new CdcState(); + final DbState dbState = new DbState() + .withCdcState(cdcState) + .withStreams(List.of(new DbStreamState().withStreamName(NAME).withStreamNamespace(NAMESPACE))); + final AirbyteStateMessage airbyteStateMessage = + new AirbyteStateMessage().withStateType(AirbyteStateType.LEGACY).withData(Jsons.jsonNode(dbState)); + + final StateManager stateManager = StateManagerFactory.createStateManager(AirbyteStateType.STREAM, List.of(airbyteStateMessage), catalog); + + Assertions.assertNotNull(stateManager); + Assertions.assertEquals(StreamStateManager.class, stateManager.getClass()); + } + + @Test + void testStreamStateManagerCreationFromGlobal() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final AirbyteGlobalState globalState = + new AirbyteGlobalState().withSharedState(Jsons.jsonNode(new DbState().withCdcState(new CdcState().withState(Jsons.jsonNode(new DbState()))))) + .withStreamStates(List.of(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withNamespace(NAMESPACE).withName(NAME)) + .withStreamState(Jsons.jsonNode(new DbStreamState())))); + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage().withStateType(AirbyteStateType.GLOBAL).withGlobal(globalState); + + Assertions.assertThrows(IllegalArgumentException.class, + () -> StateManagerFactory.createStateManager(AirbyteStateType.STREAM, List.of(airbyteStateMessage), catalog)); + } + + @Test + void testStreamStateManagerCreationWithLegacyDataPresent() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState().withStreamDescriptor(new StreamDescriptor().withName(NAME).withNamespace( + NAMESPACE)).withStreamState(Jsons.jsonNode(new DbStreamState()))) + .withData(Jsons.jsonNode(new DbState())); + + final StateManager stateManager = StateManagerFactory.createStateManager(AirbyteStateType.STREAM, List.of(airbyteStateMessage), catalog); + + Assertions.assertNotNull(stateManager); + Assertions.assertEquals(StreamStateManager.class, stateManager.getClass()); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateTestConstants.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateTestConstants.java new file mode 100644 index 000000000000..e939c9aea87d --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StateTestConstants.java @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import org.testcontainers.shaded.com.google.common.collect.Lists; + +/** + * Collection of constants for use in state management-related tests. + */ +public final class StateTestConstants { + + public static final String NAMESPACE = "public"; + public static final String STREAM_NAME1 = "cars"; + public static final AirbyteStreamNameNamespacePair NAME_NAMESPACE_PAIR1 = new AirbyteStreamNameNamespacePair(STREAM_NAME1, NAMESPACE); + public static final String STREAM_NAME2 = "bicycles"; + public static final AirbyteStreamNameNamespacePair NAME_NAMESPACE_PAIR2 = new AirbyteStreamNameNamespacePair(STREAM_NAME2, NAMESPACE); + public static final String STREAM_NAME3 = "stationary_bicycles"; + public static final String CURSOR_FIELD1 = "year"; + public static final String CURSOR_FIELD2 = "generation"; + public static final String CURSOR = "2000"; + + private StateTestConstants() {} + + @SuppressWarnings("SameParameterValue") + public static Optional getState(final String cursorField, final String cursor) { + return Optional.of(new DbStreamState() + .withStreamName(STREAM_NAME1) + .withCursorField(Lists.newArrayList(cursorField)) + .withCursor(cursor)); + } + + public static Optional getCatalog(final String cursorField) { + return Optional.of(new ConfiguredAirbyteCatalog() + .withStreams(List.of(getStream(cursorField).orElse(null)))); + } + + public static Optional getStream(final String cursorField) { + return Optional.of(new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1)) + .withCursorField(cursorField == null ? Collections.emptyList() : Lists.newArrayList(cursorField))); + } + +} diff --git a/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java new file mode 100644 index 000000000000..704dc665cf0d --- /dev/null +++ b/airbyte-integrations/connectors/source-relational-db/src/test/java/io/airbyte/integrations/source/relationaldb/state/StreamStateManagerTest.java @@ -0,0 +1,255 @@ +/* + * Copyright (c) 2022 Airbyte, Inc., all rights reserved. + */ + +package io.airbyte.integrations.source.relationaldb.state; + +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.CURSOR_FIELD2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAMESPACE; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.NAME_NAMESPACE_PAIR2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME1; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME2; +import static io.airbyte.integrations.source.relationaldb.state.StateTestConstants.STREAM_NAME3; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; + +import io.airbyte.commons.json.Jsons; +import io.airbyte.integrations.base.AirbyteStreamNameNamespacePair; +import io.airbyte.integrations.source.relationaldb.models.DbState; +import io.airbyte.integrations.source.relationaldb.models.DbStreamState; +import io.airbyte.protocol.models.AirbyteStateMessage; +import io.airbyte.protocol.models.AirbyteStateMessage.AirbyteStateType; +import io.airbyte.protocol.models.AirbyteStream; +import io.airbyte.protocol.models.AirbyteStreamState; +import io.airbyte.protocol.models.ConfiguredAirbyteCatalog; +import io.airbyte.protocol.models.ConfiguredAirbyteStream; +import io.airbyte.protocol.models.StreamDescriptor; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +/** + * Test suite for the {@link StreamStateManager} class. + */ +public class StreamStateManagerTest { + + @Test + void testCreationFromInvalidState() { + final AirbyteStateMessage airbyteStateMessage = new AirbyteStateMessage() + .withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withStreamState(Jsons.jsonNode("Not a state object"))); + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + + Assertions.assertDoesNotThrow(() -> { + final StateManager stateManager = new StreamStateManager(List.of(airbyteStateMessage), catalog); + assertNotNull(stateManager); + }); + } + + @Test + void testGetters() { + final List state = new ArrayList<>(); + state.add(createStreamState(STREAM_NAME1, NAMESPACE, List.of(CURSOR_FIELD1), CURSOR)); + state.add(createStreamState(STREAM_NAME2, NAMESPACE, List.of(), null)); + + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); + + final StateManager stateManager = new StreamStateManager(state, catalog); + + assertEquals(Optional.of(CURSOR_FIELD1), stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR1)); + assertEquals(Optional.of(CURSOR), stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR1)); + assertEquals(Optional.of(CURSOR_FIELD1), stateManager.getCursorField(NAME_NAMESPACE_PAIR1)); + assertEquals(Optional.of(CURSOR), stateManager.getCursor(NAME_NAMESPACE_PAIR1)); + + assertEquals(Optional.empty(), stateManager.getOriginalCursorField(NAME_NAMESPACE_PAIR2)); + assertEquals(Optional.empty(), stateManager.getOriginalCursor(NAME_NAMESPACE_PAIR2)); + assertEquals(Optional.empty(), stateManager.getCursorField(NAME_NAMESPACE_PAIR2)); + assertEquals(Optional.empty(), stateManager.getCursor(NAME_NAMESPACE_PAIR2)); + } + + @Test + void testToState() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD2)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + + final StateManager stateManager = new StreamStateManager(createDefaultState(), catalog); + + final DbState expectedFirstDbState = new DbState() + .withCdc(false) + .withStreams(List.of( + new DbStreamState() + .withStreamName(STREAM_NAME1) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState() + .withStreamName(STREAM_NAME2) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD2)), + new DbStreamState() + .withStreamName(STREAM_NAME3) + .withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())); + final AirbyteStateMessage expectedFirstEmission = + createStreamState(STREAM_NAME1, NAMESPACE, List.of(CURSOR_FIELD1), "a").withData(Jsons.jsonNode(expectedFirstDbState)); + + final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); + assertEquals(expectedFirstEmission, actualFirstEmission); + + final DbState expectedSecondDbState = new DbState() + .withCdc(false) + .withStreams(List.of( + new DbStreamState() + .withStreamName(STREAM_NAME1) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState() + .withStreamName(STREAM_NAME2) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD2)) + .withCursor("b"), + new DbStreamState() + .withStreamName(STREAM_NAME3) + .withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())); + final AirbyteStateMessage expectedSecondEmission = + createStreamState(STREAM_NAME2, NAMESPACE, List.of(CURSOR_FIELD2), "b").withData(Jsons.jsonNode(expectedSecondDbState)); + + final AirbyteStateMessage actualSecondEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR2, "b"); + assertEquals(expectedSecondEmission, actualSecondEmission); + } + + @Test + void testToStateWithoutCursorInfo() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD2)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + final AirbyteStreamNameNamespacePair airbyteStreamNameNamespacePair = new AirbyteStreamNameNamespacePair("other", "other"); + + final StateManager stateManager = new StreamStateManager(createDefaultState(), catalog); + final AirbyteStateMessage airbyteStateMessage = stateManager.toState(Optional.of(airbyteStreamNameNamespacePair)); + assertNotNull(airbyteStateMessage); + assertEquals(AirbyteStateType.STREAM, airbyteStateMessage.getStateType()); + assertNotNull(airbyteStateMessage.getStream()); + } + + @Test + void testToStateWithoutStreamPair() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD2)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME3).withNamespace(NAMESPACE)))); + + final StateManager stateManager = new StreamStateManager(createDefaultState(), catalog); + final AirbyteStateMessage airbyteStateMessage = stateManager.toState(Optional.empty()); + assertNotNull(airbyteStateMessage); + assertEquals(AirbyteStateType.STREAM, airbyteStateMessage.getStateType()); + assertNotNull(airbyteStateMessage.getStream()); + assertNull(airbyteStateMessage.getStream().getStreamState()); + } + + @Test + void testToStateNullCursorField() { + final ConfiguredAirbyteCatalog catalog = new ConfiguredAirbyteCatalog() + .withStreams(List.of( + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME1).withNamespace(NAMESPACE)) + .withCursorField(List.of(CURSOR_FIELD1)), + new ConfiguredAirbyteStream() + .withStream(new AirbyteStream().withName(STREAM_NAME2).withNamespace(NAMESPACE)))); + final StateManager stateManager = new StreamStateManager(createDefaultState(), catalog); + + final DbState expectedFirstDbState = new DbState() + .withCdc(false) + .withStreams(List.of( + new DbStreamState() + .withStreamName(STREAM_NAME1) + .withStreamNamespace(NAMESPACE) + .withCursorField(List.of(CURSOR_FIELD1)) + .withCursor("a"), + new DbStreamState() + .withStreamName(STREAM_NAME2) + .withStreamNamespace(NAMESPACE)) + .stream().sorted(Comparator.comparing(DbStreamState::getStreamName)).collect(Collectors.toList())); + + final AirbyteStateMessage expectedFirstEmission = + createStreamState(STREAM_NAME1, NAMESPACE, List.of(CURSOR_FIELD1), "a").withData(Jsons.jsonNode(expectedFirstDbState)); + final AirbyteStateMessage actualFirstEmission = stateManager.updateAndEmit(NAME_NAMESPACE_PAIR1, "a"); + assertEquals(expectedFirstEmission, actualFirstEmission); + } + + @Test + void testCdcStateManager() { + final ConfiguredAirbyteCatalog catalog = mock(ConfiguredAirbyteCatalog.class); + final StateManager stateManager = new StreamStateManager( + List.of(new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState())), catalog); + Assertions.assertThrows(UnsupportedOperationException.class, () -> stateManager.getCdcStateManager()); + } + + private List createDefaultState() { + return List.of(new AirbyteStateMessage().withStateType(AirbyteStateType.STREAM).withStream(new AirbyteStreamState())); + } + + private AirbyteStateMessage createStreamState(final String name, + final String namespace, + final List cursorFields, + final String cursorValue) { + final DbStreamState dbStreamState = new DbStreamState() + .withStreamName(name) + .withStreamNamespace(namespace); + + if (cursorFields != null && !cursorFields.isEmpty()) { + dbStreamState.withCursorField(cursorFields); + } + + if (cursorValue != null) { + dbStreamState.withCursor(cursorValue); + } + + return new AirbyteStateMessage() + .withStateType(AirbyteStateType.STREAM) + .withStream(new AirbyteStreamState() + .withStreamDescriptor(new StreamDescriptor().withName(name).withNamespace(namespace)) + .withStreamState(Jsons.jsonNode(dbStreamState))); + } + +} diff --git a/airbyte-integrations/connectors/source-tidb/Dockerfile b/airbyte-integrations/connectors/source-tidb/Dockerfile index 6179f1f2b654..d322630a76e5 100755 --- a/airbyte-integrations/connectors/source-tidb/Dockerfile +++ b/airbyte-integrations/connectors/source-tidb/Dockerfile @@ -17,5 +17,5 @@ ENV APPLICATION source-tidb COPY --from=build /airbyte /airbyte # Airbyte's build system uses these labels to know what to name and tag the docker images produced by this Dockerfile. -LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.version=0.1.2 LABEL io.airbyte.name=airbyte/source-tidb diff --git a/airbyte-metrics/reporter/Dockerfile b/airbyte-metrics/reporter/Dockerfile index 12ef26af9708..9b8205cce038 100644 --- a/airbyte-metrics/reporter/Dockerfile +++ b/airbyte-metrics/reporter/Dockerfile @@ -2,7 +2,7 @@ ARG JDK_VERSION=17.0.1 ARG JDK_IMAGE=openjdk:${JDK_VERSION}-slim FROM ${JDK_IMAGE} AS metrics-reporter -ARG VERSION=0.39.21-alpha +ARG VERSION=0.39.23-alpha ENV APPLICATION airbyte-metrics-reporter ENV VERSION ${VERSION} diff --git a/airbyte-server/Dockerfile b/airbyte-server/Dockerfile index deff8a86e5b9..ea9bfb4958b5 100644 --- a/airbyte-server/Dockerfile +++ b/airbyte-server/Dockerfile @@ -4,7 +4,7 @@ FROM ${JDK_IMAGE} AS server EXPOSE 8000 -ARG VERSION=0.39.21-alpha +ARG VERSION=0.39.23-alpha ENV APPLICATION airbyte-server ENV VERSION ${VERSION} diff --git a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java index 2ddec2458063..5f44b4ab547e 100644 --- a/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java +++ b/airbyte-server/src/main/java/io/airbyte/server/ServerApp.java @@ -51,6 +51,8 @@ import io.airbyte.server.handlers.DbMigrationHandler; import io.airbyte.validation.json.JsonValidationException; import io.airbyte.workers.temporal.TemporalClient; +import io.airbyte.workers.temporal.TemporalUtils; +import io.temporal.serviceclient.WorkflowServiceStubs; import java.io.IOException; import java.net.http.HttpClient; import java.util.Map; @@ -193,13 +195,17 @@ public static ServerRunnable getServer(final ServerFactory apiFactory, final TrackingClient trackingClient = TrackingClientSingleton.get(); final JobTracker jobTracker = new JobTracker(configRepository, jobPersistence, trackingClient); - final TemporalClient temporalClient = TemporalClient.production(configs.getTemporalHost(), configs.getWorkspaceRoot(), configs); + final WorkflowServiceStubs temporalService = TemporalUtils.createTemporalService(); + final TemporalClient temporalClient = new TemporalClient( + TemporalUtils.createWorkflowClient(temporalService, TemporalUtils.getNamespace()), + configs.getWorkspaceRoot(), + temporalService); + final OAuthConfigSupplier oAuthConfigSupplier = new OAuthConfigSupplier(configRepository, trackingClient); final DefaultSynchronousSchedulerClient syncSchedulerClient = new DefaultSynchronousSchedulerClient(temporalClient, jobTracker, oAuthConfigSupplier); final HttpClient httpClient = HttpClient.newBuilder().version(HttpClient.Version.HTTP_1_1).build(); - final EventRunner eventRunner = new TemporalEventRunner( - TemporalClient.production(configs.getTemporalHost(), configs.getWorkspaceRoot(), configs)); + final EventRunner eventRunner = new TemporalEventRunner(temporalClient); // It is important that the migration to the temporal scheduler is performed before the server // accepts any requests. diff --git a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java index 7b98ca7a4a8a..2eb06c836188 100644 --- a/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java +++ b/airbyte-tests/src/acceptanceTests/java/io/airbyte/test/acceptance/BasicAcceptanceTests.java @@ -831,7 +831,9 @@ public void testFailureTimeout() throws Exception { } private WorkflowClient getWorkflowClient() { - final WorkflowServiceStubs temporalService = TemporalUtils.createTemporalService("localhost:7233"); + final WorkflowServiceStubs temporalService = TemporalUtils.createTemporalService( + TemporalUtils.getAirbyteTemporalOptions("localhost:7233"), + TemporalUtils.DEFAULT_NAMESPACE); return WorkflowClient.newInstance(temporalService); } diff --git a/airbyte-webapp/.eslintrc b/airbyte-webapp/.eslintrc index f132a520cbe5..679e2b04e46c 100644 --- a/airbyte-webapp/.eslintrc +++ b/airbyte-webapp/.eslintrc @@ -4,9 +4,10 @@ "plugin:@typescript-eslint/recommended", "plugin:jest/recommended", "prettier", - "plugin:prettier/recommended" + "plugin:prettier/recommended", + "plugin:css-modules/recommended" ], - "plugins": ["react", "@typescript-eslint", "prettier", "unused-imports"], + "plugins": ["react", "@typescript-eslint", "prettier", "unused-imports", "css-modules"], "parserOptions": { "ecmaVersion": 2020, "sourceType": "module", @@ -15,11 +16,29 @@ } }, "rules": { - "curly": "error", - "prettier/prettier": "error", - "unused-imports/no-unused-imports": "error", + "curly": "warn", + "css-modules/no-undef-class": ["warn", { "camelCase": true }], + "css-modules/no-unused-class": ["warn", { "camelCase": true }], + "dot-location": "warn", + "eqeqeq": "error", + "prettier/prettier": "warn", + "unused-imports/no-unused-imports": "warn", + "no-else-return": "warn", + "no-lonely-if": "warn", + "no-inner-declarations": "off", + "no-unused-vars": "off", + "no-useless-computed-key": "warn", + "no-useless-return": "warn", + "no-var": "warn", + "object-shorthand": ["warn", "always"], + "prefer-arrow-callback": "warn", + "prefer-const": "warn", + "prefer-destructuring": ["warn", { "AssignmentExpression": { "array": true } }], + "prefer-object-spread": "warn", + "prefer-template": "warn", + "yoda": "warn", "import/order": [ - "error", + "warn", { "newlines-between": "always", "groups": ["type", "builtin", "external", "internal", ["parent", "sibling"], "index"], @@ -41,6 +60,7 @@ } } ], + "@typescript-eslint/array-type": ["warn", { "default": "array-simple" }], "@typescript-eslint/ban-ts-comment": [ "warn", { @@ -48,8 +68,22 @@ "ts-expect-error": "allow-with-description" } ], - "@typescript-eslint/consistent-type-definitions": ["error", "interface"], - "@typescript-eslint/ban-types": ["warn"] + "@typescript-eslint/ban-types": "warn", + "@typescript-eslint/consistent-indexed-object-style": ["warn", "record"], + "@typescript-eslint/consistent-type-definitions": ["warn", "interface"], + "@typescript-eslint/no-unused-vars": "warn", + "react/function-component-definition": [ + "warn", + { + "namedComponents": "arrow-function", + "unnamedComponents": "arrow-function" + } + ], + "react/jsx-boolean-value": "warn", + "react/jsx-curly-brace-presence": "warn", + "react/jsx-fragments": "warn", + "react/jsx-no-useless-fragment": ["warn", { "allowExpressions": true }], + "react/self-closing-comp": "warn" }, "parser": "@typescript-eslint/parser", "overrides": [ diff --git a/airbyte-webapp/package-lock.json b/airbyte-webapp/package-lock.json index 8a33a44f8c01..e35b9a5ec760 100644 --- a/airbyte-webapp/package-lock.json +++ b/airbyte-webapp/package-lock.json @@ -1,12 +1,12 @@ { "name": "airbyte-webapp", - "version": "0.39.21-alpha", + "version": "0.39.23-alpha", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "airbyte-webapp", - "version": "0.39.21-alpha", + "version": "0.39.23-alpha", "dependencies": { "@fortawesome/fontawesome-svg-core": "^6.1.1", "@fortawesome/free-brands-svg-icons": "^6.1.1", @@ -83,6 +83,7 @@ "@typescript-eslint/parser": "^5.27.1", "eslint-config-prettier": "^8.5.0", "eslint-config-react-app": "^7.0.1", + "eslint-plugin-css-modules": "^2.11.0", "eslint-plugin-jest": "^26.5.3", "eslint-plugin-prettier": "^4.0.0", "eslint-plugin-unused-imports": "^2.0.0", @@ -21143,6 +21144,22 @@ "node": ">=4" } }, + "node_modules/eslint-plugin-css-modules": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-css-modules/-/eslint-plugin-css-modules-2.11.0.tgz", + "integrity": "sha512-CLvQvJOMlCywZzaI4HVu7QH/ltgNXvCg7giJGiE+sA9wh5zQ+AqTgftAzrERV22wHe1p688wrU/Zwxt1Ry922w==", + "dev": true, + "dependencies": { + "gonzales-pe": "^4.0.3", + "lodash": "^4.17.2" + }, + "engines": { + "node": ">=4.0.0" + }, + "peerDependencies": { + "eslint": ">=2.0.0" + } + }, "node_modules/eslint-plugin-flowtype": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/eslint-plugin-flowtype/-/eslint-plugin-flowtype-8.0.3.tgz", @@ -23633,6 +23650,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/gonzales-pe": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/gonzales-pe/-/gonzales-pe-4.3.0.tgz", + "integrity": "sha512-otgSPpUmdWJ43VXyiNgEYE4luzHCL2pz4wQ0OnDluC6Eg4Ko3Vexy/SrSynglw/eR+OhkzmqFCZa/OFa/RgAOQ==", + "dev": true, + "dependencies": { + "minimist": "^1.2.5" + }, + "bin": { + "gonzales": "bin/gonzales.js" + }, + "engines": { + "node": ">=0.6.0" + } + }, "node_modules/got": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", @@ -62795,6 +62827,16 @@ } } }, + "eslint-plugin-css-modules": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-css-modules/-/eslint-plugin-css-modules-2.11.0.tgz", + "integrity": "sha512-CLvQvJOMlCywZzaI4HVu7QH/ltgNXvCg7giJGiE+sA9wh5zQ+AqTgftAzrERV22wHe1p688wrU/Zwxt1Ry922w==", + "dev": true, + "requires": { + "gonzales-pe": "^4.0.3", + "lodash": "^4.17.2" + } + }, "eslint-plugin-flowtype": { "version": "8.0.3", "resolved": "https://registry.npmjs.org/eslint-plugin-flowtype/-/eslint-plugin-flowtype-8.0.3.tgz", @@ -64591,6 +64633,15 @@ "slash": "^3.0.0" } }, + "gonzales-pe": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/gonzales-pe/-/gonzales-pe-4.3.0.tgz", + "integrity": "sha512-otgSPpUmdWJ43VXyiNgEYE4luzHCL2pz4wQ0OnDluC6Eg4Ko3Vexy/SrSynglw/eR+OhkzmqFCZa/OFa/RgAOQ==", + "dev": true, + "requires": { + "minimist": "^1.2.5" + } + }, "got": { "version": "9.6.0", "resolved": "https://registry.npmjs.org/got/-/got-9.6.0.tgz", diff --git a/airbyte-webapp/package.json b/airbyte-webapp/package.json index 8cf582e88a06..c6893d6c348f 100644 --- a/airbyte-webapp/package.json +++ b/airbyte-webapp/package.json @@ -1,6 +1,6 @@ { "name": "airbyte-webapp", - "version": "0.39.21-alpha", + "version": "0.39.23-alpha", "private": true, "engines": { "node": ">=16.0.0" @@ -94,6 +94,7 @@ "@typescript-eslint/parser": "^5.27.1", "eslint-config-prettier": "^8.5.0", "eslint-config-react-app": "^7.0.1", + "eslint-plugin-css-modules": "^2.11.0", "eslint-plugin-jest": "^26.5.3", "eslint-plugin-prettier": "^4.0.0", "eslint-plugin-unused-imports": "^2.0.0", diff --git a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx index 6e40572339a1..e1ace8fcbdda 100644 --- a/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx +++ b/airbyte-webapp/src/components/ArrayOfObjectsEditor/ArrayOfObjectsEditor.tsx @@ -42,7 +42,7 @@ export interface ArrayOfObjectsEditorProps { disabled?: boolean; } -export function ArrayOfObjectsEditor({ +export const ArrayOfObjectsEditor = ({ onStartEdit, onDone, onRemove, @@ -54,7 +54,7 @@ export function ArrayOfObjectsEditor): JSX.Element { +}: ArrayOfObjectsEditorProps): JSX.Element => { const onAddItem = React.useCallback(() => onStartEdit(items.length), [onStartEdit, items]); const isEditable = editableItemIndex !== null && editableItemIndex !== undefined; @@ -108,4 +108,4 @@ export function ArrayOfObjectsEditor ); -} +}; diff --git a/airbyte-webapp/src/components/BarChart/BarChart.tsx b/airbyte-webapp/src/components/BarChart/BarChart.tsx index 41561da351ab..166a8d52db38 100644 --- a/airbyte-webapp/src/components/BarChart/BarChart.tsx +++ b/airbyte-webapp/src/components/BarChart/BarChart.tsx @@ -3,10 +3,10 @@ import { Bar, BarChart as BasicBarChart, CartesianGrid, Label, ResponsiveContain import { barChartColors, theme } from "theme"; interface BarChartProps { - data: { + data: Array<{ name: string; value: number; - }[]; + }>; legendLabels: string[]; xLabel?: string; yLabel?: string; diff --git a/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx b/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx index 86ce6e73dca1..a8ac818ece2e 100644 --- a/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx +++ b/airbyte-webapp/src/components/ConnectorBlocks/TableItemTitle.tsx @@ -57,7 +57,7 @@ const TableItemTitle: React.FC = ({ }) => { const { hasFeature } = useFeatureService(); const allowCreateConnection = hasFeature(FeatureItem.AllowCreateConnection); - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const options = [ { label: formatMessage({ diff --git a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx b/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx index 29ba103d073e..78cbd8cd3827 100644 --- a/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx +++ b/airbyte-webapp/src/components/ConnectorCard/ConnectorCard.tsx @@ -52,7 +52,7 @@ const ConnectorName = styled.div` text-align: left; `; -function ConnectorCard(props: Props) { +const ConnectorCard = (props: Props) => { const { connectionName, connectorName, icon, releaseStage } = props; return ( @@ -67,6 +67,6 @@ function ConnectorCard(props: Props) { ); -} +}; export default ConnectorCard; diff --git a/airbyte-webapp/src/components/DeleteBlock/DeleteBlock.tsx b/airbyte-webapp/src/components/DeleteBlock/DeleteBlock.tsx index 5c3fe2dab6e6..631b030dc106 100644 --- a/airbyte-webapp/src/components/DeleteBlock/DeleteBlock.tsx +++ b/airbyte-webapp/src/components/DeleteBlock/DeleteBlock.tsx @@ -48,19 +48,17 @@ const DeleteBlock: React.FC = ({ type, onDelete }) => { }, [closeConfirmationModal, onDelete, openConfirmationModal, push, type]); return ( - <> - - -
- -
- -
- -
- + + +
+ +
+ +
+ +
); }; diff --git a/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx b/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx index f19ee12b5cc7..ff10ce5b5f1e 100644 --- a/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx +++ b/airbyte-webapp/src/components/DocumentationPanel/DocumentationPanel.tsx @@ -43,12 +43,11 @@ export const DocumentationPanel: React.FC = () => { if (element.tagName === "img") { // In images replace relative URLs with links to our bundled assets return url.path.replace("../../", `${config.integrationUrl}/`); - } else { - // In links replace with a link to the external documentation instead - // The external path is the markdown URL without the "../../" prefix and the .md extension - const docPath = url.path.replace(/^\.\.\/\.\.\/(.*?)(\.md)?$/, "$1"); - return `${config.links.docsLink}/${docPath}`; } + // In links replace with a link to the external documentation instead + // The external path is the markdown URL without the "../../" prefix and the .md extension + const docPath = url.path.replace(/^\.\.\/\.\.\/(.*?)(\.md)?$/, "$1"); + return `${config.links.docsLink}/${docPath}`; } return url.href; }; diff --git a/airbyte-webapp/src/components/EmptyResourceBlock/EmptyResourceBlock.tsx b/airbyte-webapp/src/components/EmptyResourceBlock/EmptyResourceBlock.tsx index 110cd48e3723..fea79684ef70 100644 --- a/airbyte-webapp/src/components/EmptyResourceBlock/EmptyResourceBlock.tsx +++ b/airbyte-webapp/src/components/EmptyResourceBlock/EmptyResourceBlock.tsx @@ -35,7 +35,7 @@ const Description = styled.div` const EmptyResourceBlock: React.FC = ({ text, description }) => ( - {"cactus"} + cactus {text} {description} diff --git a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx index 82afc70c1a84..a567fbfcb379 100644 --- a/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx +++ b/airbyte-webapp/src/components/EntityTable/ConnectionTable.tsx @@ -46,7 +46,7 @@ const ConnectionTable: React.FC = ({ data, entity, onClickRow, onChangeS search: queryString.stringify( { sortBy: field, - order: order, + order, }, { skipNull: true } ), diff --git a/airbyte-webapp/src/components/EntityTable/ImplementationTable.tsx b/airbyte-webapp/src/components/EntityTable/ImplementationTable.tsx index f2b0e3177ab2..fd21f1fae209 100644 --- a/airbyte-webapp/src/components/EntityTable/ImplementationTable.tsx +++ b/airbyte-webapp/src/components/EntityTable/ImplementationTable.tsx @@ -39,7 +39,7 @@ const ImplementationTable: React.FC = ({ data, entity, onClickRow }) => search: queryString.stringify( { sortBy: field, - order: order, + order, }, { skipNull: true } ), diff --git a/airbyte-webapp/src/components/EntityTable/components/AllConnectionsStatusCell.tsx b/airbyte-webapp/src/components/EntityTable/components/AllConnectionsStatusCell.tsx index 2479bb7035ec..700f4aa4f2a1 100644 --- a/airbyte-webapp/src/components/EntityTable/components/AllConnectionsStatusCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/AllConnectionsStatusCell.tsx @@ -6,7 +6,7 @@ import { StatusIconStatus } from "components/StatusIcon/StatusIcon"; import { Status } from "../types"; -const _statusConfig: { status: Status; statusIconStatus?: StatusIconStatus; titleId: string }[] = [ +const _statusConfig: Array<{ status: Status; statusIconStatus?: StatusIconStatus; titleId: string }> = [ { status: Status.ACTIVE, statusIconStatus: "success", titleId: "connection.successSync" }, { status: Status.INACTIVE, statusIconStatus: "inactive", titleId: "connection.disabledConnection" }, { status: Status.FAILED, titleId: "connection.failedSync" }, diff --git a/airbyte-webapp/src/components/EntityTable/components/ConnectEntitiesCell.tsx b/airbyte-webapp/src/components/EntityTable/components/ConnectEntitiesCell.tsx index f1e8109ca7d5..d35fcefc8ffe 100644 --- a/airbyte-webapp/src/components/EntityTable/components/ConnectEntitiesCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/ConnectEntitiesCell.tsx @@ -5,10 +5,10 @@ import styled from "styled-components"; import ImageBlock from "components/ImageBlock"; interface IProps { - values: { + values: Array<{ name: string; connector: string; - }[]; + }>; enabled?: boolean; entity: "source" | "destination"; } diff --git a/airbyte-webapp/src/components/EntityTable/components/NameCell.tsx b/airbyte-webapp/src/components/EntityTable/components/NameCell.tsx index a628e9915c78..fc71eb2387cf 100644 --- a/airbyte-webapp/src/components/EntityTable/components/NameCell.tsx +++ b/airbyte-webapp/src/components/EntityTable/components/NameCell.tsx @@ -41,7 +41,7 @@ const Image = styled(ConnectorIcon)` `; const NameCell: React.FC = ({ value, enabled, status, icon, img }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const statusIconStatus = useMemo( () => status === Status.EMPTY diff --git a/airbyte-webapp/src/components/EntityTable/types.ts b/airbyte-webapp/src/components/EntityTable/types.ts index 97d22273f254..6ebbf3053c19 100644 --- a/airbyte-webapp/src/components/EntityTable/types.ts +++ b/airbyte-webapp/src/components/EntityTable/types.ts @@ -4,12 +4,12 @@ interface EntityTableDataItem { entityId: string; entityName: string; connectorName: string; - connectEntities: { + connectEntities: Array<{ name: string; connector: string; status: string; lastSyncStatus: string | null; - }[]; + }>; enabled: boolean; lastSync?: number | null; connectorIcon?: string; diff --git a/airbyte-webapp/src/components/EntityTable/utils.tsx b/airbyte-webapp/src/components/EntityTable/utils.tsx index dab499bc0136..b394dc02f3c5 100644 --- a/airbyte-webapp/src/components/EntityTable/utils.tsx +++ b/airbyte-webapp/src/components/EntityTable/utils.tsx @@ -64,7 +64,7 @@ export function getEntityTableData< enabled: true, connectorName: entitySoDName, lastSync: sortBySync?.[0].latestSyncJobCreatedAt, - connectEntities: connectEntities, + connectEntities, connectorIcon: definition?.icon, }; }); diff --git a/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx b/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx index 38f2cd311f45..487242688b9f 100644 --- a/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx +++ b/airbyte-webapp/src/components/JobItem/components/DownloadButton.tsx @@ -13,7 +13,7 @@ interface DownloadButtonProps { } const DownloadButton: React.FC = ({ jobDebugInfo, fileName }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const downloadFileWithLogs = () => { const element = document.createElement("a"); diff --git a/airbyte-webapp/src/components/JobItem/components/Logs.tsx b/airbyte-webapp/src/components/JobItem/components/Logs.tsx index b88d505bae0d..6ccaf295aa1d 100644 --- a/airbyte-webapp/src/components/JobItem/components/Logs.tsx +++ b/airbyte-webapp/src/components/JobItem/components/Logs.tsx @@ -46,7 +46,7 @@ const Logs: React.FC = ({ logsArray }) => { lineClassName="logLine" highlightLineClassName="highlightLogLine" selectableLines - follow={true} + follow style={{ background: "transparent" }} scrollToLine={undefined} highlight={[]} diff --git a/airbyte-webapp/src/components/StatusIcon/StatusIcon.test.tsx b/airbyte-webapp/src/components/StatusIcon/StatusIcon.test.tsx index 23f989ffac89..0003a0705873 100644 --- a/airbyte-webapp/src/components/StatusIcon/StatusIcon.test.tsx +++ b/airbyte-webapp/src/components/StatusIcon/StatusIcon.test.tsx @@ -14,7 +14,7 @@ describe("", () => { expect(component.getByText(`${value}`)).toBeDefined(); }); - const statusCases: { status: StatusIconStatus; icon: string }[] = [ + const statusCases: Array<{ status: StatusIconStatus; icon: string }> = [ { status: "success", icon: "check" }, { status: "inactive", icon: "pause" }, { status: "sleep", icon: "moon" }, diff --git a/airbyte-webapp/src/components/base/DropDown/SelectContainer.tsx b/airbyte-webapp/src/components/base/DropDown/SelectContainer.tsx index 1b0346733700..e16af1234810 100644 --- a/airbyte-webapp/src/components/base/DropDown/SelectContainer.tsx +++ b/airbyte-webapp/src/components/base/DropDown/SelectContainer.tsx @@ -8,5 +8,5 @@ export const SelectContainer: React.FC> = (pro "data-testid": props.selectProps["data-testid"], role: props.selectProps["role"] || "combobox", }; - return ; + return ; }; diff --git a/airbyte-webapp/src/components/base/TagInput/TagInput.tsx b/airbyte-webapp/src/components/base/TagInput/TagInput.tsx index 810cd21c8dc6..9921d41df578 100644 --- a/airbyte-webapp/src/components/base/TagInput/TagInput.tsx +++ b/airbyte-webapp/src/components/base/TagInput/TagInput.tsx @@ -155,7 +155,7 @@ export const TagInput: React.FC = ({ {...inputProps} name={name} disabled={disabled} - autoComplete={"off"} + autoComplete="off" placeholder={inputPlaceholder} ref={inputElement} onBlur={handleInputBlur} diff --git a/airbyte-webapp/src/components/base/TagInput/TagItem.tsx b/airbyte-webapp/src/components/base/TagInput/TagItem.tsx index f655fc00670f..8580d705f8cf 100644 --- a/airbyte-webapp/src/components/base/TagInput/TagItem.tsx +++ b/airbyte-webapp/src/components/base/TagInput/TagItem.tsx @@ -56,7 +56,7 @@ const TagItem: React.FC = ({ item, onDeleteTag, isSelected, disabled }) return ( {item.value} - + ); }; diff --git a/airbyte-webapp/src/config/configProviders.test.ts b/airbyte-webapp/src/config/configProviders.test.ts index da9b7e1da9b8..9800734338b1 100644 --- a/airbyte-webapp/src/config/configProviders.test.ts +++ b/airbyte-webapp/src/config/configProviders.test.ts @@ -14,7 +14,7 @@ interface Value { innerProp: string; }; } -describe("applyProviders", function () { +describe("applyProviders", () => { test("should deepMerge config returned from providers", async () => { const defaultValue: Value = { prop1: { @@ -29,7 +29,7 @@ describe("applyProviders", function () { innerProp: "1", }, }; - const providers: ProviderAsync>[] = [ + const providers: Array>> = [ async () => ({ prop1: { innerProp: "John", diff --git a/airbyte-webapp/src/config/types.ts b/airbyte-webapp/src/config/types.ts index 006259c8ada4..29054ef08046 100644 --- a/airbyte-webapp/src/config/types.ts +++ b/airbyte-webapp/src/config/types.ts @@ -40,6 +40,6 @@ export type DeepPartial = { export type ProviderAsync = () => Promise; export type Provider = () => T; -export type ValueProvider = ProviderAsync>[]; +export type ValueProvider = Array>>; export type ConfigProvider = ProviderAsync>; diff --git a/airbyte-webapp/src/core/form/types.ts b/airbyte-webapp/src/core/form/types.ts index 79de48cbfa3b..3df1d164cc7c 100644 --- a/airbyte-webapp/src/core/form/types.ts +++ b/airbyte-webapp/src/core/form/types.ts @@ -33,7 +33,7 @@ type FormGroupItem = { type FormConditionItem = { _type: "formCondition"; - conditions: { [key: string]: FormGroupItem | FormBaseItem }; + conditions: Record; } & FormItem; type FormObjectArrayItem = { @@ -46,12 +46,8 @@ type FormBlock = FormGroupItem | FormBaseItem | FormConditionItem | FormObjectAr export type { FormBlock, FormConditionItem, FormGroupItem, FormObjectArrayItem }; // eslint-disable-next-line @typescript-eslint/no-explicit-any -export interface WidgetConfig { - [key: string]: any; -} -export interface WidgetConfigMap { - [key: string]: WidgetConfig; -} +export type WidgetConfig = Record; +export type WidgetConfigMap = Record; // eslint-disable-next-line @typescript-eslint/no-explicit-any export type FormComponentOverrideProps = Record; diff --git a/airbyte-webapp/src/core/form/uiWidget.ts b/airbyte-webapp/src/core/form/uiWidget.ts index 3beb07f21f4b..f3d6f695bafc 100644 --- a/airbyte-webapp/src/core/form/uiWidget.ts +++ b/airbyte-webapp/src/core/form/uiWidget.ts @@ -7,9 +7,9 @@ import { FormBlock, WidgetConfigMap } from "./types"; export const buildPathInitialState = ( formBlock: FormBlock[], - formValues: { [key: string]: unknown }, + formValues: Record, widgetState: WidgetConfigMap = {} -): { [key: string]: WidgetConfigMap } => +): Record => formBlock.reduce((widgetStateBuilder, formItem) => { switch (formItem._type) { case "formGroup": diff --git a/airbyte-webapp/src/core/jsonSchema/types.ts b/airbyte-webapp/src/core/jsonSchema/types.ts index f258aea94c2a..b2bbedba5328 100644 --- a/airbyte-webapp/src/core/jsonSchema/types.ts +++ b/airbyte-webapp/src/core/jsonSchema/types.ts @@ -15,12 +15,10 @@ export type AirbyteJSONSchema = { [Property in keyof JSONSchema7]+?: JSONSchema7[Property] extends boolean ? boolean : Property extends "properties" | "patternProperties" | "definitions" - ? { - [key: string]: AirbyteJSONSchemaDefinition; - } + ? Record : JSONSchema7[Property] extends JSONSchema7Definition ? AirbyteJSONSchemaDefinition - : JSONSchema7[Property] extends Array + : JSONSchema7[Property] extends JSONSchema7Definition[] ? AirbyteJSONSchemaDefinition[] : JSONSchema7[Property] extends JSONSchema7Definition | JSONSchema7Definition[] ? AirbyteJSONSchemaDefinition | AirbyteJSONSchemaDefinition[] diff --git a/airbyte-webapp/src/core/jsonSchema/utils.ts b/airbyte-webapp/src/core/jsonSchema/utils.ts index 3bc9a81346c2..99d8d36e52ff 100644 --- a/airbyte-webapp/src/core/jsonSchema/utils.ts +++ b/airbyte-webapp/src/core/jsonSchema/utils.ts @@ -26,7 +26,7 @@ function removeNestedPaths( } if (schema.properties) { - const properties = schema.properties; + const { properties } = schema; const filteredProperties: Record = {}; for (const propertiesKey in properties) { @@ -63,7 +63,7 @@ function removeNestedPaths( function applyFuncAt( schema: JSONSchema7Definition, - path: (string | number)[], + path: Array, f: (schema: JSONSchema7Definition) => JSONSchema7 ): JSONSchema7Definition { if (typeof schema === "boolean") { diff --git a/airbyte-webapp/src/core/request/apiOverride.ts b/airbyte-webapp/src/core/request/apiOverride.ts index 03e6e022beb3..570e178508bd 100644 --- a/airbyte-webapp/src/core/request/apiOverride.ts +++ b/airbyte-webapp/src/core/request/apiOverride.ts @@ -51,7 +51,7 @@ export const apiOverride = async ( const requestUrl = `${apiUrl.replace(/\/v1\/?$/, "")}${url.startsWith("/") ? "" : "/"}${url}`; for (const middleware of options.middlewares) { - headers = (await middleware({ headers })).headers; + ({ headers } = await middleware({ headers })); } const response = await fetch(`${requestUrl}${new URLSearchParams(params)}`, { diff --git a/airbyte-webapp/src/core/servicesProvider.tsx b/airbyte-webapp/src/core/servicesProvider.tsx index f244ea255b28..e974f3e80fd9 100644 --- a/airbyte-webapp/src/core/servicesProvider.tsx +++ b/airbyte-webapp/src/core/servicesProvider.tsx @@ -1,9 +1,7 @@ import React, { useContext, useEffect, useMemo } from "react"; import { useMap } from "react-use"; -interface ServiceContainer { - [key: string]: Service; -} +type ServiceContainer = Record; // eslint-disable-next-line @typescript-eslint/no-explicit-any type Service = any; diff --git a/airbyte-webapp/src/hooks/services/Analytics/useAnalyticsService.tsx b/airbyte-webapp/src/hooks/services/Analytics/useAnalyticsService.tsx index 6729f4df0107..4268c96f439a 100644 --- a/airbyte-webapp/src/hooks/services/Analytics/useAnalyticsService.tsx +++ b/airbyte-webapp/src/hooks/services/Analytics/useAnalyticsService.tsx @@ -15,7 +15,7 @@ export interface AnalyticsServiceProviderValue { export const analyticsServiceContext = React.createContext(null); -function AnalyticsServiceProvider({ +const AnalyticsServiceProvider = ({ children, version, initialContext = {}, @@ -23,7 +23,7 @@ function AnalyticsServiceProvider({ children: React.ReactNode; version?: string; initialContext?: AnalyticsContext; -}) { +}) => { const [analyticsContext, { set, setAll, remove }] = useMap(initialContext); const analyticsService: AnalyticsService = useMemo( @@ -50,7 +50,7 @@ function AnalyticsServiceProvider({ {children} ); -} +}; export const useAnalyticsService = (): AnalyticsService => { return useAnalytics().service; @@ -89,13 +89,13 @@ export const useAnalyticsRegisterValues = (props?: AnalyticsContext | null): voi const { addContextProps, removeContextProps } = useAnalytics(); useEffect(() => { - if (props) { - addContextProps(props); - - return () => removeContextProps(Object.keys(props)); + if (!props) { + return; } - return; + addContextProps(props); + return () => removeContextProps(Object.keys(props)); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [props]); }; diff --git a/airbyte-webapp/src/hooks/services/BulkEdit/BulkEditService.tsx b/airbyte-webapp/src/hooks/services/BulkEdit/BulkEditService.tsx index e3c569f1d8e8..bfe620e4b89c 100644 --- a/airbyte-webapp/src/hooks/services/BulkEdit/BulkEditService.tsx +++ b/airbyte-webapp/src/hooks/services/BulkEdit/BulkEditService.tsx @@ -58,10 +58,10 @@ const BatchEditProvider: React.FC<{ const allChecked = selectedBatchNodes.size === nodes.length; const ctx: BatchContext = { - isActive: isActive, + isActive, toggleNode: toggle, onCheckAll: () => (allChecked ? reset() : nodes.forEach((n) => add(n.id))), - allChecked: allChecked, + allChecked, selectedBatchNodeIds: Array.from(selectedBatchNodes).filter((node): node is string => node !== undefined), selectedBatchNodes: nodes.filter((n) => selectedBatchNodes.has(n.id)), onChangeOption: (newOptions) => setOptions({ ...options, ...newOptions }), diff --git a/airbyte-webapp/src/hooks/services/Feature/FeatureService.tsx b/airbyte-webapp/src/hooks/services/Feature/FeatureService.tsx index b0ed27df4758..cbf53962af24 100644 --- a/airbyte-webapp/src/hooks/services/Feature/FeatureService.tsx +++ b/airbyte-webapp/src/hooks/services/Feature/FeatureService.tsx @@ -7,7 +7,7 @@ import { Feature, FeatureItem, FeatureServiceApi } from "./types"; const featureServiceContext = React.createContext(null); -export function FeatureService({ children }: { children: React.ReactNode }) { +export const FeatureService = ({ children }: { children: React.ReactNode }) => { const [additionFeatures, setAdditionFeatures] = useState([]); const { features: instanceWideFeatures } = useConfig(); @@ -38,7 +38,7 @@ export function FeatureService({ children }: { children: React.ReactNode }) { ); return {children}; -} +}; export const useFeatureService: () => FeatureServiceApi = () => { const featureService = useContext(featureServiceContext); @@ -57,13 +57,14 @@ export const useFeatureRegisterValues = (props?: Feature[] | null): void => { const { registerFeature, unregisterFeature } = useFeatureService(); useDeepCompareEffect(() => { - if (props) { - registerFeature(props); - - return () => unregisterFeature(props.map((feature: Feature) => feature.id)); + if (!props) { + return; } - return; + registerFeature(props); + + return () => unregisterFeature(props.map((feature: Feature) => feature.id)); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [props]); }; diff --git a/airbyte-webapp/src/hooks/services/Notification/NotificationService.tsx b/airbyte-webapp/src/hooks/services/Notification/NotificationService.tsx index d7cad7397e74..7c3eb85e7356 100644 --- a/airbyte-webapp/src/hooks/services/Notification/NotificationService.tsx +++ b/airbyte-webapp/src/hooks/services/Notification/NotificationService.tsx @@ -9,7 +9,7 @@ import { Notification, NotificationServiceApi, NotificationServiceState } from " const notificationServiceContext = React.createContext(null); -function NotificationService({ children }: { children: React.ReactNode }) { +const NotificationService = ({ children }: { children: React.ReactNode }) => { const [state, { addNotification, clearAll, deleteNotificationById }] = useTypesafeReducer< NotificationServiceState, typeof actions @@ -48,7 +48,7 @@ function NotificationService({ children }: { children: React.ReactNode }) { ) : null} ); -} +}; export const useNotificationService: ( notification?: Notification, diff --git a/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx b/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx index b9620766b154..239dcef4516b 100644 --- a/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx +++ b/airbyte-webapp/src/hooks/services/useConnectorAuth.tsx @@ -77,18 +77,17 @@ export function useConnectorAuth(): { }; const response = await sourceAuthService.getConsentUrl(payload); - return { consentUrl: response.consentUrl, payload }; - } else { - const payload = { - workspaceId, - destinationDefinitionId: ConnectorSpecification.id(connector), - redirectUrl: `${oauthRedirectUrl}/auth_flow`, - oAuthInputConfiguration, - }; - const response = await destinationAuthService.getConsentUrl(payload); - return { consentUrl: response.consentUrl, payload }; } + const payload = { + workspaceId, + destinationDefinitionId: ConnectorSpecification.id(connector), + redirectUrl: `${oauthRedirectUrl}/auth_flow`, + oAuthInputConfiguration, + }; + const response = await destinationAuthService.getConsentUrl(payload); + + return { consentUrl: response.consentUrl, payload }; }, completeOauthRequest: async ( params: SourceOauthConsentRequest | DestinationOauthConsentRequest, diff --git a/airbyte-webapp/src/hooks/services/useDocumentation.ts b/airbyte-webapp/src/hooks/services/useDocumentation.ts index ade75ef9cf3a..cb7d10d2c5a5 100644 --- a/airbyte-webapp/src/hooks/services/useDocumentation.ts +++ b/airbyte-webapp/src/hooks/services/useDocumentation.ts @@ -13,7 +13,7 @@ const DOCS_URL = /^https:\/\/docs\.airbyte\.(io|com)/; export const useDocumentation = (documentationUrl: string): UseDocumentationResult => { const { integrationUrl } = useConfig(); - const url = documentationUrl.replace(DOCS_URL, integrationUrl) + ".md"; + const url = `${documentationUrl.replace(DOCS_URL, integrationUrl)}.md`; return useQuery(documentationKeys.text(documentationUrl), () => fetchDocumentation(url), { enabled: !!documentationUrl, diff --git a/airbyte-webapp/src/hooks/useTypesafeReducer.ts b/airbyte-webapp/src/hooks/useTypesafeReducer.ts index 8b80336c62b0..9cd34532496d 100644 --- a/airbyte-webapp/src/hooks/useTypesafeReducer.ts +++ b/airbyte-webapp/src/hooks/useTypesafeReducer.ts @@ -2,7 +2,7 @@ import { Reducer, useReducer, useMemo } from "react"; import { ActionType } from "typesafe-actions"; -function useTypesafeReducer any }>( +function useTypesafeReducer any>>( reducer: Reducer>, initialState: StateShape, actions: Actions @@ -21,7 +21,7 @@ function useTypesafeReducer { a[action] = bindActionCreator(actions[action], dispatch); return a; - }, {} as { [key: string]: (...args: any[]) => any }); + }, {} as Record any>); return newActions; }, [dispatch, actions]); return [state, boundActions as Actions]; diff --git a/airbyte-webapp/src/packages/cloud/lib/domain/cloudWorkspaces/types.ts b/airbyte-webapp/src/packages/cloud/lib/domain/cloudWorkspaces/types.ts index 619fa8250531..ba6740ac585f 100644 --- a/airbyte-webapp/src/packages/cloud/lib/domain/cloudWorkspaces/types.ts +++ b/airbyte-webapp/src/packages/cloud/lib/domain/cloudWorkspaces/types.ts @@ -30,8 +30,8 @@ export interface CreditConsumptionByConnector { export interface CloudWorkspaceUsage { workspaceId: string; creditConsumptionByConnector: CreditConsumptionByConnector[]; - creditConsumptionByDay: { + creditConsumptionByDay: Array<{ date: [number, number, number]; creditsConsumed: number; - }[]; + }>; } diff --git a/airbyte-webapp/src/packages/cloud/lib/domain/users/UserService.ts b/airbyte-webapp/src/packages/cloud/lib/domain/users/UserService.ts index c6578ddb6624..9f1f4c9a2af3 100644 --- a/airbyte-webapp/src/packages/cloud/lib/domain/users/UserService.ts +++ b/airbyte-webapp/src/packages/cloud/lib/domain/users/UserService.ts @@ -59,9 +59,9 @@ export class UserService extends AirbyteRequestService { } public async invite( - users: { + users: Array<{ email: string; - }[], + }>, workspaceId: string ): Promise { return Promise.all( diff --git a/airbyte-webapp/src/packages/cloud/services/config/index.ts b/airbyte-webapp/src/packages/cloud/services/config/index.ts index f051a749e203..ff2df46ad56f 100644 --- a/airbyte-webapp/src/packages/cloud/services/config/index.ts +++ b/airbyte-webapp/src/packages/cloud/services/config/index.ts @@ -33,12 +33,11 @@ const cloudConfigExtensionDefault: CloudConfigExtension = { }, }; -export const defaultConfig: CloudConfig = Object.assign( - {}, - coreDefaultConfig, - coreDefaultConfigOverrites, - cloudConfigExtensionDefault -); +export const defaultConfig: CloudConfig = { + ...coreDefaultConfig, + ...coreDefaultConfigOverrites, + ...cloudConfigExtensionDefault, +}; export * from "./configProviders"; export * from "./types"; diff --git a/airbyte-webapp/src/packages/cloud/services/users/UseUserHook.ts b/airbyte-webapp/src/packages/cloud/services/users/UseUserHook.ts index 9fa5c4c9b451..5cb707a6cb0a 100644 --- a/airbyte-webapp/src/packages/cloud/services/users/UseUserHook.ts +++ b/airbyte-webapp/src/packages/cloud/services/users/UseUserHook.ts @@ -36,9 +36,9 @@ export const useUserHook = () => { ), inviteUserLogic: useMutation( async (payload: { - users: { + users: Array<{ email: string; - }[]; + }>; workspaceId: string; }) => service.invite(payload.users, payload.workspaceId), { diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx index a8c5d0abd688..be2339bde757 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ConfirmPasswordResetPage/ConfirmPasswordResetPage.tsx @@ -22,7 +22,7 @@ const ResetPasswordConfirmPage: React.FC = () => { const { confirmPasswordReset } = useAuthService(); const { registerNotification } = useNotificationService(); const { push, query } = useRouterHook<{ oobCode: string }>(); - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); return (
@@ -86,7 +86,7 @@ const ResetPasswordConfirmPage: React.FC = () => { } } }} - validateOnBlur={true} + validateOnBlur validateOnChange={false} > {({ isSubmitting }) => ( diff --git a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx index ab0bc67e8810..7dc558bc3289 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/LoginPage/LoginPage.tsx @@ -19,7 +19,7 @@ const LoginPageValidationSchema = yup.object().shape({ }); const LoginPage: React.FC = () => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { login } = useAuthService(); const { query, replace } = useRouter(); diff --git a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx index 5b8308fc344c..92b1a7dae1e6 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/ResetPasswordPage/ResetPasswordPage.tsx @@ -48,7 +48,7 @@ const ResetPasswordPage: React.FC = () => { : FormikBag.setFieldError("email", "login.unknownError"); } }} - validateOnBlur={true} + validateOnBlur validateOnChange={false} > {({ isSubmitting }) => ( diff --git a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx index 0f417bf14041..0018b9a12e7b 100644 --- a/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx +++ b/airbyte-webapp/src/packages/cloud/views/auth/SignupPage/components/SignupForm.tsx @@ -196,8 +196,8 @@ export const SignupForm: React.FC = () => { } }) } - validateOnBlur={true} - validateOnChange={true} + validateOnBlur + validateOnChange > {({ isValid, isSubmitting, values }) => (
diff --git a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.tsx b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.tsx index df4c3a7ddc04..6948103aa0f3 100644 --- a/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.tsx +++ b/airbyte-webapp/src/packages/cloud/views/credits/CreditsPage/components/UsagePerConnectionTable.tsx @@ -73,7 +73,7 @@ const UsagePerConnectionTable: React.FC = ({ credi search: queryString.stringify( { sortBy: field, - order: order, + order, }, { skipNull: true } ), diff --git a/airbyte-webapp/src/packages/cloud/views/layout/MainView/InsufficientPermissionsErrorBoundary.tsx b/airbyte-webapp/src/packages/cloud/views/layout/MainView/InsufficientPermissionsErrorBoundary.tsx index e6a1b27270dc..cb792d2c381c 100644 --- a/airbyte-webapp/src/packages/cloud/views/layout/MainView/InsufficientPermissionsErrorBoundary.tsx +++ b/airbyte-webapp/src/packages/cloud/views/layout/MainView/InsufficientPermissionsErrorBoundary.tsx @@ -19,9 +19,8 @@ export class InsufficientPermissionsErrorBoundary extends React.Component< static getDerivedStateFromError(error: CommonRequestError): BoundaryState { if (error.message.startsWith("Insufficient permissions")) { return { hasError: true, message: error.message }; - } else { - throw error; } + throw error; } state = initialState; diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx index 435ef5769201..ef879bf8b29e 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/AccountSettingsView.tsx @@ -17,7 +17,7 @@ const Header = styled.div` `; const AccountSettingsView: React.FC = () => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { logout } = useAuthService(); const user = useCurrentUser(); @@ -41,7 +41,7 @@ const AccountSettingsView: React.FC = () => { } - disabled={true} + disabled placeholder={formatMessage({ id: "settings.accountSettings.fullName.placeholder", })} diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection/EmailSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection/EmailSection.tsx index a5464d175d28..50008d90cdca 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection/EmailSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/EmailSection/EmailSection.tsx @@ -29,7 +29,7 @@ const TextInputsSection = styled.div` `; const EmailSection: React.FC = () => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const user = useCurrentUser(); const emailService = useEmail(); diff --git a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx index 91990ec68de4..9568d38eb73d 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/AccountSettingsView/components/PasswordSection/PasswordSection.tsx @@ -36,7 +36,7 @@ const PasswordSection: React.FC = () => { {...field} label={} disabled={isSubmitting} - required={true} + required type="password" error={!!meta.error && meta.touched} message={meta.touched && meta.error && formatMessage({ id: meta.error })} @@ -51,7 +51,7 @@ const PasswordSection: React.FC = () => { {...field} label={} disabled={isSubmitting || values.currentPassword.length === 0} - required={true} + required type="password" error={!!meta.error && meta.touched} message={meta.touched && meta.error && formatMessage({ id: meta.error })} @@ -66,7 +66,7 @@ const PasswordSection: React.FC = () => { {...field} label={} disabled={isSubmitting || values.currentPassword.length === 0} - required={true} + required type="password" error={!!meta.error && meta.touched} message={meta.touched && meta.error && formatMessage({ id: meta.error })} diff --git a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx index 6b7a625fe413..bc97bc519cce 100644 --- a/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx +++ b/airbyte-webapp/src/packages/cloud/views/users/InviteUsersModal/InviteUsersModal.tsx @@ -52,7 +52,7 @@ const ROLE_OPTIONS = [ export const InviteUsersModal: React.FC<{ onClose: () => void; }> = (props) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { workspaceId } = useCurrentWorkspace(); const { inviteUserLogic } = useUserHook(); const { mutateAsync: invite } = inviteUserLogic; @@ -62,8 +62,8 @@ export const InviteUsersModal: React.FC<{ return ( } onClose={props.onClose}> { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { exitWorkspace } = useWorkspaceService(); const workspace = useCurrentWorkspace(); diff --git a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx index fb94d8cfb27f..ef09fa5b5efc 100644 --- a/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx +++ b/airbyte-webapp/src/packages/cloud/views/workspaces/WorkspacesPage/components/CreateWorkspaceForm.tsx @@ -43,7 +43,7 @@ const CreateWorkspaceForm: React.FC = ({ onSubmit }) = }} validationSchema={CreateWorkspaceFormValidationSchema} onSubmit={onSubmit} - validateOnBlur={true} + validateOnBlur > {({ isSubmitting }) => ( diff --git a/airbyte-webapp/src/packages/firebaseReact/firebaseApp.tsx b/airbyte-webapp/src/packages/firebaseReact/firebaseApp.tsx index b7f4335545c3..bbd49d459072 100644 --- a/airbyte-webapp/src/packages/firebaseReact/firebaseApp.tsx +++ b/airbyte-webapp/src/packages/firebaseReact/firebaseApp.tsx @@ -20,7 +20,7 @@ interface FirebaseAppProviderProps { suspense?: boolean; } -export function FirebaseAppProvider(props: React.PropsWithChildren): JSX.Element { +export const FirebaseAppProvider = (props: React.PropsWithChildren): JSX.Element => { const { firebaseConfig, appName, suspense } = props; const firebaseApp: FirebaseApp = React.useMemo(() => { @@ -32,13 +32,12 @@ export function FirebaseAppProvider(props: React.PropsWithChildren ); -} +}; export function useFirebaseApp(): FirebaseApp { const firebaseApp = React.useContext(FirebaseAppContext); diff --git a/airbyte-webapp/src/packages/firebaseReact/sdk.tsx b/airbyte-webapp/src/packages/firebaseReact/sdk.tsx index 2fabf2ca0186..d9278ba757e1 100644 --- a/airbyte-webapp/src/packages/firebaseReact/sdk.tsx +++ b/airbyte-webapp/src/packages/firebaseReact/sdk.tsx @@ -12,7 +12,7 @@ const AuthSdkContext = React.createContext(undefined); type FirebaseSdks = Auth; function getSdkProvider(SdkContext: React.Context) { - return function SdkProvider(props: React.PropsWithChildren<{ sdk: Sdk }>) { + return (props: React.PropsWithChildren<{ sdk: Sdk }>) => { if (!props.sdk) { throw new Error("no sdk provided"); } diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx index 474e01c05a20..d47f8054c41b 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/ConnectionItemPage.tsx @@ -72,9 +72,7 @@ const ConnectionItemPage: React.FC = () => { /> } error={ - isConnectionDeleted ? ( - - ) : null + isConnectionDeleted ? : null } > }> @@ -95,7 +93,7 @@ const ConnectionItemPage: React.FC = () => { path={ConnectionSettingsRoutes.SETTINGS} element={isConnectionDeleted ? : } /> - } /> + } /> diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx index 13cbee57fac6..0918dfb70eec 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionName.tsx @@ -109,7 +109,7 @@ const ConnectionName: React.FC = ({ connection }) => { }; const inputChange = (event: ChangeEvent) => { - const value = event.currentTarget.value; + const { value } = event.currentTarget; if (value) { setConnectionName(event.currentTarget.value); } diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx index 8fcdb8cd8f9c..86002856f3fd 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/ConnectionPageTitle.tsx @@ -54,7 +54,7 @@ const ConnectionPageTitle: React.FC = ({ }, { id: ConnectionSettingsRoutes.TRANSFORMATION, - name: , + name: , }, ]; diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StateBlock.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StateBlock.tsx index 4560c0329b96..29b1c6122efe 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StateBlock.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StateBlock.tsx @@ -22,7 +22,7 @@ export const StateBlock: React.FC = ({ connectionId }) => { return (
- +
{stateString}
diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx index 187ea59f77c0..7b0fdf638d47 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/StatusView.tsx @@ -87,7 +87,7 @@ const StatusView: React.FC = ({ connection, isStatusUpdating }) const resetDataBtn = ( ); @@ -99,11 +99,11 @@ const StatusView: React.FC = ({ connection, isStatusUpdating }) onClick={() => startAction({ action: onSync })} > {showFeedback ? ( - + ) : ( <> - + )} @@ -114,14 +114,14 @@ const StatusView: React.FC = ({ connection, isStatusUpdating }) - + {connection.status === ConnectionStatus.active && (
- + - +
)} diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx index f07c4a1f0af0..a1854da52a14 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/ConnectionItemPage/components/TransformationView.tsx @@ -126,7 +126,7 @@ const TransformationView: React.FC = ({ connection }) = const workspace = useCurrentWorkspace(); const { hasFeature } = useFeatureService(); - const supportsNormalization = definition.supportsNormalization; + const { supportsNormalization } = definition; const supportsDbt = hasFeature(FeatureItem.AllowCustomDBT) && definition.supportsDbt; const mode = connection.status === ConnectionStatus.deprecated ? "readonly" : "edit"; @@ -147,7 +147,7 @@ const TransformationView: React.FC = ({ connection }) = await updateConnection( buildConnectionUpdate(connection, { - operations: operations, + operations, }) ); diff --git a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/ExistingEntityForm.tsx b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/ExistingEntityForm.tsx index f4f98d207ce6..ac184dcc4706 100644 --- a/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/ExistingEntityForm.tsx +++ b/airbyte-webapp/src/pages/ConnectionPage/pages/CreationFormPage/components/ExistingEntityForm.tsx @@ -41,7 +41,7 @@ const existingEntityValidationSchema = yup.object().shape({ }); const ExistingEntityForm: React.FC = ({ type, onSubmit }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { sources } = useSourceList(); const { sourceDefinitions } = useSourceDefinitionList(); @@ -59,18 +59,18 @@ const ExistingEntityForm: React.FC = ({ type, onSubmit }) => { img: , }; }); - } else { - return destinations.map((item) => { - const destinationDef = destinationDefinitions.find( - (dd) => dd.destinationDefinitionId === item.destinationDefinitionId - ); - return { - label: item.name, - value: item.destinationId, - img: , - }; - }); } + return destinations.map((item) => { + const destinationDef = destinationDefinitions.find( + (dd) => dd.destinationDefinitionId === item.destinationDefinitionId + ); + return { + label: item.name, + value: item.destinationId, + img: , + }; + }); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [type]); diff --git a/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx index 6bafe48355d4..22892b7b2e00 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/components/ProgressBlock.tsx @@ -86,7 +86,7 @@ const ProgressBlock: React.FC = ({ connection, onSync }) =>

{showMessage(connection.latestSyncJobStatus)}

- +
); @@ -94,7 +94,7 @@ const ProgressBlock: React.FC = ({ connection, onSync }) => return ( - + ; currentStep: StepType; } diff --git a/airbyte-webapp/src/pages/OnboardingPage/useStepsConfig.tsx b/airbyte-webapp/src/pages/OnboardingPage/useStepsConfig.tsx index 551a89d62d9b..c95d5cb84b7b 100644 --- a/airbyte-webapp/src/pages/OnboardingPage/useStepsConfig.tsx +++ b/airbyte-webapp/src/pages/OnboardingPage/useStepsConfig.tsx @@ -11,7 +11,7 @@ const useStepsConfig = ( ): { currentStep: StepType; setCurrentStep: (step: StepType) => void; - steps: { name: JSX.Element; id: StepType }[]; + steps: Array<{ name: JSX.Element; id: StepType }>; } => { const getInitialStep = () => { if (hasSources) { diff --git a/airbyte-webapp/src/pages/PreferencesPage/PreferencesPage.tsx b/airbyte-webapp/src/pages/PreferencesPage/PreferencesPage.tsx index 4d1b3ea7ccf7..a1b2e049bd27 100644 --- a/airbyte-webapp/src/pages/PreferencesPage/PreferencesPage.tsx +++ b/airbyte-webapp/src/pages/PreferencesPage/PreferencesPage.tsx @@ -23,7 +23,7 @@ const PreferencesPage: React.FC = () => { - <FormattedMessage id={"preferences.title"} /> + <FormattedMessage id="preferences.title" /> diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx index 7ab879ef8eb7..0aed616a86ac 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/AccountPage/components/AccountForm.tsx @@ -52,12 +52,12 @@ interface AccountFormProps { } const AccountForm: React.FC = ({ email, onSubmit, successMessage, errorMessage }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); return ( { const [isUpdateSuccess, setIsUpdateSuccess] = useState(false); - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { destinationDefinitions } = useDestinationDefinitionList(); const { destinations } = useDestinationList(); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx index c05ea7159d0d..775b0900cabb 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/SourcesPage.tsx @@ -13,7 +13,7 @@ const SourcesPage: React.FC = () => { const [isUpdateSuccess, setIsUpdateSucces] = useState(false); const [feedbackList, setFeedbackList] = useState>({}); - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { sources } = useSourceList(); const { sourceDefinitions } = useSourceDefinitionList(); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx index e912eea39a01..73a47e16be81 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnector.tsx @@ -30,7 +30,7 @@ const CreateConnector: React.FC = ({ type }) => { setErrorMessage(""); }; - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { mutateAsync: createSourceDefinition } = useCreateSourceDefinition(); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx index 594fd45a2cc5..1c3f06318ac6 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/CreateConnectorModal.tsx @@ -91,7 +91,7 @@ const validationSchema = yup.object().shape({ const CreateConnectorModal: React.FC = ({ onClose, onSubmit, errorMessage }) => { const config = useConfig(); - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); return ( }> @@ -115,8 +115,8 @@ const CreateConnectorModal: React.FC = ({ onClose, onSubmit, errorMessag dockerImageTag: "", dockerRepository: "", }} - validateOnBlur={true} - validateOnChange={true} + validateOnBlur + validateOnChange validationSchema={validationSchema} onSubmit={async (values, { setSubmitting }) => { await onSubmit(values); diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.tsx index 702ecece29dc..008fa1f34337 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/ConnectorsPage/components/VersionCell.tsx @@ -64,7 +64,7 @@ const ErrorMessage = styled(SuccessMessage)` `; const VersionCell: React.FC = ({ id, version, onChange, feedback, currentVersion }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const renderFeedback = (dirty: boolean, feedback?: string) => { if (feedback && !dirty) { @@ -74,9 +74,8 @@ const VersionCell: React.FC = ({ id, version, onChange, feedback, curren ); - } else { - return {feedback}; } + return {feedback}; } return null; diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/components/MetricsForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/components/MetricsForm.tsx index 0d0c565c7f6e..ead11580ebbf 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/components/MetricsForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/MetricsPage/components/MetricsForm.tsx @@ -58,7 +58,7 @@ const MetricsForm: React.FC = ({ ( diff --git a/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/components/WebHookForm.tsx b/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/components/WebHookForm.tsx index c72d4547bead..78bb19b91fd4 100644 --- a/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/components/WebHookForm.tsx +++ b/airbyte-webapp/src/pages/SettingsPage/pages/NotificationPage/components/WebHookForm.tsx @@ -60,7 +60,7 @@ interface WebHookFormProps { } const WebHookForm: React.FC = ({ webhook, onSubmit, successMessage, errorMessage, onTest }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const feedBackBlock = (dirty: boolean, isSubmitting: boolean, webhook?: string) => { if (successMessage) { @@ -93,8 +93,8 @@ const WebHookForm: React.FC = ({ webhook, onSubmit, successMes return ( { diff --git a/airbyte-webapp/src/pages/routes.tsx b/airbyte-webapp/src/pages/routes.tsx index 7484bb1e0ae6..b5a983b947a9 100644 --- a/airbyte-webapp/src/pages/routes.tsx +++ b/airbyte-webapp/src/pages/routes.tsx @@ -93,7 +93,7 @@ export const AutoSelectFirstWorkspace: React.FC<{ includePath?: boolean }> = ({ return ( ); }; diff --git a/airbyte-webapp/src/utils/testutils.tsx b/airbyte-webapp/src/utils/testutils.tsx index 8d825120b48f..0a639de8ce71 100644 --- a/airbyte-webapp/src/utils/testutils.tsx +++ b/airbyte-webapp/src/utils/testutils.tsx @@ -18,7 +18,7 @@ export async function render< Q extends Queries = typeof queries, Container extends Element | DocumentFragment = HTMLElement >(ui: React.ReactNode, renderOptions?: RenderOptions): Promise> { - function Wrapper({ children }: WrapperProps) { + const Wrapper = ({ children }: WrapperProps) => { const queryClient = new QueryClient(); return ( @@ -36,7 +36,7 @@ export async function render< ); - } + }; let renderResult: RenderResult; await act(async () => { diff --git a/airbyte-webapp/src/utils/useTranslateDataType.test.tsx b/airbyte-webapp/src/utils/useTranslateDataType.test.tsx index f30318a53686..037bdd6a957f 100644 --- a/airbyte-webapp/src/utils/useTranslateDataType.test.tsx +++ b/airbyte-webapp/src/utils/useTranslateDataType.test.tsx @@ -6,7 +6,7 @@ import messages from "../locales/en.json"; import { AirbyteConnectorData, useTranslateDataType } from "./useTranslateDataType"; const wrapper: React.FC = ({ children }) => ( - + {children} ); diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx index 55b242a282dd..ae5aa4ebaa84 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/CatalogTree.tsx @@ -21,7 +21,7 @@ const CatalogTree: React.FC = ({ streams, destinationSupported const streamNode = streams.find((streamNode) => streamNode.id === id); if (streamNode) { - const newStreamNode = setIn(streamNode, "config", Object.assign({}, streamNode.config, newConfig)); + const newStreamNode = setIn(streamNode, "config", { ...streamNode.config, ...newConfig }); onChangeStream(newStreamNode); } diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.module.scss b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.module.scss index be75d57ce842..8b3f269db514 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.module.scss +++ b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.module.scss @@ -2,10 +2,6 @@ @use "../../../scss/variables"; @forward "./CatalogTree.module.scss"; -.removedStream { - color: colors.$red; -} - .icon { margin-right: 7px; margin-top: -1px; diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx index 0921ea7e8a4c..f87b7c31b4c6 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/StreamHeader.tsx @@ -31,9 +31,9 @@ interface StreamHeaderProps { stream: SyncSchemaStream; destName: string; destNamespace: string; - availableSyncModes: { + availableSyncModes: Array<{ value: SyncSchema; - }[]; + }>; onSelectSyncMode: (selectedMode: DropDownRow.IDataItem) => void; onSelectStream: () => void; primitiveFields: SyncSchemaField[]; @@ -95,7 +95,8 @@ export const StreamHeader: React.FC = ({ [styles.purpleBackground]: isSelected, [styles.redBorder]: hasError, }); - + // FIXME: find out why checkboxCell warns as unused + // eslint-disable-next-line css-modules/no-undef-class const checkboxCellCustomStyle = classnames(styles.checkboxCell, { [styles.streamRowCheckboxCell]: true }); return ( @@ -157,7 +158,7 @@ export const StreamHeader: React.FC = ({ pathType={pkType} paths={paths} path={primaryKey} - isMulti={true} + isMulti placeholder={} onPathChange={onPrimaryKeyChange} /> diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/components/BulkHeader.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/components/BulkHeader.tsx index 82259d649114..8d4612019bda 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/components/BulkHeader.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/components/BulkHeader.tsx @@ -125,7 +125,7 @@ export const BulkHeader: React.FC = ({ destinationSupportedSync {pkType && ( onChangeOption({ primaryKey: path })} pathType={pkType} paths={paths} diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopout.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopout.tsx index f0e98810ba06..a659efd1ef15 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopout.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/components/PathPopout.tsx @@ -73,7 +73,7 @@ export const PathPopout: React.FC = (props) => { // @ts-expect-error need to solve issue with typings isMulti={props.isMulti} isSearchable - onChange={(options: PathPopoutProps["isMulti"] extends true ? { value: Path }[] : { value: Path }) => { + onChange={(options: PathPopoutProps["isMulti"] extends true ? Array<{ value: Path }> : { value: Path }) => { const finalValues = Array.isArray(options) ? options.map((op) => op.value) : options.value; props.onPathChange(finalValues); diff --git a/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx b/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx index 6b3a692718c8..e435b8ca7c8f 100644 --- a/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx +++ b/airbyte-webapp/src/views/Connection/CatalogTree/components/SyncSettingsDropdown.tsx @@ -102,8 +102,8 @@ const SyncSettingsDropdown: React.FC = (props) => ( = ({ const [submitError, setSubmitError] = useState(null); const [editingTransformation, toggleEditingTransformation] = useToggle(false); - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const isEditMode: boolean = mode !== "create"; const initialValues = useInitialValues(connection, destDefinition, isEditMode); @@ -198,7 +198,7 @@ const ConnectionForm: React.FC = ({ {({ isSubmitting, setFieldValue, isValid, dirty, resetForm, values }) => ( diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx index 769b35f16aac..b4ca5ea89100 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/OperationsSection.tsx @@ -27,10 +27,10 @@ export const OperationsSection: React.FC = ({ onStartEditTransformation, onEndEditTransformation, }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const { hasFeature } = useFeatureService(); - const supportsNormalization = destDefinition.supportsNormalization; + const { supportsNormalization } = destDefinition; const supportsTransformations = destDefinition.supportsDbt && hasFeature(FeatureItem.AllowCustomDBT); const defaultTransformation = useDefaultTransformation(); diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/Search.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/components/Search.tsx index 0ccda61dae1a..e92f3c28598e 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/Search.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/Search.tsx @@ -22,7 +22,7 @@ const SearchContent = styled.div` `; const Search: React.FC = ({ onSearch }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); return ( diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/components/SyncCatalogField.module.scss b/airbyte-webapp/src/views/Connection/ConnectionForm/components/SyncCatalogField.module.scss index ca68d3df90b0..3d6a9ec00a75 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/components/SyncCatalogField.module.scss +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/components/SyncCatalogField.module.scss @@ -31,10 +31,3 @@ padding-top: 10px; margin-left: 115px; } - -.treeViewContainer { - margin-bottom: 29px; - max-height: 600px; - overflow-y: auto; - width: 100%; -} diff --git a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx index 6b1f790df46b..2e6f6e3e568f 100644 --- a/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx +++ b/airbyte-webapp/src/views/Connection/ConnectionForm/formConfig.tsx @@ -41,7 +41,7 @@ interface FormikConnectionFormValues { type ConnectionFormValues = ValuesProps; -const SUPPORTED_MODES: [SyncMode, DestinationSyncMode][] = [ +const SUPPORTED_MODES: Array<[SyncMode, DestinationSyncMode]> = [ [SyncMode.incremental, DestinationSyncMode.append_dedup], [SyncMode.full_refresh, DestinationSyncMode.overwrite], [SyncMode.incremental, DestinationSyncMode.append], @@ -114,7 +114,7 @@ const connectionValidationSchema = yup name: "connectionSchema.config.validator", // eslint-disable-next-line no-template-curly-in-string message: "${path} is wrong", - test: function (value) { + test(value) { if (!value.selected) { return true; } @@ -204,7 +204,7 @@ const getInitialTransformations = (operations: OperationCreate[]): OperationRead operations?.filter(isDbtTransformation) ?? []; const getInitialNormalization = ( - operations?: (OperationRead | OperationCreate)[], + operations?: Array, isEditMode?: boolean ): NormalizationType => { const initialNormalization = diff --git a/airbyte-webapp/src/views/Connection/FormCard.tsx b/airbyte-webapp/src/views/Connection/FormCard.tsx index ec18e057c23f..51946825b882 100644 --- a/airbyte-webapp/src/views/Connection/FormCard.tsx +++ b/airbyte-webapp/src/views/Connection/FormCard.tsx @@ -23,14 +23,14 @@ interface FormCardProps extends CollapsibleCardProps { submitDisabled?: boolean; } -export function FormCard({ +export const FormCard = ({ children, form, bottomSeparator = true, mode, submitDisabled, ...props -}: React.PropsWithChildren>) { +}: React.PropsWithChildren>) => { const { formatMessage } = useIntl(); const { mutateAsync, error, reset, isSuccess } = useMutation< @@ -73,4 +73,4 @@ export function FormCard({ )} ); -} +}; diff --git a/airbyte-webapp/src/views/Connection/TransformationForm/TransformationForm.tsx b/airbyte-webapp/src/views/Connection/TransformationForm/TransformationForm.tsx index a6856085f051..74c0f470d5bb 100644 --- a/airbyte-webapp/src/views/Connection/TransformationForm/TransformationForm.tsx +++ b/airbyte-webapp/src/views/Connection/TransformationForm/TransformationForm.tsx @@ -86,14 +86,14 @@ const TransformationForm: React.FC = ({ onDone, isNewTransformation, }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const operationService = useGetService("OperationService"); const { clearFormChange } = useFormChangeTrackerService(); const formId = useUniqueFormId(); const formik = useFormik({ initialValues: transformation, - validationSchema: validationSchema, + validationSchema, onSubmit: async (values) => { await operationService.check(values); clearFormChange(formId); diff --git a/airbyte-webapp/src/views/Connector/ConnectorCard/useTestConnector.tsx b/airbyte-webapp/src/views/Connector/ConnectorCard/useTestConnector.tsx index 337ff7940520..4c586af3fa39 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorCard/useTestConnector.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorCard/useTestConnector.tsx @@ -60,15 +60,13 @@ export const useTestConnector = ( signal: controller.signal, }; } - } else { + } else if (values) { // creating new connection - if (values) { - payload = { - connectionConfiguration: values.connectionConfiguration, - signal: controller.signal, - selectedConnectorDefinitionId: values.serviceType, - }; - } + payload = { + connectionConfiguration: values.connectionConfiguration, + signal: controller.signal, + selectedConnectorDefinitionId: values.serviceType, + }; } if (!payload) { diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss index f07e2fc80605..edf62f045bbe 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss +++ b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.module.scss @@ -34,10 +34,6 @@ height: 100%; } -.scroll { - overflow: scroll; -} - .lightOverlay { height: 100%; width: 100%; diff --git a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx index 2d39122d4423..d4ca7803ed9d 100644 --- a/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx +++ b/airbyte-webapp/src/views/Connector/ConnectorDocumentationLayout/ConnectorDocumentationLayout.tsx @@ -72,7 +72,7 @@ export const ConnectorDocumentationLayout: React.FC = ({ children }) => { {documentationPanelOpen && (
- +
)} diff --git a/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx b/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx index a4798ab734bd..8f497b2790ea 100644 --- a/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx +++ b/airbyte-webapp/src/views/Connector/RequestConnectorModal/components/ConnectorForm.tsx @@ -43,7 +43,7 @@ const requestConnectorValidationSchema = yup.object().shape({ }); const ConnectorForm: React.FC = ({ onSubmit, onCancel, currentValues, hasFeedback }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const dropdownData = [ { value: "source", label: }, { @@ -60,8 +60,8 @@ const ConnectorForm: React.FC = ({ onSubmit, onCancel, curre additionalInfo: currentValues?.additionalInfo || "", email: currentValues?.email || "", }} - validateOnBlur={true} - validateOnChange={true} + validateOnBlur + validateOnChange validationSchema={requestConnectorValidationSchema} onSubmit={onSubmit} > diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.tsx index 1d343b13407b..dbaf36e2b3c7 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/ServiceForm.tsx @@ -82,15 +82,17 @@ const SetDefaultName: React.FC = () => { const { selectedService } = useServiceForm(); useEffect(() => { - if (selectedService) { - const timeout = setTimeout(() => { - // We need to push this out one execution slot, so the form isn't still in its - // initialization status and won't react to this call but would just take the initialValues instead. - setFieldValue("name", selectedService.name); - }); - return () => clearTimeout(timeout); + if (!selectedService) { + return; } - return; + + const timeout = setTimeout(() => { + // We need to push this out one execution slot, so the form isn't still in its + // initialization status and won't react to this call but would just take the initialValues instead. + setFieldValue("name", selectedService.name); + }); + return () => clearTimeout(timeout); + // eslint-disable-next-line react-hooks/exhaustive-deps }, [selectedService]); @@ -156,9 +158,9 @@ const ServiceForm: React.FC = (props) => { const { formFields, initialValues } = useBuildForm(jsonSchema, formValues); const { setDocumentationUrl, setDocumentationPanelOpen } = useDocumentationPanelContext(); - useMemo(() => { + useEffect(() => { if (!selectedConnectorDefinitionSpecification) { - return undefined; + return; } const selectedServiceDefinition = availableServices.find((service) => { @@ -168,17 +170,15 @@ const ServiceForm: React.FC = (props) => { isSourceDefinitionSpecification(selectedConnectorDefinitionSpecification) && serviceDefinitionId === selectedConnectorDefinitionSpecification.sourceDefinitionId ); - } else { - const serviceDefinitionId = service.destinationDefinitionId; - return ( - isDestinationDefinitionSpecification(selectedConnectorDefinitionSpecification) && - serviceDefinitionId === selectedConnectorDefinitionSpecification.destinationDefinitionId - ); } + const serviceDefinitionId = service.destinationDefinitionId; + return ( + isDestinationDefinitionSpecification(selectedConnectorDefinitionSpecification) && + serviceDefinitionId === selectedConnectorDefinitionSpecification.destinationDefinitionId + ); }); setDocumentationUrl(selectedServiceDefinition?.documentationUrl ?? ""); setDocumentationPanelOpen(true); - return; }, [availableServices, selectedConnectorDefinitionSpecification, setDocumentationPanelOpen, setDocumentationUrl]); const uiOverrides = useMemo( @@ -232,8 +232,8 @@ const ServiceForm: React.FC = (props) => { return ( = return priorityB - priorityA; } else if (a.releaseStage !== b.releaseStage) { return getOrderForReleaseStage(a.releaseStage) - getOrderForReleaseStage(b.releaseStage); - } else { - return naturalComparator(a.label, b.label); } + return naturalComparator(a.label, b.label); }), // eslint-disable-next-line react-hooks/exhaustive-deps [availableServices, orderOverwrite] diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/Control.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/Control.tsx index 43780896b444..5ad41a433120 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/Control.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Property/Control.tsx @@ -33,7 +33,7 @@ export const Control: React.FC = ({ switch (typeof property.examples) { case "object": if (Array.isArray(property.examples)) { - placeholder = property.examples[0] + ""; + placeholder = `${property.examples[0]}`; } break; case "number": @@ -152,18 +152,17 @@ export const Control: React.FC = ({ disabled={disabled} /> ); - } else { - const inputType = property.type === "integer" ? "number" : "text"; - - return ( - - ); } + const inputType = property.type === "integer" ? "number" : "text"; + + return ( + + ); }; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/FormSection.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/FormSection.tsx index f4b84365b143..8e55e3461bdd 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/FormSection.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/FormSection.tsx @@ -27,13 +27,12 @@ const FormNode: React.FC = ({ sectionPath, formField, disabled }) return ; } else if (formField.const !== undefined) { return null; - } else { - return ( - - - - ); } + return ( + + + + ); }; interface FormSectionProps { diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx index e8ee5a8b29ee..7a758b47f67d 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthButton.tsx @@ -39,17 +39,15 @@ function isGoogleConnector(connectorDefinitionId: string): boolean { function getButtonComponent(connectorDefinitionId: string) { if (isGoogleConnector(connectorDefinitionId)) { return GoogleAuthButton; - } else { - return Button; } + return Button; } function getAuthenticateMessageId(connectorDefinitionId: string): string { if (isGoogleConnector(connectorDefinitionId)) { return "connectorForm.signInWithGoogle"; - } else { - return "connectorForm.authenticate"; } + return "connectorForm.authenticate"; } export const AuthButton: React.FC = () => { diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthSection.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthSection.tsx index 0b710e05ef16..0e5c55ced993 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthSection.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/AuthSection.tsx @@ -8,11 +8,9 @@ import { AuthButton } from "./AuthButton"; export const AuthSection: React.FC = () => { return ( - { - - - - } + + + ); }; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/GoogleAuthButton.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/GoogleAuthButton.tsx index 75efe97b826d..37225a1d3b31 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/GoogleAuthButton.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/GoogleAuthButton.tsx @@ -43,7 +43,7 @@ const Img = styled.img` const GoogleAuthButton: React.FC = (props) => ( - {"Sign + Sign in with Google {props.children} ); diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/useOauthFlowAdapter.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/useOauthFlowAdapter.tsx index 612433f1f7c7..92f4a181b4e7 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/useOauthFlowAdapter.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/components/Sections/auth/useOauthFlowAdapter.tsx @@ -50,7 +50,7 @@ function useFormikOauthAdapter(connector: ConnectorDefinitionSpecification): { const oauthInputProperties = ( connector?.advancedAuth?.oauthConfigSpecification?.oauthUserInputFromConnectorConfigSpecification as { - properties: { path_in_connector_config: string[] }[]; + properties: Array<{ path_in_connector_config: string[] }>; } )?.properties ?? {}; diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/serviceFormContext.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/serviceFormContext.tsx index 42912d2969dc..3937e364c6a3 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/serviceFormContext.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/serviceFormContext.tsx @@ -58,7 +58,7 @@ const ServiceFormContextProvider: React.FC<{ const { values } = useFormikContext(); const { hasFeature } = useFeatureService(); - const serviceType = values.serviceType; + const { serviceType } = values; const selectedService = useMemo( () => availableServices.find((s) => Connector.id(s) === serviceType), [availableServices, serviceType] diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/useBuildForm.tsx b/airbyte-webapp/src/views/Connector/ServiceForm/useBuildForm.tsx index 3e51b20c1e9e..fe5e9808af4a 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/useBuildForm.tsx +++ b/airbyte-webapp/src/views/Connector/ServiceForm/useBuildForm.tsx @@ -24,7 +24,7 @@ function upgradeSchemaLegacyAuth( const spec = connectorSpecification.authSpecification.oauth2Specification; return applyFuncAt( connectorSpecification.connectionSpecification as JSONSchema7Definition, - (spec?.rootObject ?? []) as (string | number)[], + (spec?.rootObject ?? []) as Array, (schema) => { // Very hacky way to allow placing button within section // @ts-expect-error json schema diff --git a/airbyte-webapp/src/views/Connector/ServiceForm/utils.ts b/airbyte-webapp/src/views/Connector/ServiceForm/utils.ts index d5369f620507..06e31d982df3 100644 --- a/airbyte-webapp/src/views/Connector/ServiceForm/utils.ts +++ b/airbyte-webapp/src/views/Connector/ServiceForm/utils.ts @@ -20,9 +20,9 @@ export interface OauthOutputSpec { type OAuthOutputSpec = { properties: Record } | undefined; -export function serverProvidedOauthPaths(connector?: ConnectorDefinitionSpecification): { - [key: string]: { path_in_connector_config: string[] }; -} { +export function serverProvidedOauthPaths( + connector?: ConnectorDefinitionSpecification +): Record { return { ...((connector?.advancedAuth?.oauthConfigSpecification?.completeOAuthOutputSpecification as OAuthOutputSpec) ?.properties ?? {}), diff --git a/airbyte-webapp/src/views/Settings/PreferencesForm/PreferencesForm.tsx b/airbyte-webapp/src/views/Settings/PreferencesForm/PreferencesForm.tsx index 3dd4cfd52ad7..26abc7d515cb 100644 --- a/airbyte-webapp/src/views/Settings/PreferencesForm/PreferencesForm.tsx +++ b/airbyte-webapp/src/views/Settings/PreferencesForm/PreferencesForm.tsx @@ -69,7 +69,7 @@ const PreferencesForm: React.FC = ({ successMessage, errorMessage, }) => { - const formatMessage = useIntl().formatMessage; + const { formatMessage } = useIntl(); const config = useConfig(); return ( @@ -80,7 +80,7 @@ const PreferencesForm: React.FC = ({ news: preferencesValues?.news || false, securityUpdates: preferencesValues?.securityUpdates || false, }} - validateOnBlur={true} + validateOnBlur validateOnChange={false} validationSchema={preferencesValidationSchema} onSubmit={async (values) => { @@ -122,7 +122,7 @@ const PreferencesForm: React.FC = ({ ( @@ -184,7 +184,7 @@ const PreferencesForm: React.FC = ({ ) : ( - + )} diff --git a/airbyte-webapp/src/views/Settings/PreferencesForm/components/EditControls.tsx b/airbyte-webapp/src/views/Settings/PreferencesForm/components/EditControls.tsx index 2136f76712c1..272801059d11 100644 --- a/airbyte-webapp/src/views/Settings/PreferencesForm/components/EditControls.tsx +++ b/airbyte-webapp/src/views/Settings/PreferencesForm/components/EditControls.tsx @@ -64,7 +64,7 @@ const EditControls: React.FC = ({ isSubmitting, isValid, dirty, resetFor {showStatusMessage()} diff --git a/airbyte-webapp/src/views/common/ResorceNotFoundErrorBoundary.tsx b/airbyte-webapp/src/views/common/ResorceNotFoundErrorBoundary.tsx index 55d3699bf8b1..7b3c471e79fe 100644 --- a/airbyte-webapp/src/views/common/ResorceNotFoundErrorBoundary.tsx +++ b/airbyte-webapp/src/views/common/ResorceNotFoundErrorBoundary.tsx @@ -24,9 +24,8 @@ export class ResourceNotFoundErrorBoundary extends React.Component< hasError: true, message: , }; - } else { - throw error; } + throw error; } state = initialState; diff --git a/airbyte-webapp/src/views/layout/SideBar/components/SidebarPopout.tsx b/airbyte-webapp/src/views/layout/SideBar/components/SidebarPopout.tsx index a6836d2c91cb..b261acf380ee 100644 --- a/airbyte-webapp/src/views/layout/SideBar/components/SidebarPopout.tsx +++ b/airbyte-webapp/src/views/layout/SideBar/components/SidebarPopout.tsx @@ -30,7 +30,7 @@ export const Icon = styled.div` const SidebarPopout: React.FC<{ children: (props: { onOpen: () => void }) => React.ReactNode; - options: { value: string; label?: React.ReactNode }[]; + options: Array<{ value: string; label?: React.ReactNode }>; }> = ({ children, options }) => { const config = useConfig(); diff --git a/airbyte-workers/Dockerfile b/airbyte-workers/Dockerfile index 177dd18748e4..eba8457e8c8c 100644 --- a/airbyte-workers/Dockerfile +++ b/airbyte-workers/Dockerfile @@ -27,7 +27,7 @@ RUN curl -fsSLo /usr/share/keyrings/kubernetes-archive-keyring.gpg https://packa RUN echo "deb [signed-by=/usr/share/keyrings/kubernetes-archive-keyring.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" | tee /etc/apt/sources.list.d/kubernetes.list RUN apt-get update && apt-get install -y kubectl -ARG VERSION=0.39.21-alpha +ARG VERSION=0.39.23-alpha ENV APPLICATION airbyte-workers ENV VERSION ${VERSION} diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java index a896b126e6da..28153a05a47c 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/WorkerApp.java @@ -115,7 +115,7 @@ public class WorkerApp { private final ProcessFactory discoverProcessFactory; private final ProcessFactory replicationProcessFactory; private final SecretsHydrator secretsHydrator; - private final WorkflowServiceStubs temporalService; + private final WorkflowClient workflowClient; private final ConfigRepository configRepository; private final MaxWorkersConfig maxWorkers; private final WorkerEnvironment workerEnvironment; @@ -148,7 +148,7 @@ public void start() { } }); - final WorkerFactory factory = WorkerFactory.newInstance(WorkflowClient.newInstance(temporalService)); + final WorkerFactory factory = WorkerFactory.newInstance(workflowClient); if (configs.shouldRunGetSpecWorkflows()) { registerGetSpec(factory); @@ -377,19 +377,12 @@ private static void launchWorkerApp(final Configs configs, final DSLContext conf final Path workspaceRoot = configs.getWorkspaceRoot(); LOGGER.info("workspaceRoot = " + workspaceRoot); - final String temporalHost = configs.getTemporalHost(); - LOGGER.info("temporalHost = " + temporalHost); - final SecretsHydrator secretsHydrator = SecretPersistence.getSecretsHydrator(configsDslContext, configs); if (configs.getWorkerEnvironment().equals(WorkerEnvironment.KUBERNETES)) { KubePortManagerSingleton.init(configs.getTemporalWorkerPorts()); } - final WorkflowServiceStubs temporalService = TemporalUtils.createTemporalService(temporalHost); - - TemporalUtils.configureTemporalNamespace(temporalService); - final Database configDatabase = new Database(configsDslContext); final FeatureFlags featureFlags = new EnvVariableFeatureFlags(); final JsonSecretsProcessor jsonSecretsProcessor = JsonSecretsProcessor.builder() @@ -415,7 +408,10 @@ private static void launchWorkerApp(final Configs configs, final DSLContext conf configRepository, new OAuthConfigSupplier(configRepository, trackingClient)); - final TemporalClient temporalClient = TemporalClient.production(temporalHost, workspaceRoot, configs); + final WorkflowServiceStubs temporalService = TemporalUtils.createTemporalService(); + final WorkflowClient workflowClient = TemporalUtils.createWorkflowClient(temporalService, TemporalUtils.getNamespace()); + final TemporalClient temporalClient = new TemporalClient(workflowClient, configs.getWorkspaceRoot(), temporalService); + TemporalUtils.configureTemporalNamespace(temporalService); final TemporalWorkerRunFactory temporalWorkerRunFactory = new TemporalWorkerRunFactory( temporalClient, @@ -449,7 +445,7 @@ private static void launchWorkerApp(final Configs configs, final DSLContext conf discoverProcessFactory, replicationProcessFactory, secretsHydrator, - temporalService, + workflowClient, configRepository, configs.getMaxWorkers(), configs.getWorkerEnvironment(), diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/ConnectionManagerUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/ConnectionManagerUtils.java index ee4d9bf7d38f..e3a82a9649cf 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/ConnectionManagerUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/ConnectionManagerUtils.java @@ -144,7 +144,6 @@ static void safeTerminateWorkflow(final WorkflowClient client, final UUID connec static ConnectionManagerWorkflow startConnectionManagerNoSignal(final WorkflowClient client, final UUID connectionId) { final ConnectionManagerWorkflow connectionManagerWorkflow = newConnectionManagerWorkflowStub(client, connectionId); final ConnectionUpdaterInput input = buildStartWorkflowInput(connectionId); - WorkflowClient.start(connectionManagerWorkflow::run, input); return connectionManagerWorkflow; @@ -206,9 +205,10 @@ static boolean isWorkflowStateRunning(final WorkflowClient client, final UUID co static WorkflowExecutionStatus getConnectionManagerWorkflowStatus(final WorkflowClient workflowClient, final UUID connectionId) { final DescribeWorkflowExecutionRequest describeWorkflowExecutionRequest = DescribeWorkflowExecutionRequest.newBuilder() - .setExecution(WorkflowExecution.newBuilder().setWorkflowId(getConnectionManagerName(connectionId)).build()) - .setNamespace(workflowClient.getOptions().getNamespace()) - .build(); + .setExecution(WorkflowExecution.newBuilder() + .setWorkflowId(getConnectionManagerName(connectionId)) + .build()) + .setNamespace(workflowClient.getOptions().getNamespace()).build(); final DescribeWorkflowExecutionResponse describeWorkflowExecutionResponse = workflowClient.getWorkflowServiceStubs().blockingStub() .describeWorkflowExecution(describeWorkflowExecutionRequest); diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java index a6d1c025894e..7c261679ae5d 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalClient.java @@ -6,7 +6,6 @@ import com.google.common.annotations.VisibleForTesting; import com.google.protobuf.ByteString; -import io.airbyte.config.Configs; import io.airbyte.config.JobCheckConnectionConfig; import io.airbyte.config.JobDiscoverCatalogConfig; import io.airbyte.config.JobGetSpecConfig; @@ -33,6 +32,8 @@ import io.temporal.client.WorkflowClient; import io.temporal.serviceclient.WorkflowServiceStubs; import java.nio.file.Path; +import java.time.Duration; +import java.time.Instant; import java.util.HashSet; import java.util.Optional; import java.util.Set; @@ -54,7 +55,6 @@ public class TemporalClient { private final Path workspaceRoot; private final WorkflowClient client; private final WorkflowServiceStubs service; - private final Configs configs; /** * This is use to sleep between 2 temporal queries. The query are needed to ensure that the cancel @@ -63,23 +63,21 @@ public class TemporalClient { */ private static final int DELAY_BETWEEN_QUERY_MS = 10; - private static final int MAXIMUM_SEARCH_PAGE_SIZE = 50; - - public static TemporalClient production(final String temporalHost, final Path workspaceRoot, final Configs configs) { - final WorkflowServiceStubs temporalService = TemporalUtils.createTemporalService(temporalHost); - return new TemporalClient(WorkflowClient.newInstance(temporalService), workspaceRoot, temporalService, configs); - } - - // todo (cgardens) - there are two sources of truth on workspace root. we need to get this down to - // one. either temporal decides and can report it or it is injected into temporal runs. public TemporalClient(final WorkflowClient client, final Path workspaceRoot, - final WorkflowServiceStubs workflowServiceStubs, - final Configs configs) { + final WorkflowServiceStubs workflowServiceStubs) { this.client = client; this.workspaceRoot = workspaceRoot; this.service = workflowServiceStubs; - this.configs = configs; + } + + /** + * Direct termination of Temporal Workflows should generally be avoided. This method exists for some + * rare circumstances where this may be required. Originally added to facilitate Airbyte's migration + * to Temporal Cloud. TODO consider deleting this after Temporal Cloud migration + */ + public void dangerouslyTerminateWorkflow(final String workflowId, final String reason) { + this.client.newUntypedWorkflowStub(workflowId).terminate(reason); } public TemporalResponse submitGetSpec(final UUID jobId, final int attempt, final JobGetSpecConfig config) { @@ -213,10 +211,23 @@ void refreshRunningWorkflow() { } while (token != null && token.size() > 0); } + /** + * Refreshes the cache of running workflows, and returns their names. Currently called by the + * Temporal Cloud migrator to generate a list of workflows that should be migrated. After the + * Temporal Migration is complete, this could be removed, though it may be handy for a future use + * case. + */ + public Set getAllRunningWorkflows() { + final var startTime = Instant.now(); + refreshRunningWorkflow(); + final var endTime = Instant.now(); + log.info("getAllRunningWorkflows took {} milliseconds", Duration.between(startTime, endTime).toMillis()); + return workflowNames; + } + public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connectionId) { log.info("Starting the scheduler temporal wf"); final ConnectionManagerWorkflow connectionManagerWorkflow = ConnectionManagerUtils.startConnectionManagerNoSignal(client, connectionId); - try { CompletableFuture.supplyAsync(() -> { try { @@ -224,7 +235,6 @@ public ConnectionManagerWorkflow submitConnectionUpdaterAsync(final UUID connect Thread.sleep(DELAY_BETWEEN_QUERY_MS); } while (!isWorkflowReachable(connectionId)); } catch (final InterruptedException e) {} - return null; }).get(60, TimeUnit.SECONDS); } catch (final InterruptedException | ExecutionException e) { diff --git a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java index 33e1dcba1dc1..5e2ab9cd89b8 100644 --- a/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java +++ b/airbyte-workers/src/main/java/io/airbyte/workers/temporal/TemporalUtils.java @@ -4,8 +4,7 @@ package io.airbyte.workers.temporal; -import static java.util.stream.Collectors.toSet; - +import com.google.common.annotations.VisibleForTesting; import io.airbyte.commons.lang.Exceptions; import io.airbyte.config.Configs; import io.airbyte.config.EnvConfigs; @@ -15,20 +14,22 @@ import io.temporal.api.namespace.v1.NamespaceConfig; import io.temporal.api.namespace.v1.NamespaceInfo; import io.temporal.api.workflowservice.v1.DescribeNamespaceRequest; -import io.temporal.api.workflowservice.v1.DescribeNamespaceResponse; -import io.temporal.api.workflowservice.v1.ListNamespacesRequest; import io.temporal.api.workflowservice.v1.UpdateNamespaceRequest; import io.temporal.client.ActivityCompletionException; import io.temporal.client.WorkflowClient; +import io.temporal.client.WorkflowClientOptions; import io.temporal.client.WorkflowOptions; import io.temporal.client.WorkflowStub; import io.temporal.common.RetryOptions; +import io.temporal.serviceclient.SimpleSslContextBuilder; import io.temporal.serviceclient.WorkflowServiceStubs; import io.temporal.serviceclient.WorkflowServiceStubsOptions; import io.temporal.workflow.Functions; +import java.io.ByteArrayInputStream; +import java.io.InputStream; import java.io.Serializable; +import java.nio.charset.StandardCharsets; import java.time.Duration; -import java.util.Set; import java.util.UUID; import java.util.concurrent.Callable; import java.util.concurrent.CompletableFuture; @@ -37,60 +38,113 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Supplier; +import javax.net.ssl.SSLException; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.time.DurationFormatUtils; import org.apache.commons.lang3.tuple.ImmutablePair; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; +@Slf4j public class TemporalUtils { - private static final Logger LOGGER = LoggerFactory.getLogger(TemporalUtils.class); + private static final Configs configs = new EnvConfigs(); + private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(configs.getTemporalRetentionInDays()); + private static final Duration WAIT_INTERVAL = Duration.ofSeconds(2); + private static final Duration MAX_TIME_TO_CONNECT = Duration.ofMinutes(2); + private static final Duration WAIT_TIME_AFTER_CONNECT = Duration.ofSeconds(5); + private static final String HUMAN_READABLE_WORKFLOW_EXECUTION_TTL = + DurationFormatUtils.formatDurationWords(WORKFLOW_EXECUTION_TTL.toMillis(), true, true); + public static final String DEFAULT_NAMESPACE = "default"; public static final Duration SEND_HEARTBEAT_INTERVAL = Duration.ofSeconds(10); public static final Duration HEARTBEAT_TIMEOUT = Duration.ofSeconds(30); - - public static WorkflowServiceStubs createTemporalService(final String temporalHost) { - final WorkflowServiceStubsOptions options = WorkflowServiceStubsOptions.newBuilder() - .setTarget(temporalHost) // todo: move to EnvConfigs - .build(); - - return getTemporalClientWhenConnected( - Duration.ofSeconds(2), - Duration.ofMinutes(2), - Duration.ofSeconds(5), - () -> WorkflowServiceStubs.newInstance(options)); - } - public static final RetryOptions NO_RETRY = RetryOptions.newBuilder().setMaximumAttempts(1).build(); - - private static final Configs configs = new EnvConfigs(); public static final RetryOptions RETRY = RetryOptions.newBuilder() .setMaximumAttempts(configs.getActivityNumberOfAttempt()) .setInitialInterval(Duration.ofSeconds(configs.getInitialDelayBetweenActivityAttemptsSeconds())) .setMaximumInterval(Duration.ofSeconds(configs.getMaxDelayBetweenActivityAttemptsSeconds())) .build(); - public static final String DEFAULT_NAMESPACE = "default"; + public static WorkflowServiceStubs createTemporalService(final WorkflowServiceStubsOptions options, final String namespace) { + return getTemporalClientWhenConnected( + WAIT_INTERVAL, + MAX_TIME_TO_CONNECT, + WAIT_TIME_AFTER_CONNECT, + () -> WorkflowServiceStubs.newInstance(options), + namespace); + } - private static final Duration WORKFLOW_EXECUTION_TTL = Duration.ofDays(configs.getTemporalRetentionInDays()); - private static final String HUMAN_READABLE_WORKFLOW_EXECUTION_TTL = - DurationFormatUtils.formatDurationWords(WORKFLOW_EXECUTION_TTL.toMillis(), true, true); + // TODO consider consolidating this method's logic into createTemporalService() after the Temporal + // Cloud migration is complete. + // The Temporal Migration migrator is the only reason this public method exists. + public static WorkflowServiceStubs createTemporalService(final boolean isCloud) { + final WorkflowServiceStubsOptions options = isCloud ? getCloudTemporalOptions() : getAirbyteTemporalOptions(configs.getTemporalHost()); + final String namespace = isCloud ? configs.getTemporalCloudNamespace() : DEFAULT_NAMESPACE; + return createTemporalService(options, namespace); + } + + public static WorkflowServiceStubs createTemporalService() { + return createTemporalService(configs.temporalCloudEnabled()); + } + + private static WorkflowServiceStubsOptions getCloudTemporalOptions() { + final InputStream clientCert = new ByteArrayInputStream(configs.getTemporalCloudClientCert().getBytes(StandardCharsets.UTF_8)); + final InputStream clientKey = new ByteArrayInputStream(configs.getTemporalCloudClientKey().getBytes(StandardCharsets.UTF_8)); + try { + return WorkflowServiceStubsOptions.newBuilder() + .setSslContext(SimpleSslContextBuilder.forPKCS8(clientCert, clientKey).build()) + .setTarget(configs.getTemporalCloudHost()) + .build(); + } catch (final SSLException e) { + log.error("SSL Exception occurred attempting to establish Temporal Cloud options."); + throw new RuntimeException(e); + } + } + + @VisibleForTesting + public static WorkflowServiceStubsOptions getAirbyteTemporalOptions(final String temporalHost) { + return WorkflowServiceStubsOptions.newBuilder() + .setTarget(temporalHost) + .build(); + } + + public static WorkflowClient createWorkflowClient(final WorkflowServiceStubs workflowServiceStubs, final String namespace) { + return WorkflowClient.newInstance( + workflowServiceStubs, + WorkflowClientOptions.newBuilder() + .setNamespace(namespace) + .build()); + } + + public static String getNamespace() { + return configs.temporalCloudEnabled() ? configs.getTemporalCloudNamespace() : DEFAULT_NAMESPACE; + } + + /** + * Modifies the retention period for on-premise deployment of Temporal at the default namespace. + * This should not be called when using Temporal Cloud, because Temporal Cloud does not allow + * programmatic modification of workflow execution retention TTL. + */ public static void configureTemporalNamespace(final WorkflowServiceStubs temporalService) { + if (configs.temporalCloudEnabled()) { + log.info("Skipping Temporal Namespace configuration because Temporal Cloud is in use."); + return; + } + final var client = temporalService.blockingStub(); final var describeNamespaceRequest = DescribeNamespaceRequest.newBuilder().setNamespace(DEFAULT_NAMESPACE).build(); final var currentRetentionGrpcDuration = client.describeNamespace(describeNamespaceRequest).getConfig().getWorkflowExecutionRetentionTtl(); final var currentRetention = Duration.ofSeconds(currentRetentionGrpcDuration.getSeconds()); if (currentRetention.equals(WORKFLOW_EXECUTION_TTL)) { - LOGGER.info("Workflow execution TTL already set for namespace " + DEFAULT_NAMESPACE + ". Remains unchanged as: " + log.info("Workflow execution TTL already set for namespace " + DEFAULT_NAMESPACE + ". Remains unchanged as: " + HUMAN_READABLE_WORKFLOW_EXECUTION_TTL); } else { final var newGrpcDuration = com.google.protobuf.Duration.newBuilder().setSeconds(WORKFLOW_EXECUTION_TTL.getSeconds()).build(); final var humanReadableCurrentRetention = DurationFormatUtils.formatDurationWords(currentRetention.toMillis(), true, true); final var namespaceConfig = NamespaceConfig.newBuilder().setWorkflowExecutionRetentionTtl(newGrpcDuration).build(); final var updateNamespaceRequest = UpdateNamespaceRequest.newBuilder().setNamespace(DEFAULT_NAMESPACE).setConfig(namespaceConfig).build(); - LOGGER.info("Workflow execution TTL differs for namespace " + DEFAULT_NAMESPACE + ". Changing from (" + humanReadableCurrentRetention + ") to (" + log.info("Workflow execution TTL differs for namespace " + DEFAULT_NAMESPACE + ". Changing from (" + humanReadableCurrentRetention + ") to (" + HUMAN_READABLE_WORKFLOW_EXECUTION_TTL + "). "); client.updateNamespace(updateNamespaceRequest); } @@ -115,6 +169,8 @@ public static WorkflowOptions getWorkflowOptionsWithWorkflowId(final TemporalJob public static WorkflowOptions getWorkflowOptions(final TemporalJobType jobType) { return WorkflowOptions.newBuilder() .setTaskQueue(jobType.name()) + .setWorkflowTaskTimeout(Duration.ofSeconds(27)) // TODO parker - temporarily increasing this to a recognizable number to see if it changes + // error I'm seeing // todo (cgardens) we do not leverage Temporal retries. .setRetryOptions(RetryOptions.newBuilder().setMaximumAttempts(1).build()) .build(); @@ -163,7 +219,7 @@ public static ImmutablePair * This function uses a supplier as input since the creation of a WorkflowServiceStubs can result in * connection exceptions as well. */ @@ -171,47 +227,45 @@ public static WorkflowServiceStubs getTemporalClientWhenConnected( final Duration waitInterval, final Duration maxTimeToConnect, final Duration waitAfterConnection, - final Supplier temporalServiceSupplier) { - LOGGER.info("Waiting for temporal server..."); + final Supplier temporalServiceSupplier, + final String namespace) { + log.info("Waiting for temporal server..."); - boolean temporalStatus = false; + boolean temporalNamespaceInitialized = false; WorkflowServiceStubs temporalService = null; long millisWaited = 0; - while (!temporalStatus) { + while (!temporalNamespaceInitialized) { if (millisWaited >= maxTimeToConnect.toMillis()) { throw new RuntimeException("Could not create Temporal client within max timeout!"); } - LOGGER.warn("Waiting for default namespace to be initialized in temporal..."); + log.warn("Waiting for namespace {} to be initialized in temporal...", namespace); Exceptions.toRuntime(() -> Thread.sleep(waitInterval.toMillis())); millisWaited = millisWaited + waitInterval.toMillis(); try { temporalService = temporalServiceSupplier.get(); - temporalStatus = getNamespaces(temporalService).contains("default"); + final var namespaceInfo = getNamespaceInfo(temporalService, namespace); + temporalNamespaceInitialized = namespaceInfo.isInitialized(); } catch (final Exception e) { // Ignore the exception because this likely means that the Temporal service is still initializing. - LOGGER.warn("Ignoring exception while trying to request Temporal namespaces:", e); + log.warn("Ignoring exception while trying to request Temporal namespace:", e); } } // sometimes it takes a few additional seconds for workflow queue listening to be available Exceptions.toRuntime(() -> Thread.sleep(waitAfterConnection.toMillis())); - LOGGER.info("Found temporal default namespace!"); + log.info("Temporal namespace {} initialized!", namespace); return temporalService; } - protected static Set getNamespaces(final WorkflowServiceStubs temporalService) { + protected static NamespaceInfo getNamespaceInfo(final WorkflowServiceStubs temporalService, final String namespace) { return temporalService.blockingStub() - .listNamespaces(ListNamespacesRequest.newBuilder().build()) - .getNamespacesList() - .stream() - .map(DescribeNamespaceResponse::getNamespaceInfo) - .map(NamespaceInfo::getName) - .collect(toSet()); + .describeNamespace(DescribeNamespaceRequest.newBuilder().setNamespace(namespace).build()) + .getNamespaceInfo(); } /** @@ -229,12 +283,12 @@ public static T withBackgroundHeartbeat(final Callable callable, return callable.call(); } catch (final ActivityCompletionException e) { - LOGGER.warn("Job either timed out or was cancelled."); + log.warn("Job either timed out or was cancelled."); throw new RuntimeException(e); } catch (final Exception e) { throw new RuntimeException(e); } finally { - LOGGER.info("Stopping temporal heartbeating..."); + log.info("Stopping temporal heartbeating..."); scheduledExecutor.shutdown(); } } @@ -260,12 +314,12 @@ public static T withBackgroundHeartbeat(final AtomicReference canc return callable.call(); } catch (final ActivityCompletionException e) { - LOGGER.warn("Job either timed out or was cancelled."); + log.warn("Job either timed out or was cancelled."); throw new RuntimeException(e); } catch (final Exception e) { throw new RuntimeException(e); } finally { - LOGGER.info("Stopping temporal heartbeating..."); + log.info("Stopping temporal heartbeating..."); scheduledExecutor.shutdown(); } } diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java index 97ca7faedd73..d876bc8359a5 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalClientTest.java @@ -21,7 +21,6 @@ import com.google.common.collect.Sets; import io.airbyte.commons.json.Jsons; -import io.airbyte.config.Configs; import io.airbyte.config.JobCheckConnectionConfig; import io.airbyte.config.JobDiscoverCatalogConfig; import io.airbyte.config.JobGetSpecConfig; @@ -92,7 +91,6 @@ class TemporalClientTest { private Path logPath; private WorkflowServiceStubs workflowServiceStubs; private WorkflowServiceBlockingStub workflowServiceBlockingStub; - private Configs configs; @BeforeEach void setup() throws IOException { @@ -105,7 +103,7 @@ void setup() throws IOException { workflowServiceBlockingStub = mock(WorkflowServiceBlockingStub.class); when(workflowServiceStubs.blockingStub()).thenReturn(workflowServiceBlockingStub); mockWorkflowStatus(WorkflowExecutionStatus.WORKFLOW_EXECUTION_STATUS_RUNNING); - temporalClient = spy(new TemporalClient(workflowClient, workspaceRoot, workflowServiceStubs, configs)); + temporalClient = spy(new TemporalClient(workflowClient, workspaceRoot, workflowServiceStubs)); } @Nested diff --git a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalUtilsTest.java b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalUtilsTest.java index af26788c0324..d203c88dab06 100644 --- a/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalUtilsTest.java +++ b/airbyte-workers/src/test/java/io/airbyte/workers/temporal/TemporalUtilsTest.java @@ -100,16 +100,18 @@ void testWaitForTemporalServerAndLogThrowsException() { final DescribeNamespaceResponse describeNamespaceResponse = mock(DescribeNamespaceResponse.class); final NamespaceInfo namespaceInfo = mock(NamespaceInfo.class); final Supplier serviceSupplier = mock(Supplier.class); + final String namespace = "default"; - when(namespaceInfo.getName()).thenReturn("default"); + when(namespaceInfo.isInitialized()).thenReturn(true); + when(namespaceInfo.getName()).thenReturn(namespace); when(describeNamespaceResponse.getNamespaceInfo()).thenReturn(namespaceInfo); when(serviceSupplier.get()) .thenThrow(RuntimeException.class) .thenReturn(workflowServiceStubs); - when(workflowServiceStubs.blockingStub().listNamespaces(any()).getNamespacesList()) + when(workflowServiceStubs.blockingStub().describeNamespace(any())) .thenThrow(RuntimeException.class) - .thenReturn(List.of(describeNamespaceResponse)); - getTemporalClientWhenConnected(Duration.ofMillis(10), Duration.ofSeconds(1), Duration.ofSeconds(0), serviceSupplier); + .thenReturn(describeNamespaceResponse); + getTemporalClientWhenConnected(Duration.ofMillis(10), Duration.ofSeconds(1), Duration.ofSeconds(0), serviceSupplier, namespace); } @Test @@ -118,8 +120,9 @@ void testWaitThatTimesOut() { final DescribeNamespaceResponse describeNamespaceResponse = mock(DescribeNamespaceResponse.class); final NamespaceInfo namespaceInfo = mock(NamespaceInfo.class); final Supplier serviceSupplier = mock(Supplier.class); + final String namespace = "default"; - when(namespaceInfo.getName()).thenReturn("default"); + when(namespaceInfo.getName()).thenReturn(namespace); when(describeNamespaceResponse.getNamespaceInfo()).thenReturn(namespaceInfo); when(serviceSupplier.get()) .thenThrow(RuntimeException.class) @@ -128,7 +131,7 @@ void testWaitThatTimesOut() { .thenThrow(RuntimeException.class) .thenReturn(List.of(describeNamespaceResponse)); assertThrows(RuntimeException.class, () -> { - getTemporalClientWhenConnected(Duration.ofMillis(100), Duration.ofMillis(10), Duration.ofSeconds(0), serviceSupplier); + getTemporalClientWhenConnected(Duration.ofMillis(100), Duration.ofMillis(10), Duration.ofSeconds(0), serviceSupplier, namespace); }); } diff --git a/charts/airbyte/Chart.yaml b/charts/airbyte/Chart.yaml index 59380a85ac96..388eec68cc02 100644 --- a/charts/airbyte/Chart.yaml +++ b/charts/airbyte/Chart.yaml @@ -21,7 +21,7 @@ version: 0.3.5 # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. -appVersion: "0.39.21-alpha" +appVersion: "0.39.23-alpha" dependencies: - name: common diff --git a/charts/airbyte/README.md b/charts/airbyte/README.md index 61446d41a554..5790cf4b71a3 100644 --- a/charts/airbyte/README.md +++ b/charts/airbyte/README.md @@ -30,7 +30,7 @@ Helm charts for Airbyte. | `webapp.replicaCount` | Number of webapp replicas | `1` | | `webapp.image.repository` | The repository to use for the airbyte webapp image. | `airbyte/webapp` | | `webapp.image.pullPolicy` | the pull policy to use for the airbyte webapp image | `IfNotPresent` | -| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.39.21-alpha` | +| `webapp.image.tag` | The airbyte webapp image tag. Defaults to the chart's AppVersion | `0.39.23-alpha` | | `webapp.podAnnotations` | Add extra annotations to the webapp pod(s) | `{}` | | `webapp.containerSecurityContext` | Security context for the container | `{}` | | `webapp.livenessProbe.enabled` | Enable livenessProbe on the webapp | `true` | @@ -103,7 +103,7 @@ Helm charts for Airbyte. | `server.replicaCount` | Number of server replicas | `1` | | `server.image.repository` | The repository to use for the airbyte server image. | `airbyte/server` | | `server.image.pullPolicy` | the pull policy to use for the airbyte server image | `IfNotPresent` | -| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.39.21-alpha` | +| `server.image.tag` | The airbyte server image tag. Defaults to the chart's AppVersion | `0.39.23-alpha` | | `server.podAnnotations` | Add extra annotations to the server pod | `{}` | | `server.containerSecurityContext` | Security context for the container | `{}` | | `server.livenessProbe.enabled` | Enable livenessProbe on the server | `true` | @@ -138,7 +138,7 @@ Helm charts for Airbyte. | `worker.replicaCount` | Number of worker replicas | `1` | | `worker.image.repository` | The repository to use for the airbyte worker image. | `airbyte/worker` | | `worker.image.pullPolicy` | the pull policy to use for the airbyte worker image | `IfNotPresent` | -| `worker.image.tag` | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.39.21-alpha` | +| `worker.image.tag` | The airbyte worker image tag. Defaults to the chart's AppVersion | `0.39.23-alpha` | | `worker.podAnnotations` | Add extra annotations to the worker pod(s) | `{}` | | `worker.containerSecurityContext` | Security context for the container | `{}` | | `worker.livenessProbe.enabled` | Enable livenessProbe on the worker | `true` | @@ -170,7 +170,7 @@ Helm charts for Airbyte. | ------------------------------- | -------------------------------------------------------------------- | -------------------- | | `bootloader.image.repository` | The repository to use for the airbyte bootloader image. | `airbyte/bootloader` | | `bootloader.image.pullPolicy` | the pull policy to use for the airbyte bootloader image | `IfNotPresent` | -| `bootloader.image.tag` | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.39.21-alpha` | +| `bootloader.image.tag` | The airbyte bootloader image tag. Defaults to the chart's AppVersion | `0.39.23-alpha` | | `bootloader.podAnnotations` | Add extra annotations to the bootloader pod | `{}` | | `bootloader.nodeSelector` | Node labels for pod assignment | `{}` | | `bootloader.tolerations` | Tolerations for worker pod assignment. | `[]` | diff --git a/charts/airbyte/values.yaml b/charts/airbyte/values.yaml index 712619b40415..0a6a64c66ccf 100644 --- a/charts/airbyte/values.yaml +++ b/charts/airbyte/values.yaml @@ -41,7 +41,7 @@ webapp: image: repository: airbyte/webapp pullPolicy: IfNotPresent - tag: 0.39.21-alpha + tag: 0.39.23-alpha ## @param webapp.podAnnotations [object] Add extra annotations to the webapp pod(s) ## @@ -315,7 +315,7 @@ server: image: repository: airbyte/server pullPolicy: IfNotPresent - tag: 0.39.21-alpha + tag: 0.39.23-alpha ## @param server.podAnnotations [object] Add extra annotations to the server pod ## @@ -442,7 +442,7 @@ worker: image: repository: airbyte/worker pullPolicy: IfNotPresent - tag: 0.39.21-alpha + tag: 0.39.23-alpha ## @param worker.podAnnotations [object] Add extra annotations to the worker pod(s) ## @@ -560,7 +560,7 @@ bootloader: image: repository: airbyte/bootloader pullPolicy: IfNotPresent - tag: 0.39.21-alpha + tag: 0.39.23-alpha ## @param bootloader.podAnnotations [object] Add extra annotations to the bootloader pod ## diff --git a/docs/connector-development/README.md b/docs/connector-development/README.md index da91130ca49a..ca18ddc8f81e 100644 --- a/docs/connector-development/README.md +++ b/docs/connector-development/README.md @@ -136,12 +136,13 @@ Once you've finished iterating on the changes to a connector as specified in its ### The /publish command Publishing a connector can be done using the `/publish` command as outlined in the above section. The command runs a [github workflow](https://github.com/airbytehq/airbyte/actions/workflows/publish-command.yml), and has the following configurable parameters: -* **connector** - Required. This tells the workflow which connector to publish. e.g. `connector=connectors/source-amazon-ads` +* **connector** - Required. This tells the workflow which connector to publish. e.g. `connector=connectors/source-amazon-ads`. This can also be a comma-separated list of many connectors, e.g. `connector=connectors/source-s3,connectors/destination-postgres,connectors/source-facebook-marketing`. See the parallel flag below if publishing multiple connectors. * **repo** - Defaults to the main airbyte repo. Set this when building connectors from forked repos. e.g. `repo=userfork/airbyte` * **gitref** - Defaults to the branch of the PR where the /publish command is run as a comment. If running manually, set this to your branch where you made changes e.g. `gitref=george/s3-update` * **run-tests** - Defaults to true. Should always run the tests as part of the publish flow so that if tests fail, the connector is not published. * **comment-id** - This is automatically filled if you run /publish from a comment and enables the workflow to write back success/fail logs to the git comment. * **auto-bump-version** - Defaults to true, automates the post-publish process of bumping the connector's version in the yaml seed definitions and generating spec. +* **parallel** - Defaults to false. If set to true, a pool of runner agents will be spun up to allow publishing multiple connectors in parallel. Only switch this to true if publishing multiple connectors at once to avoid wasting $$$. ## Using credentials in CI diff --git a/docs/integrations/destinations/mqtt.md b/docs/integrations/destinations/mqtt.md index 36e03c29a485..b1e638243be0 100644 --- a/docs/integrations/destinations/mqtt.md +++ b/docs/integrations/destinations/mqtt.md @@ -82,4 +82,4 @@ _NOTE_: MQTT version 5 is not supported yet. | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | -| 0.1.2 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests | +| 0.1.1 | 2022-05-24 | [13099](https://github.com/airbytehq/airbyte/pull/13099) | Fixed build's tests | diff --git a/docs/integrations/destinations/mssql.md b/docs/integrations/destinations/mssql.md index a855693f3f74..4c23d97ec648 100644 --- a/docs/integrations/destinations/mssql.md +++ b/docs/integrations/destinations/mssql.md @@ -138,6 +138,7 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.9 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.8 | 2022-05-25 | [13054](https://github.com/airbytehq/airbyte/pull/13054) | Destination MSSQL: added custom JDBC parameters support. | | 0.1.6 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.5 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | diff --git a/docs/integrations/destinations/mysql.md b/docs/integrations/destinations/mysql.md index fd2938f3626e..671f957b4888 100644 --- a/docs/integrations/destinations/mysql.md +++ b/docs/integrations/destinations/mysql.md @@ -110,6 +110,7 @@ Using this feature requires additional configuration, when creating the destinat | Version | Date | Pull Request | Subject | |:--------| :--- | :--- |:----------------------------------------------------------------------------------------------------| +| 0.1.20 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.19 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.18 | 2022-02-25 | [10421](https://github.com/airbytehq/airbyte/pull/10421) | Refactor JDBC parameters handling | | 0.1.17 | 2022-02-16 | [10362](https://github.com/airbytehq/airbyte/pull/10362) | Add jdbc_url_params support for optional JDBC parameters | diff --git a/docs/integrations/destinations/rockset.md b/docs/integrations/destinations/rockset.md index 601bcd1be6ba..28f1ce653fc5 100644 --- a/docs/integrations/destinations/rockset.md +++ b/docs/integrations/destinations/rockset.md @@ -30,6 +30,7 @@ | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.3 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.2 | 2022-05-17 | [12820](https://github.com/airbytehq/airbyte/pull/12820) | Improved 'check' operation performance | | 0.1.1 | 2022-02-14 | [10256](https://github.com/airbytehq/airbyte/pull/10256) | Add `-XX:+ExitOnOutOfMemoryError` JVM option | | 0.1.0 | 2021-11-15 | [\#8006](https://github.com/airbytehq/airbyte/pull/8006) | Initial release| diff --git a/docs/integrations/sources/cockroachdb.md b/docs/integrations/sources/cockroachdb.md index 3de1b54f3b0f..7bf2b0a53094 100644 --- a/docs/integrations/sources/cockroachdb.md +++ b/docs/integrations/sources/cockroachdb.md @@ -111,6 +111,7 @@ Your database user should now be ready for use with Airbyte. | Version | Date | Pull Request | Subject | |:--------| :--- | :--- | :--- | +| 0.1.13 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.12 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.8 | 2022-04-06 | [11729](https://github.com/airbytehq/airbyte/pull/11729) | Bump mina-sshd from 2.7.0 to 2.8.0 | | 0.1.6 | 2022-02-21 | [10242](https://github.com/airbytehq/airbyte/pull/10242) | Fixed cursor for old connectors that use non-microsecond format. Now connectors work with both formats | diff --git a/docs/integrations/sources/marketo.md b/docs/integrations/sources/marketo.md index e1d5d5742495..53d4574c404d 100644 --- a/docs/integrations/sources/marketo.md +++ b/docs/integrations/sources/marketo.md @@ -19,22 +19,22 @@ This connector can be used to sync the following tables from Marketo: ### Data type mapping -| Integration Type | Airbyte Type | Notes | -| :--- | :--- | :--- | -| `array` | `array` | primitive arrays are converted into arrays of the types described in this table | -| `int`, `long` | `number` | | -| `object` | `object` | | -| `string` | `string` | \`\` | -| Namespaces | No | | +| Integration Type | Airbyte Type | Notes | +|:-----------------|:-------------|:--------------------------------------------------------------------------------| +| `array` | `array` | primitive arrays are converted into arrays of the types described in this table | +| `int`, `long` | `number` | | +| `object` | `object` | | +| `string` | `string` | \`\` | +| Namespaces | No | | ### Features Feature -| Supported?\(Yes/No\) | Notes | -| :--- | :--- | -| Full Refresh Sync | Yes | -| Incremental - Append Sync | Yes | +| Supported?\(Yes/No\) | Notes | +|:--------------------------|:------| +| Full Refresh Sync | Yes | +| Incremental - Append Sync | Yes | ### Performance considerations @@ -89,10 +89,11 @@ We're almost there! Armed with your Endpoint & Identity URLs and your Client ID ## CHANGELOG -| Version | Date | Pull Request | Subject | -| :--- | :--- | :--- | :--- | -| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | -| `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483) | Improve field conversion to conform schema | -| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Fix timestamp value format issue | -| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector | +| Version | Date | Pull Request | Subject | +|:--------|:-----------|:---------------------------------------------------------|:-------------------------------------------| +| `0.1.4` | 2022-06-20 | [13930](https://github.com/airbytehq/airbyte/pull/13930) | Process failing creation of export jobs | +| `0.1.3` | 2021-12-10 | [8429](https://github.com/airbytehq/airbyte/pull/8578) | Updated titles and descriptions | +| `0.1.2` | 2021-12-03 | [8483](https://github.com/airbytehq/airbyte/pull/8483) | Improve field conversion to conform schema | +| `0.1.1` | 2021-11-29 | [0000](https://github.com/airbytehq/airbyte/pull/0000) | Fix timestamp value format issue | +| `0.1.0` | 2021-09-06 | [5863](https://github.com/airbytehq/airbyte/pull/5863) | Release Marketo CDK Connector | diff --git a/docs/integrations/sources/openweather.md b/docs/integrations/sources/openweather.md index 8609d916597b..05e4e2ae9e2a 100644 --- a/docs/integrations/sources/openweather.md +++ b/docs/integrations/sources/openweather.md @@ -34,6 +34,7 @@ The free plan allows 60 calls per minute and 1,000,000 calls per month, you won' | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.5 | 2022-06-21 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | No changes. Used connector to test publish workflow changes. | | 0.1.4 | 2022-04-27 | [12397](https://github.com/airbytehq/airbyte/pull/12397) | No changes. Used connector to test publish workflow changes. | | 0.1.0 | 2021-10-27 | [7434](https://github.com/airbytehq/airbyte/pull/7434) | Initial release | diff --git a/docs/integrations/sources/postgres.md b/docs/integrations/sources/postgres.md index 9d4338cdbf07..311fa4a905da 100644 --- a/docs/integrations/sources/postgres.md +++ b/docs/integrations/sources/postgres.md @@ -125,17 +125,7 @@ We recommend using a user specifically for Airbyte's replication so you can mini We recommend using a `pgoutput` plugin as it is the standard logical decoding plugin in Postgres. In case the replication table contains a lot of big JSON blobs and table size exceeds 1 GB, we recommend using a `wal2json` instead. Please note that `wal2json` may require additional installation for Bare Metal, VMs \(EC2/GCE/etc\), Docker, etc. For more information read [wal2json documentation](https://github.com/eulerto/wal2json). -#### 4. Create replication slot - -Next, you will need to create a replication slot. Here is the query used to create a replication slot called `airbyte_slot`: - -```text -SELECT pg_create_logical_replication_slot('airbyte_slot', 'pgoutput'); -``` - -If you would like to use `wal2json` plugin, please change `pgoutput` to `wal2json` value in the above query. - -#### 5. Create publications and replication identities for tables +#### 4. Create publications and replication identities for tables For each table you want to replicate with CDC, you should add the replication identity \(the method of distinguishing between rows\) first. We recommend using `ALTER TABLE tbl1 REPLICA IDENTITY DEFAULT;` to use primary keys to distinguish between rows. After setting the replication identity, you will need to run `CREATE PUBLICATION airbyte_publication FOR TABLE ;`. This publication name is customizable. Please refer to the [Postgres docs](https://www.postgresql.org/docs/10/sql-alterpublication.html) if you need to add or remove tables from your publication in the future. @@ -145,6 +135,18 @@ Please note that: The UI currently allows selecting any tables for CDC. If a table is selected that is not part of the publication, it will not replicate even though it is selected. If a table is part of the publication but does not have a replication identity, that replication identity will be created automatically on the first run if the Airbyte user has the necessary permissions. +#### 5. Create replication slot + +Next, you will need to create a replication slot. It's important to create the publication first (as in step 4) before creating the replication slot. Otherwise, you can run into exceptions if there is any update to the database between the creation of the two. + +Here is the query used to create a replication slot called `airbyte_slot`: + +```text +SELECT pg_create_logical_replication_slot('airbyte_slot', 'pgoutput'); +``` + +If you would like to use `wal2json` plugin, please change `pgoutput` to `wal2json` value in the above query. + #### 6. Start syncing When configuring the source, select CDC and provide the replication slot and publication you just created. You should be ready to sync data with CDC! diff --git a/docs/integrations/sources/salesforce.md b/docs/integrations/sources/salesforce.md index fa95649a37ab..89f0f1980c04 100644 --- a/docs/integrations/sources/salesforce.md +++ b/docs/integrations/sources/salesforce.md @@ -58,7 +58,7 @@ To set up Salesforce as a source in Airbyte Open Source: 2. When running a curl command, run it with the `-L` option to follow any redirects. 3. If you [created a read-only user](https://docs.google.com/document/d/1wZR8pz4MRdc2zUculc9IqoF8JxN87U40IqVnTtcqdrI/edit#heading=h.w5v6h7b2a9y4), use the user credentials when logging in to generate OAuth tokens. -2. Navigate to the Airbute Open Source dashboard and follow the same steps as [setting up Salesforce as a source in Airbyte Cloud](link to previous section). +2. Navigate to the Airbute Open Source dashboard and follow the same steps as [setting up Salesforce as a source in Airbyte Cloud](#for-airbyte-cloud). ## Supported sync modes diff --git a/docs/integrations/sources/tidb.md b/docs/integrations/sources/tidb.md index 5cea0d66f733..4b82a1728354 100644 --- a/docs/integrations/sources/tidb.md +++ b/docs/integrations/sources/tidb.md @@ -120,5 +120,6 @@ Using this feature requires additional configuration, when creating the source. | Version | Date | Pull Request | Subject | | :------ | :--- | :----------- | ------- | +| 0.1.2 | 2022-06-17 | [13864](https://github.com/airbytehq/airbyte/pull/13864) | Updated stacktrace format for any trace message errors | | 0.1.1 | 2022-04-29 | [12480](https://github.com/airbytehq/airbyte/pull/12480) | Query tables with adaptive fetch size to optimize JDBC memory consumption | | 0.1.0 | 2022-04-19 | [11283](https://github.com/airbytehq/airbyte/pull/11283) | Initial Release | diff --git a/docs/operator-guides/upgrading-airbyte.md b/docs/operator-guides/upgrading-airbyte.md index 0afba5abfa7f..87790be18fe9 100644 --- a/docs/operator-guides/upgrading-airbyte.md +++ b/docs/operator-guides/upgrading-airbyte.md @@ -103,7 +103,7 @@ If you are upgrading from (i.e. your current version of Airbyte is) Airbyte vers Here's an example of what it might look like with the values filled in. It assumes that the downloaded `airbyte_archive.tar.gz` is in `/tmp`. ```bash - docker run --rm -v /tmp:/config airbyte/migration:0.39.21-alpha --\ + docker run --rm -v /tmp:/config airbyte/migration:0.39.23-alpha --\ --input /config/airbyte_archive.tar.gz\ --output /config/airbyte_archive_migrated.tar.gz ``` diff --git a/kube/overlays/stable-with-resource-limits/.env b/kube/overlays/stable-with-resource-limits/.env index 8e65dfaf6aab..5b5dfc6f19a8 100644 --- a/kube/overlays/stable-with-resource-limits/.env +++ b/kube/overlays/stable-with-resource-limits/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.39.21-alpha +AIRBYTE_VERSION=0.39.23-alpha # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable-with-resource-limits/kustomization.yaml b/kube/overlays/stable-with-resource-limits/kustomization.yaml index 5dd6666b78e7..315710a1227d 100644 --- a/kube/overlays/stable-with-resource-limits/kustomization.yaml +++ b/kube/overlays/stable-with-resource-limits/kustomization.yaml @@ -8,15 +8,15 @@ bases: images: - name: airbyte/db - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: airbyte/bootloader - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: airbyte/server - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: airbyte/webapp - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: airbyte/worker - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: temporalio/auto-setup newTag: 1.7.0 diff --git a/kube/overlays/stable/.env b/kube/overlays/stable/.env index 4c7fb78448b5..10217c576e01 100644 --- a/kube/overlays/stable/.env +++ b/kube/overlays/stable/.env @@ -1,4 +1,4 @@ -AIRBYTE_VERSION=0.39.21-alpha +AIRBYTE_VERSION=0.39.23-alpha # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db DATABASE_HOST=airbyte-db-svc diff --git a/kube/overlays/stable/kustomization.yaml b/kube/overlays/stable/kustomization.yaml index cd18db3a1fe4..600143f2d78a 100644 --- a/kube/overlays/stable/kustomization.yaml +++ b/kube/overlays/stable/kustomization.yaml @@ -8,15 +8,15 @@ bases: images: - name: airbyte/db - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: airbyte/bootloader - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: airbyte/server - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: airbyte/webapp - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: airbyte/worker - newTag: 0.39.21-alpha + newTag: 0.39.23-alpha - name: temporalio/auto-setup newTag: 1.7.0 diff --git a/octavia-cli/Dockerfile b/octavia-cli/Dockerfile index 87643815637a..d1cc030e8f72 100644 --- a/octavia-cli/Dockerfile +++ b/octavia-cli/Dockerfile @@ -14,5 +14,5 @@ USER octavia-cli WORKDIR /home/octavia-project ENTRYPOINT ["octavia"] -LABEL io.airbyte.version=0.39.21-alpha +LABEL io.airbyte.version=0.39.23-alpha LABEL io.airbyte.name=airbyte/octavia-cli diff --git a/octavia-cli/README.md b/octavia-cli/README.md index 2578d4ad5af7..e472fb3259d4 100644 --- a/octavia-cli/README.md +++ b/octavia-cli/README.md @@ -105,7 +105,7 @@ This script: ```bash touch ~/.octavia # Create a file to store env variables that will be mapped the octavia-cli container mkdir my_octavia_project_directory # Create your octavia project directory where YAML configurations will be stored. -docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.21-alpha +docker run --name octavia-cli -i --rm -v my_octavia_project_directory:/home/octavia-project --network host --user $(id -u):$(id -g) --env-file ~/.octavia airbyte/octavia-cli:0.39.23-alpha ``` ### Using `docker-compose` diff --git a/octavia-cli/install.sh b/octavia-cli/install.sh index af6c32d4db56..92a603749609 100755 --- a/octavia-cli/install.sh +++ b/octavia-cli/install.sh @@ -3,7 +3,7 @@ # This install scripts currently only works for ZSH and Bash profiles. # It creates an octavia alias in your profile bound to a docker run command and your current user. -VERSION=0.39.21-alpha +VERSION=0.39.23-alpha OCTAVIA_ENV_FILE=${HOME}/.octavia detect_profile() { diff --git a/octavia-cli/setup.py b/octavia-cli/setup.py index 662296e29084..ec87ab4e20c4 100644 --- a/octavia-cli/setup.py +++ b/octavia-cli/setup.py @@ -15,7 +15,7 @@ setup( name="octavia-cli", - version="0.39.21", + version="0.39.23", description="A command line interface to manage Airbyte configurations", long_description=README, author="Airbyte",