Merge branch 'master' into leti/add-segment-cross-domain-tracking

* master: (200 commits)
  🪟 🧹 Display returned error messages on replication view (#16280)
  🎉 Source mixpanel: Use "Retry-After" header for backoff (#16770)
  🐛 Source google ads: mark custom query fields required (#15858)
  🪟 🔧Remove useRouter hook (#16598)
  CDK: improve TypeTransformer to convert simple types to array of simple types (#16636)
  CDK: TypeTransformer - warning message more informative (#16695)
  Source MySQL: Add Python SAT to detect backwards breaking changes (#16445)
  remove eager (#16756)
  bump com.networknt:json-schema-validator to latest version (#16619)
  Remove Cloud from Kafka docs (#16753)
  Normalization Summaries table and read/write methods (#16655)
  comment out flaky test suite while it is being investigated (#16752)
  Update ConfigRepository to read protocol version (#16670)
  Use LOG4J2 to wrap connectors logs to JSON format (#15668)
  Update connector catalog (#16749)
  🪟 🎨 Remove feedback modal from UI (#16548)
  Add missing env var for Kube overlays (#16747)
  Prepare for React v18 upgrade (#16694)
  🪟 🐛 Fix direct job linking to work with pagination (#16517)
  Fix formatting (#16743)
  ...

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.40.3
+current_version = 0.40.6
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -14,6 +14,8 @@ serialize =
 
 [bumpversion:file:airbyte-container-orchestrator/Dockerfile]
 
+[bumpversion:file:airbyte-cron/Dockerfile]
+
 [bumpversion:file:airbyte-metrics/reporter/Dockerfile]
 
 [bumpversion:file:airbyte-server/Dockerfile]

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.40.3
+VERSION=0.40.6
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
@@ -81,6 +81,7 @@ LOG_LEVEL=INFO
 
 ### APPLICATIONS ###
 # Worker #
+WORKERS_MICRONAUT_ENVIRONMENTS=control
 # Relevant to scaling.
 MAX_SYNC_WORKERS=5
 MAX_SPEC_WORKERS=5
@@ -92,7 +93,6 @@ ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS=
 ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS=
 WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=
 
-
 ### FEATURE FLAGS ###
 AUTO_DISABLE_FAILING_CONNECTIONS=false
 EXPOSE_SECRETS_IN_EXPORT=false
@@ -104,4 +104,4 @@ METRIC_CLIENT=
 # Useful only when metric client is set to be otel. Must start with http:// or https://.
 OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"
 
-USE_STREAM_CAPABLE_STATE=true
+USE_STREAM_CAPABLE_STATE=true

.env.dev

@@ -4,6 +4,7 @@ VERSION=dev
 DATABASE_USER=docker
 DATABASE_PASSWORD=docker
 DATABASE_DB=airbyte
+DATABASE_URL=jdbc:postgresql://db:5432/airbyte
 CONFIG_ROOT=/data
 WORKSPACE_ROOT=/tmp/workspace
 DATA_DOCKER_MOUNT=airbyte_data_dev
@@ -24,6 +25,11 @@ API_URL=/api/v1/
 INTERNAL_API_HOST=airbyte-server:8001
 SYNC_JOB_MAX_ATTEMPTS=3
 SYNC_JOB_MAX_TIMEOUT_DAYS=3
+WORKERS_MICRONAUT_ENVIRONMENTS=control
 
 # Sentry
 SENTRY_DSN=""
+
+# Migration Configuration
+CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.15.001
+JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001

.github/CODEOWNERS

@@ -20,6 +20,7 @@
 # JDBC-based connectors
 /airbyte-integrations/bases/base-java/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/source-jdbc/ @airbytehq/jdbc-connectors
+/airbyte-integrations/connectors/source-alloydb/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/source-bigquery/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/source-clickhouse/ @airbytehq/jdbc-connectors
 /airbyte-integrations/connectors/source-cockroachdb/ @airbytehq/jdbc-connectors

.github/workflows/gke-kube-test-command.yml

@@ -1,7 +1,7 @@
 name: GKE Kube Acceptance Test
 on:
   schedule:
-    - cron '0 0 * * 0' # runs at midnight UTC every Sunday
+    - cron: '0 0 * * 0' # runs at midnight UTC every Sunday
   workflow_dispatch:
     inputs:
       repo:

.github/workflows/gradle.yml

@@ -298,11 +298,15 @@ jobs:
           github-token: ${{ needs.find_valid_pat.outputs.pat }}
   frontend-build:
     name: "Frontend: Build"
-    needs: start-frontend-runner
+    needs:
+      - start-frontend-runner
+      - find_valid_pat
     runs-on: ${{ needs.start-frontend-runner.outputs.label }}
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
 
       - name: Cache Build Artifacts
         uses: ./.github/actions/cache-build-artifacts
@@ -333,6 +337,18 @@ jobs:
 
       - name: Build :airbyte-webapp
         run: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:build --scan
+
+      - name: Publish Storybook to Chromatic
+        uses: chromaui/action@v1
+        with:
+          projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+          token: ${{ needs.find_valid_pat.outputs.pat }}
+          workingDir: ./airbyte-webapp/
+          storybookBuildDir: build/storybook/
+          autoAcceptChanges: true
+          exitOnceUploaded: true
+
+
   frontend-test:
     name: "Frontend: Run End-to-End Tests"
     needs:
@@ -685,6 +701,169 @@ jobs:
           label: ${{ needs.start-kube-acceptance-test-runner.outputs.label }}
           ec2-instance-id: ${{ needs.start-kube-acceptance-test-runner.outputs.ec2-instance-id }}
 
+
+  ## Kube Acceptance Tests
+  # Docker acceptance tests run as part of the build job.
+  # In case of self-hosted EC2 errors, remove this block.
+  # TODO:
+  # Kyryl to fix issue
+  # start-helm-acceptance-test-runner:
+  #   name: "Platform: Start Helm Acceptance Test Runner"
+  #   needs:
+  #     - changes
+  #     - find_valid_pat
+  #   # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
+  #   if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
+  #   timeout-minutes: 10
+  #   runs-on: ubuntu-latest
+  #   outputs:
+  #     label: ${{ steps.start-ec2-runner.outputs.label }}
+  #     ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+  #   steps:
+  #     - name: Checkout Airbyte
+  #       uses: actions/checkout@v2
+  #     - name: Start AWS Runner
+  #       id: start-ec2-runner
+  #       uses: ./.github/actions/start-aws-runner
+  #       with:
+  #         # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
+  #         ec2-image-id: ami-0c1a9bc22624339d8
+  #         aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+  #         aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+  #         github-token: ${{ needs.find_valid_pat.outputs.pat }}
+
+
+  # Todo: Kyrylo turn this on.
+#  helm-acceptance-test:
+#    name: "Platform: Acceptance Tests (Helm)"
+#    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
+#    needs: start-helm-acceptance-test-runner # required to start the main job when the runner is ready
+#    runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
+#    environment: more-secrets
+#    timeout-minutes: 90
+#    steps:
+#      - name: Checkout Airbyte
+#        uses: actions/checkout@v2
+#
+#      - name: Cache Build Artifacts
+#        uses: ./.github/actions/cache-build-artifacts
+#        with:
+#          cache-key: ${{ secrets.CACHE_VERSION }}
+#          cache-python: "false"
+#
+#      - uses: actions/setup-java@v1
+#        with:
+#          java-version: "17"
+#
+#      - uses: actions/setup-node@v2
+#        with:
+#          node-version: "lts/gallium"
+#
+#      - name: Install unzip
+#        shell: bash
+#        run: sudo apt-get update && sudo apt-get install -y unzip
+#
+#      - uses: azure/setup-helm@v3
+#        with:
+#          version: 'latest'
+#          token: ${{ secrets.GITHUB_TOKEN }}
+#        id: install
+#
+#      - name: Fix EC-2 Runner
+#        run: |
+#          mkdir -p /home/runner
+#
+#      - name: Set up CI Gradle Properties
+#        run: |
+#          mkdir -p ~/.gradle/
+#          cat > ~/.gradle/gradle.properties <<EOF
+#          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+#          org.gradle.workers.max=8
+#          org.gradle.vfs.watch=false
+#          EOF
+#
+#      - name: Create cluster config file
+#        run: |
+#          cat > /tmp/kind-config.yaml <<EOF
+#          kind: Cluster
+#          apiVersion: kind.x-k8s.io/v1alpha4
+#          nodes:
+#          - role: control-plane
+#          - role: worker
+#          EOF
+#
+#      - name: KIND Kubernetes Cluster Setup
+#        uses: helm/kind-action@v1.2.0
+#        with:
+#          node_image: kindest/node:v1.21.2
+#          config: /tmp/kind-config.yaml
+#        # In case of self-hosted EC2 errors, remove this env block.
+#        env:
+#          USER: root
+#          HOME: /home/runner
+#          CHANGE_MINIKUBE_NONE_USER: true
+#
+#      - name: Build Platform Docker Images
+#        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+#
+#      - name: Run Helm End-to-End Acceptance Tests
+#        env:
+#          USER: root
+#          HOME: /home/runner
+#          # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
+#          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+#          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
+#          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
+#        run: |
+#          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
+#      - name: Generate Test Report
+#        uses: dorny/test-reporter@v1
+#        if: always()    # run this step even if previous step failed
+#        with:
+#          name: Platform Helm E2E Test Report
+#          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
+#          reporter: java-junit
+#
+#      - uses: actions/upload-artifact@v2
+#        if: failure()
+#        with:
+#          name: Kubernetes Logs
+#          path: /tmp/kubernetes_logs/*
+
+  # In case of self-hosted EC2 errors, remove this block.
+  # Todo: Kyryl turn this on.
+#  stop-helm-acceptance-test-runner:
+#    name: "Platform: Stop Helm Acceptance Test EC2 Runner"
+#    timeout-minutes: 10
+#    needs:
+#      - start-helm-acceptance-test-runner # required to get output from the start-runner job
+#      - helm-acceptance-test # required to wait when the main job is done
+#      - find_valid_pat
+#    runs-on: ubuntu-latest
+#    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
+#    # Thus, we check for skipped here.
+#    if: ${{ always() && needs.start-helm-acceptance-test-runner.result != 'skipped'}}
+#    steps:
+#      - name: Configure AWS credentials
+#        uses: aws-actions/configure-aws-credentials@v1
+#        with:
+#          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+#          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+#          aws-region: us-east-2
+#      - name: Stop EC2 runner
+#        uses: supertopher/ec2-github-runner@base64v1.0.10
+#        with:
+#          mode: stop
+#          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+#          label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }}
+#          ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }}
+
+
+
   notify-failure-slack-channel:
     name: "Notify Slack Channel on Build Failures"
     runs-on: ubuntu-latest
@@ -694,6 +873,8 @@ jobs:
       - octavia-cli-build
       - platform-build
       - kube-acceptance-test
+      # Todo: Kyryl turn this on.
+      # - helm-acceptance-test
     if: ${{ failure() && github.ref == 'refs/heads/master' }}
     steps:
       - name: Publish to OSS Build Failure Slack Channel
@@ -718,6 +899,8 @@ jobs:
       - octavia-cli-build
       - platform-build
       - kube-acceptance-test
+      # Todo: Kyryl turn this on.
+      # - helm-acceptance-test
     if: success()
     steps:
       - name: Get Previous Workflow Status

.github/workflows/publish-helm-charts.yml

@@ -56,7 +56,7 @@ jobs:
       - name: "Helm package"
         shell: bash
         run: |
-          declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker")
+          declare -a StringArray=("airbyte-bootloader" "airbyte-server" "airbyte-temporal" "airbyte-webapp" "airbyte-pod-sweeper" "airbyte-worker" "airbyte-metrics")
           for val in ${StringArray[@]}; do
             cd ./airbyte/charts/${val} && helm dep update && cd $GITHUB_WORKSPACE
             helm package ./airbyte/charts/${val} -d airbyte-oss  --version ${{ needs.generate-semantic-version.outputs.next-version }}

.github/workflows/slash-commands.yml

@@ -8,17 +8,29 @@ jobs:
     if: ${{ github.event.issue.pull_request }}
     runs-on: ubuntu-latest
     steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      - name: Check PAT rate limits
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.AIRBYTEIO_PAT }} \
+            ${{ secrets.OSS_BUILD_RUNNER_GITHUB_PAT }} \
+            ${{ secrets.SUPERTOPHER_PAT }} \
+            ${{ secrets.DAVINCHIA_PAT }}
+
       - name: Get PR repo and ref
         id: getref
         run: |
           pr_info="$(curl ${{ github.event.issue.pull_request.url }})"
           echo ::set-output name=ref::"$(echo $pr_info | jq -r '.head.ref')"
           echo ::set-output name=repo::"$(echo $pr_info | jq -r '.head.repo.full_name')"
+
       - name: Slash Command Dispatch
         id: scd
         uses: peter-evans/slash-command-dispatch@v2
         with:
-          token: ${{ secrets.SUPERTOPHER_PAT }}
+          token: ${{ env.PAT }}
           commands: |
             test
             test-performance
@@ -34,6 +46,7 @@ jobs:
             gitref=${{ steps.getref.outputs.ref }}
             comment-id=${{ github.event.comment.id }}
           dispatch-type: workflow
+
       - name: Edit comment with error message
         if: steps.scd.outputs.error-message
         uses: peter-evans/create-or-update-comment@v1

.vscode/frontend.code-workspace

@@ -18,9 +18,9 @@
   "extensions": {
     "recommendations": [
       "dbaeumer.vscode-eslint",
+      "stylelint.vscode-stylelint",
       "esbenp.prettier-vscode",
-      "ms-vsliveshare.vsliveshare",
-      "eamodio.gitlens"
+      "eamodio.gitlens",
     ]
   },
   "settings": {
@@ -33,6 +33,8 @@
     "editor.detectIndentation": true,
     "eslint.format.enable": true,
     "eslint.run": "onType",
+    "stylelint.enable": true,
+    "stylelint.validate": ["css", "scss"],
     "[javascript]": {
       "editor.formatOnSave": true,
       "editor.defaultFormatter": "dbaeumer.vscode-eslint",
@@ -57,6 +59,13 @@
     "[json]": {
       "editor.formatOnSave": true,
       "editor.defaultFormatter": "esbenp.prettier-vscode"
+    },
+    "[scss]": {
+      "editor.formatOnSave": true,
+      "editor.defaultFormatter": "esbenp.prettier-vscode",
+      "editor.codeActionsOnSave": {
+        "source.fixAll.stylelint": true
+      }
     }
   }
 }