Shrodingers/destination databricks dbt

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.39.7-alpha
+current_version = 0.39.34-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -16,8 +16,6 @@ serialize =
 
 [bumpversion:file:airbyte-metrics/reporter/Dockerfile]
 
-[bumpversion:file:airbyte-scheduler/app/Dockerfile]
-
 [bumpversion:file:airbyte-server/Dockerfile]
 
 [bumpversion:file:airbyte-webapp/package.json]

.env

@@ -10,7 +10,7 @@
 
 
 ### SHARED ###
-VERSION=0.39.7-alpha
+VERSION=0.39.34-alpha
 
 # When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
@@ -70,26 +70,32 @@ JOB_MAIN_CONTAINER_MEMORY_LIMIT=
 
 ### LOGGING/MONITORING/TRACKING ###
 TRACKING_STRATEGY=segment
+JOB_ERROR_REPORTING_STRATEGY=logging
 # Although not present as an env var, expected by Log4J configuration.
 LOG_LEVEL=INFO
-# Although not present as an env var, helps Airbyte track job healthiness.
-SENTRY_DSN="https://d4b03de0c4574c78999b8d58e55243dc@o1009025.ingest.sentry.io/6102835"
 
 
 ### APPLICATIONS ###
-# Scheduler #
-# Relevant to scaling.
-SUBMITTER_NUM_THREADS=10
-
 # Worker #
 # Relevant to scaling.
 MAX_SYNC_WORKERS=5
 MAX_SPEC_WORKERS=5
 MAX_CHECK_WORKERS=5
 MAX_DISCOVER_WORKERS=5
+# Temporal Activity configuration
+ACTIVITY_MAX_ATTEMPT=
+ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS=
+ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS=
+WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=
 
 
 ### FEATURE FLAGS ###
 AUTO_DISABLE_FAILING_CONNECTIONS=false
 EXPOSE_SECRETS_IN_EXPORT=false
 FORCE_MIGRATE_SECRET_STORE=false
+
+### MONITORING FLAGS ###
+# Accepted values are datadog and otel (open telemetry)
+METRIC_CLIENT=
+# Useful only when metric client is set to be otel. Must start with http:// or https://.
+OTEL_COLLECTOR_ENDPOINT="http://host.docker.internal:4317"

.github/CODEOWNERS

@@ -4,3 +4,13 @@
 ## Exclude the package(-lock).json from code ownership to prevent version bump PRs from triggering codeowners review
 /airbyte-webapp/package.json
 /airbyte-webapp/package-lock.json
+
+# CDK and SAT
+/airbyte-cdk/ @airbytehq/api-connectors-dx
+/airbyte-integrations/bases/source-acceptance-tests/ @airbytehq/api-connectors-dx
+/airbyte-integrations/connector-templates/ @airbytehq/api-connectors-dx
+
+
+# Protocol related items
+/airbyte-protocol/ @airbytehq/protocol-reviewers
+/docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers

.github/actions/build-branch/action.yml

@@ -24,9 +24,9 @@ runs:
       with:
         java-version: "17"
 
-    - uses: actions/setup-node@v1
+    - uses: actions/setup-node@v2
       with:
-        node-version: "16.13.0"
+        node-version: "lts/gallium"
 
     - name: Set up CI Gradle Properties
       run: |

.github/actions/start-aws-runner/action.yml

@@ -41,14 +41,17 @@ runs:
         aws-region: us-east-2
     - name: Start EC2 runner
       id: start-ec2-runner
-      uses: supertopher/ec2-github-runner@base64v1.0.10
+      uses: airbytehq/ec2-github-runner@base64v1.1.0
       with:
         mode: start
         github-token: ${{ inputs.github-token }}
         ec2-image-id: ${{ inputs.ec2-image-id }}
         ec2-instance-type: ${{ inputs.ec2-instance-type }}
         subnet-id: ${{ inputs.subnet-id }}
         security-group-id: ${{ inputs.security-group-id }}
+        # this adds a label to group any EC2 runners spun up within the same action run
+        # this enables creating a pool of runners to run multiple/matrix jobs on in parallel
+        label: runner-pool-${{ github.run_id }}
         aws-resource-tags: >
           [
             {"Key": "BuildType", "Value": "oss"},

.github/pull_request_template.md

@@ -26,7 +26,6 @@ Expand the relevant checklist and delete the others.
 - [ ] Documentation updated
     - [ ] Connector's `README.md`
     - [ ] Connector's `bootstrap.md`. See [description and examples](https://docs.google.com/document/d/1ypdgmwmEHWv-TrO4_YOQ7pAJGVrMp5BOkEVh831N260/edit?usp=sharing)
-    - [ ] `docs/SUMMARY.md`
     - [ ] `docs/integrations/<source or destination>/<name>.md` including changelog. See changelog [example](https://docs.airbyte.io/integrations/sources/stripe#changelog)
     - [ ] `docs/integrations/README.md`
     - [ ] `airbyte-integrations/builds.md`

.github/workflows/build-connector-command.yml

@@ -135,6 +135,12 @@ jobs:
         run: |
           source venv/bin/activate
           ci_credentials ${{ github.event.inputs.connector }}
+          # normalization also runs destination-specific tests, so fetch their creds also
+          if [ 'bases/base-normalization' = "${{ github.event.inputs.connector }}" ]; then
+            ci_credentials destination-bigquery
+            ci_credentials destination-postgres
+            ci_credentials destination-snowflake
+          fi
         env:
           GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }}
       # TODO: seems like this should run in post-merge workflow

.github/workflows/create-release.yml

@@ -0,0 +1,71 @@
+# This is an action that runs when an Airbyte version bump is merged into master.
+# It fetches the changelog from the version bump PR and automatically creates a
+# Release for the version bump.
+name: Create Release
+
+on:
+  push:
+    branches:
+      - master
+
+jobs:
+  create-release:
+    if: startsWith(github.event.head_commit.message, 'Bump Airbyte version')
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      pull-requests: read
+    steps:
+      - name: Fetch Version Bump PR Body
+        id: fetch_pr_body
+        env:
+          COMMIT_ID: ${{ github.event.head_commit.id }}
+        shell: bash
+        run: |-
+          set -x
+          PR=$(curl \
+          -H "Accept: application/vnd.github.v3+json" \
+          -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+          https://api.github.com/repos/${{ github.repository }}/commits/$COMMIT_ID/pulls)
+          # The printf helps escape characters so that jq can parse the output.
+          # The sed removes carriage returns so that the body is easier to parse later, and
+          #   escapes backticks so that they are not executed as commands.
+          PR_BODY=$(printf '%s' "$PR" | jq '.[0].body' | sed 's/\\r//g' | sed 's/`/\\`/g')
+          echo ::set-output name=pr_body::${PR_BODY}
+      - name: Extract Changelog
+        id: extract_changelog
+        shell: bash
+        run: |-
+          set -x
+          PR_BODY=${{ steps.fetch_pr_body.outputs.pr_body}}
+          if [[ $PR_BODY = "null" ]]; then
+            echo "No PR body exists for this commit, so a release cannot be generated."
+            exit 1
+          fi
+          # this regex extracts just the changelog contents
+          if [[ $PR_BODY =~ Changelog:(\\n)*(.*)\\n\\n ]]; then
+            CHANGELOG="${BASH_REMATCH[2]}"
+          else
+            echo "PR body does not match the changelog extraction regex"
+            exit 1
+          fi
+          # save CHANGELOG into a multiline env var on the action itself, since Github Actions do not support outputting multiline strings well
+          echo "CHANGELOG<<EOF" >> $GITHUB_ENV
+          echo -e "$CHANGELOG" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+      - name: Get Version
+        id: get_version
+        shell: bash
+        run: |
+          VERSION=$(grep -w VERSION .env | cut -d"=" -f2)
+          echo ::set-output name=VERSION::${VERSION}
+      - name: Create Release
+        id: create_release
+        uses: ncipollo/release-action@v1
+        with:
+          body: ${{ env.CHANGELOG }}
+          token: ${{ secrets.GITHUB_TOKEN }}
+          prerelease: true
+          tag: v${{ steps.get_version.outputs.VERSION }}

.github/workflows/fe-validate-links.yml

@@ -0,0 +1,50 @@
+name: Frontend Link Validation
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron:  '0 14 * * *'
+
+jobs:
+  validate-frontend-links:
+    name: "Validate frontend links"
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: "17"
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: "lts/gallium"
+
+      - name: Set up CI Gradle Properties
+        run: |
+          mkdir -p ~/.gradle/
+          cat > ~/.gradle/gradle.properties <<EOF
+          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+          org.gradle.workers.max=8
+          org.gradle.vfs.watch=false
+          EOF
+
+      - name: Run :airbyte-webapp:validateLinks
+        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon :airbyte-webapp:validateLinks --scan
+
+      - name: Publish Failures to dev-frontend channel
+        uses: abinoda/slack-action@master
+        if: failure()
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN_AIRBYTE_TEAM }}
+        with:
+          args: >-
+            {\"channel\":\"C03088BTMFC\", \"blocks\":[
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\":alarm: The periodic link validation failed!\n\n\"}},
+            {\"type\":\"section\",\"text\":{\"type\":\"mrkdwn\",\"text\":\"See details on <https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}|GitHub>\n\"}}]}

.github/workflows/gke-kube-test-command.yml

@@ -78,9 +78,9 @@ jobs:
         with:
           java-version: "17"
 
-      - uses: actions/setup-node@v1
+      - uses: actions/setup-node@v2
         with:
-          node-version: "16.13.0"
+          node-version: "lts/gallium"
 
       - name: Fix EC-2 Runner
         run: |