Merge remote-tracking branch 'origin/master' into alvova/4374-stripe-…

…source

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.27.2-alpha
+current_version = 0.27.3-alpha
 commit = True
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?
@@ -19,3 +19,7 @@ serialize =
 [bumpversion:file:kube/overlays/stable/.env]
 
 [bumpversion:file:kube/overlays/stable/kustomization.yaml]
+
+[bumpversion:file:kube/overlays/stable-with-resource-limits/.env]
+
+[bumpversion:file:kube/overlays/stable-with-resource-limits/kustomization.yaml]

.env

@@ -1,4 +1,4 @@
-VERSION=0.27.2-alpha
+VERSION=0.27.3-alpha
 
 # Airbyte Internal Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
 DATABASE_USER=docker

.github/workflows/gradle.yml

@@ -6,10 +6,10 @@ on:
   push:
 
 jobs:
-  ## Gradle Build
+  ## Gradle Build (Connectors Base)
   # In case of self-hosted EC2 errors, remove this block.
-  start-build-runner:
-    name: Start Build EC2 Runner
+  start-connectors-base-build-runner:
+    name: "Connectors Base: Start Build EC2 Runner"
     runs-on: ubuntu-latest
     outputs:
       label: ${{ steps.start-ec2-runner.outputs.label }}
@@ -31,17 +31,17 @@ jobs:
           ec2-instance-type: c5.2xlarge
           subnet-id: subnet-0469a9e68a379c1d3
           security-group-id: sg-0793f3c9413f21970
-  build:
+  build-connectors-base:
     # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
-    needs: start-build-runner # required to start the main job when the runner is ready
-    runs-on: ${{ needs.start-build-runner.outputs.label }} # run the job on the newly created runner
-    name: Build Airbyte
+    needs: start-connectors-base-build-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-connectors-base-build-runner.outputs.label }} # run the job on the newly created runner
+    name: "Connectors Base: Build"
     steps:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
 
       - name: Check images exist
-        run: ./tools/bin/check_images_exist.sh
+        run: ./tools/bin/check_images_exist.sh connectors
 
       - name: Pip Caching
         uses: actions/cache@v2
@@ -90,39 +90,181 @@ jobs:
         run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan
 
       - name: Format
-        run: ./gradlew format --scan --info --stacktrace
+        run: SUB_BUILD=CONNECTORS_BASE ./gradlew format --scan --info --stacktrace
+
+      - name: Build
+        run: SUB_BUILD=CONNECTORS_BASE ./gradlew build --scan
+
+      - name: Ensure no file change
+        run: git status --porcelain && test -z "$(git status --porcelain)"
+
+      - name: Slack Notification - Failure
+        if: failure() && github.ref == 'refs/heads/master'
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+          SLACK_USERNAME: Buildozer
+          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
+          SLACK_COLOR: DC143C
+          SLACK_TITLE: "Build failure"
+          SLACK_FOOTER: ""
+
+      - name: Slack Notification - Success
+        if: success() && github.ref == 'refs/heads/master'
+        uses: rtCamp/action-slack-notify@master
+        env:
+          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
+          SLACK_USERNAME: Buildbot
+          SLACK_TITLE: "Build Success"
+          SLACK_FOOTER: ""
+  # In case of self-hosted EC2 errors, remove this block.
+  stop-connectors-base-build-runner:
+    name: "Connectors Base: Stop Build EC2 Runner"
+    needs:
+      - start-connectors-base-build-runner # required to get output from the start-runner job
+      - build-connectors-base # required to wait when the main job is done
+    runs-on: ubuntu-latest
+    if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Stop EC2 runner
+        uses: machulav/ec2-github-runner@v2.1.0
+        with:
+          mode: stop
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          label: ${{ needs.start-connectors-base-build-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-connectors-base-build-runner.outputs.ec2-instance-id }}
+
+  ## Gradle Build (Platform)
+  # In case of self-hosted EC2 errors, remove this block.
+  start-platform-build-runner:
+    name: "Platform: Start Build EC2 Runner"
+    runs-on: ubuntu-latest
+    outputs:
+      label: ${{ steps.start-ec2-runner.outputs.label }}
+      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+    steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          aws-region: us-east-2
+      - name: Start EC2 Runner
+        id: start-ec2-runner
+        uses: machulav/ec2-github-runner@v2.2.0
+        with:
+          mode: start
+          github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
+          ec2-image-id: ami-04bd6e81239f4f3fb
+          ec2-instance-type: c5.2xlarge
+          subnet-id: subnet-0469a9e68a379c1d3
+          security-group-id: sg-0793f3c9413f21970
+  platform-build:
+    # In case of self-hosted EC2 errors, remove the next two lines and uncomment the currently commented out `runs-on` line.
+    needs: start-platform-build-runner # required to start the main job when the runner is ready
+    runs-on: ${{ needs.start-platform-build-runner.outputs.label }} # run the job on the newly created runner
+    name: "Platform: Build"
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v2
+
+      - name: Check images exist
+        run: ./tools/bin/check_images_exist.sh platform
+
+      - name: Pip Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/pip
+          key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
+          restore-keys: |
+            ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-
+
+      - name: Npm Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.npm
+          key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
+          restore-keys: |
+            ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-
+
+      # this intentionally does not use restore-keys so we don't mess with gradle caching
+      - name: Gradle and Python Caching
+        uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+            **/.venv
+          key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '14'
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.7'
+
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '3.7'
+
+      - name: Install Pyenv
+        run: python3 -m pip install virtualenv==16.7.9 --user
+
+      - name: Format
+        run: SUB_BUILD=PLATFORM ./gradlew format --scan --info --stacktrace
 
       - name: Ensure no file change
         run: git status --porcelain && test -z "$(git status --porcelain)"
 
       - name: Build
-        run: CORE_ONLY=true ./gradlew build --scan
+        run: SUB_BUILD=PLATFORM ./gradlew build --scan
 
       - name: Ensure no file change
         run: git status --porcelain && test -z "$(git status --porcelain)"
 
+      # todo (cgardens) - scope by platform.
       - name: Check documentation
         if: success() && github.ref == 'refs/heads/master'
         run: ./tools/site/link_checker.sh check_docs
 
-# This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
-#      - name: Get Docker Space
-#        run: docker run --rm busybox df -h
+      # This is only required on the usual github runner. The usual runner does not contain enough disk space for our use.
+      #      - name: Get Docker Space
+      #        run: docker run --rm busybox df -h
 
-      - name: Build Core Docker Images
+      - name: Image Cleanup
+        run: ./tools/bin/clean_images.sh
+
+      - name: Build Platform Docker Images
         if: success() && github.ref == 'refs/heads/master'
-        run: ./gradlew composeBuild --scan
+        run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
         env:
           GIT_REVISION: ${{ github.sha }}
 
-      - name: Image Cleanup
-        run: ./tools/bin/clean_images.sh
-
+      # make sure these always run before pushing platform docker images
       - name: Run End-to-End Acceptance Tests
+        if: success() && github.ref == 'refs/heads/master'
         run: ./tools/bin/acceptance_test.sh
 
       - name: Automatic Migration Acceptance Test
-        run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr  -d "VERSION=") ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
+        run: MIGRATION_TEST_VERSION=$(grep VERSION .env | tr  -d "VERSION=") SUB_BUILD=PLATFORM ./gradlew :airbyte-tests:automaticMigrationAcceptanceTest --scan -i
+
+      - name: Push Platform Docker Images
+        if: success() && github.ref == 'refs/heads/master'
+        run: |
+          docker login -u airbytebot -p ${DOCKER_PASSWORD}
+          VERSION=dev docker-compose -f docker-compose.build.yaml push
+        env:
+          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
 
       - name: Slack Notification - Failure
         if: failure() && github.ref == 'refs/heads/master'
@@ -144,11 +286,11 @@ jobs:
           SLACK_TITLE: "Build Success"
           SLACK_FOOTER: ""
   # In case of self-hosted EC2 errors, remove this block.
-  stop-build-runner:
-    name: Stop Build EC2 Runner
+  stop-platform-build-runner:
+    name: "Platform: Stop Build EC2 Runner"
     needs:
-      - start-build-runner # required to get output from the start-runner job
-      - build # required to wait when the main job is done
+      - start-platform-build-runner # required to get output from the start-runner job
+      - platform-build # required to wait when the main job is done
     runs-on: ubuntu-latest
     if: ${{ always() }} # required to stop the runner even if the error happened in the previous jobs
     steps:
@@ -163,8 +305,8 @@ jobs:
         with:
           mode: stop
           github-token: ${{ secrets.SELF_RUNNER_GITHUB_ACCESS_TOKEN }}
-          label: ${{ needs.start-build-runner.outputs.label }}
-          ec2-instance-id: ${{ needs.start-build-runner.outputs.ec2-instance-id }}
+          label: ${{ needs.start-platform-build-runner.outputs.label }}
+          ec2-instance-id: ${{ needs.start-platform-build-runner.outputs.ec2-instance-id }}
 
   ## Frontend Test
   ## Gradle Build
@@ -218,8 +360,8 @@ jobs:
       - name: Install Cypress Test Dependencies
         run: sudo apt-get update && sudo apt-get install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb
 
-      - name: Build Core Docker Images and Run Tests
-        run: CORE_ONLY=true ./gradlew --no-daemon composeBuild --scan
+      - name: Build Platform Docker Images
+        run: SUB_BUILD=PLATFORM ./gradlew --no-daemon composebuild --scan
 
       - name: Run End-to-End Frontend Tests
         run: ./tools/bin/e2e_test.sh
@@ -313,8 +455,8 @@ jobs:
           HOME: /home/runner
           CHANGE_MINIKUBE_NONE_USER: true
 
-      - name: Build Core Docker Images and Run Tests
-        run: CORE_ONLY=true ./gradlew --no-daemon composeBuild --scan
+      - name: Build Platform Docker Images
+        run: SUB_BUILD=PLATFORM ./gradlew composeBuild --scan
 
       - name: Run Logging Tests
         run: ./tools/bin/cloud_storage_logging_test.sh

.github/workflows/publish-cdk-command.yml

@@ -27,7 +27,7 @@ jobs:
       - name: Checkout Airbyte
         uses: actions/checkout@v2
       - name: Build CDK Package
-        run: ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
+        run: SUB_BUILD=CONNECTORS_BASE ./gradlew --no-daemon --no-build-cache :airbyte-cdk:python:build
       - name: Add Failure Comment
         if: github.event.inputs.comment-id && !success()
         uses: peter-evans/create-or-update-comment@v1

airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9fa5862c-da7c-11eb-8d19-0242ac130003.json

@@ -1,7 +1,7 @@
 {
   "sourceDefinitionId": "9fa5862c-da7c-11eb-8d19-0242ac130003",
   "name": "Cockroachdb",
-  "dockerRepository": "airbyte/source-postgres",
+  "dockerRepository": "airbyte/source-cockroachdb",
   "dockerImageTag": "0.1.1",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-cockroachdb"
 }

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -51,7 +51,7 @@
   icon: postgresql.svg
 - sourceDefinitionId: 9fa5862c-da7c-11eb-8d19-0242ac130003
   name: Cockroachdb
-  dockerRepository: airbyte/source-postgres
+  dockerRepository: airbyte/source-cockroachdb
   dockerImageTag: 0.1.1
   documentationUrl: https://hub.docker.com/r/airbyte/source-cockroachdb
 - sourceDefinitionId: af6d50ee-dddf-4126-a8ee-7faee990774f

airbyte-config/models/README.md

@@ -9,7 +9,7 @@ This module uses `jsonschema2pojo` to generate Java config objects from [json sc
   ```
 - Run the following command under the project root:
   ```sh
-  ./gradlew airbyte-config:models:generateJsonSchema2Pojo
+  SUB_BUILD=PLATFORM ./gradlew airbyte-config:models:generateJsonSchema2Pojo
   ```
   The generated file is under:
   ```

airbyte-integrations/bases/base-java/build.gradle

@@ -7,6 +7,8 @@ dependencies {
     implementation 'commons-cli:commons-cli:1.4'
 
     implementation project(':airbyte-protocol:models')
+    implementation project(':airbyte-queue')
+    implementation project(":airbyte-json-validation")
 
     implementation files(project(':airbyte-integrations:bases:base').airbyteDocker.outputs)
 }