Merge branch 'master' into lmossman/add-including-job-to-list-jobs-api

.github/workflows/gradle.yml

@@ -704,160 +704,162 @@ jobs:
   ## Kube Acceptance Tests
   # Docker acceptance tests run as part of the build job.
   # In case of self-hosted EC2 errors, remove this block.
-  start-helm-acceptance-test-runner:
-    name: "Platform: Start Helm Acceptance Test Runner"
-    needs:
-      - changes
-      - find_valid_pat
-    # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
-    if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
-    timeout-minutes: 10
-    runs-on: ubuntu-latest
-    outputs:
-      label: ${{ steps.start-ec2-runner.outputs.label }}
-      ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v2
-      - name: Start AWS Runner
-        id: start-ec2-runner
-        uses: ./.github/actions/start-aws-runner
-        with:
-          # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
-          ec2-image-id: ami-0c1a9bc22624339d8
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          github-token: ${{ needs.find_valid_pat.outputs.pat }}
-
-
-
-  helm-acceptance-test:
-    name: "Platform: Acceptance Tests (Helm)"
-    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
-    needs: start-helm-acceptance-test-runner # required to start the main job when the runner is ready
-    runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
-    environment: more-secrets
-    timeout-minutes: 90
-    steps:
-      - name: Checkout Airbyte
-        uses: actions/checkout@v2
-
-      - name: Cache Build Artifacts
-        uses: ./.github/actions/cache-build-artifacts
-        with:
-          cache-key: ${{ secrets.CACHE_VERSION }}
-          cache-python: "false"
-
-      - uses: actions/setup-java@v1
-        with:
-          java-version: "17"
-
-      - uses: actions/setup-node@v2
-        with:
-          node-version: "lts/gallium"
-
-      - name: Install unzip
-        shell: bash
-        run: sudo apt-get update && sudo apt-get install -y unzip
-
-      - uses: azure/setup-helm@v3
-        with:
-          version: 'latest' 
-          token: ${{ secrets.GITHUB_TOKEN }}
-        id: install
-
-      - name: Fix EC-2 Runner
-        run: |
-          mkdir -p /home/runner
-
-      - name: Set up CI Gradle Properties
-        run: |
-          mkdir -p ~/.gradle/
-          cat > ~/.gradle/gradle.properties <<EOF
-          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
-            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
-          org.gradle.workers.max=8
-          org.gradle.vfs.watch=false
-          EOF
-
-      - name: Create cluster config file
-        run: |
-          cat > /tmp/kind-config.yaml <<EOF
-          kind: Cluster
-          apiVersion: kind.x-k8s.io/v1alpha4
-          nodes:
-          - role: control-plane
-          - role: worker
-          EOF
-
-      - name: KIND Kubernetes Cluster Setup
-        uses: helm/kind-action@v1.2.0
-        with:
-          node_image: kindest/node:v1.21.2
-          config: /tmp/kind-config.yaml
-        # In case of self-hosted EC2 errors, remove this env block.
-        env:
-          USER: root
-          HOME: /home/runner
-          CHANGE_MINIKUBE_NONE_USER: true
-
-      - name: Build Platform Docker Images
-        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
-
-      - name: Run Helm End-to-End Acceptance Tests
-        env:
-          USER: root
-          HOME: /home/runner
-          # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
-          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
-          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
-          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
-        run: |
-          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
-
-      - name: Generate Test Report
-        uses: dorny/test-reporter@v1
-        if: always()    # run this step even if previous step failed
-        with:
-          name: Platform Helm E2E Test Report
-          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
-          reporter: java-junit
-
-      - uses: actions/upload-artifact@v2
-        if: failure()
-        with:
-          name: Kubernetes Logs
-          path: /tmp/kubernetes_logs/*
+  # TODO:
+  # Kyryl to fix issue
+  # start-helm-acceptance-test-runner:
+  #   name: "Platform: Start Helm Acceptance Test Runner"
+  #   needs:
+  #     - changes
+  #     - find_valid_pat
+  #   # Because scheduled builds on master require us to skip the changes job. Use always() to force this to run on master.
+  #   if: needs.changes.outputs.backend == 'true' || needs.changes.outputs.build == 'true' || (always() && github.ref == 'refs/heads/master')
+  #   timeout-minutes: 10
+  #   runs-on: ubuntu-latest
+  #   outputs:
+  #     label: ${{ steps.start-ec2-runner.outputs.label }}
+  #     ec2-instance-id: ${{ steps.start-ec2-runner.outputs.ec2-instance-id }}
+  #   steps:
+  #     - name: Checkout Airbyte
+  #       uses: actions/checkout@v2
+  #     - name: Start AWS Runner
+  #       id: start-ec2-runner
+  #       uses: ./.github/actions/start-aws-runner
+  #       with:
+  #         # github-self-hosted-runner-ubuntu-20-with-150gdisk-docker-20.10.7-and-socat
+  #         ec2-image-id: ami-0c1a9bc22624339d8
+  #         aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+  #         aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+  #         github-token: ${{ needs.find_valid_pat.outputs.pat }}
+
+
+  # Todo: Kyrylo turn this on.
+#  helm-acceptance-test:
+#    name: "Platform: Acceptance Tests (Helm)"
+#    # In case of self-hosted EC2 errors, removed the `needs` line and switch back to running on ubuntu-latest.
+#    needs: start-helm-acceptance-test-runner # required to start the main job when the runner is ready
+#    runs-on: ${{ needs.start-helm-acceptance-test-runner.outputs.label }} # run the job on the newly created runner
+#    environment: more-secrets
+#    timeout-minutes: 90
+#    steps:
+#      - name: Checkout Airbyte
+#        uses: actions/checkout@v2
+#
+#      - name: Cache Build Artifacts
+#        uses: ./.github/actions/cache-build-artifacts
+#        with:
+#          cache-key: ${{ secrets.CACHE_VERSION }}
+#          cache-python: "false"
+#
+#      - uses: actions/setup-java@v1
+#        with:
+#          java-version: "17"
+#
+#      - uses: actions/setup-node@v2
+#        with:
+#          node-version: "lts/gallium"
+#
+#      - name: Install unzip
+#        shell: bash
+#        run: sudo apt-get update && sudo apt-get install -y unzip
+#
+#      - uses: azure/setup-helm@v3
+#        with:
+#          version: 'latest'
+#          token: ${{ secrets.GITHUB_TOKEN }}
+#        id: install
+#
+#      - name: Fix EC-2 Runner
+#        run: |
+#          mkdir -p /home/runner
+#
+#      - name: Set up CI Gradle Properties
+#        run: |
+#          mkdir -p ~/.gradle/
+#          cat > ~/.gradle/gradle.properties <<EOF
+#          org.gradle.jvmargs=-Xmx8g -Xss4m --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED \
+#            --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED
+#          org.gradle.workers.max=8
+#          org.gradle.vfs.watch=false
+#          EOF
+#
+#      - name: Create cluster config file
+#        run: |
+#          cat > /tmp/kind-config.yaml <<EOF
+#          kind: Cluster
+#          apiVersion: kind.x-k8s.io/v1alpha4
+#          nodes:
+#          - role: control-plane
+#          - role: worker
+#          EOF
+#
+#      - name: KIND Kubernetes Cluster Setup
+#        uses: helm/kind-action@v1.2.0
+#        with:
+#          node_image: kindest/node:v1.21.2
+#          config: /tmp/kind-config.yaml
+#        # In case of self-hosted EC2 errors, remove this env block.
+#        env:
+#          USER: root
+#          HOME: /home/runner
+#          CHANGE_MINIKUBE_NONE_USER: true
+#
+#      - name: Build Platform Docker Images
+#        run: SUB_BUILD=PLATFORM ./gradlew assemble -x test --scan
+#
+#      - name: Run Helm End-to-End Acceptance Tests
+#        env:
+#          USER: root
+#          HOME: /home/runner
+#          # AWS_S3_INTEGRATION_TEST_CREDS can be found in LastPass as AWS_S3_INTEGRATION_TEST_CREDS
+#          AWS_S3_INTEGRATION_TEST_CREDS: ${{ secrets.AWS_S3_INTEGRATION_TEST_CREDS }}
+#          SECRET_STORE_GCP_CREDENTIALS: ${{ secrets.SECRET_STORE_GCP_CREDENTIALS }}
+#          SECRET_STORE_GCP_PROJECT_ID: ${{ secrets.SECRET_STORE_GCP_PROJECT_ID }}
+#        run: |
+#          CI=true IS_MINIKUBE=true ./tools/bin/acceptance_test_kube_helm.sh
+#      - name: Generate Test Report
+#        uses: dorny/test-reporter@v1
+#        if: always()    # run this step even if previous step failed
+#        with:
+#          name: Platform Helm E2E Test Report
+#          path: '/actions-runner/_work/airbyte/airbyte/*/build/test-results/*/*.xml'
+#          reporter: java-junit
+#
+#      - uses: actions/upload-artifact@v2
+#        if: failure()
+#        with:
+#          name: Kubernetes Logs
+#          path: /tmp/kubernetes_logs/*
 
   # In case of self-hosted EC2 errors, remove this block.
-  stop-helm-acceptance-test-runner:
-    name: "Platform: Stop Helm Acceptance Test EC2 Runner"
-    timeout-minutes: 10
-    needs:
-      - start-helm-acceptance-test-runner # required to get output from the start-runner job
-      - helm-acceptance-test # required to wait when the main job is done
-      - find_valid_pat
-    runs-on: ubuntu-latest
-    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
-    # Thus, we check for skipped here.
-    if: ${{ always() && needs.start-helm-acceptance-test-runner.result != 'skipped'}}
-    steps:
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@v1
-        with:
-          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
-          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
-          aws-region: us-east-2
-      - name: Stop EC2 runner
-        uses: supertopher/ec2-github-runner@base64v1.0.10
-        with:
-          mode: stop
-          github-token: ${{ needs.find_valid_pat.outputs.pat }}
-          label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }}
-          ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }}
+  # Todo: Kyryl turn this on.
+#  stop-helm-acceptance-test-runner:
+#    name: "Platform: Stop Helm Acceptance Test EC2 Runner"
+#    timeout-minutes: 10
+#    needs:
+#      - start-helm-acceptance-test-runner # required to get output from the start-runner job
+#      - helm-acceptance-test # required to wait when the main job is done
+#      - find_valid_pat
+#    runs-on: ubuntu-latest
+#    # Always is required to stop the runner even if the previous job has errors. However always() runs even if the previous step is skipped.
+#    # Thus, we check for skipped here.
+#    if: ${{ always() && needs.start-helm-acceptance-test-runner.result != 'skipped'}}
+#    steps:
+#      - name: Configure AWS credentials
+#        uses: aws-actions/configure-aws-credentials@v1
+#        with:
+#          aws-access-key-id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+#          aws-secret-access-key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+#          aws-region: us-east-2
+#      - name: Stop EC2 runner
+#        uses: supertopher/ec2-github-runner@base64v1.0.10
+#        with:
+#          mode: stop
+#          github-token: ${{ needs.find_valid_pat.outputs.pat }}
+#          label: ${{ needs.start-helm-acceptance-test-runner.outputs.label }}
+#          ec2-instance-id: ${{ needs.start-helm-acceptance-test-runner.outputs.ec2-instance-id }}
 
 
 
@@ -870,7 +872,8 @@ jobs:
       - octavia-cli-build
       - platform-build
       - kube-acceptance-test
-      - helm-acceptance-test
+      # Todo: Kyryl turn this on.
+      # - helm-acceptance-test
     if: ${{ failure() && github.ref == 'refs/heads/master' }}
     steps:
       - name: Publish to OSS Build Failure Slack Channel
@@ -895,7 +898,8 @@ jobs:
       - octavia-cli-build
       - platform-build
       - kube-acceptance-test
-      - helm-acceptance-test
+      # Todo: Kyryl turn this on.
+      # - helm-acceptance-test
     if: success()
     steps:
       - name: Get Previous Workflow Status

airbyte-bootloader/build.gradle

@@ -24,25 +24,15 @@ application {
     applicationDefaultJvmArgs = ['-XX:+ExitOnOutOfMemoryError', '-XX:MaxRAMPercentage=75.0']
 }
 
-task copyGeneratedTar(type: Copy) {
-    dependsOn copyDocker
-    dependsOn distTar
-
-    from('build/distributions') {
-        include 'airbyte-bootloader-*.tar'
-    }
-    into 'build/docker/bin'
-}
-
-Task dockerBuildTask = getDockerBuildTask("bootloader", "$project.projectDir", "$rootProject.ext.version", "$rootProject.ext.image_tag")
-dockerBuildTask.dependsOn(copyGeneratedTar)
-assemble.dependsOn(dockerBuildTask)
-
 // produce reproducible archives
 // (see https://docs.gradle.org/current/userguide/working_with_files.html#sec:reproducible_archives)
 tasks.withType(AbstractArchiveTask) {
     preserveFileTimestamps = false
     reproducibleFileOrder = true
 }
 
+tasks.named("buildDockerImage") {
+    dependsOn copyGeneratedTar
+}
+
 Task publishArtifactsTask = getPublishArtifactsTask("$rootProject.ext.version", project)

airbyte-bootloader/gradle.properties

@@ -0,0 +1 @@
+dockerImageName=bootloader

airbyte-cdk/python/CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.1.83
+- Fix per-stream to send legacy format for connectors that override read
+
 ## 0.1.82
 - Freeze dataclasses-jsonschema to 2.15.1
 

airbyte-cdk/python/airbyte_cdk/sources/source.py

@@ -66,7 +66,11 @@ def read_state(self, state_path: str) -> List[AirbyteStateMessage]:
                     parsed_state_messages.append(parsed_message)
                 return parsed_state_messages
             else:
-                # When the legacy JSON object format is received, always outputting an AirbyteStateMessage simplifies processing downstream
+                # Existing connectors that override read() might not be able to interpret the new state format. We temporarily
+                # send state in the old format for these connectors, but once all have been upgraded, this block can be removed
+                # vars(self.__class__) checks if the current class directly overrides the read() function
+                if "read" in vars(self.__class__):
+                    return state_obj
                 return [AirbyteStateMessage(type=AirbyteStateType.LEGACY, data=state_obj)]
         return []
 

airbyte-cdk/python/setup.py

@@ -15,7 +15,7 @@
 
 setup(
     name="airbyte-cdk",
-    version="0.1.82",
+    version="0.1.83",
     description="A framework for writing Airbyte Connectors.",
     long_description=README,
     long_description_content_type="text/markdown",