Merge branch 'master' into BEAM-13633_default_ex_for_sdks

# Conflicts: # playground/api/v1/api.proto # playground/backend/internal/api/v1/api.pb.go # playground/backend/internal/cloud_bucket/precompiled_objects.go # playground/backend/internal/utils/precompiled_objects_utils_test.go # playground/frontend/lib/api/v1/api.pb.dart # playground/frontend/lib/api/v1/api.pbjson.dart # playground/infrastructure/api/v1/api_pb2.py # playground/infrastructure/helper.py
nancyxu123 · Feb 8, 2022 · 29c28b5 · 29c28b5
2 parents ebf5888 + 56e5c2c
commit 29c28b5
Show file tree

Hide file tree

Showing 118 changed files with 4,255 additions and 2,316 deletions.
diff --git a/.github/workflows/build_playground_backend.yml b/.github/workflows/build_playground_backend.yml
@@ -81,6 +81,9 @@ jobs:
       - name: Prepare Build Go
         run: ./gradlew playground:backend:containers:go:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
         if: startsWith(github.ref, 'refs/tags/')
+      - name: Prepare Build SCIO
+        run: ./gradlew playground:backend:containers:scio:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
+        if: startsWith(github.ref, 'refs/tags/')
       - name: Prepare Build Router
         run: ./gradlew playground:backend:containers:router:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{ env.DOCKERTAG }}
         if: startsWith(github.ref, 'refs/tags/')
@@ -104,6 +107,12 @@ jobs:
         run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PLAYGROUND_PROJECT_ID }}" -var="docker_registry_address=${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" -var="docker_image_tag=${{ env.DOCKERTAG }}" -var="service_name=backend-python${{env.STAND_SUFFIX}}"
         working-directory: playground/terraform/applications/backend-python
         if: startsWith(github.ref, 'refs/tags/')
+      - name: Deploy Backend Application SCIO
+        env:
+          GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json
+        run: terraform init && terraform apply -auto-approve -var="project_id=${{ secrets.PLAYGROUND_PROJECT_ID }}" -var="docker_registry_address=${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository" -var="docker_image_tag=${{ env.DOCKERTAG }}" -var="service_name=backend-scio${{env.STAND_SUFFIX}}"
+        working-directory: playground/terraform/applications/backend-scio
+        if: startsWith(github.ref, 'refs/tags/')
       - name: Deploy Backend Application Router
         env:
           GOOGLE_APPLICATION_CREDENTIALS: /tmp/gcp_access.json

diff --git a/.github/workflows/build_playground_frontend.yml b/.github/workflows/build_playground_frontend.yml
@@ -32,6 +32,7 @@ jobs:
       TERRAFORM_VERSION: 1.0.9
       FLUTTER_VERSION: 2.5.2-stable
       STAND_SUFFIX: ''
+      GOOGLE_DOMAIN: '-dot-apache-beam-testing.appspot.com'
     steps:
       - name: Check out the repo
         uses: actions/checkout@v2
@@ -79,9 +80,9 @@ jobs:
         run: cat /tmp/gcp_access.json | docker login -u _json_key --password-stdin https://${{ secrets.PLAYGROUND_REGISTRY_NAME }}
         if: startsWith(github.ref, 'ref/tags/')
       - name: Create config
-        run: ./gradlew playground:frontend:createConfig -PplaygroundBackendJavaRouteUrl="https://backend-java${{env.STAND_SUFFIX}}-dot-apache-beam-testing.appspot.com/" -PplaygroundBackendGoRouteUrl="https://backend-go${{env.STAND_SUFFIX}}-dot-apache-beam-testing.appspot.com/" -PplaygroundBackendPythonRouteUrl="https://backend-python${{env.STAND_SUFFIX}}-dot-apache-beam-testing.appspot.com/" -PplaygroundBackendUrl="https://backend-router${{env.STAND_SUFFIX}}-dot-apache-beam-testing.appspot.com/"
+        run: ./gradlew playground:frontend:createConfig -PplaygroundBackendJavaRouteUrl="https://backend-java${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendGoRouteUrl="https://backend-go${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendPythonRouteUrl="https://backend-python${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendUrl="https://backend-router${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendScioRouteUrl="https://backend-scio${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/" -PplaygroundBackendUrl="https://backend-router${{env.STAND_SUFFIX}}${{env.GOOGLE_DOMAIN}}/"
         if: startsWith(github.ref, 'refs/tags/')
-      - name: Prepare Build
+      - name: Prepare Build Frontend
         run: ./gradlew playground:frontend:dockerTagPush -Pdocker-repository-root='${{ secrets.PLAYGROUND_REGISTRY_NAME}}/${{ secrets.PLAYGROUND_PROJECT_ID }}/playground-repository' -Pdocker-tag=${{env.DOCKERTAG}}
         if: startsWith(github.ref, 'refs/tags/')
       - name: Deploy Frontend Application

diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Dataflow.groovy
@@ -17,30 +17,27 @@
  */
 
 import CommonJobProperties as commonJobProperties
+import PostcommitJobBuilder
 
 // This job runs the Python examples tests with DataflowRunner.
-job('beam_PostCommit_Python_Examples_Dataflow') {
-  description('Runs the Python Examples with DataflowRunner.')
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Dataflow',
+    'Run Python Examples_Dataflow', 'Python Dataflow Runner Examples', this) {
 
-  // Set common parameters.
-  commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
+      description('Runs the Python Examples with DataflowRunner')
 
-  // Allows triggering this build against pull requests.
-  commonJobProperties.enablePhraseTriggeringFromPullRequest(
-      delegate,
-      'Python Dataflow Runner Examples',
-      'Run Python Examples_Dataflow')
+      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
 
-  publishers {
-    archiveJunit('**/pytest*.xml')
-  }
+      // Publish all test results to Jenkins
+      publishers {
+        archiveJunit('**/pytest*.xml')
+      }
 
-  // Execute shell command to run examples.
-  steps {
-    gradle {
-      rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(":sdks:python:test-suites:dataflow:examplesPostCommit")
-      commonJobProperties.setGradleSwitches(delegate)
+      // Execute shell command to run examples.
+      steps {
+        gradle {
+          rootBuildScriptDir(commonJobProperties.checkoutDir)
+          tasks(":sdks:python:test-suites:dataflow:examplesPostCommit")
+          commonJobProperties.setGradleSwitches(delegate)
+        }
+      }
     }
-  }
-}
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Direct.groovy
@@ -17,30 +17,27 @@
  */
 
 import CommonJobProperties as commonJobProperties
+import PostcommitJobBuilder
 
 // This job runs the Python examples tests with DirectRunner.
-job('beam_PostCommit_Python_Examples_Direct') {
-  description('Runs the Python Examples with DirectRunner.')
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Direct',
+    'Run Python Examples_Direct', 'Python Direct Runner Examples', this) {
 
-  // Set common parameters.
-  commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
+      description('Runs the Python Examples with DirectRunner')
 
-  // Allows triggering this build against pull requests.
-  commonJobProperties.enablePhraseTriggeringFromPullRequest(
-      delegate,
-      'Python Direct Runner Examples',
-      'Run Python Examples_Direct')
+      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
 
-  publishers {
-    archiveJunit('**/pytest*.xml')
-  }
+      // Publish all test results to Jenkins
+      publishers {
+        archiveJunit('**/pytest*.xml')
+      }
 
-  // Execute shell command to run examples.
-  steps {
-    gradle {
-      rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(":sdks:python:test-suites:direct:examplesPostCommit")
-      commonJobProperties.setGradleSwitches(delegate)
-    }
-  }
-}
+      // Execute shell command to run examples.
+      steps {
+        gradle {
+          rootBuildScriptDir(commonJobProperties.checkoutDir)
+          tasks(":sdks:python:test-suites:direct:examplesPostCommit")
+          commonJobProperties.setGradleSwitches(delegate)
+        }
+      }
+    }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Flink.groovy
@@ -17,30 +17,27 @@
  */
 
 import CommonJobProperties as commonJobProperties
+import PostcommitJobBuilder
 
 // This job runs the Python examples tests with FlinkRunner.
-job('beam_PostCommit_Python_Examples_Flink') {
-  description('Runs the Python Examples with Flink Runner.')
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Flink',
+    'Run Python Examples_Flink', 'Python Flink Runner Examples', this) {
 
-  // Set common parameters.
-  commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
+      description('Runs the Python Examples with Flink Runner')
 
-  // Allows triggering this build against pull requests.
-  commonJobProperties.enablePhraseTriggeringFromPullRequest(
-      delegate,
-      'Python Flink Runner Examples',
-      'Run Python Examples_Flink')
+      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
 
-  publishers {
-    archiveJunit('**/pytest*.xml')
-  }
+      // Publish all test results to Jenkins
+      publishers {
+        archiveJunit('**/pytest*.xml')
+      }
 
-  // Execute shell command to run examples.
-  steps {
-    gradle {
-      rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(":sdks:python:test-suites:portable:flinkExamplesPostCommit")
-      commonJobProperties.setGradleSwitches(delegate)
-    }
-  }
-}
+      // Execute shell command to run examples.
+      steps {
+        gradle {
+          rootBuildScriptDir(commonJobProperties.checkoutDir)
+          tasks(":sdks:python:test-suites:portable:flinkExamplesPostCommit")
+          commonJobProperties.setGradleSwitches(delegate)
+        }
+      }
+    }
diff --git a/.test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy b/.test-infra/jenkins/job_PostCommit_Python_Examples_Spark.groovy
@@ -20,28 +20,26 @@ import CommonJobProperties as commonJobProperties
 import PostcommitJobBuilder
 
 // This job runs the Python examples tests with SparkRunner.
-job('beam_PostCommit_Python_Examples_Spark') {
-  description('Runs the Python Examples with Spark Runner.')
-
-  // Set common parameters.
-  commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
-
-  // Allows triggering this build against pull requests.
-  commonJobProperties.enablePhraseTriggeringFromPullRequest(
-      delegate,
-      'Python Spark Runner Examples',
-      'Run Python Examples_Spark')
-
-  publishers {
-    archiveJunit('**/pytest*.xml')
-  }
-
-  // Execute shell command to run examples.
-  steps {
-    gradle {
-      rootBuildScriptDir(commonJobProperties.checkoutDir)
-      tasks(":sdks:python:test-suites:portable:sparkExamplesPostCommit")
-      commonJobProperties.setGradleSwitches(delegate)
+PostcommitJobBuilder.postCommitJob('beam_PostCommit_Python_Examples_Spark',
+    'Run Python Examples_Spark', 'Python Spark Runner Examples', this) {
+
+      description('Runs the Python Examples with Spark Runner')
+
+      commonJobProperties.setTopLevelMainJobProperties(delegate, 'master', 120)
+
+      // Publish all test results to Jenkins
+      publishers {
+        archiveJunit('**/pytest*.xml')
+      }
+
+      // Execute shell command to run examples.
+      steps {
+        gradle {
+          rootBuildScriptDir(commonJobProperties.checkoutDir)
+          tasks(":sdks:python:test-suites:portable:sparkExamplesPostCommit")
+          commonJobProperties.setGradleSwitches(delegate)
+        }
+      }
     }
-  }
-}
+
+
diff --git a/CHANGES.md b/CHANGES.md
@@ -67,9 +67,7 @@
 
 ## Known Issues
 
-# 2.36.0 - Unreleased
-
-## Highlights
+# [2.36.0] - 2022-02-07
 
 ## I/Os
 
@@ -87,15 +85,17 @@
 * Updated the jedis from 3.x to 4.x to Java RedisIO. If you are using RedisIO and using jedis directly, please refer to [this page](https://github.com/redis/jedis/blob/v4.0.0/docs/3to4.md) to update it. ([BEAM-12092](https://issues.apache.org/jira/browse/BEAM-12092)).
 * Datatype of timestamp fields in `SqsMessage` for AWS IOs for SDK v2 was changed from `String` to `long`, visibility of all fields was fixed from `package private` to `public` [BEAM-13638](https://issues.apache.org/jira/browse/BEAM-13638).
 
-## Deprecations
-
 ## Bugfixes
 
 * Properly check output timestamps on elements output from DoFns, timers, and onWindowExpiration in Java [BEAM-12931](https://issues.apache.org/jira/browse/BEAM-12931).
 * Fixed a bug with DeferredDataFrame.xs when used with a non-tuple key
   ([BEAM-13421](https://issues.apache.org/jira/browse/BEAM-13421])).
 
-## Known Issues
+# Known Issues
+
+* Users may encounter an unexpected java.lang.ArithmeticException when outputting a timestamp
+  for an element further than allowedSkew from an allowed DoFN skew set to a value more than
+  Integer.MAX_VALUE.
 
 # [2.35.0] - 2021-12-29