Merge branch 'master' into issue25255-delete-account

.github/workflows/playground_backend_precommit.yml

@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Playground PreCommit
+
+on:
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - .github/workflows/playground_backend_precommit.yml
+      - playground/backend/**
+jobs:
+  precommit_check:
+    name: precommit-check
+    runs-on: ubuntu-latest
+    env:
+      DATASTORE_EMULATOR_VERSION: '389.0.0'
+      PYTHON_VERSION: '3.8'
+      JAVA_VERSION: '8'
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '${{ env.PYTHON_VERSION }}'
+      - uses: actions/setup-java@v3.8.0
+        with:
+          distribution: 'zulu'
+          java-version: '${{ env.JAVA_VERSION }}'
+
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
+        with:
+          cache-read-only: false
+
+      - name: Add GOPATH/bin to PATH
+        run: echo "PATH=$PATH:$(go env GOPATH)/bin" >> $GITHUB_ENV
+
+      - name: Install sbt for running SCIO tests
+        run: |
+          sudo apt-get update --yes
+          sudo apt-get install apt-transport-https curl gnupg -yqq
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian all main" | sudo tee /etc/apt/sources.list.d/sbt.list
+          echo "deb https://repo.scala-sbt.org/scalasbt/debian /" | sudo tee /etc/apt/sources.list.d/sbt_old.list
+          curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo -H gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/scalasbt-release.gpg --import
+          sudo chmod 644 /etc/apt/trusted.gpg.d/scalasbt-release.gpg
+          sudo apt-get update --yes
+          sudo apt-get install sbt --yes
+      - name: Set up Cloud SDK and its components
+        uses: google-github-actions/setup-gcloud@v0
+        with:
+          install_components: 'beta,cloud-datastore-emulator'
+          version: '${{ env.DATASTORE_EMULATOR_VERSION }}'
+      - name: Run PreCommit
+        run: ./gradlew playground:backend:precommit

.test-infra/dataproc/flink_cluster.sh

@@ -46,8 +46,7 @@ set -Eeuxo pipefail
 
 # GCloud properties
 GCLOUD_ZONE="${GCLOUD_ZONE:=us-central1-a}"
-# TODO: replace preview once dataproc 2.1 released
-DATAPROC_VERSION="${DATAPROC_VERSION:=preview-debian11}"
+DATAPROC_VERSION="${DATAPROC_VERSION:=2.1-debian}"
 GCLOUD_REGION=`echo $GCLOUD_ZONE | sed -E "s/(-[a-z])?$//"`
 
 MASTER_NAME="$CLUSTER_NAME-m"

.test-infra/jenkins/CommonJobProperties.groovy

@@ -69,6 +69,10 @@ class CommonJobProperties {
         extensions {
           wipeOutWorkspace()
           relativeTargetDirectory(checkoutDir)
+          cloneOptions {
+            shallow()
+            noTags()
+          }
           if (!allowRemotePoll) {
             disableRemotePoll()
           }

.test-infra/jenkins/PrecommitJobBuilder.groovy

@@ -47,13 +47,27 @@ class PrecommitJobBuilder {
   /** Whether to trigger on new PR commits. Useful to set to false when testing new jobs. */
   boolean commitTriggering = true
 
+  /**
+   * Whether to trigger on cron run. Useful to set jobs that runs tasks covered by
+   * other test suites but are deemed to triggered on pull request only.
+   */
+  boolean cronTriggering = true
+
+  /**
+   * Whether to configure defaultPathTriggers.
+   * Set to false for PreCommit only runs on certain code path change.
+   */
+  boolean defaultPathTriggering = true
+
   /**
    * Define a set of pre-commit jobs.
    *
    * @param additionalCustomization Job DSL closure with additional customization to apply to the job.
    */
   void build(Closure additionalCustomization = {}) {
-    defineCronJob additionalCustomization
+    if (cronTriggering) {
+      defineCronJob additionalCustomization
+    }
     if (commitTriggering) {
       defineCommitJob additionalCustomization
     }
@@ -82,7 +96,7 @@ class PrecommitJobBuilder {
       '^gradle.bat$',
       '^settings.gradle.kts$'
     ]
-    if (triggerPathPatterns) {
+    if (defaultPathTriggering && triggerPathPatterns) {
       triggerPathPatterns.addAll defaultPathTriggers
     }
     job.with {

.test-infra/jenkins/job_PerformanceTests_MongoDBIO_IT.groovy

@@ -24,7 +24,7 @@ String jobName = "beam_PerformanceTests_MongoDBIO_IT"
 
 job(jobName) {
   common.setTopLevelMainJobProperties(delegate)
-  common.setAutoJob(delegate,'H H/6 * * *')
+  common.setAutoJob(delegate,'H H/12 * * *')
   common.enablePhraseTriggeringFromPullRequest(
       delegate,
       'Java MongoDBIO Performance Test',
@@ -51,6 +51,8 @@ job(jobName) {
     mongoDBDatabaseName : 'beam',
     mongoDBHostName     : "\$${mongoHostName}",
     mongoDBPort         : 27017,
+    mongoDBUsername     : 'root',
+    mongoDBPassword     : 'uuinkkS',
     runner              : 'DataflowRunner',
     autoscalingAlgorithm: 'NONE',
     numWorkers          : '5'

.test-infra/jenkins/job_PostCommit_Go.groovy

@@ -22,7 +22,7 @@ import PostcommitJobBuilder
 // This is the Go postcommit which runs a gradle build, and the current set
 // of postcommit tests.
 PostcommitJobBuilder.postCommitJob('beam_PostCommit_Go', 'Run Go PostCommit',
-    './gradlew :goPostCommit', this) {
+    'Go PostCommit (\"Run Go PostCommit\")', this) {
       description('Runs Go PostCommit tests against master.')
       previousNames(/beam_PostCommit_Go_GradleBuild/)
 

.test-infra/jenkins/job_PostCommit_Python_MongoDBIO_Load_Test.groovy

@@ -22,7 +22,7 @@ String jobName = "beam_python_mongoio_load_test"
 
 job(jobName) {
   common.setTopLevelMainJobProperties(delegate)
-  common.setAutoJob(delegate, 'H H/6 * * *')
+  common.setAutoJob(delegate, 'H H/12 * * *')
   common.enablePhraseTriggeringFromPullRequest(
       delegate,
       'Python MongoDBIO Load Test',
@@ -40,7 +40,7 @@ job(jobName) {
     temp_location: 'gs://temp-storage-for-perf-tests/loadtests',
     project      : 'apache-beam-testing',
     region       : 'us-central1',
-    mongo_uri    : "mongodb://\$${mongoHostName}:27017",
+    mongo_uri    : "mongodb://root:uuinkkS@\$${mongoHostName}:27017",
     num_documents: '1000000',
     batch_size   : '10000',
     runner       : 'DataflowRunner',