Merge branch 'master' into link_flinkrunner_to_dataproc

.github/REVIEWERS.yml

@@ -0,0 +1,21 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+labels:
+- name: "Go"
+  reviewers: ["damccorm", "lostluck", "jrmccluskey", "youngoli", "riteshghorse"]
+  exclusionList: [] # These users will never be suggested as reviewers
+# I don't know the other areas well enough to assess who the normal committers/contributors who might want to be reviewers are
+fallbackReviewers: [] # List of committers to use when no label matches

.github/codecov.yml

@@ -29,12 +29,16 @@ coverage:
   status:
     project:
       python:
+        flags: 
+          - python
         target: auto
         threshold: 5%
         base: auto
         paths:
           - "sdks/python"
       go:
+        flags: 
+          - go
         target: auto
         threshold: 5%
         base: auto
@@ -59,3 +63,8 @@ ignore:
   - "**/*_test.py"
   - "**/*_test_py3*.py"
   - "**/*_microbenchmark.py"
+
+# See https://docs.codecov.com/docs/flags for options.
+flag_management:
+  default_rules: # the rules that will be followed for any flag added, generally
+    carryforward: true # recommended for multi-lang mono-repos.

.github/workflows/go_tests.yml

@@ -44,7 +44,10 @@ jobs:
         run: "cd sdks/go/pkg && rm -rf .coverage || :"
       - name: Run coverage
         run: cd sdks/go/pkg && go test -coverprofile=coverage.txt -covermode=atomic ./...
-      - name: Upload to codecov
-        run: bash <(curl -s https://codecov.io/bash)
+      - uses: codecov/codecov-action@v2
+        with:
+          flags: go 
+          files: ./sdks/go/pkg/coverage.txt
+          name: go-unittests
       - name: Run vet
         run: cd sdks/go/pkg/beam && go vet --copylocks=false --unsafeptr=false ./...

.gitignore

@@ -126,3 +126,4 @@ website/www/yarn-error.log
 **/*.tfstate
 **/*.tfstate.*
 **/*.hcl
+**/*.tfvars

.test-infra/jenkins/job_PreCommit_Java.groovy

@@ -23,7 +23,6 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     nameBase: 'Java',
     gradleTask: ':javaPreCommit',
     gradleSwitches: [
-      '-PretryFlakyTest=true',
       '-PdisableSpotlessCheck=true'
     ], // spotless checked in separate pre-commit
     triggerPathPatterns: [
@@ -40,11 +39,7 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
     )
 builder.build {
   publishers {
-    archiveJunit('**/build/test-results/**/*.xml') {
-      testDataPublishers {
-        publishFlakyTestsReport()
-      }
-    }
+    archiveJunit('**/build/test-results/**/*.xml')
     recordIssues {
       tools {
         errorProne()

.test-infra/jenkins/job_PreCommit_Python.groovy

@@ -21,9 +21,6 @@ import PrecommitJobBuilder
 PrecommitJobBuilder builder = new PrecommitJobBuilder(
     scope: this,
     nameBase: 'Python',
-    gradleSwitches: [
-      '-PretryFlakyTest=true',
-    ],
     gradleTask: ':pythonPreCommit',
     timeoutMins: 180,
     triggerPathPatterns: [
@@ -35,10 +32,6 @@ PrecommitJobBuilder builder = new PrecommitJobBuilder(
 builder.build {
   // Publish all test results to Jenkins.
   publishers {
-    archiveJunit('**/pytest*.xml'){
-      testDataPublishers {
-        publishFlakyTestsReport()
-      }
-    }
+    archiveJunit('**/pytest*.xml')
   }
 }

README.md

@@ -25,6 +25,7 @@
 
 [![Maven Version](https://maven-badges.herokuapp.com/maven-central/org.apache.beam/beam-sdks-java-core/badge.svg)](http://search.maven.org/#search|gav|1|g:"org.apache.beam")
 [![PyPI version](https://badge.fury.io/py/apache-beam.svg)](https://badge.fury.io/py/apache-beam)
+[![Go version](https://pkg.go.dev/badge/github.com/apache/beam/sdks/v2/go.svg)](https://pkg.go.dev/github.com/apache/beam/sdks/v2/go)
 [![Python coverage](https://codecov.io/gh/apache/beam/branch/master/graph/badge.svg)](https://codecov.io/gh/apache/beam)
 [![Build python source distribution and wheels](https://github.com/apache/beam/workflows/Build%20python%20source%20distribution%20and%20wheels/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Build+python+source+distribution+and+wheels%22+branch%3Amaster+event%3Aschedule)
 [![Python tests](https://github.com/apache/beam/workflows/Python%20tests/badge.svg?branch=master&event=schedule)](https://github.com/apache/beam/actions?query=workflow%3A%22Python+Tests%22+branch%3Amaster+event%3Aschedule)

build.gradle.kts

@@ -193,6 +193,7 @@ tasks.register("javaPostCommit") {
   dependsOn(":sdks:java:io:amazon-web-services2:integrationTest")
   dependsOn(":sdks:java:extensions:ml:postCommit")
   dependsOn(":sdks:java:io:kafka:kafkaVersionsCompatibilityTest")
+  dependsOn(":sdks:java:io:neo4j:integrationTest")
 }
 
 tasks.register("javaHadoopVersionsTest") {

buildSrc/build.gradle.kts

@@ -54,7 +54,6 @@ dependencies {
   runtimeOnly("ca.cutterslade.gradle:gradle-dependency-analyze:1.8.3")                                     // Enable dep analysis
   runtimeOnly("gradle.plugin.net.ossindex:ossindex-gradle-plugin:0.4.11")                                  // Enable dep vulnerability analysis
   runtimeOnly("org.checkerframework:checkerframework-gradle-plugin:0.5.16")                                 // Enable enhanced static checking plugin
-  runtimeOnly("org.gradle:test-retry-gradle-plugin:1.3.1")                                                 // Enable flaky tests mitigation
 }
 
 // Because buildSrc is built and tested automatically _before_ gradle

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -409,23 +409,6 @@ class BeamModulePlugin implements Plugin<Project> {
       project.tasks.withType(Test) { jacoco.enabled = enabled }
     }
 
-    // Add Retry Gradle Plugin to mitigate flaky tests
-    if (project.hasProperty("retryFlakyTest")) {
-      project.apply plugin: "org.gradle.test-retry"
-      project.tasks.withType(Test) {
-        reports{
-          junitXml{
-            mergeReruns = true
-          }
-        }
-        retry {
-          failOnPassedAfterRetry = false
-          maxFailures = 10
-          maxRetries = 3
-        }
-      }
-    }
-
     // Apply a plugin which provides tasks for dependency / property / task reports.
     // See https://docs.gradle.org/current/userguide/project_reports_plugin.html
     // for further details. This is typically very useful to look at the "htmlDependencyReport"
@@ -471,8 +454,12 @@ class BeamModulePlugin implements Plugin<Project> {
     def checkerframework_version = "3.10.0"
     def classgraph_version = "4.8.104"
     def errorprone_version = "2.10.0"
+    // Try to keep gax_version consistent with gax-grpc version in google_cloud_platform_libraries_bom
+    def gax_version = "2.8.1"
     def google_clients_version = "1.32.1"
     def google_cloud_bigdataoss_version = "2.2.4"
+    // Try to keep google_cloud_spanner_version consistent with google_cloud_spanner_bom in google_cloud_platform_libraries_bom
+    def google_cloud_spanner_version = "6.17.4"
     def google_code_gson_version = "2.8.9"
     def google_oauth_clients_version = "1.32.1"
     // Try to keep grpc_version consistent with gRPC version in google_cloud_platform_libraries_bom
@@ -560,6 +547,7 @@ class BeamModulePlugin implements Plugin<Project> {
         flogger_system_backend                      : "com.google.flogger:flogger-system-backend:0.7.3",
         gax                                         : "com.google.api:gax", // google_cloud_platform_libraries_bom sets version
         gax_grpc                                    : "com.google.api:gax-grpc", // google_cloud_platform_libraries_bom sets version
+        gax_grpc_test                               : "com.google.api:gax-grpc:$gax_version:testlib", // google_cloud_platform_libraries_bom sets version
         gax_httpjson                                : "com.google.api:gax-httpjson", // google_cloud_platform_libraries_bom sets version
         google_api_client                           : "com.google.api-client:google-api-client:$google_clients_version", // for the libraries using $google_clients_version below.
         google_api_client_jackson2                  : "com.google.api-client:google-api-client-jackson2:$google_clients_version",
@@ -591,6 +579,7 @@ class BeamModulePlugin implements Plugin<Project> {
         // Update libraries-bom version on sdks/java/container/license_scripts/dep_urls_java.yaml
         google_cloud_platform_libraries_bom         : "com.google.cloud:libraries-bom:24.2.0",
         google_cloud_spanner                        : "com.google.cloud:google-cloud-spanner", // google_cloud_platform_libraries_bom sets version
+        google_cloud_spanner_test                   : "com.google.cloud:google-cloud-spanner:$google_cloud_spanner_version:tests",
         google_code_gson                            : "com.google.code.gson:gson:$google_code_gson_version",
         // google-http-client's version is explicitly declared for sdks/java/maven-archetypes/examples
         // This version should be in line with the one in com.google.cloud:libraries-bom.
@@ -1333,7 +1322,8 @@ class BeamModulePlugin implements Plugin<Project> {
         }
 
         if (configuration.validateShadowJar) {
-          project.task('validateShadedJarDoesntLeakNonProjectClasses', dependsOn: 'shadowJar') {
+          def validateShadedJarDoesntLeakNonProjectClasses = project.tasks.register('validateShadedJarDoesntLeakNonProjectClasses') {
+            dependsOn 'shadowJar'
             ext.outFile = project.file("${project.reportsDir}/${name}.out")
             inputs.files(project.configurations.shadow.artifacts.files)
                 .withPropertyName("shadowArtifactsFiles")
@@ -1357,10 +1347,10 @@ class BeamModulePlugin implements Plugin<Project> {
               }
             }
           }
-          project.tasks.check.dependsOn project.tasks.validateShadedJarDoesntLeakNonProjectClasses
+          project.tasks.check.dependsOn validateShadedJarDoesntLeakNonProjectClasses
         }
       } else {
-        project.task("testJar", type: Jar, {
+        project.tasks.register("testJar", Jar) {
           group = "Jar"
           description = "Create a JAR of test classes"
           classifier = "tests"
@@ -1370,7 +1360,7 @@ class BeamModulePlugin implements Plugin<Project> {
           exclude "META-INF/*.SF"
           exclude "META-INF/*.DSA"
           exclude "META-INF/*.RSA"
-        })
+        }
         project.artifacts.testRuntimeMigration project.testJar
       }
 
@@ -1392,7 +1382,8 @@ class BeamModulePlugin implements Plugin<Project> {
           }
         }
 
-        project.task("jmh", type: JavaExec, dependsOn: project.classes, {
+        project.tasks.register("jmh", JavaExec)  {
+          dependsOn project.classes
           mainClass = "org.openjdk.jmh.Main"
           classpath = project.sourceSets.main.runtimeClasspath
           // For a list of arguments, see
@@ -1424,12 +1415,13 @@ class BeamModulePlugin implements Plugin<Project> {
             args 'org.apache.beam'
           }
           args '-foe=true'
-        })
+        }
 
         // Single shot of JMH benchmarks ensures that they can execute.
         //
         // Note that these tests will fail on JVMs that JMH doesn't support.
-        project.task("jmhTest", type: JavaExec, dependsOn: project.classes, {
+        def jmhTest = project.tasks.register("jmhTest", JavaExec) {
+          dependsOn project.classes
           mainClass = "org.openjdk.jmh.Main"
           classpath = project.sourceSets.main.runtimeClasspath
 
@@ -1442,8 +1434,8 @@ class BeamModulePlugin implements Plugin<Project> {
           args '-f=0'
           args '-wf=0'
           args '-foe=true'
-        })
-        project.check.dependsOn("jmhTest")
+        }
+        project.check.dependsOn jmhTest
       }
 
       project.ext.includeInJavaBom = configuration.publish

playground/README.md

@@ -22,7 +22,7 @@
 The Beam Playground is a web application to run Beam code snippets in a modern browser. This directory holds code to
 build, test, and deploy the frontend and backend services.
 
-# Requirements
+# Development Requirements
 
 The following requirements are needed for development, testing, and deploying.
 
@@ -55,12 +55,7 @@ cd beam
 ./gradlew playground:generateProto
 ```
 
-# Backend application
+# Deployment
 
-All backend's files are placed into the [backend](./backend) folder. For more details about the backend there are
-also [Contribution](./backend/CONTRIBUTE.md) guide and [README](./backend/README.md) file.
-
-# Frontend application
-
-All frontend's files are placed into the [frontend](./frontend) folder. For more details about the frontend there are
-also [Contribution](./frontend/CONTRIBUTE.md) guide and [README](./frontend/README.md) file.
+See [terraform](./terraform/README.md) for details on how to build and deploy
+the application and its dependent infrastructure.