Clean up integration test manual run parameters (#31938)

Previously, gcpProject is used in some projects, while dataflowProject
is used in some others

This change dedup the parameters used to run integration tests manually
and align these additional parameters

* Change dataflowProject -> gcpProject

* Change dataflowRegion -> gcpRegion

* Change dataflowTempRoot -> gcpTempRoot

contributor-docs/code-change-guide.md

@@ -286,14 +286,27 @@ Integration tests differ from standard pipelines in the following ways:
 * They have a default timeout of 15 minutes.
 * The pipeline options are set in the system property `beamTestPipelineOptions`.
 
-To configure the test, you need to set the property `-DbeamTestPipelineOptions=[...]`. This property sets the runner that the test uses.
-
-The following example demonstrates how to run an integration test by using the command line. This example includes the options required to run the pipeline on the Dataflow runner.
+To configure the test pipeline, you need to set the property `-DbeamTestPipelineOptions=[...]`. This property sets the pipeline option that the test uses, for example,
 
 ```
 -DbeamTestPipelineOptions='["--runner=TestDataflowRunner","--project=mygcpproject","--region=us-central1","--stagingLocation=gs://mygcsbucket/path"]'
 ```
 
+For some projects, `beamTestPipelineOptions` is explicitly configured in `build.gradle`.
+Checkout the sources of the corresponding build file for setting. For example,
+in `sdks/java/io/google-cloud-platform/build.gradle`, it sets `beamTestPipelineOptions`
+from project properties 'gcpProject', 'gcpTempRoot', etc, and when not assigned,
+it defaults to `apache-beam-testing` GCP project. To run the test in your own project,
+assign these project properties with command line:
+
+```
+./gradlew :sdks:java:io:google-cloud-platform:integrationTest -PgcpProject=<mygcpproject> -PgcpTempRoot=<gs://mygcsbucket/path>
+```
+
+Some other projects (e.g. `sdks/java/io/jdbc`, `sdks/java/io/kafka`) does not
+assemble (overwrite) `beamTestPipelineOptions` in `build.gradle`, then just set
+it explicitly with `-DbeamTestPipelineOptions='[...]'`, as aforementioned.
+
 #### Write integration tests
 
 To set up a `TestPipeline` object in an integration test, use the following code:

runners/google-cloud-dataflow-java/arm/build.gradle

@@ -84,19 +84,19 @@ def javaVer = "java8"
 if (project.hasProperty('testJavaVersion')) {
   javaVer = "java${project.getProperty('testJavaVersion')}"
 }
-def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
-def dataflowRegion = project.findProperty('dataflowRegion') ?: 'us-central1'
-def dataflowValidatesTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-validates-runner-tests'
+def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
+def gcpRegion = project.findProperty('gcpRegion') ?: 'us-central1'
+def dataflowValidatesTempRoot = project.findProperty('gcpTempRoot') ?: 'gs://temp-storage-for-validates-runner-tests'
 def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb'
-def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it"
+def dockerImageRoot = project.findProperty('docker-repository-root') ?: "us.gcr.io/${gcpProject}/java-postcommit-it"
 def DockerJavaMultiarchImageContainer = "${dockerImageRoot}/${project.docker_image_default_repo_prefix}${javaVer}_sdk"
 def dockerTag = project.findProperty('docker-tag') ?: new Date().format('yyyyMMddHHmmss')
 ext.DockerJavaMultiarchImageName = "${DockerJavaMultiarchImageContainer}:${dockerTag}" as String
 
 def runnerV2PipelineOptionsARM = [
   "--runner=TestDataflowRunner",
-  "--project=${dataflowProject}",
-  "--region=${dataflowRegion}",
+  "--project=${gcpProject}",
+  "--region=${gcpRegion}",
   "--tempRoot=${dataflowValidatesTempRoot}",
   "--sdkContainerImage=${project.ext.DockerJavaMultiarchImageName}",
   "--experiments=use_unified_worker,use_runner_v2",

runners/google-cloud-dataflow-java/build.gradle

@@ -138,18 +138,18 @@ dependencies {
   googleCloudPlatformIntegrationTest project(path: ":sdks:java:io:google-cloud-platform", configuration: "testRuntimeMigration")
 }
 
-def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
-def dataflowRegion = project.findProperty('dataflowRegion') ?: 'us-central1'
-def dataflowValidatesTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-validates-runner-tests'
-def dataflowPostCommitTempRoot = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests'
-def dataflowPostCommitTempRootKms = project.findProperty('dataflowTempRootKms') ?: 'gs://temp-storage-for-end-to-end-tests-cmek'
-def dataflowUploadTemp = project.findProperty('dataflowTempRoot') ?: 'gs://temp-storage-for-upload-tests'
+def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
+def gcpRegion = project.findProperty('gcpRegion') ?: 'us-central1'
+def dataflowValidatesTempRoot = project.findProperty('gcpTempRoot') ?: 'gs://temp-storage-for-validates-runner-tests'
+def dataflowPostCommitTempRoot = project.findProperty('gcpTempRoot') ?: 'gs://temp-storage-for-end-to-end-tests'
+def dataflowPostCommitTempRootKms = project.findProperty('gcpTempRootKms') ?: 'gs://temp-storage-for-end-to-end-tests-cmek'
+def dataflowUploadTemp = project.findProperty('gcpTempRoot') ?: 'gs://temp-storage-for-upload-tests'
 def testFilesToStage = project.findProperty('filesToStage') ?: 'test.txt'
 def dataflowLegacyWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath
 def dataflowKmsKey = project.findProperty('dataflowKmsKey') ?: "projects/apache-beam-testing/locations/global/keyRings/beam-it/cryptoKeys/test"
 def firestoreDb = project.findProperty('firestoreDb') ?: 'firestoredb'
 
-def dockerImageRoot = project.findProperty('dockerImageRoot') ?: "us.gcr.io/${dataflowProject}/java-postcommit-it"
+def dockerImageRoot = project.findProperty('dockerImageRoot') ?: "us.gcr.io/${gcpProject}/java-postcommit-it"
 def dockerJavaImageContainer = "${dockerImageRoot}/java"
 def dockerPythonImageContainer = "${dockerImageRoot}/python"
 def dockerTag = new Date().format('yyyyMMddHHmmss')
@@ -158,17 +158,17 @@ ext.dockerPythonImageName = "${dockerPythonImageContainer}:${dockerTag}"
 
 def legacyPipelineOptions = [
   "--runner=TestDataflowRunner",
-  "--project=${dataflowProject}",
-  "--region=${dataflowRegion}",
+  "--project=${gcpProject}",
+  "--region=${gcpRegion}",
   "--tempRoot=${dataflowValidatesTempRoot}",
   "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
   "--workerHarnessContainerImage=",
 ]
 
 def runnerV2PipelineOptions = [
   "--runner=TestDataflowRunner",
-  "--project=${dataflowProject}",
-  "--region=${dataflowRegion}",
+  "--project=${gcpProject}",
+  "--region=${gcpRegion}",
   "--tempRoot=${dataflowValidatesTempRoot}",
   "--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}",
   "--experiments=use_unified_worker,use_runner_v2",
@@ -435,14 +435,14 @@ createCrossLanguageValidatesRunnerTask(
   semiPersistDir: "/var/opt/google",
   pythonPipelineOptions: [
     "--runner=TestDataflowRunner",
-    "--project=${dataflowProject}",
-    "--region=${dataflowRegion}",
+    "--project=${gcpProject}",
+    "--region=${gcpRegion}",
     "--sdk_harness_container_image_overrides=.*java.*,${dockerJavaImageContainer}:${dockerTag}",
   ],
   javaPipelineOptions: [
     "--runner=TestDataflowRunner",
-    "--project=${dataflowProject}",
-    "--region=${dataflowRegion}",
+    "--project=${gcpProject}",
+    "--region=${gcpRegion}",
     "--tempRoot=${dataflowValidatesTempRoot}",
     "--sdkContainerImage=${dockerJavaImageContainer}:${dockerTag}",
     "--sdkHarnessContainerImageOverrides=.*python.*,${dockerPythonImageContainer}:${dockerTag}",
@@ -455,9 +455,9 @@ createCrossLanguageValidatesRunnerTask(
   ],
   goScriptOptions: [
     "--runner dataflow",
-    "--project ${dataflowProject}",
-    "--dataflow_project ${dataflowProject}",
-    "--region ${dataflowRegion}",
+    "--project ${gcpProject}",
+    "--dataflow_project ${gcpProject}",
+    "--region ${gcpRegion}",
     "--tests \"./test/integration/xlang ./test/integration/io/xlang/...\"",
     "--sdk_overrides \".*java.*,${dockerJavaImageContainer}:${dockerTag}\"",
   ],
@@ -554,8 +554,8 @@ task googleCloudPlatformLegacyWorkerIntegrationTest(type: Test, dependsOn: copyG
   dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
   systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
           "--runner=TestDataflowRunner",
-          "--project=${dataflowProject}",
-          "--region=${dataflowRegion}",
+          "--project=${gcpProject}",
+          "--region=${gcpRegion}",
           "--tempRoot=${dataflowPostCommitTempRoot}",
           "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
           "--workerHarnessContainerImage=",
@@ -582,8 +582,8 @@ task googleCloudPlatformLegacyWorkerKmsIntegrationTest(type: Test) {
   dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar"
   systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
           "--runner=TestDataflowRunner",
-          "--project=${dataflowProject}",
-          "--region=${dataflowRegion}",
+          "--project=${gcpProject}",
+          "--region=${gcpRegion}",
           "--tempRoot=${dataflowPostCommitTempRootKms}",
           "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
           "--workerHarnessContainerImage=",
@@ -668,8 +668,8 @@ task coreSDKJavaLegacyWorkerIntegrationTest(type: Test) {
 
   systemProperty "beamTestPipelineOptions", JsonOutput.toJson([
           "--runner=TestDataflowRunner",
-          "--project=${dataflowProject}",
-          "--region=${dataflowRegion}",
+          "--project=${gcpProject}",
+          "--region=${gcpRegion}",
           "--tempRoot=${dataflowPostCommitTempRoot}",
           "--dataflowWorkerJar=${dataflowLegacyWorkerJar}",
           "--workerHarnessContainerImage=",
@@ -714,8 +714,6 @@ task postCommitRunnerV2 {
   dependsOn coreSDKJavaRunnerV2IntegrationTest
 }
 
-def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
-def gcpRegion = project.findProperty('gcpRegion') ?: 'us-central1'
 def gcsBucket = project.findProperty('gcsBucket') ?: 'temp-storage-for-release-validation-tests/nightly-snapshot-validation'
 def bqDataset = project.findProperty('bqDataset') ?: 'beam_postrelease_mobile_gaming'
 def pubsubTopic = project.findProperty('pubsubTopic') ?: 'java_mobile_gaming_topic'
@@ -745,7 +743,7 @@ createJavaExamplesArchetypeValidationTask(type: 'MobileGaming',
         bqDataset: bqDataset,
         pubsubTopic: pubsubTopic)
 
-// Standalone task for testing GCS upload, use with -PfilesToStage and -PdataflowTempRoot.
+// Standalone task for testing GCS upload, use with -PfilesToStage and -PgcpTempRoot.
 task GCSUpload(type: JavaExec) {
   mainClass = 'org.apache.beam.runners.dataflow.util.GCSUploadMain'
   classpath = sourceSets.test.runtimeClasspath

sdks/python/test-suites/dataflow/common.gradle

@@ -523,8 +523,8 @@ project.tasks.register("inferencePostCommitIT") {
 
 
 // Create cross-language tasks for running tests against Java expansion service(s)
-def dataflowProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
-def dataflowRegion = project.findProperty('dataflowRegion') ?: 'us-central1'
+def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
+def gcpRegion = project.findProperty('gcpRegion') ?: 'us-central1'
 
 project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata ->
     createCrossLanguageUsingJavaExpansionTask(
@@ -533,8 +533,8 @@ project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata ->
       collectMarker: taskMetadata.collectMarker,
       pythonPipelineOptions: [
         "--runner=TestDataflowRunner",
-        "--project=${dataflowProject}",
-        "--region=${dataflowRegion}",
+        "--project=${gcpProject}",
+        "--region=${gcpRegion}",
         "--sdk_container_image=gcr.io/apache-beam-testing/beam-sdk/beam_python${project.ext.pythonVersion}_sdk:latest",
         "--sdk_harness_container_image_overrides=.*java.*,gcr.io/apache-beam-testing/beam-sdk/beam_java8_sdk:latest"
       ],

sdks/python/test-suites/direct/common.gradle

@@ -436,7 +436,7 @@ project.tasks.register("inferencePostCommitIT") {
 }
 
 // Create cross-language tasks for running tests against Java expansion service(s)
-def gcpProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
+def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
 
 project(":sdks:python:test-suites:xlang").ext.xlangTasks.each { taskMetadata ->
     createCrossLanguageUsingJavaExpansionTask(

sdks/python/test-suites/direct/xlang/build.gradle

@@ -44,7 +44,7 @@ def cleanupTask = project.tasks.register("fnApiJobServerCleanup", Exec) {
   args '-c', ". ${envDir}/bin/activate && python -m apache_beam.runners.portability.local_job_service_main --pid_file ${pidFile} --stop"
 }
 
-def gcpProject = project.findProperty('dataflowProject') ?: 'apache-beam-testing'
+def gcpProject = project.findProperty('gcpProject') ?: 'apache-beam-testing'
 
 createCrossLanguageValidatesRunnerTask(
   startJobServer: setupTask,