Revert "Generate external transform wrappers using a script (apache#2…

…9834)"

This reverts commit 11f9bce.

.github/trigger_files/beam_PostCommit_Python_Xlang_Gcp_Dataflow.json

@@ -1 +0,0 @@
-

.github/workflows/README.md

@@ -271,7 +271,6 @@ PreCommit Jobs run in a schedule and also get triggered in a PR if relevant sour
 | [ PreCommit Website ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml) | N/A |`Run Website PreCommit`| [![.github/workflows/beam_PreCommit_Website.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website.yml?query=event%3Aschedule) |
 | [ PreCommit Website Stage GCS ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml) | N/A |`Run Website_Stage_GCS PreCommit`| [![.github/workflows/beam_PreCommit_Website_Stage_GCS.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Website_Stage_GCS.yml?query=event%3Aschedule) |
 | [ PreCommit Whitespace ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml) | N/A |`Run Whitespace PreCommit`| [![.github/workflows/beam_PreCommit_Whitespace.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Whitespace.yml?query=event%3Aschedule) |
-| [ PreCommit Xlang Generated Transforms ](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml) | N/A |`Run Xlang_Generated_Transforms PreCommit`| [![.github/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml/badge.svg?event=schedule)](https://github.com/apache/beam/actions/workflows/beam_PreCommit_Xlang_Generated_Transforms.yml?query=event%3Aschedule) |
 
 ### PostCommit Jobs
 

.gitignore

@@ -52,7 +52,6 @@ sdks/python/**/*.egg
 sdks/python/LICENSE
 sdks/python/NOTICE
 sdks/python/README.md
-sdks/python/apache_beam/transforms/xlang/*
 sdks/python/apache_beam/portability/api/*
 sdks/python/apache_beam/yaml/docs/*
 sdks/python/nosetests*.xml

CHANGES.md

@@ -58,7 +58,6 @@
 
 * New highly anticipated feature X added to Python SDK ([#X](https://github.com/apache/beam/issues/X)).
 * New highly anticipated feature Y added to Java SDK ([#Y](https://github.com/apache/beam/issues/Y)).
-* The Python SDK will now include automatically generated wrappers for external Java transforms! ([#29834](https://github.com/apache/beam/pull/29834))
 
 ## I/Os
 

build.gradle.kts

@@ -647,11 +647,6 @@ tasks.register("checkSetup") {
   dependsOn(":examples:java:wordCount")
 }
 
-// Generates external transform config
-project.tasks.register("generateExternalTransformsConfig") {
-  dependsOn(":sdks:python:generateExternalTransformsConfig")
-}
-
 // Configure the release plugin to do only local work; the release manager determines what, if
 // anything, to push. On failure, the release manager can reset the branch without pushing.
 release {

buildSrc/src/main/groovy/org/apache/beam/gradle/BeamModulePlugin.groovy

@@ -321,17 +321,19 @@ class BeamModulePlugin implements Plugin<Project> {
 
   // A class defining the common properties in a given suite of cross-language tests
   // Properties are shared across runners and are used when creating a CrossLanguageUsingJavaExpansionConfiguration object
-  static class CrossLanguageTask {
+  static class CrossLanguageTaskCommon {
     // Used as the task name for cross-language
     String name
-    // List of project paths for required expansion services
-    List<String> expansionProjectPaths
+    // The expansion service's project path (required)
+    String expansionProjectPath
     // Collect Python pipeline tests with this marker
     String collectMarker
-    // Additional environment variables to set before running tests
+    // Job server startup task.
+    TaskProvider startJobServer
+    // Job server cleanup task.
+    TaskProvider cleanupJobServer
+    // any additional environment variables specific to the suite of tests
     Map<String,String> additionalEnvs
-    // Additional Python dependencies to install before running tests
-    List<String> additionalDeps
   }
 
   // A class defining the configuration for CrossLanguageUsingJavaExpansion.
@@ -347,16 +349,18 @@ class BeamModulePlugin implements Plugin<Project> {
     ]
     // Additional pytest options
     List<String> pytestOptions = []
+    // Job server startup task.
+    TaskProvider startJobServer
+    // Job server cleanup task.
+    TaskProvider cleanupJobServer
     // Number of parallel test runs.
     Integer numParallelTests = 1
-    // List of project paths for required expansion services
-    List<String> expansionProjectPaths
+    // Project path for the expansion service to start up
+    String expansionProjectPath
     // Collect Python pipeline tests with this marker
     String collectMarker
     // any additional environment variables to be exported
     Map<String,String> additionalEnvs
-    // Additional Python dependencies to install before running tests
-    List<String> additionalDeps
   }
 
   // A class defining the configuration for CrossLanguageValidatesRunner.
@@ -2572,7 +2576,7 @@ class BeamModulePlugin implements Plugin<Project> {
     /** ***********************************************************************************************/
     // Method to create the createCrossLanguageUsingJavaExpansionTask.
     // The method takes CrossLanguageUsingJavaExpansionConfiguration as parameter.
-    // This method creates a task that runs Python SDK test-suites that use external Java transforms
+    // This method creates a task that runs Python SDK pipeline tests that use Java transforms via an input expansion service
     project.ext.createCrossLanguageUsingJavaExpansionTask = {
       // This task won't work if the python build file doesn't exist.
       if (!project.project(":sdks:python").buildFile.exists()) {
@@ -2582,29 +2586,49 @@ class BeamModulePlugin implements Plugin<Project> {
       def config = it ? it as CrossLanguageUsingJavaExpansionConfiguration : new CrossLanguageUsingJavaExpansionConfiguration()
 
       project.evaluationDependsOn(":sdks:python")
-      for (path in config.expansionProjectPaths) {
-        project.evaluationDependsOn(path)
-      }
+      project.evaluationDependsOn(config.expansionProjectPath)
       project.evaluationDependsOn(":sdks:java:extensions:python")
 
+      // Setting up args to launch the expansion service
       def pythonDir = project.project(":sdks:python").projectDir
+      def javaExpansionPort = -1 // will be populated in setupTask
+      def expansionJar = project.project(config.expansionProjectPath).shadowJar.archivePath
+      def javaClassLookupAllowlistFile = project.project(config.expansionProjectPath).projectDir.getPath()
+      def expansionServiceOpts = [
+        "group_id": project.name,
+        "java_expansion_service_jar": expansionJar,
+        "java_expansion_service_allowlist_file": javaClassLookupAllowlistFile,
+      ]
       def usesDataflowRunner = config.pythonPipelineOptions.contains("--runner=TestDataflowRunner") || config.pythonPipelineOptions.contains("--runner=DataflowRunner")
       def javaContainerSuffix = getSupportedJavaVersion()
 
-      // Sets up, collects, and runs Python pipeline tests
-      project.tasks.register(config.name+"PythonUsingJava") {
-        group = "Verification"
-        description = "Runs Python SDK pipeline tests that use a Java expansion service"
-        // Each expansion service we use needs to be built before running these tests
-        // The built jars will be started up automatically using the BeamJarExpansionService utility
-        for (path in config.expansionProjectPaths) {
-          dependsOn project.project(path).shadowJar.getPath()
-        }
-        dependsOn ":sdks:java:container:$javaContainerSuffix:docker"
-        dependsOn "installGcpTest"
+      // 1. Builds the chosen expansion service jar and launches it
+      def setupTask = project.tasks.register(config.name+"Setup") {
+        dependsOn ':sdks:java:container:' + javaContainerSuffix + ':docker'
+        dependsOn project.project(config.expansionProjectPath).shadowJar.getPath()
+        dependsOn 'installGcpTest'
         if (usesDataflowRunner) {
           dependsOn ":sdks:python:test-suites:dataflow:py${project.ext.pythonVersion.replace('.', '')}:initializeForDataflowJob"
         }
+        doLast {
+          project.exec {
+            // Prepare a port to use for the expansion service
+            javaExpansionPort = getRandomPort()
+            expansionServiceOpts.put("java_port", javaExpansionPort)
+            // setup test env
+            def serviceArgs = project.project(':sdks:python').mapToArgString(expansionServiceOpts)
+            executable 'sh'
+            args '-c', ". ${project.ext.envdir}/bin/activate && $pythonDir/scripts/run_expansion_services.sh stop --group_id ${project.name} && $pythonDir/scripts/run_expansion_services.sh start $serviceArgs"
+          }
+        }
+      }
+
+      // 2. Sets up, collects, and runs Python pipeline tests
+      def pythonTask = project.tasks.register(config.name+"PythonUsingJava") {
+        group = "Verification"
+        description = "Runs Python SDK pipeline tests that use a Java expansion service"
+        dependsOn setupTask
+        dependsOn config.startJobServer
         doLast {
           def beamPythonTestPipelineOptions = [
             "pipeline_opts": config.pythonPipelineOptions + (usesDataflowRunner ? [
@@ -2617,19 +2641,29 @@ class BeamModulePlugin implements Plugin<Project> {
           def cmdArgs = project.project(':sdks:python').mapToArgString(beamPythonTestPipelineOptions)
 
           project.exec {
-            // environment variable to indicate that jars have been built
-            environment "EXPANSION_JARS", config.expansionProjectPaths
-            String additionalDependencyCmd = ""
-            if (config.additionalDeps != null && !config.additionalDeps.isEmpty()){
-              additionalDependencyCmd = "&& pip install ${config.additionalDeps.join(' ')} "
+            environment "EXPANSION_JAR", expansionJar
+            environment "EXPANSION_PORT", javaExpansionPort
+            for (envs in config.additionalEnvs){
+              environment envs.getKey(), envs.getValue()
             }
             executable 'sh'
-            args '-c', ". ${project.ext.envdir}/bin/activate " +
-                additionalDependencyCmd +
-                "&& cd $pythonDir && ./scripts/run_integration_test.sh $cmdArgs"
+            args '-c', ". ${project.ext.envdir}/bin/activate && cd $pythonDir && ./scripts/run_integration_test.sh $cmdArgs"
           }
         }
       }
+
+      // 3. Shuts down the expansion service
+      def cleanupTask = project.tasks.register(config.name+'Cleanup', Exec) {
+        // teardown test env
+        executable 'sh'
+        args '-c', ". ${project.ext.envdir}/bin/activate && $pythonDir/scripts/run_expansion_services.sh stop --group_id ${project.name}"
+      }
+
+      setupTask.configure {finalizedBy cleanupTask}
+      config.startJobServer.configure {finalizedBy config.cleanupJobServer}
+
+      cleanupTask.configure{mustRunAfter pythonTask}
+      config.cleanupJobServer.configure{mustRunAfter pythonTask}
     }
 
     /** ***********************************************************************************************/

scripts/ci/release/test/resources/mass_comment.txt

@@ -79,7 +79,6 @@ Run PythonDocs PreCommit
 Run PythonFormatter PreCommit
 Run PythonLint PreCommit
 Run Python_PVR_Flink PreCommit
-Run Python_Xlang_Gcp_Direct PostCommit
 Run RAT PreCommit
 Run SQL PostCommit
 Run SQL PreCommit
@@ -95,7 +94,6 @@ Run Twister2 ValidatesRunner
 Run Typescript PreCommit
 Run ULR Loopback ValidatesRunner
 Run Whitespace PreCommit
-Run Xlang_Generated_Transforms PreCommit
 Run XVR_Direct PostCommit
 Run XVR_Flink PostCommit
 Run XVR_JavaUsingPython_Dataflow PostCommit

sdks/java/core/src/main/java/org/apache/beam/sdk/providers/GenerateSequenceSchemaTransformProvider.java

@@ -66,7 +66,7 @@ public List<String> outputCollectionNames() {
   public String description() {
     return String.format(
         "Outputs a PCollection of Beam Rows, each containing a single INT64 "
-            + "number called \"value\". The count is produced from the given \"start\" "
+            + "number called \"value\". The count is produced from the given \"start\""
             + "value and either up to the given \"end\" or until 2^63 - 1.%n"
             + "To produce an unbounded PCollection, simply do not specify an \"end\" value. "
             + "Unbounded sequences can specify a \"rate\" for output elements.%n"

sdks/python/MANIFEST.in

@@ -16,7 +16,6 @@
 #
 
 include gen_protos.py
-include gen_xlang_wrappers.py
 include README.md
 include NOTICE
 include LICENSE

sdks/python/apache_beam/io/__init__.py

@@ -36,7 +36,6 @@
   from apache_beam.io.gcp.bigquery import *
   from apache_beam.io.gcp.pubsub import *
   from apache_beam.io.gcp import gcsio
-  from apache_beam.transforms.xlang.io import *
 except ImportError:
   pass
 # pylint: enable=wrong-import-order, wrong-import-position