Generate external transform wrappers using a script

.github/workflows/beam_PostCommit_Python_Xlang_IO_Direct.yml

@@ -0,0 +1,95 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: PostCommit Python Xlang IO Direct
+
+on:
+  schedule:
+    - cron: '30 5/6 * * *'
+  pull_request_target:
+    paths: ['release/trigger_all_tests.json', '.github/trigger_files/beam_PostCommit_Python_Xlang_IO_Direct.json']
+  workflow_dispatch:
+
+#Setting explicit permissions for the action to avoid the default permissions which are `write-all` in case of pull_request_target event
+permissions:
+  actions: write
+  pull-requests: write
+  checks: write
+  contents: read
+  deployments: read
+  id-token: none
+  issues: write
+  discussions: read
+  packages: read
+  pages: read
+  repository-projects: read
+  security-events: read
+  statuses: read
+
+# This allows a subsequently queued workflow run to interrupt previous runs
+concurrency:
+  group: '${{ github.workflow }} @ ${{ github.event.issue.number || github.sha || github.head_ref || github.ref }}-${{ github.event.schedule || github.event.comment.id || github.event.sender.login }}'
+  cancel-in-progress: true
+
+env:
+  GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
+  GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }}
+  GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }}
+
+jobs:
+  beam_PostCommit_Python_Xlang_IO_Direct:
+    if: |
+      github.event_name == 'workflow_dispatch' ||
+      github.event_name == 'pull_request_target' ||
+      (github.event_name == 'schedule' && github.repository == 'apache/beam') ||
+      github.event.comment.body == 'Run Python_Xlang_IO_Direct PostCommit'
+    runs-on: [self-hosted, ubuntu-20.04, main]
+    timeout-minutes: 100
+    name: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+    strategy:
+      matrix:
+        job_name: ["beam_PostCommit_Python_Xlang_IO_Direct"]
+        job_phrase: ["Run Python_Xlang_IO_Direct PostCommit"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup repository
+        uses: ./.github/actions/setup-action
+        with:
+          comment_phrase: ${{ matrix.job_phrase }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }})
+      - name: Setup environment
+        uses: ./.github/actions/setup-environment-action
+        with:
+          python-version: |
+            3.8
+            3.11
+      - name: run PostCommit Python Xlang IO Direct script
+        uses: ./.github/actions/gradle-command-self-hosted-action
+        with:
+          gradle-command: :sdks:python:test-suites:direct:ioCrossLanguagePostCommit
+      - name: Archive Python Test Results
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: Python Test Results
+          path: '**/pytest*.xml'
+      - name: Publish Python Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        if: always()
+        with:
+          commit: '${{ env.prsha || env.GITHUB_SHA }}'
+          comment_mode: ${{ github.event_name == 'issue_comment'  && 'always' || 'off' }}
+          files: '**/pytest*.xml'

.gitignore

@@ -52,6 +52,7 @@ sdks/python/**/*.egg
 sdks/python/LICENSE
 sdks/python/NOTICE
 sdks/python/README.md
+sdks/python/apache_beam/**/*_et.py
 sdks/python/apache_beam/portability/api/*
 sdks/python/nosetests*.xml
 sdks/python/pytest*.xml

sdks/python/MANIFEST.in

@@ -16,6 +16,7 @@
 #
 
 include gen_protos.py
+include gen_xlang_wrappers.py
 include README.md
 include NOTICE
 include LICENSE

sdks/python/apache_beam/transforms/external_schematransform_provider.py

@@ -142,9 +142,9 @@ class ExternalSchemaTransformProvider:
   """Dynamically discovers Schema-aware external transforms from a given list
   of expansion services and provides them as ready PTransforms.
 
-  A :class:`ExternalSchemaTransform` subclass is generated for each external
+  An :class:`ExternalSchemaTransform` subclass is generated for each external
   transform, and is named based on what can be inferred from the URN
-  (see :param urn_pattern).
+  (see the `urn_pattern` parameter).
 
   These classes are generated when :class:`ExternalSchemaTransformProvider` is
   initialized. We need to give it one or more expansion service addresses that
@@ -256,7 +256,7 @@ def _create_wrappers(self):
 
       if skipped_urns:
         logging.info(
-            "Skipped URN(s) in %s that don't follow the pattern [%s]: %s",
+            "Skipped URN(s) in %s that don't follow the pattern \"%s\": %s",
             target,
             self._urn_pattern,
             skipped_urns)
@@ -268,6 +268,10 @@ def get_available(self) -> List[Tuple[str, str]]:
     """Get a list of available ExternalSchemaTransform names and identifiers"""
     return list(self._name_to_urn.items())
 
+  def get_all(self) -> Dict[str, ExternalSchemaTransform]:
+    """Get all ExternalSchemaTransforms"""
+    return self._transforms
+
   def get(self, name) -> ExternalSchemaTransform:
     """Get an ExternalSchemaTransform by its inferred class name"""
     return self._transforms[self._name_to_urn[name]]