test_base.py: use bazel-runfiles to respect repo mappings

Use rules_python to fetch the bazel-runfiles pip package, we can later migrate other checked-in Python deps to rules_python as well.

On linux and macOS, we are using a hermetic Python toolchain fetched by rules_python, which helped avoid issues with older system installed Python version (<=3.6) on centos7 and ubuntu1804.

On windows, the hermetic Python toolchain doesn't work with `pywin32` pip package, therefore we still use the auto-detected python toolchain.

For bootstrapping test, we need to use the auto-detected python toolchain to avoid downloading Python.

Working towards: #18957

RELNOTES:
PiperOrigin-RevId: 553731631
Change-Id: I605399067e21653cab707694810de24fd2109c41

.bazelrc

@@ -27,6 +27,11 @@ build:remote --config=ubuntu1804_java11
 
 build:macos --macos_minimum_os=10.11
 
+# On Windows, we need pywin32 pip package, which doesn't work with the Python hermetic toolchain.
+# See https://github.com/bazelbuild/rules_python/issues/1356
+# Therefore, use the local detected Python toolchain on Windows.
+build:windows --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain
+
 # Enable Bzlmod
 build:bzlmod --enable_bzlmod
 build:bzlmod --check_direct_dependencies=error

MODULE.bazel

@@ -120,3 +120,11 @@ bazel_dep(name = "googletest", version = "1.12.1", repo_name = "com_google_googl
 
 bazel_dev_deps = use_extension("//:extensions.bzl", "bazel_dev_deps")
 use_repo(bazel_dev_deps, "local_bazel_source_list")
+
+pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+pip.parse(
+    hub_name = "bazel_pip_dev_deps",
+    python_version = "3.8",
+    requirements_lock = "//:requirements.txt",
+)
+use_repo(pip, "bazel_pip_dev_deps")

WORKSPACE

@@ -182,10 +182,6 @@ dist_http_archive(
     patch_cmds_win = EXPORT_WORKSPACE_IN_BUILD_BAZEL_FILE_WIN,
 )
 
-load("@rules_python//python:repositories.bzl", "py_repositories")
-
-py_repositories()
-
 dist_http_archive(
     name = "zstd-jni",
     build_file = "//third_party:zstd-jni/zstd-jni.BUILD",
@@ -560,3 +556,23 @@ maven_install(
 load("@maven_android//:defs.bzl", pinned_maven_install_android = "pinned_maven_install")
 
 pinned_maven_install_android()
+
+load("@rules_python//python:repositories.bzl", "py_repositories", "python_register_toolchains")
+
+py_repositories()
+
+python_register_toolchains(
+    name = "python_3_8",
+    python_version = "3.8",
+)
+
+load("@python_3_8//:defs.bzl", "interpreter")
+
+load("@rules_python//python:pip.bzl", "pip_parse")
+pip_parse(
+   name = "bazel_pip_dev_deps",
+   requirements_lock = "//:requirements.txt",
+   python_interpreter_target = interpreter,
+)
+load("@bazel_pip_dev_deps//:requirements.bzl", "install_deps")
+install_deps()

requirements.txt

@@ -0,0 +1 @@
+bazel-runfiles==0.24.0

scripts/bootstrap/bootstrap.sh

@@ -37,11 +37,17 @@ _BAZEL_ARGS="--spawn_strategy=standalone \
       --compilation_mode=opt \
       --distdir=derived/distdir \
       --extra_toolchains=//scripts/bootstrap:all \
+      --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain \
+      --override_repository=maven="$(get_cwd)/maven" \
       ${DIST_BOOTSTRAP_ARGS:-} \
       ${EXTRA_BAZEL_ARGS:-}"
 
 cp scripts/bootstrap/BUILD.bootstrap scripts/bootstrap/BUILD
 
+# Remove lines containing 'install_deps' to avoid loading @bazel_pip_dev_deps,
+# which requires fetching the python toolchain.
+sed -i.bak '/install_deps/d' WORKSPACE && rm WORKSPACE.bak
+
 if [ -z "${BAZEL-}" ]; then
   function _run_bootstrapping_bazel() {
     local command=$1

scripts/bootstrap/compile.sh

@@ -468,7 +468,6 @@ function run_bazel_jar() {
       --startup_time=329 --extract_data_time=523 \
       --rc_source=/dev/null --isatty=1 \
       --build_python_zip \
-      --override_repository=maven="$(get_cwd)/maven" \
       "${client_env[@]}" \
       --client_cwd="$(get_cwd)" \
       "${@}"

src/test/py/bazel/BUILD

@@ -1,5 +1,6 @@
 load("//:distdir_deps.bzl", "gen_workspace_stanza")
 load("@rules_python//python:defs.bzl", "py_library", "py_test")
+load("@bazel_pip_dev_deps//:requirements.bzl", "requirement")
 
 package(default_visibility = ["//visibility:private"])
 
@@ -35,6 +36,7 @@ py_library(
         "//tools/ctexplain:__pkg__",
         "//tools/python:__pkg__",
     ],
+    deps = [requirement("bazel-runfiles")],
 )
 
 gen_workspace_stanza(

src/test/py/bazel/test_base.py

@@ -14,6 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+"""Bazel Python integration test framework."""
+
 import locale
 import os
 import shutil
@@ -23,6 +25,7 @@
 import sys
 import tempfile
 import unittest
+import runfiles
 
 
 class Error(Exception):
@@ -43,6 +46,7 @@ def __init__(self, name):
 
 
 class TestBase(unittest.TestCase):
+  """Bazel Python integration test base."""
 
   _runfiles = None
   _temp = None
@@ -103,7 +107,7 @@ class TestBase(unittest.TestCase):
   def setUp(self):
     unittest.TestCase.setUp(self)
     if self._runfiles is None:
-      self._runfiles = TestBase._LoadRunfiles()
+      self._runfiles = runfiles.Create()
     test_tmpdir = TestBase._CreateDirs(TestBase.GetEnv('TEST_TMPDIR'))
     self._tests_root = TestBase._CreateDirs(
         os.path.join(test_tmpdir, 'tests_root'))
@@ -258,10 +262,7 @@ def Path(self, path):
 
   def Rlocation(self, runfile):
     """Returns the absolute path to a runfile."""
-    if TestBase.IsWindows():
-      return self._runfiles.get(runfile)
-    else:
-      return os.path.join(self._runfiles, runfile)
+    return self._runfiles.Rlocation(runfile)
 
   def ScratchDir(self, path):
     """Creates directories under the test's scratch directory.
@@ -569,30 +570,6 @@ def _EnvMap(self, env_remove=None, env_add=None):
         env[e] = env_add[e]
     return env
 
-  @staticmethod
-  def _LoadRunfiles():
-    """Loads the runfiles manifest from ${TEST_SRCDIR}/MANIFEST.
-
-    Only necessary to use on Windows, where runfiles are not symlinked in to the
-    runfiles directory, but are written to a MANIFEST file instead.
-
-    Returns:
-      on Windows: {string: string} dictionary, keys are runfiles-relative paths,
-        values are absolute paths that the runfiles entry is mapped to;
-      on other platforms: string; value of $TEST_SRCDIR
-    """
-    test_srcdir = TestBase.GetEnv('TEST_SRCDIR')
-    if not TestBase.IsWindows():
-      return test_srcdir
-
-    result = {}
-    with open(os.path.join(test_srcdir, 'MANIFEST'), 'r') as f:
-      for l in f:
-        tokens = l.strip().split(' ')
-        if len(tokens) == 2:
-          result[tokens[0]] = tokens[1]
-    return result
-
   @staticmethod
   def _CreateDirs(path):
     if not os.path.exists(path):

src/test/shell/bazel/bazel_determinism_test.sh

@@ -61,10 +61,18 @@ function test_determinism()  {
     # Set up the maven repository properly.
     cp maven/BUILD.vendor maven/BUILD
 
+    # Remove lines containing 'install_deps' to avoid loading @bazel_pip_dev_deps,
+    # which requires fetching the python toolchain.
+    sed -i.bak '/install_deps/d' WORKSPACE && rm WORKSPACE.bak
+
+    # Use @bazel_tools//tools/python:autodetecting_toolchain to avoid
+    # downloading python toolchain.
+
     # Build Bazel once.
     bazel \
       --output_base="${TEST_TMPDIR}/out1" \
       build \
+      --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain \
       --distdir=$distdir \
       --override_repository=maven=$maven \
       --nostamp \
@@ -77,6 +85,7 @@ function test_determinism()  {
       --install_base="${TEST_TMPDIR}/install_base2" \
       --output_base="${TEST_TMPDIR}/out2" \
       build \
+      --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain \
       --distdir=$distdir \
       --override_repository=maven=$maven \
       --nostamp \