WIP: enable Bzlmod for Bazel

bazelbuild · Aug 9, 2023 · c0fad89 · c0fad89
1 parent 4e525df
commit c0fad89
Show file tree

Hide file tree

Showing 9 changed files with 85 additions and 81 deletions.
diff --git a/.bazelci/presubmit.yml b/.bazelci/presubmit.yml
@@ -1,44 +1,36 @@
 ---
-matrix:
-  bzlmod_platforms: ["centos7_java11_devtoolset10", "ubuntu2004", "macos", "windows"]
+
 
 tasks:
-  bzlmod_builds:
-    name: "Bazel Bzlmod build"
-    platform: ${{ bzlmod_platforms }}
-    build_flags:
-      - "--config=bzlmod"
-    build_targets:
-      - "//src:bazel"
-    test_targets:
-      - "//src/test/cpp/..."
-    include_json_profile:
-      - build
-      - test
   centos7_java11_devtoolset10:
     shards: 4
     shell_commands:
       - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
-        android_ndk_repository/android_ndk_repository/' WORKSPACE
-      - rm -f WORKSPACE.bak
+        android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
+      - rm -f WORKSPACE.bzlmod.bak
       - rm -rf $HOME/bazeltest
       - mkdir $HOME/bazeltest
     build_flags:
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
+      # TODO(pcloudy): figure out a way on how to share repo cache for integration test
+      # - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
     build_targets:
-      - "//:bazel-distfile.zip"
+      # TODO(pcloudy): how to make the bootstrap build work with Bzlmod enabled.
+      # - "//:bazel-distfile.zip"
       - "//scripts/packages/debian:bazel-debian.deb"
       - "//scripts/packages:with-jdk/install.sh"
       - "//src:bazel"
       - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
+      # TODO(pcloudy): figure out a way on how to share repo cache for integration test
+      # - "//src:test_repos"
       - "//src/main/java/..."
     test_flags:
-      - "--sandbox_default_allow_network=false"
+      # TODO(pcloudy): Re-enable this when figuring out sharing repo cache
+      # - "--sandbox_default_allow_network=false"
       - "--sandbox_writable_path=$HOME/bazeltest"
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
+      # TODO(pcloudy): figure out a way on how to share repo cache for integration test
+      # - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
     test_targets:
       - "//scripts/..."
       - "//src/java_tools/..."
@@ -52,6 +44,7 @@ tasks:
       - "//tools/aquery_differ/..."
       - "//tools/compliance/..."
       - "//tools/python/..."
+      - "//tools/bash/..."
       # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/8162
       - "-//src/java_tools/buildjar/..."
       - "-//src/java_tools/import_deps_checker/..."
@@ -74,24 +67,24 @@ tasks:
     shards: 4
     shell_commands:
       - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
-        android_ndk_repository/android_ndk_repository/' WORKSPACE
-      - rm -f WORKSPACE.bak
+        android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
+      - rm -f WORKSPACE.bzlmod.bak
       - rm -rf $HOME/bazeltest
       - mkdir $HOME/bazeltest
     build_flags:
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
       - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80"
     build_targets:
       - "//src:bazel"
       - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
+      # - "//src:test_repos"
       - "//src/main/java/..."
     test_flags:
-      - "--sandbox_default_allow_network=false"
+      # - "--sandbox_default_allow_network=false"
       - "--sandbox_writable_path=$HOME/bazeltest"
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
       - "--test_tag_filters=-no_1804"
       # Configure and enable tests that require access to the network.
       - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80"
@@ -107,6 +100,7 @@ tasks:
       - "//tools/android/..."
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
+      - "//tools/bash/..."
       # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/8162
       - "-//src/java_tools/import_deps_checker/..."
       # https://github.com/bazelbuild/bazel/issues/18776
@@ -127,24 +121,26 @@ tasks:
     name: "Clang"
     shell_commands:
       - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
-        android_ndk_repository/android_ndk_repository/' WORKSPACE
-      - rm -f WORKSPACE.bak
+        android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
+      - rm -f WORKSPACE.bzlmod.bak
       - rm -rf $HOME/bazeltest
       - mkdir $HOME/bazeltest
     build_flags:
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
+      - "--features=-layering_check"
+      # - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
     build_targets:
       - "//src:bazel"
       - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
+      # - "//src:test_repos"
       - "//src/main/java/..."
     test_flags:
-      - "--sandbox_default_allow_network=false"
+      # - "--sandbox_default_allow_network=false"
       - "--sandbox_writable_path=$HOME/bazeltest"
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
       - "--test_tag_filters=-no_1804"
+      - "--features=-layering_check"
     test_targets:
       - "//src/test/shell/bazel:cc_integration_test"
     include_json_profile:
@@ -154,24 +150,24 @@ tasks:
     shards: 4
     shell_commands:
       - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
-        android_ndk_repository/android_ndk_repository/' WORKSPACE
-      - rm -f WORKSPACE.bak
+        android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
+      - rm -f WORKSPACE.bzlmod.bak
       - rm -rf $HOME/bazeltest
       - mkdir $HOME/bazeltest
     build_flags:
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
       - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80"
     build_targets:
       - "//src:bazel"
       - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
+      # - "//src:test_repos"
       - "//src/main/java/..."
     test_flags:
-      - "--sandbox_default_allow_network=false"
+      # - "--sandbox_default_allow_network=false"
       - "--sandbox_writable_path=$HOME/bazeltest"
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$OUTPUT_BASE/external"
       # Configure and enable tests that require access to the network.
       - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80"
     test_targets:
@@ -186,6 +182,7 @@ tasks:
       - "//tools/android/..."
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
+      - "//tools/bash/..."
       # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/8162
       - "-//src/java_tools/import_deps_checker/..."
       # https://github.com/bazelbuild/bazel/issues/18776
@@ -202,25 +199,25 @@ tasks:
     shards: 5
     shell_commands:
       - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
-        android_ndk_repository/android_ndk_repository/' WORKSPACE
-      - rm -f WORKSPACE.bak
+        android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
+      - rm -f WORKSPACE.bzlmod.bak
       - rm -rf $HOME/bazeltest
       - mkdir $HOME/bazeltest
       - ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external
     build_flags:
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$HOME/bazeltest/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$HOME/bazeltest/external"
       - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80"
     build_targets:
       - "//src:bazel"
       - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
+      # - "//src:test_repos"
       - "//src/main/java/..."
     test_flags:
-      - "--sandbox_default_allow_network=false"
+      # - "--sandbox_default_allow_network=false"
       - "--sandbox_writable_path=$HOME/bazeltest"
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$HOME/bazeltest/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$HOME/bazeltest/external"
       # Configure and enable tests that require access to the network.
       - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80"
       # Fine tune the number of test jobs running in parallel to avoid timeout
@@ -236,6 +233,7 @@ tasks:
       - "//tools/android/..."
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
+      - "//tools/bash/..."
       # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/8162
       - "-//src/java_tools/buildjar/..."
       - "-//src/java_tools/import_deps_checker/..."
@@ -270,28 +268,28 @@ tasks:
       - build
       - test
   macos_arm64:
-    shards: 2
+    shards: 1
     shell_commands:
       - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/' -e 's/^#
-        android_ndk_repository/android_ndk_repository/' WORKSPACE
-      - rm -f WORKSPACE.bak
+        android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
+      - rm -f WORKSPACE.bzlmod.bak
       - rm -rf $HOME/bazeltest
       - mkdir $HOME/bazeltest
       - ln -sf $OUTPUT_BASE/external $HOME/bazeltest/external
     build_flags:
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$HOME/bazeltest/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$HOME/bazeltest/external"
       - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80"
     build_targets:
       - "//src:bazel"
       - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
+      # - "//src:test_repos"
       - "//src/main/java/..."
     test_flags:
-      - "--sandbox_default_allow_network=false"
+      # - "--sandbox_default_allow_network=false"
       - "--sandbox_writable_path=$HOME/bazeltest"
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest/install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=$HOME/bazeltest/external"
+      # - "--test_env=TEST_REPOSITORY_HOME=$HOME/bazeltest/external"
       # Configure and enable tests that require access to the network.
       - "--test_env=REMOTE_NETWORK_ADDRESS=bazel.build:80"
     test_targets:
@@ -305,6 +303,7 @@ tasks:
       - "//tools/android/..."
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
+      - "//tools/bash/..."
       # Re-enable once fixed: https://github.com/bazelbuild/bazel/issues/8162
       - "-//src/java_tools/buildjar/..."
       - "-//src/java_tools/import_deps_checker/..."
@@ -390,17 +389,17 @@ tasks:
       - mkdir C:\b
       - mklink /J C:\b\bazeltest_external %OUTPUT_BASE:/=\%\external
     batch_commands:
-      - powershell -Command "(Get-Content WORKSPACE) -Replace '# android_', 'android_' | Set-Content WORKSPACE"
+      - powershell -Command "(Get-Content WORKSPACE.bzlmod) -Replace '# android_', 'android_' | Set-Content WORKSPACE.bzlmod"
     build_flags:
       - "--copt=-w"
       - "--host_copt=-w"
       - "--test_env=JAVA_HOME"
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest_install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=C:/b/bazeltest_external"
+      # - "--test_env=TEST_REPOSITORY_HOME=C:/b/bazeltest_external"
     build_targets:
       - "//src:bazel.exe"
       - "//src:bazel_jdk_minimal"
-      - "//src:test_repos"
+      # - "//src:test_repos"
       - "//src/main/java/..."
     test_flags:
       - "--copt=-w"
@@ -409,7 +408,7 @@ tasks:
       - "--test_env=JAVA_HOME"
       - "--test_env=BAZEL_VC"
       - "--test_env=TEST_INSTALL_BASE=$HOME/bazeltest_install_base"
-      - "--test_env=TEST_REPOSITORY_HOME=C:/b/bazeltest_external"
+      # - "--test_env=TEST_REPOSITORY_HOME=C:/b/bazeltest_external"
     test_targets:
       - "//src:embedded_tools_size_test"
       - "//src/main/starlark/tests/builtins_bzl/..."
@@ -453,8 +452,8 @@ tasks:
     shell_commands:
       - sed -i.bak
         -e 's/^# android_sdk_repository/android_sdk_repository/'
-        -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE
-      - rm -f WORKSPACE.bak
+        -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
+      - rm -f WORKSPACE.bzlmod.bak
     build_flags:
       - "--config=ubuntu1804_java11"
       - "--remote_executor=grpcs://remotebuildexecution.googleapis.com"
@@ -485,6 +484,7 @@ tasks:
       - "//third_party/ijar/..."
       - "//tools/aquery_differ/..."
       - "//tools/python/..."
+      - "//tools/bash/..."
       - "//tools/android/..."
       # See https://github.com/bazelbuild/bazel/issues/8033
       - "-//src/tools/singlejar:output_jar_simple_test"
@@ -518,8 +518,8 @@ tasks:
   kythe_ubuntu2004:
     shell_commands:
     - sed -i.bak -e 's/^# android_sdk_repository/android_sdk_repository/'
-      -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE
-    - rm -f WORKSPACE.bak
+      -e 's/^# android_ndk_repository/android_ndk_repository/' WORKSPACE.bzlmod
+    - rm -f WORKSPACE.bzlmod.bak
     index_flags:
     - "--define=kythe_corpus=github.com/bazelbuild/bazel"
     index_targets_query: "kind(\"cc_(binary|library|test|proto_library) rule\", ...) union kind(\"java_(binary|import|library|plugin|test|proto_library) rule\", ...) union kind(\"proto_library rule\", ...)"

diff --git a/.bazelrc b/.bazelrc
@@ -33,8 +33,9 @@ build:macos --macos_minimum_os=10.11
 build:windows --extra_toolchains=@bazel_tools//tools/python:autodetecting_toolchain
 
 # Enable Bzlmod
-build:bzlmod --enable_bzlmod
-build:bzlmod --check_direct_dependencies=error
+common:bzlmod --enable_bzlmod
+common:bzlmod --check_direct_dependencies=error
+common --config=bzlmod
 
 # Enable Java 11 language features (https://github.com/bazelbuild/bazel/issues/14592)
 build --java_language_version=11

diff --git a/scripts/docs/rewriter_test.py b/scripts/docs/rewriter_test.py
@@ -34,7 +34,7 @@ def testCanRewrite(self, path, expected_can_rewrite):
 
 def read_data_file(basename, in_or_out_fragment):
   path = os.path.join(
-      os.getenv("TEST_SRCDIR"), "io_bazel/scripts/docs/testdata",
+      os.getenv("TEST_SRCDIR"), "_main/scripts/docs/testdata",
       in_or_out_fragment, basename)
   with open(path, "rt", encoding="utf-8") as f:
     return path, f.read()

diff --git a/src/main/starlark/tests/builtins_bzl/builtin_test_setup.sh b/src/main/starlark/tests/builtins_bzl/builtin_test_setup.sh
@@ -39,10 +39,10 @@ function setup_tests() {
 }
 
 function get_runfiles_dir() {
-  src="$TEST_SRCDIR/io_bazel/$1"
+  src="$TEST_SRCDIR/_main/$1"
   if [ -e "$src" ]; then
       echo $src
   elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
-      echo $(grep -m1 "io_bazel/$1" "${RUNFILES_MANIFEST_FILE}" | cut -d' ' -f2 | sed "s|$1.*|$1|")
+      echo $(grep -m1 "_main/$1" "${RUNFILES_MANIFEST_FILE}" | cut -d' ' -f2 | sed "s|$1.*|$1|")
   fi
 }