From aab2d14e85f4af4faea1b108bb6df35e142caee6 Mon Sep 17 00:00:00 2001 From: Yun Peng Date: Wed, 14 Feb 2024 19:20:10 +0100 Subject: [PATCH] [7.1.0] Cherry-pick recent changes to fix CI flakiness and breakages (#21349) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes //src/test/py/bazel:bazel_vendor_test in ipv6-only environment … …on macOS https://buildkite.com/bazel/bazel-bazel-macos-ninja/builds/514 PiperOrigin-RevId: 606599015 Change-Id: I36a8534d6676bc5c3c9a0157eea28fb033e9cf3e ------- Fixes //src/test/shell/bazel:bazel_test_test in ipv6-only environment… … on macOS https://buildkite.com/bazel/bazel-bazel-macos-ninja/builds/517 PiperOrigin-RevId: 606603953 Change-Id: Id730a0457e2a6bc1ac5371cbbce25c4acd25ab9d ----- Fixes _module_repo_name when building with Bazel@HEAD or Bazel 7.1 If `get_canonical_repo_name` no longer returns the repo name with version due to containing a54a393, the `_module_repo_name` should not either. Fixes: #21292 Closes #21324. PiperOrigin-RevId: 606646238 Change-Id: I8835a84842c2c66929586b39156eb9f5a541652f ----- Make sure generate_dist_lockfile works in ipv6-only environment Fixes https://buildkite.com/bazel/bazel-bazel-macos-ninja/builds/534#018da46c-5aff-45ea-8b66-937e676b09b2 PiperOrigin-RevId: 606951013 Change-Id: I9336c9b8a173ed464b0fd3dab22a5dc1614b4c62 ---- Fix googleapis and remoteapis sources in bootstrap distfile Previously, their sources are included as external repos and mapped to third_party, after this change, their sources are directly included under the third_party directory. This change is needed because https://github.com/bazelbuild/bazel/commit/a54a393d209ab9c8cf5e80b2a0ef092196c17df3 broke the mapping mechanism which depends on the canonical repository name. Fixes //src/test/shell/bazel:bazel_bootstrap_distfile_tar_test with Bazel@HEAD and Bazel@7.1 https://buildkite.com/bazel/bazel-at-head-plus-downstream/builds/3657#018da594-e743-44da-8f76-782a8e5c86b1 Related https://github.com/bazelbuild/bazel/issues/21292 PiperOrigin-RevId: 606960533 Change-Id: Ia4a81d5730e04964bc06c8f8ee2685364ce8623b --- BUILD | 5 +---- MODULE.bazel.lock | 6 +++--- src/test/py/bazel/test_base.py | 8 ++++++++ src/test/shell/bazel/bazel_test_test.sh | 10 +++++++--- src/tools/bzlmod/utils.bzl | 5 +++++ third_party/BUILD | 4 ++-- 6 files changed, 26 insertions(+), 12 deletions(-) diff --git a/BUILD b/BUILD index ebf481121853e2..170689bfeec994 100644 --- a/BUILD +++ b/BUILD @@ -111,7 +111,7 @@ genrule( # Instead of `bazel mod deps`, we run a simpler command like `bazel query :all` here # so that we only trigger module resolution, not extension eval. # Also use `--batch` so that Bazel doesn't keep a server process alive. - "$(location //src:bazel) --batch --output_user_root=$$PWD/tmp_bazel_root query --check_direct_dependencies=error --lockfile_mode=update :all && " + + "$(location //src:bazel) --batch --host_jvm_args=-Djava.net.preferIPv6Addresses=true --output_user_root=$$PWD/tmp_bazel_root query --check_direct_dependencies=error --lockfile_mode=update :all && " + "mv MODULE.bazel.lock $@", tags = ["requires-network"], tools = ["//src:bazel"], @@ -174,9 +174,6 @@ pkg_tar( remap_paths = { "MODULE.bazel.lock.dist": "MODULE.bazel.lock", "WORKSPACE.bzlmod.filtered": "WORKSPACE.bzlmod", - # Rewrite paths coming from local repositories back into third_party. - "external/googleapis~override": "third_party/googleapis", - "external/remoteapis~override": "third_party/remoteapis", }, strip_prefix = ".", # Public but bazel-only visibility. diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index cfa61045a404e8..1c1682ce6a504b 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -2160,7 +2160,7 @@ "moduleExtensions": { "//:extensions.bzl%bazel_android_deps": { "general": { - "bzlTransitiveDigest": "QEIqMr4FzAzaC9gUQqkVGRYfqdCS9MXzPlDCt5zC8H0=", + "bzlTransitiveDigest": "ViQGEDr/pPfdaylbQ9kIMC61dAyi2clQRXxliJle+HM=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { @@ -2291,7 +2291,7 @@ }, "//:extensions.bzl%bazel_build_deps": { "general": { - "bzlTransitiveDigest": "QEIqMr4FzAzaC9gUQqkVGRYfqdCS9MXzPlDCt5zC8H0=", + "bzlTransitiveDigest": "ViQGEDr/pPfdaylbQ9kIMC61dAyi2clQRXxliJle+HM=", "accumulatedFileDigests": { "@@//src/test/tools/bzlmod:MODULE.bazel.lock": "69df266de27085a01662606690412507656529aa965f177330f46eafea22c456", "@@//:MODULE.bazel": "f0f6c040c50ad1d3555157b29dea32260bdaf5cc7205dfc346d4b1b6b008baca" @@ -2661,7 +2661,7 @@ }, "//:extensions.bzl%bazel_test_deps": { "general": { - "bzlTransitiveDigest": "QEIqMr4FzAzaC9gUQqkVGRYfqdCS9MXzPlDCt5zC8H0=", + "bzlTransitiveDigest": "ViQGEDr/pPfdaylbQ9kIMC61dAyi2clQRXxliJle+HM=", "accumulatedFileDigests": {}, "envVariables": {}, "generatedRepoSpecs": { diff --git a/src/test/py/bazel/test_base.py b/src/test/py/bazel/test_base.py index 959cb9342ccc85..fb7b6ec43e4ba1 100644 --- a/src/test/py/bazel/test_base.py +++ b/src/test/py/bazel/test_base.py @@ -584,6 +584,14 @@ def _EnvMap(self, env_remove=None, env_add=None): # that by checking for TEST_TMPDIR. env['TEST_TMPDIR'] = TestBase.GetEnv('TEST_TMPDIR') env['TMP'] = self._temp + + if TestBase.IsDarwin(): + # Make sure rules_jvm_external works in ipv6 only environment + # https://github.com/bazelbuild/rules_jvm_external?tab=readme-ov-file#ipv6-support + env['COURSIER_OPTS'] = TestBase.GetEnv( + 'COURSIER_OPTS', '-Djava.net.preferIPv6Addresses=true' + ) + if env_remove: for e in env_remove: if e in env: diff --git a/src/test/shell/bazel/bazel_test_test.sh b/src/test/shell/bazel/bazel_test_test.sh index c017bea9b35794..309c4c0885ce40 100755 --- a/src/test/shell/bazel/bazel_test_test.sh +++ b/src/test/shell/bazel/bazel_test_test.sh @@ -705,8 +705,12 @@ exit 1 EOF chmod +x true.sh flaky.sh false.sh + # The next two lines ensure that the test passes in IPv6-only networks. + export JAVA_TOOL_OPTIONS="-Djava.net.preferIPv6Addresses=true" + export STARTUP_OPTS="--host_jvm_args=-Djava.net.preferIPv6Addresses=true" + # We do not use sandboxing so we can trick to be deterministically flaky - bazel --ignore_all_rc_files test --experimental_ui_debug_all_events \ + bazel --ignore_all_rc_files "$STARTUP_OPTS" test --experimental_ui_debug_all_events \ --spawn_strategy=standalone //:flaky &> $TEST_log \ || fail "//:flaky should have passed with flaky support" [ -f "${FLAKE_FILE}" ] || fail "Flaky test should have created the flake-file!" @@ -720,7 +724,7 @@ EOF cat bazel-testlogs/flaky/test.log &> $TEST_log assert_equals "pass" "$(awk "NR == $(wc -l < $TEST_log)" $TEST_log)" - bazel --ignore_all_rc_files test --experimental_ui_debug_all_events //:pass \ + bazel --ignore_all_rc_files "$STARTUP_OPTS" test --experimental_ui_debug_all_events //:pass \ &> $TEST_log || fail "//:pass should have passed" expect_log_once "PASS.*: //:pass" expect_log_once "PASSED" @@ -729,7 +733,7 @@ EOF cat bazel-testlogs/flaky/test.log &> $TEST_log assert_equals "pass" "$(tail -1 bazel-testlogs/flaky/test.log)" - bazel --ignore_all_rc_files test --experimental_ui_debug_all_events //:fail \ + bazel --ignore_all_rc_files "$STARTUP_OPTS" test --experimental_ui_debug_all_events //:fail \ &> $TEST_log && fail "//:fail should have failed" \ || true expect_log_n "FAIL.*: //:fail (.*/fail/test_attempts/attempt_..log)" 2 diff --git a/src/tools/bzlmod/utils.bzl b/src/tools/bzlmod/utils.bzl index a78f84dee1ae7d..82a4d58b2354ab 100644 --- a/src/tools/bzlmod/utils.bzl +++ b/src/tools/bzlmod/utils.bzl @@ -135,4 +135,9 @@ def _module_repo_name(module): module_name = module["name"] if module_name in _WELL_KNOWN_MODULES: return module_name + + # TODO(pcloudy): Simplify the following logic after we upgrade to 7.1 + if get_canonical_repo_name("rules_cc").endswith("~"): + return "{}~".format(module_name) + return "{}~{}".format(module_name, module["version"]) diff --git a/third_party/BUILD b/third_party/BUILD index 912beec0fe1947..c923ef9a8ce739 100644 --- a/third_party/BUILD +++ b/third_party/BUILD @@ -12,6 +12,7 @@ filegroup( "//third_party/android_dex:srcs", "//third_party/cncf_udpa:srcs", "//third_party/def_parser:srcs", + "//third_party/googleapis:srcs", "//third_party/grpc:srcs", "//third_party/grpc-java:srcs", "//third_party/ijar:srcs", @@ -33,9 +34,8 @@ filegroup( "//third_party/py/frozendict:srcs", "//third_party/py/mock:srcs", "//third_party/py/six:srcs", + "//third_party/remoteapis:srcs", "//third_party/zlib:srcs", - "@googleapis//:srcs", - "@remoteapis//:srcs", ], )