Polish Bazel external dependencies (#20097)

Clean up and polish Bazel's external dependencies declarations: - Gathered all repository declarations in `repositories.bzl` file. - Removed most of repo definitions in `distdir_deps.bzl` and renamed it to `workspace_deps.bzl`, the rest repo definitions are only used in WORKSPACE suffix and test setup. - Added a repo cache for the workspace repos so that we can decouple repo cache used by - dependencies used to build and test Bazel: cached by `//src:test_repos` - dependencies embedded in MODULE.tools: cached by `@bazel_tools_repo_cache//:files` - dependencies embedded in WORKSPACE suffix: cached by `@workspace_repo_cache//:files` - Removed unused macros from `distdir.bzl` This PR largely simplifies `distdir_deps.bzl`, which is also loaded in internal codebase because `gen_workspace_stanza` is a build rule. Closes #20042. PiperOrigin-RevId: 580212706 Change-Id: I91a9f7bbf89b9af15fdb98f387d95a40a89e7700
bazelbuild · Nov 8, 2023 · 33a313c · 33a313c
1 parent cb91b9e
commit 33a313c
Show file tree

Hide file tree

Showing 17 changed files with 371 additions and 180 deletions.
diff --git a/BUILD b/BUILD
@@ -70,7 +70,7 @@ filegroup(
     srcs = [
         ":WORKSPACE",
         ":distdir.bzl",
-        ":distdir_deps.bzl",
+        ":workspace_deps.bzl",
     ],
     visibility = [
         "//src/test/shell/bazel:__subpackages__",

diff --git a/MODULE.bazel b/MODULE.bazel
@@ -52,7 +52,7 @@ local_path_override(
 )
 
 # The following Bazel modules are not direct dependencies for building Bazel,
-# but are required for visibility from DIST_ARCHIVE_REPOS in distdir_deps.bzl
+# but are required for visibility from DIST_ARCHIVE_REPOS in repositories.bzl
 bazel_dep(name = "apple_support", version = "1.5.0")
 bazel_dep(name = "abseil-cpp", version = "20220623.1")
 bazel_dep(name = "c-ares", version = "1.15.0")
@@ -211,7 +211,7 @@ java_toolchains = use_extension("@rules_java//java:extensions.bzl", "toolchains"
 use_repo(
     java_toolchains,
     "local_jdk",
-    # The following are required for visibility from TEST_REPOS in distdir_deps.bzl
+    # The following are required for visibility in //src:test_repos
     "remote_java_tools",
     "remote_java_tools_darwin_arm64",
     "remote_java_tools_darwin_x86_64",
@@ -273,6 +273,7 @@ use_repo(
     "openjdk_macos_x86_64_vanilla",
     "openjdk_win_arm64_vanilla",
     "openjdk_win_vanilla",
+    "workspace_repo_cache",
 )
 
 # Required only by `--extra_toolchains=@local_config_cc//:cc-toolchain-arm64_windows` from .bazelrc

diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
diff --git a/distdir.bzl b/distdir.bzl
@@ -13,9 +13,7 @@
 # limitations under the License.
 """Defines a repository rule that generates an archive consisting of the specified files to fetch"""
 
-load("//:distdir_deps.bzl", "DEPS_BY_NAME")
 load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts")
-load("//tools/build_defs/repo:http.bzl", "http_archive", "http_file", "http_jar")
 
 _BUILD = """
 load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
@@ -57,29 +55,26 @@ _distdir_tar = repository_rule(
     attrs = _distdir_tar_attrs,
 )
 
-def distdir_tar(name, archives, sha256, urls, dirname, dist_deps = None):
+def distdir_tar(name, dist_deps):
     """Creates a repository whose content is a set of tar files.
 
     Args:
       name: repo name.
-      archives: list of tar file names.
-      sha256: map of tar file names to SHAs.
-      urls: map of tar file names to URL lists.
-      dirname: output directory in repo.
       dist_deps: map of repo names to dict of archive, sha256, and urls.
     """
-    if dist_deps:
-        for dep, info in dist_deps.items():
-            archive_file = info["archive"]
-            archives.append(archive_file)
-            sha256[archive_file] = info["sha256"]
-            urls[archive_file] = info["urls"]
+    archives = []
+    sha256 = {}
+    urls = {}
+    for _, info in dist_deps.items():
+        archive_file = info["archive"]
+        archives.append(archive_file)
+        sha256[archive_file] = info["sha256"]
+        urls[archive_file] = info["urls"]
     _distdir_tar(
         name = name,
         archives = archives,
         sha256 = sha256,
         urls = urls,
-        dirname = dirname,
     )
 
 def _repo_cache_tar_impl(ctx):
@@ -133,63 +128,3 @@ repo_cache_tar = repository_rule(
     implementation = _repo_cache_tar_impl,
     attrs = _repo_cache_tar_attrs,
 )
-
-def dist_http_archive(name, **kwargs):
-    """Wraps http_archive, providing attributes like sha and urls from the central list.
-
-    dist_http_archive wraps an http_archive invocation, but looks up relevant attributes
-    from distdir_deps.bzl so the user does not have to specify them.
-
-    Args:
-      name: repo name
-      **kwargs: see http_archive for allowed args.
-    """
-    info = DEPS_BY_NAME[name]
-    if "patch_args" not in kwargs:
-        kwargs["patch_args"] = info.get("patch_args")
-    if "patches" not in kwargs:
-        kwargs["patches"] = info.get("patches")
-    if "strip_prefix" not in kwargs:
-        kwargs["strip_prefix"] = info.get("strip_prefix")
-    http_archive(
-        name = name,
-        sha256 = info["sha256"],
-        urls = info["urls"],
-        **kwargs
-    )
-
-def dist_http_file(name, **kwargs):
-    """Wraps http_file, providing attributes like sha and urls from the central list.
-
-    dist_http_file wraps an http_file invocation, but looks up relevant attributes
-    from distdir_deps.bzl so the user does not have to specify them.
-
-    Args:
-      name: repo name
-      **kwargs: see http_file for allowed args.
-    """
-    info = DEPS_BY_NAME[name]
-    http_file(
-        name = name,
-        sha256 = info["sha256"],
-        urls = info["urls"],
-        **kwargs
-    )
-
-def dist_http_jar(name, **kwargs):
-    """Wraps http_jar, providing attributes like sha and urls from the central list.
-
-    dist_http_jar wraps an http_jar invocation, but looks up relevant attributes
-    from distdir_deps.bzl so the user does not have to specify them.
-
-    Args:
-      name: repo name
-      **kwargs: see http_jar for allowed args.
-    """
-    info = DEPS_BY_NAME[name]
-    http_jar(
-        name = name,
-        sha256 = info["sha256"],
-        urls = info["urls"],
-        **kwargs
-    )
diff --git a/extensions.bzl b/extensions.bzl
@@ -16,9 +16,9 @@
 
 """
 
-load("//:distdir.bzl", "dist_http_archive", "repo_cache_tar")
-load("//:distdir_deps.bzl", "DIST_ARCHIVE_REPOS")
-load("//:repositories.bzl", "embedded_jdk_repositories")
+load("//:distdir.bzl", "distdir_tar", "repo_cache_tar")
+load("//:repositories.bzl", "DIST_ARCHIVE_REPOS", "android_deps_repos", "bazelci_rules_repo", "embedded_jdk_repositories")
+load("//:workspace_deps.bzl", "WORKSPACE_REPOS")
 load("//src/main/res:winsdk_configure.bzl", "winsdk_configure")
 load("//src/test/shell/bazel:list_source_repository.bzl", "list_source_repository")
 load("//src/tools/bzlmod:utils.bzl", "parse_bazel_module_repos")
@@ -31,19 +31,20 @@ def _bazel_build_deps(_ctx):
     repo_cache_tar(name = "bootstrap_repo_cache", repos = DIST_ARCHIVE_REPOS, dirname = "derived/repository_cache")
     BAZEL_TOOLS_DEPS_REPOS = parse_bazel_module_repos(_ctx, _ctx.path(Label("//src/test/tools/bzlmod:MODULE.bazel.lock")))
     repo_cache_tar(name = "bazel_tools_repo_cache", repos = BAZEL_TOOLS_DEPS_REPOS, lockfile = "//src/test/tools/bzlmod:MODULE.bazel.lock")
+    distdir_tar(name = "workspace_repo_cache", dist_deps = WORKSPACE_REPOS)
 
 bazel_build_deps = module_extension(implementation = _bazel_build_deps)
 
 ### Dependencies for testing Bazel
 def _bazel_test_deps(_ctx):
+    bazelci_rules_repo()
     list_source_repository(name = "local_bazel_source_list")
-    dist_http_archive(name = "bazelci_rules")
     winsdk_configure(name = "local_config_winsdk")
 
 bazel_test_deps = module_extension(implementation = _bazel_test_deps)
 
 ### Dependencies for Bazel Android tools
 def _bazel_android_deps(_ctx):
-    dist_http_archive(name = "desugar_jdk_libs")
+    android_deps_repos()
 
 bazel_android_deps = module_extension(implementation = _bazel_android_deps)
diff --git a/repositories.bzl b/repositories.bzl
@@ -15,46 +15,123 @@
 
 """
 
-load("//:distdir.bzl", "dist_http_file")
+load("//src/tools/bzlmod:utils.bzl", "get_canonical_repo_name")
+load("//tools/build_defs/repo:http.bzl", "http_archive", "http_file")
+
+##################################################################################
+#
+# The list of repositories required while bootstrapping Bazel offline
+#
+##################################################################################
+DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [
+    "abseil-cpp",
+    "apple_support",
+    "bazel_skylib",
+    "blake3",
+    "c-ares",
+    "com_github_grpc_grpc",
+    "com_google_protobuf",
+    "io_bazel_skydoc",
+    "platforms",
+    "rules_cc",
+    "rules_go",
+    "rules_java",
+    "rules_jvm_external",
+    "rules_license",
+    "rules_pkg",
+    "rules_proto",
+    "rules_python",
+    "upb",
+    "zlib",
+    "zstd-jni",
+]] + [(get_canonical_repo_name("com_github_grpc_grpc") + suffix) for suffix in [
+    # Extra grpc dependencies introduced via its module extension
+    "~grpc_repo_deps_ext~bazel_gazelle",  # TODO: Should be a bazel_dep
+    "~grpc_repo_deps_ext~bazel_skylib",  # TODO: Should be removed
+    "~grpc_repo_deps_ext~com_envoyproxy_protoc_gen_validate",
+    "~grpc_repo_deps_ext~com_github_cncf_udpa",
+    "~grpc_repo_deps_ext~com_google_googleapis",
+    "~grpc_repo_deps_ext~envoy_api",
+    "~grpc_repo_deps_ext~rules_cc",  # TODO: Should be removed
+]]
+
+##################################################################################
+#
+# Make sure all URLs below are mirrored to https://mirror.bazel.build
+#
+##################################################################################
 
 def embedded_jdk_repositories():
     """OpenJDK distributions used to create a version of Bazel bundled with the OpenJDK."""
-    dist_http_file(
+    http_file(
         name = "openjdk_linux_vanilla",
+        sha256 = "0c0eadfbdc47a7ca64aeab51b9c061f71b6e4d25d2d87674512e9b6387e9e3a6",
         downloaded_file_path = "zulu-linux-vanilla.tar.gz",
+        url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_x64.tar.gz",
     )
-
-    dist_http_file(
+    http_file(
         name = "openjdk_linux_aarch64_vanilla",
+        sha256 = "1fb64b8036c5d463d8ab59af06bf5b6b006811e6012e3b0eb6bccf57f1c55835",
         downloaded_file_path = "zulu-linux-aarch64-vanilla.tar.gz",
+        url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-linux_aarch64.tar.gz",
     )
 
-    dist_http_file(
-        name = "openjdk_linux_ppc64le_vanilla",
-        downloaded_file_path = "adoptopenjdk-ppc64le-vanilla.tar.gz",
-    )
-
-    dist_http_file(
+    # JDK21 unavailable so use JDK19 instead for linux s390x.
+    http_file(
         name = "openjdk_linux_s390x_vanilla",
+        sha256 = "f2512f9a8e9847dd5d3557c39b485a8e7a1ef37b601dcbcb748d22e49f44815c",
         downloaded_file_path = "adoptopenjdk-s390x-vanilla.tar.gz",
+        url = "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_s390x_linux_hotspot_19.0.2_7.tar.gz",
     )
 
-    dist_http_file(
+    # JDK21 unavailable so use JDK19 instead for linux ppc64le.
+    http_file(
+        name = "openjdk_linux_ppc64le_vanilla",
+        sha256 = "45dde71faf8cbb78fab3c976894259655c8d3de827347f23e0ebe5710921dded",
+        downloaded_file_path = "adoptopenjdk-ppc64le-vanilla.tar.gz",
+        url = "https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20%2B36/OpenJDK20U-jdk_ppc64le_linux_hotspot_20_36.tar.gz",
+    )
+    http_file(
         name = "openjdk_macos_x86_64_vanilla",
+        sha256 = "9639b87db586d0c89f7a9892ae47f421e442c64b97baebdff31788fbe23265bd",
         downloaded_file_path = "zulu-macos-vanilla.tar.gz",
+        url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_x64.tar.gz",
     )
-
-    dist_http_file(
+    http_file(
         name = "openjdk_macos_aarch64_vanilla",
+        sha256 = "2a7a99a3ea263dbd8d32a67d1e6e363ba8b25c645c826f5e167a02bbafaff1fa",
         downloaded_file_path = "zulu-macos-aarch64-vanilla.tar.gz",
+        url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-macosx_aarch64.tar.gz",
     )
-
-    dist_http_file(
+    http_file(
         name = "openjdk_win_vanilla",
+        sha256 = "e9959d500a0d9a7694ac243baf657761479da132f0f94720cbffd092150bd802",
         downloaded_file_path = "zulu-win-vanilla.zip",
+        url = "https://cdn.azul.com/zulu/bin/zulu21.28.85-ca-jdk21.0.0-win_x64.zip",
     )
 
-    dist_http_file(
+    # JDK21 unavailable from zulu, we'll use Microsoft's OpenJDK build instead.
+    http_file(
         name = "openjdk_win_arm64_vanilla",
+        sha256 = "975603e684f2ec5a525b3b5336d6aa0b09b5b7d2d0d9e271bd6a9892ad550181",
         downloaded_file_path = "zulu-win-arm64.zip",
+        url = "https://aka.ms/download-jdk/microsoft-jdk-21.0.0-windows-aarch64.zip",
+    )
+
+def bazelci_rules_repo():
+    """Required by the Bazel CI jobs."""
+    http_archive(
+        name = "bazelci_rules",
+        sha256 = "eca21884e6f66a88c358e580fd67a6b148d30ab57b1680f62a96c00f9bc6a07e",
+        strip_prefix = "bazelci_rules-1.0.0",
+        url = "https://github.com/bazelbuild/continuous-integration/releases/download/rules-1.0.0/bazelci_rules-1.0.0.tar.gz",
+    )
+
+def android_deps_repos():
+    """Required by building the android tools."""
+    http_archive(
+        name = "desugar_jdk_libs",
+        sha256 = "ef71be474fbb3b3b7bd70cda139f01232c63b9e1bbd08c058b00a8d538d4db17",
+        strip_prefix = "desugar_jdk_libs-24dcd1dead0b64aae3d7c89ca9646b5dc4068009",
+        url = "https://github.com/google/desugar_jdk_libs/archive/24dcd1dead0b64aae3d7c89ca9646b5dc4068009.zip",
     )
diff --git a/src/BUILD b/src/BUILD
@@ -587,5 +587,8 @@ filegroup(
         "@remotejdk%s_%s//:WORKSPACE" % (version, os)
         for version in ("17", "21")
         for os in ("macos", "macos_aarch64", "linux", "win")
-    ] + ["@bazel_tools_repo_cache//:files"],
+    ] + [
+        "@bazel_tools_repo_cache//:files",
+        "@workspace_repo_cache//:files",
+    ],
 )
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/BUILD
@@ -1,4 +1,4 @@
-load("//:distdir_deps.bzl", "gen_workspace_stanza")
+load("//:workspace_deps.bzl", "gen_workspace_stanza")
 load("@rules_java//java:defs.bzl", "java_library")
 
 package(
@@ -125,7 +125,7 @@ genrule(
 # 1. bazel build tools/test/CoverageOutputGenerator/java/com/google/devtools/coverageoutputgenerator:coverage_output_generator_zip
 # 2. Copy and rename the zip file with a new version locally.
 # 3. Upload the file under https://mirror.bazel.build/bazel_coverage_output_generator/releases.
-# 4. Update //distdir_deps.bzl and //tools/test/extensions.bzl to point to the new release.
+# 4. Update //workspace_deps.bzl and //tools/test/extensions.bzl to point to the new release.
 gen_workspace_stanza(
     name = "workspace_with_coverage_output_generator",
     out = "coverage.WORKSPACE",

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/cpp/BUILD
@@ -1,5 +1,5 @@
+load("//:workspace_deps.bzl", "gen_workspace_stanza")
 load("@rules_java//java:defs.bzl", "java_library")
-load("//:distdir_deps.bzl", "gen_workspace_stanza")
 
 package(
     default_applicable_licenses = ["//:license"],

diff --git a/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD b/src/main/java/com/google/devtools/build/lib/bazel/rules/java/BUILD
@@ -1,4 +1,4 @@
-load("//:distdir_deps.bzl", "gen_workspace_stanza")
+load("//:workspace_deps.bzl", "gen_workspace_stanza")
 load("@rules_java//java:defs.bzl", "java_library")
 
 package(

diff --git a/src/test/java/com/google/devtools/build/lib/blackbox/framework/BUILD b/src/test/java/com/google/devtools/build/lib/blackbox/framework/BUILD
@@ -1,5 +1,5 @@
 load("@rules_java//java:defs.bzl", "java_library", "java_test")
-load("//:distdir_deps.bzl", "gen_workspace_stanza")
+load("//:workspace_deps.bzl", "gen_workspace_stanza")
 
 java_library(
     name = "deps",

diff --git a/src/test/py/bazel/BUILD b/src/test/py/bazel/BUILD
@@ -1,6 +1,6 @@
 load("@bazel_pip_dev_deps//:requirements.bzl", "requirement")
 load("@rules_python//python:defs.bzl", "py_library", "py_test")
-load("//:distdir_deps.bzl", "gen_workspace_stanza")
+load("//:workspace_deps.bzl", "gen_workspace_stanza")
 
 package(default_visibility = ["//visibility:private"])
 

diff --git a/src/test/shell/BUILD b/src/test/shell/BUILD
@@ -1,5 +1,5 @@
 load("@rules_python//python:defs.bzl", "py_test")
-load("//:distdir_deps.bzl", "gen_workspace_stanza")
+load("//:workspace_deps.bzl", "gen_workspace_stanza")
 
 package(default_visibility = ["//visibility:private"])
 

diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
@@ -1,4 +1,4 @@
-load("//:distdir_deps.bzl", "gen_workspace_stanza")
+load("//:workspace_deps.bzl", "gen_workspace_stanza")
 
 package(default_visibility = ["//visibility:private"])
 

diff --git a/src/test/shell/bazel/verify_workspace.sh b/src/test/shell/bazel/verify_workspace.sh
@@ -40,7 +40,7 @@ fi
 source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
   || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
 
-WORKSPACE_FILES=("$(rlocation io_bazel/WORKSPACE)" "$(rlocation io_bazel/distdir_deps.bzl)")
+WORKSPACE_FILES=("$(rlocation io_bazel/WORKSPACE)" "$(rlocation io_bazel/workspace_deps.bzl)")
 
 # base maven repository URLs can return 404s.
 URL_ALLOWLIST=("https://dl.google.com/android/maven2" "https://repo1.maven.org/maven2")