Skip to content

Commit

Permalink
[build] Use MODULE.bazel for all dependencies
Browse files Browse the repository at this point in the history
The WORKSPACE.bzlmod is no longer load-bearing, so as of this commit
it should be possible to consume Drake as module external of other
downstream projects (instead of as a repository external). However,
note that the Drake module only provides Drake itself -- it doesn't
yet export any public externals (e.g., eigen) for re-use. That is
still future work (along with other customizations).

Detailed changes:

Adjust the pkg_config repository rule to convert canonical repository
names back into their apparent names.

Adjust some of our hard-coded runfiles paths (header_lint test, wheel
build snopt, drake_models parse_test) to align with the new canonical
repository names. Since we're doing surgery on the parse_test anyway,
we also refactor it to use sharding for improved latency.

Adjust our lcm native code loader to accommodate the new runfiles
layout.

Adjust labels used by our (non-symbolic) macros to only ever refer to
drake labels, not external labels. Textual macros resolve labels in
the workspace context of the code calling them, not Drake. Therefore
they must only ever refer to Drake, since Drake's externals are now
invisible (by default) with bzlmod. We introduce Drake aliases for the
externals so that we can use a safe labels in our macros. (This fix is
only necessary for macros which we expect downstream code to call,
i.e., macros with out a "drake_..." prefix in their name. We still
have plenty of other drake-specific macros that refer to non-drake
labels, but that's not a problem.) The longer term fix for this will
probably be switching from textual macros to symbolic macros, but we
don't attempt that here.

Adjust CMake logic to edit MODULE.bazel to opt-out of any overridden
externals, so that the WORKSPACE.bzlmod replacements take effect.
  • Loading branch information
jwnimmer-tri committed Dec 20, 2024
1 parent 56b0160 commit 3f76c2f
Show file tree
Hide file tree
Showing 18 changed files with 352 additions and 173 deletions.
19 changes: 15 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,8 @@ project(drake
# (e.g., `-DCMAKE_BUILD_TYPE=Release`) and install Drake using those settings.
#
# We'll do that by converting the settings to generated Bazel inputs:
# - a `WORKSPACE.bazel` file that specifies dependencies; and
# - a tweaked copy of our `MODULE.bazel` to opt-out of some dependencies; and
# - a completely new `WORKSPACE.bzlmod` to replace what we opted-out of; and
# - a `.bazelrc` file that specifies configuration choices.
# and then running the `@drake//:install` program from that temporary workspace.

Expand Down Expand Up @@ -77,8 +78,6 @@ else()
endif()
endif()

# The version passed to find_package(Bazel) should match the
# minimum_bazel_version value in the call to versions.check() in WORKSPACE.
set(MINIMUM_BAZEL_VERSION 7.4)
find_package(Bazel ${MINIMUM_BAZEL_VERSION} MODULE)
if(NOT Bazel_FOUND)
Expand Down Expand Up @@ -350,6 +349,9 @@ endfunction()
set(BAZEL_WORKSPACE_EXTRA)
set(BAZEL_WORKSPACE_EXCLUDES)

# Our cmake/WORKSPACE.bzlmod always provides @python.
list(APPEND BAZEL_WORKSPACE_EXCLUDES "python")

macro(override_repository NAME)
set(repo "${CMAKE_CURRENT_BINARY_DIR}/external/workspace/${NAME}")
string(APPEND BAZEL_WORKSPACE_EXTRA
Expand Down Expand Up @@ -541,10 +543,19 @@ endforeach()
# name `drake_build_cwd` isn't important, it just needs to be unique. Note,
# however, that the macOS wheel builds also need to know this path, so if it
# ever changes, tools/wheel/macos/build-wheel.sh will also need to be updated.
file(READ "${PROJECT_SOURCE_DIR}/MODULE.bazel" BAZEL_MODULE_CONTENTS)
foreach(BAZEL_WORKSPACE_EXCLUDE ${BAZEL_WORKSPACE_EXCLUDES})
string(REGEX REPLACE
"\"${BAZEL_WORKSPACE_EXCLUDE}\","
"# ${BAZEL_WORKSPACE_EXCLUDE} comes from WORKSPACE.bzlmod"
BAZEL_MODULE_CONTENTS "${BAZEL_MODULE_CONTENTS}")
endforeach()
file(WRITE
"${CMAKE_CURRENT_BINARY_DIR}/drake_build_cwd/MODULE.bazel"
"${BAZEL_MODULE_CONTENTS}")
configure_file(cmake/bazel.rc.in drake_build_cwd/.bazelrc @ONLY)
configure_file(cmake/WORKSPACE.bzlmod.in drake_build_cwd/WORKSPACE.bzlmod @ONLY)
file(CREATE_LINK "${PROJECT_SOURCE_DIR}/.bazeliskrc" drake_build_cwd/.bazeliskrc SYMBOLIC)
file(CREATE_LINK "${PROJECT_SOURCE_DIR}/MODULE.bazel" drake_build_cwd/MODULE.bazel SYMBOLIC)
file(CREATE_LINK "${PROJECT_SOURCE_DIR}/WORKSPACE" drake_build_cwd/WORKSPACE SYMBOLIC)

find_package(Git)
Expand Down
166 changes: 162 additions & 4 deletions MODULE.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,165 @@ register_toolchains(
"//tools/py_toolchain:exec_tools_toolchain",
)

# TODO(#20731) Move all of our dependencies from WORKSPACE.bzlmod into this
# file, so that downstream projects can consume Drake exclusively via bzlmod
# (and so that we can delete our WORKSPACE files prior to Bazel 9 which drops
# suppose for it).
# TODO(#20731) More improvements are still needed to our MODULE organization:
# - Switch public API dependencies (e.g., eigen) to use modules.
# - Export public API and ABI dependencies (e.g., eigen, zlib) as extensions.
# - Provide better configuation options for choosing dependencies.
# - Adjust the wheel build to build more dependencies as Bazel modules.
# - Deprecate non-bzlmod use of Drake downstream.

drake_dep_repositories = use_extension(
"//tools/workspace:default.bzl",
"drake_dep_repositories",
)
use_repo(
drake_dep_repositories,
"abseil_cpp_internal",
"bazelisk",
"blas",
"buildifier",
"cc",
"ccd_internal",
"clang_cindex_python3_internal",
"clarabel_cpp_internal",
"clp_internal",
"coinutils_internal",
"com_jidesoft_jide_oss",
"common_robotics_utilities_internal",
"commons_io",
"conex_internal",
"csdp_internal",
"curl_internal",
"dm_control_internal",
"doxygen",
"drake_models",
"eigen",
"fcl_internal",
"fmt",
"gflags",
"gfortran",
"github3_py_internal",
"gklib_internal",
"glib",
"glx",
"googlebenchmark",
"gtest",
"gurobi",
"gymnasium_py",
"gz_math_internal",
"gz_utils_internal",
"highway_internal",
"ipopt",
"ipopt_internal_fromsource",
"lapack",
"lcm",
"libblas",
"libjpeg_turbo_internal",
"libpng_internal",
"libtiff_internal",
"meshcat",
"metis_internal",
"mosek",
"mpmath_py_internal",
"msgpack_internal",
"mujoco_menagerie_internal",
"mumps_internal",
"mypy_extensions_internal",
"mypy_internal",
"nanoflann_internal",
"nasm",
"net_sf_jchart2d",
"nlohmann_internal",
"nlopt_internal",
"onetbb_internal",
"opencl",
"opengl",
"openusd_internal",
"org_apache_xmlgraphics_commons",
"osqp_internal",
"picosha2_internal",
"poisson_disk_sampling_internal",
"pybind11",
"pycodestyle",
"python",
"qdldl_internal",
"qhull_internal",
"ros_xacro_internal",
"rules_python_drake_constants",
"scs_internal",
"sdformat_internal",
"snopt",
"spdlog",
"spgrid_internal",
"spral_internal",
"stable_baselines3_internal",
"statsjs",
"stduuid_internal",
"styleguide",
"suitesparse_internal",
"sympy_py_internal",
"tinygltf_internal",
"tinyobjloader_internal",
"tinyxml2_internal",
"tomli_internal",
"typing_extensions_internal",
"uritemplate_py_internal",
"usockets_internal",
"uwebsockets_internal",
"voxelized_geometry_tools_internal",
"vtk_internal",
"x11",
"xmlrunner_py",
"yaml_cpp_internal",
"zlib",
)

crate_universe_repositories = use_extension(
"//tools/workspace:default.bzl",
"crate_universe_repositories",
)
use_repo(
crate_universe_repositories,
"crate__amd-0.2.2",
"crate__autocfg-1.4.0",
"crate__blas-0.22.0",
"crate__blas-sys-0.7.1",
"crate__cfg-if-1.0.0",
"crate__clarabel-0.9.0",
"crate__darling-0.14.4",
"crate__darling_core-0.14.4",
"crate__darling_macro-0.14.4",
"crate__derive_builder-0.11.2",
"crate__derive_builder_core-0.11.2",
"crate__derive_builder_macro-0.11.2",
"crate__either-1.13.0",
"crate__enum_dispatch-0.3.13",
"crate__equivalent-1.0.1",
"crate__fnv-1.0.7",
"crate__hashbrown-0.15.2",
"crate__ident_case-1.0.1",
"crate__indexmap-2.7.0",
"crate__itertools-0.11.0",
"crate__itoa-1.0.14",
"crate__lapack-0.19.0",
"crate__lapack-sys-0.14.0",
"crate__lazy_static-1.5.0",
"crate__libc-0.2.168",
"crate__memchr-2.7.4",
"crate__num-complex-0.4.6",
"crate__num-traits-0.2.19",
"crate__once_cell-1.19.0",
"crate__paste-1.0.15",
"crate__proc-macro2-1.0.92",
"crate__quote-1.0.37",
"crate__ryu-1.0.18",
"crate__serde-1.0.216",
"crate__serde_derive-1.0.216",
"crate__serde_json-1.0.133",
"crate__strsim-0.10.0",
"crate__syn-1.0.109",
"crate__syn-2.0.90",
"crate__thiserror-1.0.69",
"crate__thiserror-impl-1.0.69",
"crate__unicode-ident-1.0.14",
)
11 changes: 0 additions & 11 deletions WORKSPACE.bzlmod
Original file line number Diff line number Diff line change
Expand Up @@ -5,18 +5,7 @@

workspace(name = "drake")

load("//tools/workspace:default.bzl", "add_default_workspace")

add_default_workspace(bzlmod = True)

# Add some special heuristic logic for using CLion with Drake.
load("//tools/clion:repository.bzl", "drake_clion_environment")

drake_clion_environment()

load("@bazel_skylib//lib:versions.bzl", "versions")

# This needs to be in WORKSPACE or a repository rule for native.bazel_version
# to actually be defined. The minimum_bazel_version value should match the
# version passed to the find_package(Bazel) call in the root CMakeLists.txt.
versions.check(minimum_bazel_version = "7.4")
5 changes: 1 addition & 4 deletions bindings/pydrake/common/test_utilities/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ load(

package(default_visibility = [
"//bindings/pydrake:__subpackages__",
"//tools/workspace/drake_models:__pkg__",
])

# This determines how `PYTHONPATH` is configured.
Expand All @@ -31,10 +32,6 @@ drake_py_library(
name = "algebra_test_util_py",
testonly = True,
srcs = ["algebra_test_util.py"],
visibility = [
"//bindings/pydrake/autodiffutils:__pkg__",
"//bindings/pydrake/symbolic:__pkg__",
],
deps = [
":module_py",
],
Expand Down
11 changes: 0 additions & 11 deletions cmake/WORKSPACE.bzlmod.in
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
workspace(name = "drake")

load("//:cmake/external/workspace/conversion.bzl", "split_cmake_list")
load("//tools/workspace/python:repository.bzl", "python_repository")
load("//tools/workspace:default.bzl", "add_default_workspace")

# Use Drake's python repository rule to interrogate the interpreter chosen by
# the CMake find_program stanza, in support of compiling our C++ bindings.
Expand All @@ -15,12 +13,3 @@ python_repository(

# Custom repository rules injected by CMake.
@BAZEL_WORKSPACE_EXTRA@

# The list of repositories already provided via BAZEL_WORKSPACE_EXTRA.
_BAZEL_WORKSPACE_EXCLUDES = split_cmake_list("@BAZEL_WORKSPACE_EXCLUDES@")

# For anything not already overridden, use Drake's default externals.
add_default_workspace(
repository_excludes = ["python"] + _BAZEL_WORKSPACE_EXCLUDES,
bzlmod = True,
)
11 changes: 4 additions & 7 deletions tools/install/libdrake/header_lint.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,10 @@ load("//tools/skylark:sh.bzl", "sh_test")
# without consulting Drake's build system maintainers (see #7451). Keep this
# list in sync with test/header_dependency_test.py.
_ALLOWED_EXTERNALS = [
"eigen",
"fmt",
"lcm",
"spdlog",

# The entries that follow are defects; we should work to remove them.
"zlib",
"+drake_dep_repositories+eigen",
"+drake_dep_repositories+fmt",
"+drake_dep_repositories+lcm",
"+drake_dep_repositories+spdlog",
]

# Drake's allowed list of public preprocessor definitions. The only things
Expand Down
5 changes: 2 additions & 3 deletions tools/skylark/py.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,10 @@ load(
)

# All of Drake's Python code should depend on our requirements.txt pins, so we
# add it as a data dependency to every python rule. If this particular build
# doesn't use a requirements.txt, then the file will be empty (and thus inert).
# add it as a data dependency to every python rule.

def _add_requirements(data):
return (data or []) + ["@python//:requirements.txt"]
return (data or []) + ["@drake//tools/workspace/python:requirements"]

def py_binary(name, *, data = None, **kwargs):
_py_binary(
Expand Down
2 changes: 1 addition & 1 deletion tools/skylark/pybind.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ def pybind_py_library(
copts = cc_copts + EXTRA_PYBIND_COPTS,
# Always link to pybind11.
deps = [
"@pybind11",
"@drake//tools/workspace/pybind11",
] + cc_deps,
**kwargs
)
Expand Down
3 changes: 2 additions & 1 deletion tools/wheel/wheel_builder/common.py
Original file line number Diff line number Diff line change
Expand Up @@ -88,7 +88,8 @@ def create_snopt_tgz(*, snopt_path, output):
output_base = subprocess.check_output(
command, cwd=resource_root,
stderr=subprocess.DEVNULL, encoding='utf-8').strip()
bazel_snopt = os.path.join(output_base, 'external/snopt')
bazel_snopt = os.path.join(
output_base, 'external/+drake_dep_repositories+snopt')

# Ask Bazel to fetch SNOPT from its default git pin.
command = [
Expand Down
Loading

0 comments on commit 3f76c2f

Please sign in to comment.