From 81104a5ab5802787702e0567330a4141efc1d7e9 Mon Sep 17 00:00:00 2001 From: Jeremy Nimmer Date: Sat, 21 Dec 2024 11:40:16 -0800 Subject: [PATCH] [build] Use MODULE.bazel for all dependencies The WORKSPACE.bzlmod is no longer load-bearing, so as of this commit it should be possible to consume Drake as module external of other downstream projects (instead of as a repository external). Notable changes: Adjust the pkg_config and java repository rules to handle the new distinction between canonical vs apparent repository names. Adjust some of our hard-coded runfiles paths (header_lint test, wheel build snopt, drake_models parse_test) to align with the new canonical repository names. Adjust our lcm native code loader to accommodate the new runfiles layout. Add test coverage for EventLog (which uses a distinctive spelling of its import paths in upstream code). Adjust labels used by our (non-symbolic) macros to only ever refer to drake labels, not external labels. Textual macros resolve labels in the workspace context of the code calling them, not Drake. Therefore they must only ever refer to Drake, since Drake's externals are now invisible (by default) with bzlmod. We introduce Drake aliases for the externals so that we can use a safe labels in our macros. (This fix is only necessary for macros which we expect downstream code to call, i.e., macros without a "drake_..." prefix in their name. We still have plenty of other drake-specific macros that refer to non-drake labels, but that's not a problem.) The longer term fix for this will probably be switching from textual macros to symbolic macros, but we don't attempt that here. Adjust CMake logic to edit MODULE.bazel to opt-out of any overridden externals, so that the WORKSPACE.bzlmod replacements take effect. Document our stability promises for our module extension. --- CMakeLists.txt | 20 +- MODULE.bazel | 201 +++++++++++++++++- WORKSPACE.bzlmod | 16 +- cmake/WORKSPACE.bzlmod.in | 11 - doc/_pages/stable.md | 27 ++- tools/install/libdrake/header_lint.bzl | 11 +- tools/skylark/py.bzl | 5 +- tools/skylark/pybind.bzl | 2 +- tools/wheel/wheel_builder/common.py | 3 +- tools/workspace/default.bzl | 165 ++++++++++---- .../workspace/drake_models/test/parse_test.py | 2 +- tools/workspace/java.bzl | 7 +- tools/workspace/lcm/package.BUILD.bazel | 59 ++--- .../lcm/test/no_lcm_warnings_test.py | 11 +- tools/workspace/pkg_config.BUILD.tpl | 2 +- tools/workspace/pkg_config.bzl | 14 +- tools/workspace/pybind11/BUILD.bazel | 8 + tools/workspace/python/BUILD.bazel | 15 +- tools/workspace/workspace_bzlmod_sync_test.py | 72 +++++-- 19 files changed, 489 insertions(+), 162 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 13531879aa47..1e8769e19a1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,8 @@ project(drake # (e.g., `-DCMAKE_BUILD_TYPE=Release`) and install Drake using those settings. # # We'll do that by converting the settings to generated Bazel inputs: -# - a `WORKSPACE.bazel` file that specifies dependencies; and +# - a tweaked copy of our `MODULE.bazel` to opt-out of some dependencies; and +# - a completely new `WORKSPACE.bzlmod` to replace what we opted-out of; and # - a `.bazelrc` file that specifies configuration choices. # and then running the `@drake//:install` program from that temporary workspace. @@ -77,8 +78,7 @@ else() endif() endif() -# The version passed to find_package(Bazel) should match the -# minimum_bazel_version value in the call to versions.check() in WORKSPACE. +# This version number should match bazel_compatibility in MODULE.bazel. set(MINIMUM_BAZEL_VERSION 7.4) find_package(Bazel ${MINIMUM_BAZEL_VERSION} MODULE) if(NOT Bazel_FOUND) @@ -350,6 +350,9 @@ endfunction() set(BAZEL_WORKSPACE_EXTRA) set(BAZEL_WORKSPACE_EXCLUDES) +# Our cmake/WORKSPACE.bzlmod always provides @python. +list(APPEND BAZEL_WORKSPACE_EXCLUDES "python") + macro(override_repository NAME) set(repo "${CMAKE_CURRENT_BINARY_DIR}/external/workspace/${NAME}") string(APPEND BAZEL_WORKSPACE_EXTRA @@ -541,10 +544,19 @@ endforeach() # name `drake_build_cwd` isn't important, it just needs to be unique. Note, # however, that the macOS wheel builds also need to know this path, so if it # ever changes, tools/wheel/macos/build-wheel.sh will also need to be updated. +file(READ "${PROJECT_SOURCE_DIR}/MODULE.bazel" BAZEL_MODULE_CONTENTS) +foreach(BAZEL_WORKSPACE_EXCLUDE ${BAZEL_WORKSPACE_EXCLUDES}) + string(REGEX REPLACE + "\"${BAZEL_WORKSPACE_EXCLUDE}\"," + "# ${BAZEL_WORKSPACE_EXCLUDE} comes from WORKSPACE.bzlmod" + BAZEL_MODULE_CONTENTS "${BAZEL_MODULE_CONTENTS}") +endforeach() +file(WRITE + "${CMAKE_CURRENT_BINARY_DIR}/drake_build_cwd/MODULE.bazel" + "${BAZEL_MODULE_CONTENTS}") configure_file(cmake/bazel.rc.in drake_build_cwd/.bazelrc @ONLY) configure_file(cmake/WORKSPACE.bzlmod.in drake_build_cwd/WORKSPACE.bzlmod @ONLY) file(CREATE_LINK "${PROJECT_SOURCE_DIR}/.bazeliskrc" drake_build_cwd/.bazeliskrc SYMBOLIC) -file(CREATE_LINK "${PROJECT_SOURCE_DIR}/MODULE.bazel" drake_build_cwd/MODULE.bazel SYMBOLIC) file(CREATE_LINK "${PROJECT_SOURCE_DIR}/WORKSPACE" drake_build_cwd/WORKSPACE SYMBOLIC) find_package(Git) diff --git a/MODULE.bazel b/MODULE.bazel index 390b5780574a..6a186bbc91ed 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -4,7 +4,13 @@ # This file lists Drake's external dependencies as known to bzlmod. It is used # in concert with WORKSPACE.bzlmod (which has the workspace-style externals). -module(name = "drake") +module( + name = "drake", + # This version number should match MINIMUM_BAZEL_VERSION in CMakeLists.txt. + bazel_compatibility = [">=7.4.0"], +) + +# Add starlark rules. bazel_dep(name = "apple_support", version = "1.17.1", repo_name = "build_bazel_apple_support") # noqa bazel_dep(name = "bazel_features", version = "1.22.0") @@ -17,6 +23,8 @@ bazel_dep(name = "rules_python", version = "0.40.0") bazel_dep(name = "rules_rust", version = "0.56.0") bazel_dep(name = "rules_shell", version = "0.3.0") +# Customize our toolchains. + cc_configure = use_extension( "@rules_cc//cc:extensions.bzl", "cc_configure_extension", @@ -24,11 +32,190 @@ cc_configure = use_extension( use_repo(cc_configure, "local_config_cc") register_toolchains( - "//tools/py_toolchain:toolchain", - "//tools/py_toolchain:exec_tools_toolchain", + "@drake//tools/py_toolchain:toolchain", + "@drake//tools/py_toolchain:exec_tools_toolchain", +) + +# Load dependencies which are "public", i.e., made available to downstream +# projects. +# +# Downstream projects may load the same `drake_dep_repositories` module +# extension shown below and call its `use_repo` with whatever list of +# repositories they desire to cite from their project. It's safe to call +# `use_repo` on a subset of this list, or not call it at all downstream. +# Its only effect on a downstream project is to make the repository name +# visible to BUILD rules; Drake's own use of the repository is unaffected. + +drake_dep_repositories = use_extension( + "@drake//tools/workspace:default.bzl", + "drake_dep_repositories", +) +use_repo( + drake_dep_repositories, + "blas", + "buildifier", + "drake_models", + "eigen", + "fmt", + "gflags", + "glib", + "glx", + "gtest", + "gurobi", + "lapack", + "lcm", + "libblas", + "liblapack", + "meshcat", + "mosek", + "opencl", + "opengl", + "pybind11", + "pycodestyle", + "python", + "spdlog", + "styleguide", + "x11", + "zlib", +) + +# Load dependencies which are "private", i.e., not available for use by +# downstream projects. These are all "internal use only". + +internal_repositories = use_extension( + "@drake//tools/workspace:default.bzl", + "internal_repositories", +) +use_repo( + internal_repositories, + "abseil_cpp_internal", + "bazelisk", + "cc", + "ccd_internal", + "clang_cindex_python3_internal", + "clarabel_cpp_internal", + "clp_internal", + "coinutils_internal", + "com_jidesoft_jide_oss", + "common_robotics_utilities_internal", + "commons_io", + "conex_internal", + "csdp_internal", + "curl_internal", + "dm_control_internal", + "doxygen", + "fcl_internal", + "gfortran", + "github3_py_internal", + "gklib_internal", + "googlebenchmark", + "gymnasium_py", + "gz_math_internal", + "gz_utils_internal", + "highway_internal", + "ipopt", + "ipopt_internal_fromsource", + "libjpeg_turbo_internal", + "libpng_internal", + "libtiff_internal", + "metis_internal", + "mpmath_py_internal", + "msgpack_internal", + "mujoco_menagerie_internal", + "mumps_internal", + "mypy_extensions_internal", + "mypy_internal", + "nanoflann_internal", + "nasm", + "net_sf_jchart2d", + "nlohmann_internal", + "nlopt_internal", + "onetbb_internal", + "openusd_internal", + "org_apache_xmlgraphics_commons", + "osqp_internal", + "picosha2_internal", + "poisson_disk_sampling_internal", + "qdldl_internal", + "qhull_internal", + "ros_xacro_internal", + "rules_python_drake_constants", + "scs_internal", + "sdformat_internal", + "snopt", + "spgrid_internal", + "spral_internal", + "stable_baselines3_internal", + "statsjs", + "stduuid_internal", + "suitesparse_internal", + "sympy_py_internal", + "tinygltf_internal", + "tinyobjloader_internal", + "tinyxml2_internal", + "tomli_internal", + "typing_extensions_internal", + "uritemplate_py_internal", + "usockets_internal", + "uwebsockets_internal", + "voxelized_geometry_tools_internal", + "vtk_internal", + "xmlrunner_py", + "yaml_cpp_internal", +) + +internal_crate_universe_repositories = use_extension( + "//tools/workspace:default.bzl", + "internal_crate_universe_repositories", +) +use_repo( + internal_crate_universe_repositories, + "crate__amd-0.2.2", + "crate__autocfg-1.4.0", + "crate__blas-0.22.0", + "crate__blas-sys-0.7.1", + "crate__cfg-if-1.0.0", + "crate__clarabel-0.9.0", + "crate__darling-0.14.4", + "crate__darling_core-0.14.4", + "crate__darling_macro-0.14.4", + "crate__derive_builder-0.11.2", + "crate__derive_builder_core-0.11.2", + "crate__derive_builder_macro-0.11.2", + "crate__either-1.13.0", + "crate__enum_dispatch-0.3.13", + "crate__equivalent-1.0.1", + "crate__fnv-1.0.7", + "crate__hashbrown-0.15.2", + "crate__ident_case-1.0.1", + "crate__indexmap-2.7.0", + "crate__itertools-0.11.0", + "crate__itoa-1.0.14", + "crate__lapack-0.19.0", + "crate__lapack-sys-0.14.0", + "crate__lazy_static-1.5.0", + "crate__libc-0.2.168", + "crate__memchr-2.7.4", + "crate__num-complex-0.4.6", + "crate__num-traits-0.2.19", + "crate__once_cell-1.19.0", + "crate__paste-1.0.15", + "crate__proc-macro2-1.0.92", + "crate__quote-1.0.37", + "crate__ryu-1.0.18", + "crate__serde-1.0.216", + "crate__serde_derive-1.0.216", + "crate__serde_json-1.0.133", + "crate__strsim-0.10.0", + "crate__syn-1.0.109", + "crate__syn-2.0.90", + "crate__thiserror-1.0.69", + "crate__thiserror-impl-1.0.69", + "crate__unicode-ident-1.0.14", ) -# TODO(#20731) Move all of our dependencies from WORKSPACE.bzlmod into this -# file, so that downstream projects can consume Drake exclusively via bzlmod -# (and so that we can delete our WORKSPACE files prior to Bazel 9 which drops -# suppose for it). +# TODO(#20731) More improvements are still needed to our MODULE organization: +# - Switch public API dependencies (e.g., eigen) to use modules. +# - Provide better configuation options for choosing dependencies. +# - Adjust the wheel build to build more dependencies as Bazel modules. +# - Deprecate non-bzlmod use of Drake downstream. diff --git a/WORKSPACE.bzlmod b/WORKSPACE.bzlmod index df38712b8dd8..bd09b8271142 100644 --- a/WORKSPACE.bzlmod +++ b/WORKSPACE.bzlmod @@ -1,22 +1,12 @@ # -*- bazel -*- # -# This file lists Drake's workspace-style external dependencies. It is used in -# concert with MODULE.bazel (which has the module-style externals). +# This file lists Drake's workspace-style external dependencies that are only +# needed by Drake Developers instead of downstream projects. Most dependencies +# are listed in MODULE.bazel, instead. workspace(name = "drake") -load("//tools/workspace:default.bzl", "add_default_workspace") - -add_default_workspace(bzlmod = True) - # Add some special heuristic logic for using CLion with Drake. load("//tools/clion:repository.bzl", "drake_clion_environment") drake_clion_environment() - -load("@bazel_skylib//lib:versions.bzl", "versions") - -# This needs to be in WORKSPACE or a repository rule for native.bazel_version -# to actually be defined. The minimum_bazel_version value should match the -# version passed to the find_package(Bazel) call in the root CMakeLists.txt. -versions.check(minimum_bazel_version = "7.4") diff --git a/cmake/WORKSPACE.bzlmod.in b/cmake/WORKSPACE.bzlmod.in index a3eaf1edee2e..2a61dd6559c2 100644 --- a/cmake/WORKSPACE.bzlmod.in +++ b/cmake/WORKSPACE.bzlmod.in @@ -1,8 +1,6 @@ workspace(name = "drake") -load("//:cmake/external/workspace/conversion.bzl", "split_cmake_list") load("//tools/workspace/python:repository.bzl", "python_repository") -load("//tools/workspace:default.bzl", "add_default_workspace") # Use Drake's python repository rule to interrogate the interpreter chosen by # the CMake find_program stanza, in support of compiling our C++ bindings. @@ -15,12 +13,3 @@ python_repository( # Custom repository rules injected by CMake. @BAZEL_WORKSPACE_EXTRA@ - -# The list of repositories already provided via BAZEL_WORKSPACE_EXTRA. -_BAZEL_WORKSPACE_EXCLUDES = split_cmake_list("@BAZEL_WORKSPACE_EXCLUDES@") - -# For anything not already overridden, use Drake's default externals. -add_default_workspace( - repository_excludes = ["python"] + _BAZEL_WORKSPACE_EXCLUDES, - bzlmod = True, -) diff --git a/doc/_pages/stable.md b/doc/_pages/stable.md index e750faa0ba61..2fc6c2006528 100644 --- a/doc/_pages/stable.md +++ b/doc/_pages/stable.md @@ -104,20 +104,31 @@ part of the "Stable API": For Drake's dependencies: -* The `add_default_...` macros defined in `@drake//tools/workspace:default.bzl` - are all part of the Stable API. - * For any Bazel external loaded by these functions (e.g., `"@eigen"`), we - will deprecate it prior to removing our definition of the dependency. - * Excluding any items documented as "internal use only". - * Excluding any items documented with an "experimental" warning. +* When using Bazel to depend on Drake as a Bazel Module (i.e., using bzlmod): + * The extension module + `use_extension("@drake//tools/workspace:default.bzl", "drake_dep_repositories")` + is part of the Stable API, including the names of the repositories it offers + as extensions (e.g., `"eigen"`). + * For any repository provided by the extension, we will deprecate + it prior to removing it. +* When using Bazel to depend on Drake via `WORKSPACE.bazel` (i.e., without + bzlmod): + * The `add_default_...` macros defined in + `@drake//tools/workspace:default.bzl` are all part of the Stable API. + * For any Bazel external loaded by these functions (e.g., `"@eigen"`), we + will deprecate it prior to removing our definition of the dependency. + * Excluding any items documented as "internal use only". + * Excluding any items documented with an "experimental" warning. We may upgrade any of our dependencies to a newer version without prior notice. If you require an older version, you will need to rebuild Drake from source and -pin your own WORKSPACE to refer to the older version of the dependency. +customize your own `WORKSPACE.bazel` or `MODULE.bazel` file to refer to the +older version of the dependency. We may add new dependencies without prior notice. All of our dependencies will either be installed via the host system via our `install_prereqs` scripts, -and/or downloaded at build-time via our `add_default_...` macros, and/or +and/or downloaded at build-time via our `add_default_...` macros (when not +using `bzlmod`) or our `MODULE.bazel` file (when using bzlmod), and/or specified via packaging metadata in the case of `apt` or `pip`. ## LCM messages diff --git a/tools/install/libdrake/header_lint.bzl b/tools/install/libdrake/header_lint.bzl index e6f2c7729db6..a1bb76fc9075 100644 --- a/tools/install/libdrake/header_lint.bzl +++ b/tools/install/libdrake/header_lint.bzl @@ -10,13 +10,10 @@ load("//tools/skylark:sh.bzl", "sh_test") # without consulting Drake's build system maintainers (see #7451). Keep this # list in sync with test/header_dependency_test.py. _ALLOWED_EXTERNALS = [ - "eigen", - "fmt", - "lcm", - "spdlog", - - # The entries that follow are defects; we should work to remove them. - "zlib", + "+drake_dep_repositories+eigen", + "+drake_dep_repositories+fmt", + "+drake_dep_repositories+lcm", + "+drake_dep_repositories+spdlog", ] # Drake's allowed list of public preprocessor definitions. The only things diff --git a/tools/skylark/py.bzl b/tools/skylark/py.bzl index 62b622bce599..de3d4e685fd6 100644 --- a/tools/skylark/py.bzl +++ b/tools/skylark/py.bzl @@ -9,11 +9,10 @@ load( ) # All of Drake's Python code should depend on our requirements.txt pins, so we -# add it as a data dependency to every python rule. If this particular build -# doesn't use a requirements.txt, then the file will be empty (and thus inert). +# add it as a data dependency to every python rule. def _add_requirements(data): - return (data or []) + ["@python//:requirements.txt"] + return (data or []) + ["@drake//tools/workspace/python:requirements"] def py_binary(name, *, data = None, **kwargs): _py_binary( diff --git a/tools/skylark/pybind.bzl b/tools/skylark/pybind.bzl index 7fb31ed79259..4e819971bff0 100644 --- a/tools/skylark/pybind.bzl +++ b/tools/skylark/pybind.bzl @@ -69,7 +69,7 @@ def pybind_py_library( copts = cc_copts + EXTRA_PYBIND_COPTS, # Always link to pybind11. deps = [ - "@pybind11", + "@drake//tools/workspace/pybind11", ] + cc_deps, **kwargs ) diff --git a/tools/wheel/wheel_builder/common.py b/tools/wheel/wheel_builder/common.py index 2669d3e0bc11..4aaf97d178f3 100644 --- a/tools/wheel/wheel_builder/common.py +++ b/tools/wheel/wheel_builder/common.py @@ -88,7 +88,8 @@ def create_snopt_tgz(*, snopt_path, output): output_base = subprocess.check_output( command, cwd=resource_root, stderr=subprocess.DEVNULL, encoding='utf-8').strip() - bazel_snopt = os.path.join(output_base, 'external/snopt') + bazel_snopt = os.path.join( + output_base, 'external/+internal_repositories+snopt') # Ask Bazel to fetch SNOPT from its default git pin. command = [ diff --git a/tools/workspace/default.bzl b/tools/workspace/default.bzl index 4ce1df39d21b..dde4c589ab03 100644 --- a/tools/workspace/default.bzl +++ b/tools/workspace/default.bzl @@ -112,27 +112,11 @@ load("//tools/workspace/xmlrunner_py:repository.bzl", "xmlrunner_py_repository") load("//tools/workspace/yaml_cpp_internal:repository.bzl", "yaml_cpp_internal_repository") # noqa load("//tools/workspace/zlib:repository.bzl", "zlib_repository") -# This is the list of modules that our MODULE.bazel already incorporates. -# It is cross-checked by the workspace_bzlmod_sync_test.py test. -REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES = [ - "build_bazel_apple_support", - "bazel_features", - "bazel_skylib", - "platforms", - "rust_toolchain", - "rules_cc", - "rules_java", - "rules_license", - "rules_python", - "rules_rust", - "rules_shell", -] +# ============================================================================= +# For Bazel projects using Drake as a depedency via the WORKSPACE mechanism. +# ============================================================================= -def add_default_repositories( - excludes = [], - mirrors = DEFAULT_MIRRORS, - *, - bzlmod = False): +def add_default_repositories(excludes = [], mirrors = DEFAULT_MIRRORS): """Declares workspace repositories for all externals needed by drake (other than those built into Bazel, of course). This is intended to be loaded and called from a WORKSPACE file. @@ -141,11 +125,7 @@ def add_default_repositories( excludes: list of string names of repositories to exclude; this can be useful if a WORKSPACE file has already supplied its own external of a given name. - bzlmod: when True, skips repositories declared in our MODULE.bazel; - set this to True if you are using bzlmod. """ - if bzlmod: - excludes = excludes + REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES if "abseil_cpp_internal" not in excludes: abseil_cpp_internal_repository(name = "abseil_cpp_internal", mirrors = mirrors) # noqa if "bazelisk" not in excludes: @@ -377,23 +357,14 @@ def add_default_repositories( if "zlib" not in excludes: zlib_repository(name = "zlib") -def add_default_toolchains( - excludes = [], - *, - bzlmod = False): +def add_default_toolchains(excludes = []): """Register toolchains for each language (e.g., "py") not explicitly excluded and/or not using an automatically generated toolchain. Args: excludes: List of languages for which a toolchain should not be registered. - bzlmod: when True, skips toolchains declared in our MODULE.bazel; - set this to True if you are using bzlmod. """ - if bzlmod: - # All toolchains are in MODULE.bazel already. - return - if "py" not in excludes: native.register_toolchains( "//tools/py_toolchain:toolchain", @@ -407,9 +378,7 @@ def add_default_toolchains( def add_default_workspace( repository_excludes = [], toolchain_excludes = [], - mirrors = DEFAULT_MIRRORS, - *, - bzlmod = False): + mirrors = DEFAULT_MIRRORS): """Declare repositories in this WORKSPACE for each dependency of @drake (e.g., "eigen") that is not explicitly excluded, and register toolchains for each language (e.g., "py") not explicitly excluded and/or not using an @@ -423,16 +392,118 @@ def add_default_workspace( mirrors: Dictionary of mirrors from which to download repository files. See mirrors.bzl file in this directory for the file format and default values. - bzlmod: when True, skips repositories and toolchains declared in our - MODULE.bazel; set this to True if you are using bzlmod. """ - add_default_repositories( - excludes = repository_excludes, - mirrors = mirrors, - bzlmod = bzlmod, - ) - add_default_toolchains( - excludes = toolchain_excludes, - bzlmod = bzlmod, + add_default_repositories(excludes = repository_excludes, mirrors = mirrors) + add_default_toolchains(excludes = toolchain_excludes) + +# ============================================================================= +# For Bazel projects using Drake as a depedency via the MODULE mechanism. +# ============================================================================= + +# This is the list of modules that our MODULE.bazel already incorporates. +# It is cross-checked by the workspace_bzlmod_sync_test.py test. +REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES = [ + "build_bazel_apple_support", + "bazel_features", + "bazel_skylib", + "platforms", + "rust_toolchain", + "rules_cc", + "rules_java", + "rules_license", + "rules_python", + "rules_rust", + "rules_shell", +] + +# This is the list of repositories that Drake provides as a module extension +# for downstream projects; see comments in drake/MODULE.bazel for details. +# It is cross-checked by the workspace_bzlmod_sync_test.py test. +REPOS_EXPORTED = [ + "blas", + "buildifier", + "drake_models", + "eigen", + "fmt", + "gflags", + "glib", + "glx", + "gtest", + "gurobi", + "lapack", + "lcm", + "libblas", + "liblapack", + "meshcat", + "mosek", + "opencl", + "opengl", + "pybind11", + "pycodestyle", + "python", + "spdlog", + "styleguide", + "x11", + "zlib", +] + +def _drake_dep_repositories_impl(module_ctx): + # This sequence should match REPOS_EXPORTED exactly. + # Mismatches will be reported as errors by Bazel. + mirrors = DEFAULT_MIRRORS + blas_repository(name = "blas") + buildifier_repository(name = "buildifier", mirrors = mirrors) + drake_models_repository(name = "drake_models", mirrors = mirrors) + eigen_repository(name = "eigen") + fmt_repository(name = "fmt", mirrors = mirrors) + gflags_repository(name = "gflags", mirrors = mirrors) + glib_repository(name = "glib") + glx_repository(name = "glx") + gtest_repository(name = "gtest", mirrors = mirrors) + gurobi_repository(name = "gurobi") + lapack_repository(name = "lapack") + lcm_repository(name = "lcm", mirrors = mirrors) + libblas_repository(name = "libblas") + liblapack_repository(name = "liblapack") + meshcat_repository(name = "meshcat", mirrors = mirrors) + mosek_repository(name = "mosek", mirrors = mirrors) + opencl_repository(name = "opencl") + opengl_repository(name = "opengl") + pybind11_repository(name = "pybind11", mirrors = mirrors) + pycodestyle_repository(name = "pycodestyle", mirrors = mirrors) + python_repository(name = "python") + spdlog_repository(name = "spdlog", mirrors = mirrors) + styleguide_repository(name = "styleguide", mirrors = mirrors) + x11_repository(name = "x11") + zlib_repository(name = "zlib") + +drake_dep_repositories = module_extension( + implementation = _drake_dep_repositories_impl, + doc = """(Stable API) Provides access to Drake's dependencies for use by + downstream projects. See comments in drake/MODULE.bazel for details.""", +) + +def _internal_repositories_impl(module_ctx): + excludes = ( + REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES + + REPOS_EXPORTED + + ["crate_universe"] ) + add_default_repositories(excludes = excludes) + +internal_repositories = module_extension( + implementation = _internal_repositories_impl, + doc = """(Internal use only) Wraps the add_default_repositories repository + rule into a bzlmod module extension, excluding repositories that are + already covered by modules, drake_dep_repositories, and crate_universe.""", +) + +def _internal_crate_universe_repositories_impl(module_ctx): + crate_universe_repositories(mirrors = DEFAULT_MIRRORS) + +internal_crate_universe_repositories = module_extension( + implementation = _internal_crate_universe_repositories_impl, + doc = """(Internal use only) Wraps the crate_universe repository rules to + be usable as a bzlmod module extension.""", +) diff --git a/tools/workspace/drake_models/test/parse_test.py b/tools/workspace/drake_models/test/parse_test.py index 42f8a20cf339..8e9b6eb6c320 100644 --- a/tools/workspace/drake_models/test/parse_test.py +++ b/tools/workspace/drake_models/test/parse_test.py @@ -19,7 +19,7 @@ def _runfiles_inventory() -> Iterator[tuple[str, Path]]: manifest = runfiles.Create() inventory = Path(manifest.Rlocation( "drake/tools/workspace/drake_models/inventory.txt")) - repo_name = "drake_models/" + repo_name = "+drake_dep_repositories+drake_models/" for line in inventory.read_text(encoding="utf-8").splitlines(): assert line.startswith(repo_name), line filename = line[len(repo_name):].strip() diff --git a/tools/workspace/java.bzl b/tools/workspace/java.bzl index 061c0bb6e4ed..a06abb0250b5 100644 --- a/tools/workspace/java.bzl +++ b/tools/workspace/java.bzl @@ -30,7 +30,7 @@ package(default_visibility = ["//visibility:public"]) else: is_local = False name = "jar" - actual = "@drake_java_internal_maven_{}//jar".format(repo_ctx.name) + actual = "@{}//jar".format(repo_ctx.attr.maven_name) build_content += "alias(name = {name}, actual = {actual})\n".format( name = repr(name), actual = repr(actual), @@ -60,6 +60,7 @@ _internal_drake_java_import = repository_rule( "licenses": attr.string_list(mandatory = True), "local_os_targets": attr.string_list(mandatory = True), "local_jar": attr.string(mandatory = True), + "maven_name": attr.string(mandatory = True), }, implementation = _impl, ) @@ -79,8 +80,9 @@ def drake_java_import( the jar. Otherwise, the maven_jar will be used. The recognized values for OSs in the list of targets are either "linux" or "osx". """ + maven_name = "drake_java_internal_maven_{}".format(name) java_import_external( - name = "drake_java_internal_maven_{}".format(name), + name = maven_name, licenses = licenses, jar_urls = [ x.format(fulljar = maven_jar) @@ -94,4 +96,5 @@ def drake_java_import( licenses = licenses, local_os_targets = local_os_targets, local_jar = local_jar, + maven_name = maven_name, ) diff --git a/tools/workspace/lcm/package.BUILD.bazel b/tools/workspace/lcm/package.BUILD.bazel index 76d549a77041..331adf02b73c 100644 --- a/tools/workspace/lcm/package.BUILD.bazel +++ b/tools/workspace/lcm/package.BUILD.bazel @@ -201,41 +201,41 @@ cc_binary( ) # Downstream users of lcm-python expect to say "import lcm". However, in the -# sandbox the python package is located at lcm/lcm-python/lcm/__init__.py to -# match the source tree structure of LCM; without any special help the import -# would fail. +# sandbox the python package is located at lcm-python/lcm/__init__.py to match +# the source tree structure of LCM; without declaring an `imports = [...]` path +# the import would fail. # # Normally we'd add `imports = ["lcm-python"]` to establish a PYTHONPATH at the -# correct subdirectory, and that almost works. However, because the external -# is named "lcm", Bazel's auto-generated empty "lcm/__init__.py" at the root of -# the sandbox is found first, and prevents the lcm-python subdirectory from -# ever being found. +# correct subdirectory, and that almost works -- except that the native code's +# RUNPATH entries are not quite correct. Even though `./_lcm.so` (the glue) +# resolves correctly in the sandbox, it needs to then load the main library +# `liblcm.so` to operate. That happens via its RUNPATH, but because the RUNPATH +# is relative, when the lcm module is loaded from the wrong sys.path entry, the +# RUNPATH no longer works. # -# To repair this, we provide our own init file at the root of the sandbox that -# overrides the Bazel empty default. Its implementation just delegates to the -# lcm-python init file. (Note that this __init__.py shim is neither used nor -# present in the installed copy of Drake; once Drake is installed, the paths -# are standard and there is no aliasing confusion.) +# To repair this, we'll generate our own init file that pre-loads the shared +# library (using python ctypes with the realpath to the shared library) before +# calling the upstream __init__. # -# Relatedly, within the upstream __init__.py there is a `from ._lcm import` -# statement that loads the compiled C code for LCM python support. Even though -# the `./_lcm.so` (the glue) resolves correctly in the sandbox, it needs to -# then load the main library `liblcm.so` to operate. That happens via its -# RUNPATH, but because the RUNPATH is relative, when the lcm module is loaded -# from the wrong sys.path entry, the RUNPATH no longer works. To work around -# that, we pre-load the shared library before calling the upstream __init__, -# using python ctypes with the realpath to the shared library. +# Note that this generated __init__.py shim is neither used nor present in the +# installed copy of Drake; once Drake is installed, the paths are standard and +# there is no aliasing confusion. generate_file( - name = "__init__.py", + name = "gen/lcm/__init__.py", content = """ import ctypes -import os.path -ctypes.cdll.LoadLibrary(os.path.realpath( - __path__[0] + '/_lcm{extension_suffix}')) -_filename = __path__[0] + \"/lcm-python/lcm/__init__.py\" -with open(_filename) as f: - _code = compile(f.read(), _filename, 'exec') - exec(_code) +from pathlib import Path +# The base_dir refers to the base of our package.BUILD.bazel. +_base_dir = Path(__path__[0]).resolve().parent.parent +# Load the native code. +ctypes.cdll.LoadLibrary(_base_dir / '_lcm{extension_suffix}') +# We need to tweak the upstream __init__ before we run it. +_filename = _base_dir / 'lcm-python/lcm/__init__.py' +_text = _filename.read_text(encoding='utf-8') +# Respell where the native code comes from. +_text = _text.replace('from lcm import _lcm', 'import _lcm') +_text = _text.replace('from lcm._lcm import', 'from _lcm import') +exec(compile(_text, _filename, 'exec')) """.format(extension_suffix = PYTHON_EXTENSION_SUFFIX), visibility = ["//visibility:private"], ) @@ -249,7 +249,8 @@ py_library( py_library( name = "lcm-python", - srcs = ["__init__.py"], # Shim, from the genrule above. + srcs = ["gen/lcm/__init__.py"], # Shim, from the genrule above. + imports = ["gen"], deps = [":lcm-python-upstream"], ) diff --git a/tools/workspace/lcm/test/no_lcm_warnings_test.py b/tools/workspace/lcm/test/no_lcm_warnings_test.py index 8067cd13d240..993b32fa61a6 100644 --- a/tools/workspace/lcm/test/no_lcm_warnings_test.py +++ b/tools/workspace/lcm/test/no_lcm_warnings_test.py @@ -1,7 +1,8 @@ +import os import unittest import warnings -from lcm import LCM +from lcm import EventLog, LCM class Test(unittest.TestCase): @@ -14,3 +15,11 @@ def test_publish(self): with warnings.catch_warnings(): warnings.simplefilter("error", DeprecationWarning) lcm.publish("TEST_CHANNEL", b"") + + def test_event_log(self): + """ + Ensures no crashes on construction / destruction. + """ + dut = EventLog(path=f"{os.environ['TEST_TMPDIR']}/lcm.log", mode="w") + dut.close() + del dut diff --git a/tools/workspace/pkg_config.BUILD.tpl b/tools/workspace/pkg_config.BUILD.tpl index e30075603e62..7742157a5618 100644 --- a/tools/workspace/pkg_config.BUILD.tpl +++ b/tools/workspace/pkg_config.BUILD.tpl @@ -9,7 +9,7 @@ licenses(%{licenses}) package(default_visibility = ["//visibility:public"]) cc_library( - name = %{name}, + name = %{library_name}, srcs = %{srcs}, hdrs = %{hdrs}, copts = %{copts}, diff --git a/tools/workspace/pkg_config.bzl b/tools/workspace/pkg_config.bzl index 738190f88bd3..3187686f87f3 100644 --- a/tools/workspace/pkg_config.bzl +++ b/tools/workspace/pkg_config.bzl @@ -59,6 +59,12 @@ def setup_pkg_config_repository(repository_ctx): pkg_config_paths.insert(0, "/opt/drake-dependencies/share/pkgconfig") pkg_config_paths.insert(0, "/opt/drake-dependencies/lib/pkgconfig") + # Convert the canonical name (e.g., "+_repo_rules+eigen") to its apparent + # name (e.g., "eigen") so that when a BUILD file uses a label which omits + # the target name (e.g., deps = ["@eigen"]) the unabbreviated label (e.g., + # "@eigen//:eigen") will match what we provide here. + library_name = repository_ctx.name.split("+")[-1] + # Check if we can find the required *.pc file of any version. result = _run_pkg_config(repository_ctx, args, pkg_config_paths) if result.error != None: @@ -73,12 +79,12 @@ def setup_pkg_config_repository(repository_ctx): """ load("@drake//tools/skylark:cc.bzl", "cc_library") cc_library( - name = {name}, + name = {library_name}, srcs = ["pkg_config_failed.cc"], visibility = ["//visibility:public"], ) """.format( - name = repr(repository_ctx.name), + library_name = repr(library_name), ), ) return struct(value = True, error = None) @@ -264,8 +270,8 @@ cc_library( "%{licenses}": repr( getattr(repository_ctx.attr, "licenses", []), ), - "%{name}": repr( - repository_ctx.name, + "%{library_name}": repr( + library_name, ), "%{srcs}": repr( getattr(repository_ctx.attr, "extra_srcs", []), diff --git a/tools/workspace/pybind11/BUILD.bazel b/tools/workspace/pybind11/BUILD.bazel index d94f84327827..f9af2b7e8c5d 100644 --- a/tools/workspace/pybind11/BUILD.bazel +++ b/tools/workspace/pybind11/BUILD.bazel @@ -15,6 +15,14 @@ load( "generate_pybind_documentation_header", ) +# This alias provides a single point of control for defining which pybind11 +# library our tools/skylark/pybind.bzl macro should use. +alias( + name = "pybind11", + actual = "@pybind11", + visibility = ["//visibility:public"], +) + exports_files( [ "pybind11-config.cmake", diff --git a/tools/workspace/python/BUILD.bazel b/tools/workspace/python/BUILD.bazel index b77b93ae0dbc..3a88a8f2c7c1 100644 --- a/tools/workspace/python/BUILD.bazel +++ b/tools/workspace/python/BUILD.bazel @@ -1,6 +1,15 @@ -# This file exists to make our directory into a Bazel package, so that our -# neighboring *.bzl file can be loaded elsewhere. - load("//tools/lint:lint.bzl", "add_lint_tests") +# All of Drake's Python code should depend on our requirements.txt pin. This +# filegroup provides a single point of control for any targets in Drake that +# need to depend on changes to our requirements file. If this particular build +# doesn't use a requirements.txt, then the file will be empty (and thus inert). +filegroup( + name = "requirements", + srcs = [ + "@python//:requirements.txt", + ], + visibility = ["//visibility:public"], +) + add_lint_tests() diff --git a/tools/workspace/workspace_bzlmod_sync_test.py b/tools/workspace/workspace_bzlmod_sync_test.py index 3313c44284da..9bd7443e8cb9 100644 --- a/tools/workspace/workspace_bzlmod_sync_test.py +++ b/tools/workspace/workspace_bzlmod_sync_test.py @@ -12,10 +12,11 @@ def _read(self, respath): path = Path(manifest.Rlocation(respath)) return path.read_text(encoding="utf-8") - def _parse_modules(self, content): - """Given the contents of MODULE.bazel, returns a dictionary mapping - from module_name to module_version. + def _parse_modules(self): + """Parses MODULE.bazel to return a dictionary mapping from module_name + to module_version. """ + content = self._read(f"drake/MODULE.bazel") result = {} for line in content.splitlines(): # Only match bazel_dep lines. @@ -32,10 +33,12 @@ def _parse_modules(self, content): result[kwargs["name"]] = kwargs["version"] return result - def _parse_repo_rule_version(self, content): - """Given the contents of a repository.bzl that calls 'github_archive', - returns the version number it pins to. + def _parse_repo_rule_version(self, repo_name): + """Parses tools/workspace/{repo_name}/repository.bzl to find the call + to 'github_archive' and returns the version number it pins to. """ + content = self._read( + f"drake/tools/workspace/{repo_name}/repository.bzl") assert "github_archive" in content, content for line in content.splitlines(): line = line.strip() @@ -61,7 +64,7 @@ def test_version_sync(self): and WORKSPACE. This test ensures that the versions pinned in each file are correctly synchronized. """ - modules = self._parse_modules(self._read(f"drake/MODULE.bazel")) + modules = self._parse_modules() # Don't check modules that are known to be module-only. del modules["bazel_features"] @@ -73,22 +76,22 @@ def test_version_sync(self): self.assertTrue(modules) for module_name, module_version in modules.items(): repo_name = self._module_name_to_repo_name(module_name) - workspace_version = self._parse_repo_rule_version(self._read( - f"drake/tools/workspace/{repo_name}/repository.bzl")) + workspace_version = self._parse_repo_rule_version(repo_name) self.assertEqual(workspace_version, module_version) - def _parse_workspace_already_provided(self, content): - """Given the contents of default.bzl, returns the list of - REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES. + def _parse_workspace_list_constant(self, name): + """Returns the contents of the list constant named `name` in our + tools/workspace/default.bzl. """ + content = self._read("drake/tools/workspace/default.bzl") result = None for line in content.splitlines(): line = line.strip() - if line == "REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES = [": + if line == f"{name} = [": result = list() continue if result is None: - # We haven't seen the REPOS_ALREADY_... line yet. + # We haven't seen the opening line yet. continue if line == "]": break @@ -103,18 +106,49 @@ def test_default_exclude_sync(self): provided by MODULE.bazel. This test ensures that the list is correctly synchronized. """ - modules = self._parse_modules(self._read(f"drake/MODULE.bazel")) + modules = self._parse_modules() # These workspace-only repositories are irrelevant for bzlmod. modules["rust_toolchain"] = None - # Check that default.bzl's constant matches the inventory of modules. - repo_names = sorted([ + repo_names_in_module = sorted([ self._module_name_to_repo_name(module_name) for module_name in modules.keys() ]) - self.assertEqual(repo_names, self._parse_workspace_already_provided( - self._read("drake/tools/workspace/default.bzl"))) + repo_names_in_default = self._parse_workspace_list_constant( + name="REPOS_ALREADY_PROVIDED_BY_BAZEL_MODULES") + self.assertEqual(repo_names_in_module, repo_names_in_default) + + def _parse_module_drake_dep_repositories(self): + """Parses MODULE.bazel to return the list of drake_dep_repositories. + """ + content = self._read(f"drake/MODULE.bazel") + result = None + for line in content.splitlines(): + line = line.strip() + if line == "drake_dep_repositories,": + result = list() + continue + if result is None: + # We haven't seen the opening line yet. + continue + if line == ")": + break + assert line.startswith('"'), line + assert line.endswith('",'), line + result.append(line[1:-2]) + assert result, content + return sorted(result) + + def test_default_exported_sync(self): + """Our default.bzl has a list of REPOS_EXPORTED that must match the + drake_dep_repositories listed in MODULE.bazel. This test ensures that + the lists are correctly synchronized. + """ + repo_names_in_module = self._parse_module_drake_dep_repositories() + repo_names_in_default = self._parse_workspace_list_constant( + name="REPOS_EXPORTED") + self.assertEqual(repo_names_in_module, repo_names_in_default) assert __name__ == '__main__'