Use venv for Python dependencies (macOS)

Create a series of scripts to set up and manage a Python virtual
environment that can be used to build (and test) Drake. Remove Python
packages from macOS setup. Add dependencies to the virtual environment
as needed.

.gitignore

@@ -28,3 +28,4 @@
 
 # User-specified configuration
 /user.bazelrc
+/setup/mac/source_distribution/requirements.txt

bindings/pydrake/BUILD.bazel

@@ -48,6 +48,7 @@ drake_py_library(
     ],
     deps = [
         "//bindings/pydrake/common:_init_py",
+        "@venv",
     ],
 )
 
@@ -399,6 +400,7 @@ drake_py_binary(
         ":all_py",
         "@mypy_internal//:mypy",
         "@stable_baselines3_internal//:stable_baselines3",
+        "@venv",
     ],
 )
 

common/proto/BUILD.bazel

@@ -41,6 +41,7 @@ drake_py_library(
     imports = ["."],
     deps = [
         "//lcmtypes:lcmtypes_drake_py",
+        "@venv",
     ],
 )
 

doc/doxygen_cxx/BUILD.bazel

@@ -23,6 +23,7 @@ drake_py_library(
         "//doc/pydrake:__pkg__",
         "//tools/workspace/pybind11:__pkg__",
     ],
+    deps = ["@venv"],
 )
 
 drake_py_unittest(

examples/hardware_sim/BUILD.bazel

@@ -89,6 +89,7 @@ drake_py_library(
     srcs = ["test/hardware_sim_test_common.py"],
     deps = [
         "@rules_python//python/runfiles",
+        "@venv",
     ],
 )
 

geometry/BUILD.bazel

@@ -571,6 +571,7 @@ drake_py_binary(
     visibility = [
         "//bindings/pydrake/visualization:__pkg__",
     ],
+    deps = ["@venv"],
 )
 
 drake_cc_googletest(

geometry/render_gltf_client/BUILD.bazel

@@ -360,6 +360,7 @@ drake_py_unittest(
     ],
     deps = [
         "@rules_python//python/runfiles",
+        "@venv",
     ],
 )
 

multibody/plant/images/BUILD.bazel

@@ -9,6 +9,7 @@ PYTHON_SCRIPTS = [
 drake_py_unittest(
     name = "scripts_test",
     data = PYTHON_SCRIPTS,
+    deps = ["@venv"],
 )
 
 add_lint_tests(

setup/mac/binary_distribution/install_prereqs.sh

@@ -6,13 +6,18 @@
 set -euxo pipefail
 
 with_update=1
+with_python_deps=1
 
 while [ "${1:-}" != "" ]; do
   case "$1" in
     # Do NOT call brew update during execution of this script.
     --without-update)
       with_update=0
       ;;
+    # Do NOT install Python (pip) dependencies.
+    --without-python-dependencies)
+      with_python_deps=0
+      ;;
     *)
       echo 'Invalid command line argument' >&2
       exit 5
@@ -67,4 +72,6 @@ if ! command -v pip3.12 &>/dev/null; then
   exit 2
 fi
 
-pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt"
+if [[ "${with_python_deps}" -eq 1 ]]; then
+  pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt"
+fi

setup/mac/install_prereqs.sh

@@ -5,7 +5,7 @@
 
 set -euxo pipefail
 
-binary_distribution_args=()
+binary_distribution_args=(--without-python-dependencies)
 source_distribution_args=()
 
 while [ "${1:-}" != "" ]; do
@@ -48,3 +48,12 @@ source "${BASH_SOURCE%/*}/source_distribution/install_prereqs.sh" \
 # script should be run once per user who develops with source distributions.
 
 source "${BASH_SOURCE%/*}/source_distribution/install_prereqs_user_environment.sh"
+
+# Bootstrap the venv. The `setup` is required here (nothing else runs it), but
+# the `sync` is not strictly requied (bazel will run it for us). We do it here
+# anyway so that the user gets immediate, streaming feedback about problems
+# with internet downloads, or other kinds of glitches. (When run from Bazel,
+# stderr text is batched up until much later, not streamed.)
+
+"${BASH_SOURCE%/*}/../../tools/workspace/venv/setup"
+"${BASH_SOURCE%/*}/../../tools/workspace/venv/sync"

setup/mac/source_distribution/install_prereqs.sh

@@ -49,13 +49,7 @@ if [[ "${with_test_only}" -eq 1 ]]; then
   brew bundle --file="${BASH_SOURCE%/*}/Brewfile-test-only" --no-lock
 fi
 
-if ! command -v pip3.12 &>/dev/null; then
-  echo 'ERROR: pip3.12 is NOT installed. The post-install step for the python@3.12 formula may have failed.' >&2
+if ! command -v python3.12 &>/dev/null; then
+  echo 'ERROR: python3.12 is NOT installed. The post-install step for the python@3.12 formula may have failed.' >&2
   exit 2
 fi
-
-pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements.txt"
-
-if [[ "${with_test_only}" -eq 1 ]]; then
-  pip3.12 install --break-system-packages -r "${BASH_SOURCE%/*}/requirements-test-only.txt"
-fi

setup/mac/source_distribution/requirements-base.in

@@ -0,0 +1,25 @@
+# PyPI packages to make available as Drake's bazel `@venv` repository.
+
+# WARNING: If you change this file, then it is up to you to regenerate
+# the "locked" requirements.txt file via:
+#
+#     tools/workspace/venv/upgrade
+
+# Packages that we are happy to take the most recent version any time we
+# update requirements.  The versions of these packages are "locked" in the
+# requirements.txt file and change any time it updates.  This should be
+# the overwhelming majority of our requirements.
+
+ipython
+ipywidgets
+matplotlib
+notebook
+Pillow
+pydot
+PyYAML
+
+# The following are constrained or pinned version of packages.
+
+# Keep `black` and `isort` fixed until we want to explicitly update.
+black == 22.10.0
+isort == 5.1.3

setup/mac/source_distribution/requirements-test.in

@@ -0,0 +1,18 @@
+# PyPI packages to make available as Drake's bazel `@venv` repository.
+
+# WARNING: If you change this file, then it is up to you to regenerate
+# the "locked" requirements.txt file via:
+#
+#     tools/workspace/venv/upgrade
+
+# Packages that we are happy to take the most recent version any time we
+# update requirements.  The versions of these packages are "locked" in the
+# requirements.txt file and change any time it updates.  This should be
+# the overwhelming majority of our requirements.
+
+flask
+six
+u-msgpack-python
+websockets
+
+# The following are constrained or pinned version of packages.