Trac #32874: New option "./configure --enable-wheels"

When `./configure --disable-editable` is in use (the default before
#32406), the Sage library is installed using its custom incremental
implementation of `setup.py install` -- which installs its files in
`site-packages` and then removes the files that it does not know about
(the "cleaner").

Problem 1: The cleaner is incompatible with namespace packages and
trying to fix it would lead to complicated and awkward code (#32927).
(Besides, `setup.py install` is deprecated since
https://setuptools.pypa.io/en/latest/history.html#v58-3-0)

Problem 2: Because of the direct installation, we do not have wheels for
sagemath-standard available (even if [[https://trac.sagemath.org/wiki/Re
leaseTours/sage-9.7#Editableinstallationisnowthedefault|make wheels]] is
used explicitly). Such wheels can be useful for making separate venvs.

In this ticket:
- We create a new option `./configure --enable-wheels` (which can be
used both with `./configure --enable-editable` and `./configure
--disable-editable`).
- If enabled, installation of sagelib will go through wheel building and
installation - like we do with all other Python packages. This enables
modularized builds (#34587).
- If disabled, there is no change.

A trivial `./sage -b` is much slower with `./configure --disable-
editable --enable-wheels` compared to `./configure --disable-editable`
(from 28s as per timings in
https://trac.sagemath.org/ticket/32406#comment:21 to 70s).
This ticket brings a number of speed improvements to mitigate this.

URL: https://trac.sagemath.org/32874
Reported by: mkoeppe
Ticket author(s): Matthias Koeppe
Reviewer(s): Kwankyu Lee

.gitignore

@@ -131,9 +131,15 @@ __pycache__/
 # Generated Cython files
 *.so
 **/*.so
+/src/cython_debug
+# Most C and C++ files are generated by Cython and should not
+# be included in the sdist.
 /src/sage/**/*.c
 /src/sage/**/*.cpp
+# C header generated by Cython
 /src/sage/modular/arithgroup/farey_symbol.h
+# List of C and C++ files that are actual source files,
+# NOT generated by Cython. The same list appears in src/MANIFEST.in
 !/src/sage/cpython/debugimpl.c
 !/src/sage/graphs/base/boost_interface.cpp
 !/src/sage/graphs/cliquer/cl.c
@@ -156,7 +162,6 @@ __pycache__/
 !/src/sage/stats/distributions/dgs_gauss_dp.c
 !/src/sage/stats/distributions/dgs_gauss_mp.c
 !/src/sage/symbolic/ginac/*.cpp
-/src/cython_debug
 
 # Temporary build files
 build/temp.*/

Makefile

@@ -101,7 +101,7 @@ pypi-wheels:
 	    rm -f venv/var/lib/sage/installed/$$a-*; \
 	done
 	for a in $(PYPI_WHEEL_PACKAGES); do \
-	    $(MAKE) SAGE_EDITABLE=no $$a; \
+	    $(MAKE) SAGE_EDITABLE=no SAGE_WHEELS=yes $$a; \
 	done
 	@echo "Built wheels are in venv/var/lib/sage/wheels/"
 
@@ -112,7 +112,7 @@ wheels:
 	    rm -f venv/var/lib/sage/installed/$$a-*; \
 	done
 	for a in $(WHEEL_PACKAGES); do \
-	    $(MAKE) SAGE_EDITABLE=no $$a; \
+	    $(MAKE) SAGE_EDITABLE=no SAGE_WHEELS=yes $$a; \
 	done
 	@echo "Built wheels are in venv/var/lib/sage/wheels/"
 

build/bin/sage-build-env

@@ -31,6 +31,10 @@ if [ "x$SAGE_BUILD_ENV_SOURCED" = "x" ]; then
     if [ "x$SAGE_EDITABLE" = "x" ]; then
         export SAGE_EDITABLE="$CONFIGURED_SAGE_EDITABLE"
     fi
+    # Likewise for SAGE_WHEELS
+    if [ "x$SAGE_WHEELS" = "x" ]; then
+        export SAGE_WHEELS="$CONFIGURED_SAGE_WHEELS"
+    fi
 
     # This is usually blank if the system GMP is used, or $SAGE_LOCAL otherwise
     if [ -n "$SAGE_GMP_PREFIX" ]; then

build/bin/sage-build-env-config.in

@@ -58,3 +58,4 @@ export SAGE_SUITESPARSE_PREFIX="@SAGE_SUITESPARSE_PREFIX@"
 export SAGE_CONFIGURE_FFLAS_FFPACK="@SAGE_CONFIGURE_FFLAS_FFPACK@"
 
 export CONFIGURED_SAGE_EDITABLE="@SAGE_EDITABLE@"
+export CONFIGURED_SAGE_WHEELS="@SAGE_WHEELS@"

build/bin/sage-pip-install

@@ -28,7 +28,7 @@ PIP=pip3
 # We should avoid running pip while installing a package because that
 # is prone to race conditions. Therefore, we use a lockfile while
 # running pip. This is implemented in the Python script sage-flock
-LOCK="$SAGE_LOCAL/var/lock/$PIP.lock"
+LOCK="$SAGE_VENV/var/lock/$PIP.lock"
 
 # Trac #33155: Pythons installed using the python.org macOS installers
 # for Python < 3.10 identify macOS Big Sur and newer as "10.16", causing

build/bin/sage-pip-uninstall

@@ -14,7 +14,7 @@ PIP=pip3
 # We should avoid running pip while uninstalling a package because that
 # is prone to race conditions. Therefore, we use a lockfile while
 # running pip. This is implemented in the Python script sage-flock
-LOCK="$SAGE_LOCAL/var/lock/$PIP.lock"
+LOCK="$SAGE_VENV/var/lock/$PIP.lock"
 
     # --disable-pip-version-check: Don't periodically check PyPI to determine whether a new version of pip is available
     # --no-input: Disable prompting for input.

build/pkgs/sage_conf/spkg-install

@@ -12,6 +12,9 @@ fi
 cd src
 if [ "$SAGE_EDITABLE" = yes ]; then
     sdh_pip_editable_install .
+    if [ "$SAGE_WHEELS" = yes ]; then
+        sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
 else
     sdh_pip_install .
 fi

build/pkgs/sage_docbuild/spkg-install

@@ -12,6 +12,9 @@ fi
 cd src
 if [ "$SAGE_EDITABLE" = yes ]; then
     sdh_pip_editable_install .
+    if [ "$SAGE_WHEELS" = yes ]; then
+        sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
 else
     sdh_pip_install .
 fi

build/pkgs/sage_setup/spkg-install

@@ -12,6 +12,9 @@ fi
 cd src
 if [ "$SAGE_EDITABLE" = yes ]; then
     sdh_pip_editable_install .
+    if [ "$SAGE_WHEELS" = yes ]; then
+        sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
 else
     sdh_pip_install .
 fi

build/pkgs/sage_sws2rst/spkg-install

@@ -3,16 +3,21 @@
 # For type=script packages, the build rule in build/make/Makefile sources
 # sage-env but not sage-dist-helpers.
 lib="$SAGE_ROOT/build/bin/sage-dist-helpers"
-. "$lib"
+source "$lib"
 if [ $? -ne 0 ]; then
     echo >&2 "Error: failed to source $lib"
     echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?"
     exit 1
 fi
-set -e
-# We build the wheel directly with "setup.py bdist_wheel", not with "pip wheel",
-# because pip does not handle our symlinks correctly.
-(cd src && sdh_setup_bdist_wheel && sdh_store_and_pip_install_wheel .)
+cd src
+if [ "$SAGE_EDITABLE" = yes ]; then
+    sdh_pip_editable_install .
+    if [ "$SAGE_WHEELS" = yes ]; then
+        sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
+else
+    sdh_pip_install .
+fi
 # For type=script packages, spkg-check is not run
 case "$SAGE_CHECK" in
     yes)

build/pkgs/sagelib/spkg-install

@@ -1,4 +1,15 @@
 #!/usr/bin/env bash
+# From sage-spkg.
+# For type=script packages, the build rule in build/make/Makefile sources
+# sage-env but not sage-dist-helpers.
+lib="$SAGE_ROOT/build/bin/sage-dist-helpers"
+source "$lib"
+if [ $? -ne 0 ]; then
+    echo >&2 "Error: failed to source $lib"
+    echo >&2 "Is $SAGE_ROOT the correct SAGE_ROOT?"
+    exit 1
+fi
+
 if [ "$SAGE_EDITABLE" = yes ]; then
     cd "$SAGE_SRC"
 else
@@ -27,8 +38,6 @@ export SAGE_SRC_ROOT=/doesnotexist
 export SAGE_DOC_SRC=/doesnotexist
 export SAGE_BUILD_DIR=/doesnotexist
 
-export PYTHON="$SAGE_LOCAL/bin/python3"
-
 # We also poison all directories below SAGE_LOCAL.
 export SAGE_PKGCONFIG=/doesnotexist
 export SAGE_SPKG_SCRIPTS=/doesnotexist
@@ -51,17 +60,27 @@ if [ "$SAGE_EDITABLE" = yes ]; then
     # under the old distribution name "sage" (before #30912, which switched to setuptools
     # and renamed the distribution to "sagemath-standard").  There is no clean way to uninstall
     # them, so we just use rm.
-    (cd "$SITEPACKAGESDIR" && rm -rf sage sage_setup sage-[1-9]*.egg-info sage-[1-9]*.dist-info)
-    # Until https://trac.sagemath.org/ticket/34209 switches us to PEP 660 editable wheels
-    export SETUPTOOLS_ENABLE_FEATURES=legacy-editable
-    time python3 -m pip install --verbose --no-deps --no-index --no-build-isolation --isolated --editable . || exit 1
+    (cd "$SITEPACKAGESDIR" && rm -rf sage sage-[1-9]*.egg-info sage-[1-9]*.dist-info)
+    time sdh_pip_editable_install .
+
+    if [ "$SAGE_WHEELS" = yes ]; then
+        # Additionally build a wheel (for use in other venvs)
+        cd $SAGE_PKGS/sagelib/src && time sdh_setup_bdist_wheel && sdh_store_wheel .
+    fi
 else
     # Make sure that an installed old version of sagelib in which sage is an ordinary package
     # does not shadow the namespace package sage during the build.
     (cd "$SITEPACKAGESDIR" && rm -f sage/__init__.py)
     # Likewise, we should remove the egg-link that may have been installed previously.
     (cd "$SITEPACKAGESDIR" && rm -f sagemath-standard.egg-link)
-    time python3 -u setup.py --no-user-cfg build install || exit 1
+
+    if [ "$SAGE_WHEELS" = yes ]; then
+        # Use --no-build-isolation to avoid rebuilds because of dependencies:
+        # Compiling sage/interfaces/sagespawn.pyx because it depends on /private/var/folders/38/wnh4gf1552g_crsjnv2vmmww0000gp/T/pip-build-env-609n5985/overlay/lib/python3.10/site-packages/Cython/Includes/posix/unistd.pxd
+        time sdh_pip_install --no-build-isolation .
+    else
+        time python3 -u setup.py --no-user-cfg build install || exit 1
+    fi
 fi
 
 # Trac #33103: The temp.* directories are large after a full build.

configure.ac

@@ -132,6 +132,12 @@ AC_ARG_ENABLE([editable],
               [AC_SUBST([SAGE_EDITABLE], [$enableval])],
               [AC_SUBST([SAGE_EDITABLE], [yes])])
 
+AC_ARG_ENABLE([wheels],
+              [AS_HELP_STRING([--enable-wheels],
+                              [build wheels for the Sage library and update them on "sage -b"; if disabled, use "make wheels" to build wheels])],
+              [AC_SUBST([SAGE_WHEELS], [$enableval])],
+              [])
+
 # Check whether we are on a supported platform
 AC_CANONICAL_BUILD()
 AC_CANONICAL_HOST()

pkgs/sagemath-standard/setup.py

@@ -15,6 +15,15 @@
     import multiprocessing
     multiprocessing.set_start_method('fork', force=True)
 
+# If build isolation is not in use and setuptools_scm is installed,
+# then its file_finders entry point is invoked, which we don't need.
+# Workaround from ​https://github.com/pypa/setuptools_scm/issues/190#issuecomment-351181286
+try:
+    import setuptools_scm.integration
+    setuptools_scm.integration.find_files = lambda _: []
+except ImportError:
+    pass
+
 #########################################################
 ### Set source directory
 #########################################################
@@ -30,7 +39,7 @@
 ### Configuration
 #########################################################
 
-if len(sys.argv) > 1 and (sys.argv[1] == "sdist" or sys.argv[1] == "egg_info"):
+if len(sys.argv) > 1 and (sys.argv[1] in ["sdist", "egg_info", "dist_info"]):
     sdist = True
 else:
     sdist = False

src/MANIFEST.in

@@ -1,11 +1,47 @@
-global-include *.c *.cc *.cpp *.h *.hh *.hpp *.inc *.py *.pyx *.pxd *.pxi *.rst *.txt *.tex
-
-include MANIFEST.in
-include pyproject.toml
-
-prune .tox
-
 prune sage/ext/interpreters   # In particular, __init__.py must not be present in the distribution; or sage_setup.autogen.interpreters.rebuild will not generate the code
 prune sage_setup
 prune sage_docbuild
 prune doc
+
+#
+# Most C and C++ files are generated by Cython and should not
+# be included in the sdist.
+#
+global-exclude *.c
+global-exclude *.cpp
+
+#
+# List of C and C++ files that are actual source files,
+# NOT generated by Cython. The same list appears in SAGE_ROOT/.gitignore
+#
+include sage/cpython/debugimpl.c
+include sage/graphs/base/boost_interface.cpp
+include sage/graphs/cliquer/cl.c
+include sage/graphs/graph_decompositions/sage_tdlib.cpp
+include sage/libs/eclib/wrap.cpp
+include sage/libs/linkages/padics/relaxed/flint_helper.c
+include sage/misc/inherit_comparison_impl.c
+include sage/modular/arithgroup/farey.cpp
+include sage/modular/arithgroup/sl2z.cpp
+include sage/rings/bernmm/bern_modp.cpp
+include sage/rings/bernmm/bern_modp_util.cpp
+include sage/rings/bernmm/bern_rat.cpp
+include sage/rings/bernmm/bernmm-test.cpp
+include sage/rings/padics/transcendantal.c
+include sage/rings/polynomial/weil/power_sums.c
+include sage/schemes/hyperelliptic_curves/hypellfrob/hypellfrob.cpp
+include sage/schemes/hyperelliptic_curves/hypellfrob/recurrences_ntl.cpp
+include sage/schemes/hyperelliptic_curves/hypellfrob/recurrences_zn_poly.cpp
+include sage/stats/distributions/dgs_bern.c
+include sage/stats/distributions/dgs_gauss_dp.c
+include sage/stats/distributions/dgs_gauss_mp.c
+include sage/symbolic/ginac/*.cpp
+
+global-exclude __pycache__
+global-exclude *.py[co]
+global-exclude *.bak
+global-exclude *.so
+global-exclude *~
+prune .tox
+prune build
+prune dist