CI Linux: Consolidate jobs

.github/workflows/ci-linux-incremental.yml

@@ -60,9 +60,9 @@ jobs:
           uninstall_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))
           build_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))
           if [ -n "$uninstall_targets" ]; then
-              echo "build_targets=$uninstall_targets reconfigure $build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
+              echo "build_targets=$uninstall_targets reconfigure $build_targets" >> $GITHUB_OUTPUT
           else
-              echo "build_targets=$build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
+              echo "build_targets=$build_targets" >> $GITHUB_OUTPUT
           fi
           cat $GITHUB_OUTPUT
       - uses: actions/checkout@v4
@@ -92,7 +92,7 @@ jobs:
       from_docker_target: "with-targets"
       from_docker_tag: "dev"
       docker_targets: "with-targets"
-      targets: "${{needs.changed_files.outputs.build_targets}} doc-html ptest-nodoc"
+      targets: "${{needs.changed_files.outputs.build_targets}} ci-build-with-fallback doc-html ptest-nodoc"
       tox_system_factors: >-
         ["ubuntu-focal",
          "ubuntu-noble",
@@ -108,6 +108,29 @@ jobs:
       docker_push_repository: ghcr.io/${{ github.repository }}/
       max_parallel: 8
 
+  constraints_pkgs-norequirements:
+    needs: [changed_files]
+    uses: ./.github/workflows/docker.yml
+    with:
+      # Build incrementally from published Docker image
+      incremental: true
+      free_disk_space: true
+      from_docker_repository: ghcr.io/sagemath/sage/
+      from_docker_target: "with-targets-pre"
+      from_docker_tag: "dev"
+      docker_targets: "with-targets-pre"
+      targets_pre: "${{needs.changed_files.outputs.build_targets}} all-sage-local python3-ensure tox-ensure sagelib-tox-sagepython-constraints_pkgs-norequirements"
+      tox_system_factors: >-
+        ["ubuntu-focal",
+         "ubuntu-noble",
+         "debian-bookworm",
+         "fedora-40",
+         "debian-bullseye-i386"]
+      tox_packages_factors: >-
+          ["standard"]
+      docker_push_repository: ghcr.io/${{ github.repository }}/
+      max_parallel: 16
+
   site:
     needs: [changed_files]
     uses: ./.github/workflows/docker.yml

.github/workflows/ci-linux.yml

@@ -34,7 +34,7 @@ permissions:
 
 jobs:
 
-  # standard-pre and standard (without ptest) for the default platform (used by build.yml etc.)
+  # standard (without ptest) for the default platform (used by build.yml etc.)
   default:
     uses: ./.github/workflows/docker.yml
     with:
@@ -54,45 +54,43 @@ jobs:
   # All platforms. This duplicates the default platform, but why not,
   # it makes it more robust regarding random timeouts.
 
-  standard-pre:
+  standard:
     if: ${{ success() || failure() }}
     uses: ./.github/workflows/docker.yml
     with:
       # Build from scratch
-      docker_targets: "with-system-packages configured with-targets-pre"
+      free_disk_space: true
+      docker_targets: "with-system-packages configured with-targets-pre with-targets with-targets-optional"
       # FIXME: duplicated from env.TARGETS
       targets_pre: all-sage-local
+      targets: build doc-html
+      targets_optional: ptest
       tox_packages_factors: >-
           ["standard"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
-      # Make sure that all "standard-pre" jobs can start simultaneously,
+      # Make sure that all "standard" jobs can start simultaneously,
       # so that runners are available by the time that "default" starts.
       max_parallel: 50
 
-  standard:
+  standard-constraints_pkgs-norequirements:
     if: ${{ success() || failure() }}
-    needs: [standard-pre]
+    needs: [standard]
     uses: ./.github/workflows/docker.yml
     with:
       # Build incrementally from previous stage (pre)
       incremental: true
       free_disk_space: true
       from_docker_repository: ghcr.io/${{ github.repository }}/
       from_docker_target: "with-targets-pre"
-      docker_targets: "with-targets with-targets-optional"
-      # FIXME: duplicated from env.TARGETS
-      targets: build doc-html
-      targets_optional: ptest
+      docker_targets: "with-targets-pre"
+      targets_pre: all-sage-local python3-ensure tox-ensure sagelib-tox-sagepython-constraints_pkgs-norequirements
       tox_packages_factors: >-
           ["standard"]
-      docker_push_repository: ghcr.io/${{ github.repository }}/
-      # Reduce from 30 to 20 because it runs in parallel with 'standard-sitepackages'
-      # and 'minimal-pre' below
-      max_parallel: 20
+      max_parallel: 15
 
   standard-sitepackages:
     if: ${{ success() || failure() }}
-    needs: [standard-pre]
+    needs: [standard]
     uses: ./.github/workflows/docker.yml
     with:
       # Build incrementally from previous stage (pre)
@@ -128,38 +126,22 @@ jobs:
       docker_push_repository: ghcr.io/${{ github.repository }}/
       max_parallel: 8
 
-  minimal-pre:
-    if: ${{ success() || failure() }}
-    uses: ./.github/workflows/docker.yml
-    with:
-      # Build from scratch
-      docker_targets: "with-system-packages configured with-targets-pre"
-      # FIXME: duplicated from env.TARGETS
-      targets_pre: all-sage-local
-      tox_packages_factors: >-
-          ["minimal"]
-      docker_push_repository: ghcr.io/${{ github.repository }}/
-      # Reduced from 30 because it may run in parallel with 'standard' and 'standard-sitepackages' above.
-      # Calibrated for clogging the job pipeline until the "default" job has finished.
-      max_parallel: 24
-
   minimal:
     if: ${{ success() || failure() }}
-    needs: [minimal-pre]
     uses: ./.github/workflows/docker.yml
     with:
-      # Build incrementally from previous stage (pre)
-      incremental: true
+      # Build from scratch
       free_disk_space: true
-      from_docker_repository: ghcr.io/${{ github.repository }}/
-      from_docker_target: "with-targets-pre"
-      docker_targets: "with-targets with-targets-optional"
+      docker_targets: "with-system-packages configured with-targets-pre with-targets with-targets-optional"
       # FIXME: duplicated from env.TARGETS
+      targets_pre: all-sage-local
       targets: build doc-html
       targets_optional: ptest
       tox_packages_factors: >-
           ["minimal"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
+      # Reduced from 30 because it may run in parallel with 'standard' and 'standard-sitepackages' above.
+      # Calibrated for clogging the job pipeline until the "default" job has finished.
       max_parallel: 24
 
   maximal-pre:
@@ -176,7 +158,7 @@ jobs:
           ["maximal"]
       docker_push_repository: ghcr.io/${{ github.repository }}/
 
-  optional-0-o:
+  optional:
     if: ${{ success() || failure() }}
     needs: [maximal-pre]
     uses: ./.github/workflows/docker.yml
@@ -188,42 +170,12 @@ jobs:
       tox_packages_factors: >-
           ["maximal"]
       docker_targets: "with-targets-optional"
-      # [0-9a-o] excludes _, in particular package _develop
-      targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :optional: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep ^[0-9a-o]))'
-
-
-  optional-p-z:
-    if: ${{ success() || failure() }}
-    needs: [optional-0-o]
-    uses: ./.github/workflows/docker.yml
-    with:
-      incremental: true
-      free_disk_space: true
-      from_docker_repository: ghcr.io/${{ github.repository }}/
-      from_docker_target: "with-targets-pre"
-      tox_packages_factors: >-
-          ["maximal"]
-      docker_targets: "with-targets-optional"
-      # [0-9a-o] excludes _, in particular package _develop
-      targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :optional: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep ^[p-z]))'
-
-  experimental-0-o:
-    if: ${{ success() || failure() }}
-    needs: [optional-p-z]
-    uses: ./.github/workflows/docker.yml
-    with:
-      incremental: true
-      free_disk_space: true
-      from_docker_repository: ghcr.io/${{ github.repository }}/
-      from_docker_target: "with-targets-pre"
-      tox_packages_factors: >-
-          ["maximal"]
-      docker_targets: "with-targets-optional"
-      targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep ^[0-9a-o]))'
+      # We remove packages starting with _, in particular package _develop
+      targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :optional: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep -v ^_))'
 
-  experimental-p-z:
+  experimental:
     if: ${{ success() || failure() }}
-    needs: [experimental-0-o]
+    needs: [optional]
     uses: ./.github/workflows/docker.yml
     with:
       incremental: true
@@ -233,4 +185,4 @@ jobs:
       tox_packages_factors: >-
           ["maximal"]
       docker_targets: "with-targets-optional"
-      targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc | grep ^[p-z]))'
+      targets_optional: '$(echo $(export PATH=build/bin:$PATH && sage-package list :experimental: --has-file "spkg-install.in|spkg-install|requirements.txt" --no-file "huge|has_nonfree_dependencies" | grep -v sagemath_doc))'

build/make/Makefile.in

@@ -779,12 +779,14 @@ $(1)-sdist: FORCE python_build sage_setup cython
 		. '$$(SAGE_ROOT)/src/bin/sage-env' && \
 		'$$(SAGE_ROOT)/build/pkgs/$(1)/spkg-src'
 
-# Recursive tox invocation (note - we do not set the environment here).
+# Recursive tox invocation
 # Setting SAGE_SPKG_WHEELS is for the benefit of sagelib's tox.ini
 $(1)-tox-%: FORCE
 	$(AM_V_at)cd '$$(SAGE_ROOT)/build/pkgs/$(1)/src' && \
-		export PATH="$$(SAGE_ORIG_PATH)" && \
-		SAGE_SPKG_WHEELS=$$(SAGE_LOCAL)/var/lib/sage/wheels \
+		. '$$(SAGE_ROOT)/src/bin/sage-src-env-config' && \
+		. '$$(SAGE_ROOT)/src/bin/sage-env-config' && \
+		. '$$(SAGE_ROOT)/src/bin/sage-env' && \
+		SAGE_SPKG_WHEELS=$$(SAGE_VENV)/var/lib/sage/wheels \
 		tox -v -v -v -e $$*
 
 .PHONY: $(1) $(1)-uninstall $(1)-clean $(1)-build-deps $(1)-no-deps $(1)-clean

constraints_pkgs.txt

@@ -0,0 +1,39 @@
+# This "constraints file" can be used for forcing pip
+# (and any tools that delegate to pip, such as pypa/build)
+# to install the distribution packages included in
+# the SageMath monorepository only from their source trees
+# in SAGE_ROOT/pkgs/ instead of from PyPI.
+#
+# Example: Building a sagemath-standard wheel
+#
+# [alice@localhost sage]$ ./bootstrap
+# [alice@localhost sage]$ ./configure
+# [alice@localhost sage]$ export MAKE="make -j16" SAGE_NUM_THREADS=16
+# [alice@localhost sage]$ make all-sage-local
+# [alice@localhost sage]$ export PIP_CONSTRAINT="$(pwd)/constraints_pkgs.txt"
+# [alice@localhost sage]$ ./sage -sh -c 'python3 -m build -v -v pkgs/sagemath-standard'
+#
+# Non-example: Installing the built wheel using the same
+# constraints file will fail because sagemath-standard is one
+# of the distribution packages listed below. It will conflict
+# with the built wheel for sagemath-standard!
+# Use "pkgs/sagemath-standard/constraints_pkgs.txt" instead.
+
+# Reference on the format:
+# https://pip.pypa.io/en/stable/user_guide/#constraints-files
+#
+sage_conf            @ file://${SAGE_ROOT}/pkgs/sage-conf
+sage_docbuild        @ file://${SAGE_ROOT}/pkgs/sage-docbuild
+sage_setup           @ file://${SAGE_ROOT}/pkgs/sage-setup
+sage_sws2rst         @ file://${SAGE_ROOT}/pkgs/sage-sws2rst
+sagemath-bliss       @ file://${SAGE_ROOT}/pkgs/sagemath-bliss
+sagemath-categories  @ file://${SAGE_ROOT}/pkgs/sagemath-categories
+sagemath-coxeter3    @ file://${SAGE_ROOT}/pkgs/sagemath-coxeter3
+sagemath-environment @ file://${SAGE_ROOT}/pkgs/sagemath-environment
+sagemath-mcqd        @ file://${SAGE_ROOT}/pkgs/sagemath-mcqd
+sagemath-meataxe     @ file://${SAGE_ROOT}/pkgs/sagemath-meataxe
+sagemath-objects     @ file://${SAGE_ROOT}/pkgs/sagemath-objects
+sagemath-repl        @ file://${SAGE_ROOT}/pkgs/sagemath-repl
+sagemath-sirocco     @ file://${SAGE_ROOT}/pkgs/sagemath-sirocco
+sagemath-standard    @ file://${SAGE_ROOT}/pkgs/sagemath-standard
+sagemath-tdlib       @ file://${SAGE_ROOT}/pkgs/sagemath-tdlib

pkgs/sagemath-standard/README.rst

@@ -8,24 +8,26 @@ About SageMath
    "Creating a Viable Open Source Alternative to
     Magma, Maple, Mathematica, and MATLAB"
 
-   Copyright (C) 2005-2020 The Sage Development Team
+   Copyright (C) 2005-2024 The Sage Development Team
 
    https://www.sagemath.org
 
 SageMath fully supports all major Linux distributions, recent versions of macOS, and Windows (Windows Subsystem for Linux).
 
-The traditional and recommended way to install SageMath is from source via Sage-the-distribution (https://www.sagemath.org/download-source.html).  Sage-the-distribution first builds a large number of open source packages from source (unless it finds suitable versions installed in the system) and then installs the Sage Library (sagelib, implemented in Python and Cython).
+See https://doc.sagemath.org/html/en/installation/index.html
+for general installation instructions.
 
 
-About this experimental pip-installable source distribution
------------------------------------------------------------
+About this pip-installable distribution package
+-----------------------------------------------
 
-This pip-installable source distribution `sagemath-standard` is an experimental distribution of the Sage Library.  Use at your own risk.
+This pip-installable source distribution `sagemath-standard` is a
+distribution of the Sage Library.
 
 Building `sagemath-standard` has a large number of system packages as prerequisites.
 See https://doc.sagemath.org/html/en/installation/source.html#linux-recommended-installation
 for partial lists for various systems.
 
-The connection to the system environment is facilitated through the https://pypi.org/project/sage-conf/ distribution package.
+The connection to the system environment is facilitated through the https://pypi.org/project/sage-conf/ distribution package; for step-by-step installation instructions, see https://github.com/sagemath/sage/blob/develop/README.md#alternative-installation-using-pypi
 
 A modularization effort is in progress with the goal of making it possible to install parts of the Sage Library with fewer prerequisites. https://github.com/sagemath/sage/issues/29705

pkgs/sagemath-standard/constraints_pkgs.txt

@@ -0,0 +1,21 @@
+# This "constraints file" can be used for forcing pip
+# (and any tools that delegate to pip, such as pypa/build)
+# to install the dependencies of sagemath-standard that are
+# distribution packages included in the SageMath monorepository
+# only from their source trees in SAGE_ROOT/pkgs/
+# instead of from PyPI.
+#
+# Example:
+#
+# [alice@localhost sage]$ ./bootstrap
+# [alice@localhost sage]$ ./configure
+# [alice@localhost sage]$ export MAKE="make -j16" SAGE_NUM_THREADS=16
+# [alice@localhost sage]$ make all-sage-local
+# [alice@localhost sage]$ export PIP_CONSTRAINT="$(pwd)/pkgs/sagemath-standard/constraints_pkgs.txt"
+# [alice@localhost sage]$ ./sage -sh -c 'python3 -m build -v -v pkgs/sagemath-standard'
+#
+# Reference on the format:
+# https://pip.pypa.io/en/stable/user_guide/#constraints-files
+#
+sage_conf            @ file://${SAGE_ROOT}/pkgs/sage-conf
+sage_setup           @ file://${SAGE_ROOT}/pkgs/sage-setup