Merge branch 'develop' into loops

.ci/create-changes-html.sh

@@ -16,6 +16,12 @@ echo '<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/highli
 echo '<script src="https://cdnjs.cloudflare.com/ajax/libs/highlight.js/11.9.0/highlight.min.js"></script>' >> CHANGES.html
 echo '<script>hljs.highlightAll();</script>' >> CHANGES.html
 cat >> CHANGES.html << EOF
+<style>
+  p.diff a:first-child {
+    font-weight: bold;
+    font-size: x-large;
+  }
+</style>
 <script>
 document.addEventListener('DOMContentLoaded', () => {
 // This URL is hardcoded in the file .github/workflows/doc-publish.yml.

.devcontainer/onCreate-conda.sh

@@ -2,8 +2,11 @@
 set -e
 
 # Create conda environment
-conda install mamba -n base -c conda-forge -y
-mamba env create --file src/environment-dev-3.11-linux.yml || mamba env update --file src/environment-dev-3.11-linux.yml
+conda config --env --add channels conda-forge
+conda config --env --set channel_priority strict
+conda update -y --all --override-channels -c conda-forge
+conda install mamba=1 -n base -y
+mamba env create -y --file environment-dev-3.11-linux.yml || mamba env update -y --file environment-dev-3.11-linux.yml
 conda init bash
 
 # Build sage

.github/workflows/ci-conda.yml

@@ -59,7 +59,7 @@ jobs:
         with:
           path: ~/conda_pkgs_dir
           key:
-            ${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11.yml') }}
+            ${{ runner.os }}-conda-${{ hashFiles('environment-3.11.yml') }}
 
       - name: Setup Conda environment
         uses: conda-incubator/setup-miniconda@v3
@@ -70,7 +70,7 @@ jobs:
           channels: conda-forge
           channel-priority: true
           activate-environment: sage
-          environment-file: src/${{ matrix.conda-env }}-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
+          environment-file: ${{ matrix.conda-env }}-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
 
       - name: Print Conda environment
         shell: bash -l {0}

.github/workflows/ci-macos.yml

@@ -95,7 +95,7 @@ jobs:
       - name: make dist
         run: |
           ./configure --enable-download-from-upstream-url && make dist
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: "dist/*.tar.gz"
           name: dist
@@ -119,7 +119,7 @@ jobs:
     steps:
       - uses: actions/checkout@v4
         if: "!contains(matrix.tox_system_factor, 'nobootstrap')"
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           path: .
           name: dist
@@ -147,7 +147,7 @@ jobs:
         run: |
           mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
         if: always()
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: artifacts
           name: ${{ env.LOGS_ARTIFACT_NAME }}

.github/workflows/ci-meson.yml

@@ -39,7 +39,7 @@ jobs:
         with:
           path: ~/conda_pkgs_dir
           key:
-            ${{ runner.os }}-conda-${{ hashFiles('src/environment-3.11-linux.yml') }}
+            ${{ runner.os }}-conda-${{ hashFiles('environment-3.11-linux.yml') }}
 
       - name: Compiler cache
         uses: hendrikmuhs/ccache-action@v1.2
@@ -55,7 +55,7 @@ jobs:
           channels: conda-forge
           channel-priority: true
           activate-environment: sage
-          environment-file: src/environment-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
+          environment-file: environment-${{ matrix.python }}-${{ startsWith(matrix.os, 'macos') && (startsWith(runner.arch, 'ARM') && 'macos' || 'macos-x86_64') || 'linux' }}.yml
 
       - name: Print Conda environment
         shell: bash -l {0}

.github/workflows/conda-lock-update.py

@@ -4,30 +4,53 @@
 import subprocess
 
 script_dir = Path(__file__).resolve().parent
-root_dir = script_dir / '..' / '..'
+root_dir = script_dir / ".." / ".."
 
 subprocess.run([str(root_dir / "bootstrap-conda")])
 
 platforms = {
     "linux-64": "linux",
     "linux-aarch64": "linux-aarch64",
     "osx-64": "macos-x86_64",
-    "osx-arm64": "macos"
-    #"win-64": "win",
+    "osx-arm64": "macos",
+    # "win-64": "win",
 }
 pythons = ["3.9", "3.10", "3.11"]
 tags = ["", "-dev"]
-sources = ["", "src"]
 
 for platform_key, platform_value in platforms.items():
     for python in pythons:
         for tag in tags:
-            for src in sources:
-                env_file = root_dir / src / f"environment{tag}-{python}.yml"
-                lock_file = root_dir / src / f"environment{tag}-{python}-{platform_value}"
+            env_file = root_dir / f"environment{tag}-{python}.yml"
+            lock_file = root_dir / f"environment{tag}-{python}-{platform_value}"
+            lock_file_gen = root_dir / f"environment{tag}-{python}-{platform_value}.yml"
 
-                if not env_file.exists():
-                    continue
+            if not env_file.exists():
+                continue
 
-                print(f"Updating lock file for {env_file} at {lock_file}", flush=True)
-                subprocess.run(["conda-lock", "--channel", "conda-forge", "--kind", "env", "--platform", platform_key, "--file", str(env_file), "--lockfile", str(lock_file), "--filename-template", str(lock_file)])
+            print(f"Updating lock file for {env_file} at {lock_file_gen}", flush=True)
+            subprocess.run(
+                [
+                    "conda-lock",
+                    "--mamba",
+                    "--channel",
+                    "conda-forge",
+                    "--kind",
+                    "env",
+                    "--platform",
+                    platform_key,
+                    "--file",
+                    str(env_file),
+                    "--lockfile",
+                    str(lock_file),
+                    "--filename-template",
+                    str(lock_file),
+                ],
+                check=True,
+            )
+
+            # Add conda env name to lock file at beginning
+            with open(lock_file_gen, "r+") as f:
+                content = f.read()
+                f.seek(0, 0)
+                f.write(f"name: sage{tag}\n{content}")

.github/workflows/doc-build.yml

@@ -114,59 +114,77 @@ jobs:
       - name: Start container
         id: container
         # Try to continue when "exporting to GitHub Actions Cache" failed with timeout
-        if: (success() || failure())
         run: |
           docker run --name BUILD -dit \
                      --mount type=bind,src=$(pwd),dst=$(pwd) \
                      --workdir $(pwd) \
                      ${{ env.BUILD_IMAGE }} /bin/sh
 
       #
-      # On PRs and pushes to develop
+      # On pull request and push to develop events
       #
 
+      - name: Get workflow run-id
+        id: get_run_id
+        if: steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/') && github.event_name == 'pull_request'
+        run: |
+          RESPONSE=$(curl -s -L \
+            -H "Accept: application/vnd.github+json" \
+            -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
+            "https://api.github.com/repos/${{ github.repository }}/actions/runs?event=push&branch=develop&status=completed")
+          RUN_ID=$(echo "$RESPONSE" | jq -r --arg name "${{ github.workflow }}" --arg conclusion "success" \
+            '.workflow_runs[] | select(.name == $name and .conclusion == $conclusion) | .id' | head -n 1)
+          echo "RUN_ID=$RUN_ID" >> $GITHUB_ENV
+
+      - name: Download old doc
+        id: download-doc
+        if: steps.get_run_id.outcome == 'success'
+        uses: actions/download-artifact@v4
+        with:
+          name: doc-develop
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          repository: ${{ github.repository }}
+          run-id: ${{ env.RUN_ID }}
+
       - name: Store old doc
         id: worktree
-        if: (success() || failure()) && steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/')
+        if: steps.download-doc.outcome == 'success'
         run: |
           git config --global --add safe.directory $(pwd)
           git config --global user.email "ci-sage@example.com"
           git config --global user.name "Build documentation workflow"
-          mkdir -p doc
-          # Check if we are on PR
+          unzip doc.zip
           PR_NUMBER=""
-          if [[ -n "$GITHUB_REF" ]]; then
-            if [[ "$GITHUB_REF" =~ refs/pull/([0-9]+)/merge ]]; then
-              PR_NUMBER="${BASH_REMATCH[1]}"
-            fi
+          if [[ "$GITHUB_REF" =~ refs/pull/([0-9]+)/merge ]]; then
+            PR_NUMBER="${BASH_REMATCH[1]}"
           fi
-          # If so, then prepare to create CHANGES.html
+          # Create CHANGES.html
           if [[ -n "$PR_NUMBER" ]]; then
             # mathjax path in old doc (regex)
             mathjax_path_from="[-./A-Za-z_]*/tex-chtml[.]js?v=[0-9a-f]*"
             # mathjax path in new doc
             mathjax_path_to=$(docker exec -e SAGE_USE_CDNS=yes BUILD /sage/sage -python -c "from sage_docbuild.conf import mathjax_path; print(mathjax_path)")
             new_version=$(docker exec BUILD cat src/VERSION.txt)
-            docker cp BUILD:/sage/local/share/doc/sage/html doc/
             # Wipe out chronic diffs between old doc and new doc
             (cd doc && \
              find . -name "*.html" | xargs sed -i -e '/class="sidebar-brand-text"/ s/Sage [0-9a-z.]* /Sage '"$new_version"' /' \
-                                                  -e '/<link rel="stylesheet"/ s/?v=[0-9a-f]*"/"/' \
+                                                  -e 's;?v=[0-9a-f]*";";' \
                                                   -e 's;'"$mathjax_path_from"';'"$mathjax_path_to"';' \
                                                   -e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d' \
                                                   -e 's;#L[0-9]*";";' \
+                                                  -e 's;tab-set--[0-9]*-;tab-set-;' \
              && true)
             # Create git repo from old doc
             (cd doc && \
              git init && \
-             (echo "*.svg binary"; echo "*.pdf binary") >> .gitattributes && \
-             (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore; \
+             (echo "*.svg binary"; echo "*.pdf binary") > .gitattributes && \
+             (echo ".buildinfo"; echo '*.inv'; echo '.git*'; echo '*.svg'; echo '*.pdf'; echo '*.png'; echo 'searchindex.js') > .gitignore && \
              git add -A && git commit --quiet -m 'old')
           fi
 
       - name: Build doc
         id: docbuild
-        if: (success() || failure()) && steps.worktree.outcome == 'success' && !startsWith(github.ref, 'refs/tags/')
+        if: steps.container.outcome == 'success' && !startsWith(github.ref, 'refs/tags/')
         # Always non-incremental because of the concern that
         # incremental docbuild may introduce broken links (inter-file references) though build succeeds
         run: |
@@ -175,18 +193,22 @@ jobs:
           export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
           make doc-clean doc-uninstall
           export SAGE_USE_CDNS=yes
+          export SAGE_DOCBUILD_OPTS="--include-tests-blocks"
           ./config.status && make sagemath_doc_html-no-deps
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
       - name: Copy doc
         id: copy
-        if: (success() || failure()) && steps.docbuild.outcome == 'success'
+        if: steps.docbuild.outcome == 'success'
         run: |
           set -ex
-          # We copy everything to a local folder
-          docker cp --follow-link BUILD:/sage/local/share/doc/sage/html doc
-          docker cp --follow-link BUILD:/sage/local/share/doc/sage/index.html doc
-          # Check if we are on PR
+          # Simpler "docker cp --follow-link ... doc" does not work
+          mkdir -p doc
+          mkdir -p temp
+          docker cp --follow-link BUILD:/sage/local/share/doc/sage/html temp
+          docker cp --follow-link BUILD:/sage/local/share/doc/sage/index.html temp
+          cp -r -L temp/* doc/
+          # Check if we are on pull request event
           PR_NUMBER=""
           if [[ -n "$GITHUB_REF" ]]; then
             if [[ "$GITHUB_REF" =~ refs/pull/([0-9]+)/merge ]]; then
@@ -199,9 +221,10 @@ jobs:
             # Wipe out chronic diffs of new doc against old doc before creating CHANGES.html
             (cd doc && \
              find . -name "*.html" | xargs sed -i -e '/This is documentation/ s/ built with GitHub PR .* for development/ for development/' \
-                                                  -e '/<link rel="stylesheet"/ s/?v=[0-9a-f]*"/"/' \
+                                                  -e 's;?v=[0-9a-f]*";";' \
                                                   -e '\;<script type="application/vnd\.jupyter\.widget-state+json">;,\;</script>; d' \
                                                   -e 's;#L[0-9]*";";' \
+                                                  -e 's;tab-set--[0-9]*-;tab-set-;' \
              && git commit -a -m 'wipe-out')
             # Since HEAD is at commit 'wipe-out', HEAD~1 is commit 'new' (new doc), HEAD~2 is commit 'old' (old doc)
             (cd doc && git diff $(git rev-parse HEAD~2) -- "*.html") > diff.txt
@@ -221,20 +244,31 @@ jobs:
 
       - name: Upload doc
         id: upload
-        if: (success() || failure()) && steps.copy.outcome == 'success'
+        if: steps.copy.outcome == 'success'
         uses: actions/upload-artifact@v4
         with:
           name: doc
           path: doc.zip
 
+      - name: Upload doc-develop
+        # artifact doc-develop is used for doc build on pull request event
+        id: upload-push
+        if: steps.copy.outcome == 'success' && github.event_name == 'push'
+        uses: actions/upload-artifact@v4
+        with:
+          name: doc-${{ github.ref_name }}
+          path: doc.zip
+
       #
-      # On release tags: live doc and wheels
+      # On release tag event
       #
 
       - name: Build live doc
         id: buildlivedoc
-        if: (success() || failure()) && startsWith(github.ref, 'refs/tags/')
+        if: startsWith(github.ref, 'refs/tags/')
         run: |
+          # Avoid running out of disk space
+          rm -rf upstream
           export MAKE="make -j5 --output-sync=recurse" SAGE_NUM_THREADS=5
           export PATH="build/bin:$PATH"
           eval $(sage-print-system-package-command auto update)
@@ -249,7 +283,7 @@ jobs:
 
       - name: Copy live doc
         id: copylivedoc
-        if: (success() || failure()) && steps.buildlivedoc.outcome == 'success'
+        if: steps.buildlivedoc.outcome == 'success'
         run: |
           mkdir -p ./livedoc
           # We copy everything to a local folder
@@ -259,7 +293,7 @@ jobs:
           zip -r livedoc.zip livedoc
 
       - name: Upload live doc
-        if: (success() || failure()) && steps.copylivedoc.outcome == 'success'
+        if: steps.copylivedoc.outcome == 'success'
         uses: actions/upload-artifact@v4
         with:
           name: livedoc

.github/workflows/docker.yml

@@ -170,7 +170,7 @@ jobs:
           ref: ${{ inputs.sage_ref }}
           fetch-depth: 10000
       - name: Download upstream artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: upstream
           name: ${{ inputs.upstream_artifact }}
@@ -260,7 +260,7 @@ jobs:
           cp -r .tox/$TOX_ENV/* "artifacts/$LOGS_ARTIFACT_NAME"
           rm -rf "artifacts/$LOGS_ARTIFACT_NAME"/{bin,lib,pyvenv.cfg}
         if: always()
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: artifacts
           name: ${{ env.LOGS_ARTIFACT_NAME }}

.github/workflows/macos.yml

@@ -91,7 +91,7 @@ jobs:
         run: |
           "${{ steps.python.outputs.python-path }}" -m pip install pipx
       - name: Download upstream artifact
-        uses: actions/download-artifact@v3
+        uses: actions/download-artifact@v4
         with:
           path: upstream
           name: ${{ inputs.upstream_artifact }}
@@ -108,7 +108,7 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           SAGE_CI_FIXES_FROM_REPOSITORIES: ${{ vars.SAGE_CI_FIXES_FROM_REPOSITORIES }}
 
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           path: sage-local-artifact
           name: ${{ env.LOCAL_ARTIFACT_NAME }}
@@ -147,7 +147,7 @@ jobs:
         run: |
           mkdir -p "artifacts/$LOGS_ARTIFACT_NAME"; cp -r .tox/*/log "artifacts/$LOGS_ARTIFACT_NAME"
         if: always()
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: artifacts
           name: ${{ env.LOGS_ARTIFACT_NAME }}
@@ -165,7 +165,7 @@ jobs:
         run: |
           mkdir -p sage-local-artifact && (cd .tox/$TOX_ENV && rm -f "local/lib64" && tar -cf - $(pwd)) > sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ inputs.stage }}.tar
         if: contains(inputs.stage, '1')
-      - uses: actions/upload-artifact@v3
+      - uses: actions/upload-artifact@v4
         with:
           path: sage-local-artifact/sage-${{ env.TOX_ENV }}-${{ inputs.stage }}.tar
           name: ${{ env.LOCAL_ARTIFACT_NAME }}