Merge branch 'merge_ci_linux' into scip-9.0.1

sagemath · Jun 3, 2024 · c8f17be · c8f17be
2 parents b07fa20 + f1cd6c4
commit c8f17be
Show file tree

Hide file tree

Showing 2 changed files with 63 additions and 32 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -78,6 +78,8 @@ env:
 jobs:
   test-new:
     runs-on: ubuntu-latest
+    outputs:
+      build_targets:     ${{ steps.build-targets.outputs.build_targets }}
     services:
       # https://docs.docker.com/build/ci/github-actions/local-registry/
       registry:
@@ -98,6 +100,46 @@ jobs:
       - name: Checkout
         id: checkout
         uses: actions/checkout@v4
+      - name: Get changed files and packages
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+          # File extensions for doctests per sage.doctest.control.skipfile
+          files_yaml: |
+            configures:
+              - 'build/pkgs/*/spkg-configure.m4'
+            pkgs:
+              - 'build/pkgs/**'
+              - 'pkgs/**'
+            doctests:
+              - 'src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}'
+              - '!src/{setup,conftest*}.py'
+      - name: Determine targets to build
+        id: build-targets
+        run: |
+          uninstall_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))
+          build_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))
+          if [ -n "$uninstall_targets" ]; then
+              echo "build_targets=$uninstall_targets reconfigure $build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
+          else
+              echo "build_targets=$build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
+          fi
+          cat $GITHUB_OUTPUT
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref }}
+          path: worktree-base
+        if: github.base_ref && steps.changed-files.outputs.pkgs_all_changed_files
+      - name: Compute metrics
+        run: |
+          export PATH=build/bin:$PATH
+          if [ -d worktree-base ]; then
+            (echo "# $GITHUB_BASE_REF"; SAGE_ROOT=worktree-base sage-package metrics :all:) > base-metrics.txt
+            (echo "# $GITHUB_REF"; sage-package metrics :all:) > metrics.txt
+            diff --color=always --width=100 --side-by-side --left-column base-metrics.txt metrics.txt || true
+          else
+            sage-package metrics :all:
+          fi
       - name: Install test prerequisites
         # From docker.yml
         run: |
@@ -142,7 +184,7 @@ jobs:
           # more reliable than "load", for which we observed random failure
           # conditions in which the built image could not be found.
           #
-          push:       true
+          push:       ${{ steps.changed-files.outputs.doctests_all_changed_files && true || false }}
           load:       false
           context:    .
           tags:       ${{ env.BUILD_IMAGE }}
@@ -153,9 +195,10 @@ jobs:
             NUMPROC=6
             USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
             TARGETS_PRE=build/make/Makefile
-            TARGETS=ci-build-with-fallback
+            TARGETS=${{ steps.build-targets.outputs.build_targets }}
 
       - name: Start container
+        if: steps.changed-files.outputs.doctests_all_changed_files
         run: |
           docker run --name BUILD -dit \
                      --mount type=bind,src=$(pwd),dst=$(pwd) \
@@ -165,6 +208,7 @@ jobs:
       # Testing
 
       - name: Check that all modules can be imported
+        if: steps.changed-files.outputs.doctests_all_changed_files
         run: |
           # Increase the length of the lines in the "short summary"
           export COLUMNS=120
@@ -175,20 +219,12 @@ jobs:
           ./sage -python -m pytest -c tox.ini -qq --doctest --collect-only || true
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v44
-        with:
-          # File extensions per sage.doctest.control.skipfile
-          files: src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}
-          files_ignore: src/{setup,conftest*}.py
-
       - name: Test changed files (sage -t --new)
-        if: steps.changed-files.outputs.all_changed_files
+        if: steps.changed-files.outputs.doctests_all_changed_files
         run: |
           export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4
           # https://github.com/tj-actions/changed-files?tab=readme-ov-file#outputs-
-          ./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.all_changed_files }}
+          ./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.doctests_all_changed_files }}
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
   test-mod:
@@ -266,7 +302,7 @@ jobs:
             NUMPROC=6
             USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
             TARGETS_PRE=build/make/Makefile
-            TARGETS=ci-build-with-fallback
+            TARGETS=${{ needs.test-new.outputs.build_targets }}
 
       - name: Start container
         run: |
@@ -361,7 +397,7 @@ jobs:
             NUMPROC=6
             USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
             TARGETS_PRE=build/make/Makefile
-            TARGETS=ci-build-with-fallback
+            TARGETS=${{ needs.test-new.outputs.build_targets }}
 
       - name: Start container
         id: container
@@ -473,7 +509,7 @@ jobs:
             NUMPROC=6
             USE_MAKEFLAGS=-k V=0 SAGE_NUM_THREADS=4 --output-sync=recurse
             TARGETS_PRE=build/make/Makefile
-            TARGETS=ci-build-with-fallback
+            TARGETS=${{ needs.test-new.outputs.build_targets }}
 
       - name: Start container
         id: container

diff --git a/.github/workflows/ci-linux-incremental.yml b/.github/workflows/ci-linux-incremental.yml
@@ -41,7 +41,6 @@ jobs:
     runs-on: ubuntu-latest
     name: List changed packages
     outputs:
-      uninstall_targets: ${{ steps.build-targets.outputs.uninstall_targets }}
       build_targets:     ${{ steps.build-targets.outputs.build_targets }}
     steps:
       - uses: actions/checkout@v4
@@ -58,14 +57,19 @@ jobs:
       - name: Determine targets to build
         id: build-targets
         run: |
-          echo "uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))" >> $GITHUB_OUTPUT
-          echo "build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))" >> $GITHUB_OUTPUT
+          uninstall_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))
+          build_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))
+          if [ -n "$uninstall_targets" ]; then
+              echo "build_targets=$uninstall_targets reconfigure $build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
+          else
+              echo "build_targets=$build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
+          fi
           cat $GITHUB_OUTPUT
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.base_ref }}
           path: worktree-base
-        if: github.base_ref
+        if: github.base_ref && steps.changed-files.outputs.pkgs_all_changed_files
       - name: Compute metrics
         run: |
           export PATH=build/bin:$PATH
@@ -88,15 +92,14 @@ jobs:
       from_docker_target: "with-targets"
       from_docker_tag: "dev"
       docker_targets: "with-targets"
-      targets: "${{needs.changed_files.outputs.uninstall_targets}} reconfigure ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      targets: "${{needs.changed_files.outputs.build_targets}} doc-html ptest-nodoc"
       tox_system_factors: >-
         ["ubuntu-focal",
-         "ubuntu-jammy",
-         "ubuntu-mantic",
+         "ubuntu-noble",
          "debian-bullseye",
          "debian-bookworm",
          "fedora-30",
-         "fedora-38",
+         "fedora-40",
          "gentoo-python3.11",
          "debian-bullseye-i386"]
       tox_packages_factors: >-
@@ -107,14 +110,6 @@ jobs:
 
   site:
     needs: [changed_files]
-    if: |
-      github.event_name != 'pull_request' ||
-      ((github.event.action != 'labeled' &&
-         (contains(github.event.pull_request.labels.*.name, 'c: packages: standard') ||
-          contains(github.event.pull_request.labels.*.name, 'c: packages: optional'))) ||
-       (github.event.action == 'labeled' &&
-         (github.event.label.name == 'c: packages: optional' ||
-          github.event.label.name == 'c: packages: standard')))
     uses: ./.github/workflows/docker.yml
     with:
       # Build incrementally from published Docker image
@@ -124,7 +119,7 @@ jobs:
       from_docker_target: "with-targets"
       from_docker_tag: "dev"
       docker_targets: "with-targets"
-      targets: "${{needs.changed_files.outputs.uninstall_targets}} reconfigure ${{needs.changed_files.outputs.build_targets}} build doc-html ptest"
+      targets: "${{needs.changed_files.outputs.build_targets}} doc-html ptest-nodoc"
       # Only test systems with a usable system python (>= 3.9)
       # with only a small number of test failures as of 10.2.rc0
       tox_system_factors: >-