diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 9d355e29482..45a8bb56e8d 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -100,21 +100,25 @@ jobs:
       - name: Checkout
         id: checkout
         uses: actions/checkout@v4
-      - name: Get all packages that have changed
-        id: changed-packages
+      - name: Get changed files and packages
+        id: changed-files
         uses: tj-actions/changed-files@v44
         with:
+          # File extensions for doctests per sage.doctest.control.skipfile
           files_yaml: |
             configures:
               - 'build/pkgs/*/spkg-configure.m4'
             pkgs:
               - 'build/pkgs/**'
               - 'pkgs/**'
+            doctests:
+              - 'src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}'
+              - '!src/{setup,conftest*}.py'
       - name: Determine targets to build
         id: build-targets
         run: |
-          uninstall_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))
-          build_targets=$(echo $(for a in '' ${{ steps.changed-packages.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))
+          uninstall_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.configures_all_changed_files }}; do echo $a | sed -E 's,build/pkgs/([a-z0-9][_.a-z0-9]*)/spkg-configure[.]m4 *,\1-uninstall,'; done | sort -u))
+          build_targets=$(echo $(for a in '' ${{ steps.changed-files.outputs.pkgs_all_changed_files }}; do SPKG=$(echo $a | sed -E 's,-,_,g;s,(build/)?pkgs/([a-z0-9][-_.a-z0-9]*)/[^ ]* *,\2,;'); if [ -f "build/pkgs/$SPKG/checksums.ini" -o -f "build/pkgs/$SPKG/requirements.txt" -o -f "build/pkgs/$SPKG/spkg-install" ]; then echo "$SPKG-ensure"; fi; done | sort -u))
           if [ -n "$uninstall_targets" ]; then
               echo "build_targets=$uninstall_targets reconfigure $build_targets ci-build-with-fallback" >> $GITHUB_OUTPUT
           else
@@ -125,7 +129,7 @@ jobs:
         with:
           ref: ${{ github.base_ref }}
           path: worktree-base
-        if: github.base_ref && steps.changed-packages.outputs.pkgs_all_changed_files
+        if: github.base_ref && steps.changed-files.outputs.pkgs_all_changed_files
       - name: Compute metrics
         run: |
           export PATH=build/bin:$PATH
@@ -213,20 +217,12 @@ jobs:
           ./sage -python -m pytest -c tox.ini -qq --doctest --collect-only || true
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
-      - name: Get changed files
-        id: changed-files
-        uses: tj-actions/changed-files@v44
-        with:
-          # File extensions per sage.doctest.control.skipfile
-          files: src/**/*.{py,pyx,pxd,pxi,sage,spyx,rst,tex}
-          files_ignore: src/{setup,conftest*}.py
-
       - name: Test changed files (sage -t --new)
-        if: steps.changed-files.outputs.all_changed_files
+        if: steps.changed-files.outputs.doctests_all_changed_files
         run: |
           export MAKE="make -j2 --output-sync=recurse" SAGE_NUM_THREADS=4
           # https://github.com/tj-actions/changed-files?tab=readme-ov-file#outputs-
-          ./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.all_changed_files }}
+          ./sage -t --long --format github -p4 ${{ steps.changed-files.outputs.doctests_all_changed_files }}
         shell: sh .ci/docker-exec-script.sh BUILD /sage {0}
 
   test-mod: