Merge branch 'master' into power-management

.github/scripts/on-push.sh

@@ -108,7 +108,7 @@ if [ "$BUILD_PIO" -eq 0 ]; then
 
     if [ "$BUILD_LOG" -eq 1 ]; then
         #remove last comma from the last JSON object
-        sed -i '$ s/.$//' "$sizes_file"
+        sed -i '$ s/,$//' "$sizes_file"
         #echo end of JSON array
         echo "]}" >> $sizes_file
     fi

.github/scripts/set_push_chunks.sh

@@ -0,0 +1,83 @@
+#!/bin/bash
+
+build_all=false
+chunks_count=0
+
+if [[ $CORE_CHANGED == 'true' ]] || [[ $IS_PR != 'true' ]]; then
+  echo "Core files changed or not a PR. Building all."
+  build_all=true
+  chunks_count=$MAX_CHUNKS
+elif [[ $LIB_CHANGED == 'true' ]]; then
+  echo "Libraries changed. Building only affected sketches."
+  if [[ $NETWORKING_CHANGED == 'true' ]]; then
+    echo "Networking libraries changed. Building networking related sketches."
+    networking_sketches="$(find libraries/WiFi -name *.ino) "
+    networking_sketches+="$(find libraries/Ethernet -name *.ino) "
+    networking_sketches+="$(find libraries/PPP -name *.ino) "
+    networking_sketches+="$(find libraries/NetworkClientSecure -name *.ino) "
+    networking_sketches+="$(find libraries/WebServer -name *.ino) "
+  fi
+  if [[ $FS_CHANGED == 'true' ]]; then
+    echo "FS libraries changed. Building FS related sketches."
+    fs_sketches="$(find libraries/SD -name *.ino) "
+    fs_sketches+="$(find libraries/SD_MMC -name *.ino) "
+    fs_sketches+="$(find libraries/SPIFFS -name *.ino) "
+    fs_sketches+="$(find libraries/LittleFS -name *.ino) "
+    fs_sketches+="$(find libraries/FFat -name *.ino) "
+  fi
+  sketches="$networking_sketches $fs_sketches"
+  for file in $LIB_FILES; do
+      if [[ $file == *.ino ]]; then
+          # If file ends with .ino, add it to the list of sketches
+          echo "Sketch found: $file"
+          sketches+="$file "
+      elif [[ $(basename $(dirname $file)) == "src" ]]; then
+          # If file is in a src directory, find all sketches in the parent/examples directory
+          echo "Library src file found: $file"
+          lib=$(dirname $(dirname $file))
+          if [[ -d $lib/examples ]]; then
+              lib_sketches=$(find $lib/examples -name *.ino)
+              sketches+="$lib_sketches "
+              echo "Library sketches: $lib_sketches"
+          fi
+      else
+          # If file is in a example folder but it is not a sketch, find all sketches in the current directory
+          echo "File in example folder found: $file"
+          sketch=$(find $(dirname $file) -name *.ino)
+          sketches+="$sketch "
+          echo "Sketch in example folder: $sketch"
+      fi
+      echo ""
+  done
+fi
+
+if [[ -n $sketches ]]; then
+  # Remove duplicates
+  sketches=$(echo $sketches | tr ' ' '\n' | sort | uniq)
+  for sketch in $sketches; do
+      echo $sketch >> sketches_found.txt
+      chunks_count=$((chunks_count+1))
+  done
+  echo "Number of sketches found: $chunks_count"
+  echo "Sketches:"
+  echo "$sketches"
+
+  if [[ $chunks_count -gt $MAX_CHUNKS ]]; then
+    echo "More sketches than the allowed number of chunks found. Limiting to $MAX_CHUNKS chunks."
+    chunks_count=$MAX_CHUNKS
+  fi
+fi
+
+chunks='["0"'
+for i in $(seq 1 $(( $chunks_count - 1 )) ); do
+  chunks+=",\"$i\""
+done
+chunks+="]"
+
+echo "build_all=$build_all" >> $GITHUB_OUTPUT
+echo "build_libraries=$BUILD_LIBRARIES" >> $GITHUB_OUTPUT
+echo "build_static_sketches=$BUILD_STATIC_SKETCHES" >> $GITHUB_OUTPUT
+echo "build_idf=$BUILD_IDF" >> $GITHUB_OUTPUT
+echo "build_platformio=$BUILD_PLATFORMIO" >> $GITHUB_OUTPUT
+echo "chunk_count=$chunks_count" >> $GITHUB_OUTPUT
+echo "chunks=$chunks" >> $GITHUB_OUTPUT

.github/scripts/sketch_utils.sh

@@ -192,7 +192,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
 
             exit_status=$?
             if [ $exit_status -ne 0 ]; then
-                echo ""ERROR: Compilation failed with error code $exit_status""
+                echo "ERROR: Compilation failed with error code $exit_status"
                 exit $exit_status
             fi
 
@@ -236,7 +236,7 @@ function build_sketch(){ # build_sketch <ide_path> <user_path> <path-to-ino> [ex
 
             exit_status=$?
             if [ $exit_status -ne 0 ]; then
-                echo ""ERROR: Compilation failed with error code $exit_status""
+                echo "ERROR: Compilation failed with error code $exit_status"
                 exit $exit_status
             fi
             # $ide_path/arduino-builder -compile -logger=human -core-api-version=10810 \
@@ -398,6 +398,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
     else
         start_index=$(( $chunk_index * $chunk_size ))
         if [ "$sketchcount" -le "$start_index" ]; then
+            echo "No sketches to build for $target in this chunk"
             return 0
         fi
 
@@ -437,7 +438,7 @@ function build_sketches(){ # build_sketches <ide_path> <user_path> <target> <pat
             continue
         fi
         echo ""
-        echo "Building Sketch Index $(($sketchnum - 1)) - $sketchdirname"
+        echo "Building Sketch Index $sketchnum - $sketchdirname"
         build_sketch $args -s $sketchdir $xtra_opts
         local result=$?
         if [ $result -ne 0 ]; then

.github/scripts/tests_run.sh

@@ -95,13 +95,12 @@ function run_test() {
             printf "\033[95mpytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file $extra_args\033[0m\n"
             bash -c "set +e; pytest tests --build-dir $build_dir -k test_$sketchname --junit-xml=$report_file $extra_args; exit \$?" || result=$?
             printf "\n"
-            result=$?
             if [ $result -ne 0 ]; then
+              printf "\033[91mFailed test: $sketchname -- Config: $i\033[0m\n\n"
               error=$result
             fi
         fi
     done
-    printf "\n"
     return $error
 }
 

.github/workflows/boards.yml

@@ -59,6 +59,19 @@ jobs:
             exit 1;
           fi
 
+      - name: Get libs cache
+        uses: actions/cache@v4
+        with:
+          key: libs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package/package_esp32_index.template.json', 'tools/get.py') }}
+          path: |
+            ./tools/dist
+            ./tools/esp32-arduino-libs
+            ./tools/esptool
+            ./tools/mk*
+            ./tools/openocd-esp32
+            ./tools/riscv32-*
+            ./tools/xtensa-*
+            
       - name: Compile sketch
         uses: P-R-O-C-H-Y/compile-sketches@main
         with:
@@ -73,3 +86,4 @@ jobs:
           exit-on-fail: true
           sketch-paths:
             "- ./libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino"
+          verbose: true

.github/workflows/build_tests.yml

@@ -12,10 +12,6 @@ on:
         description: 'Chip to build tests for'
         required: true
 
-concurrency:
-  group: tests-build-${{ github.event.pull_request.number || github.ref }}-${{ inputs.chip }}-${{ inputs.type }}
-  cancel-in-progress: true
-
 jobs:
   build-tests:
     name: Build ${{ inputs.type }} tests for ${{ inputs.chip }}
@@ -47,11 +43,24 @@ jobs:
 
           echo "enabled=$enabled" >> $GITHUB_OUTPUT
 
-      - name: Checkout Repository
-        uses: actions/checkout@v4
+      - name: Checkout user repository
         if: ${{ steps.check-build.outputs.enabled == 'true' }}
+        uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+            /*
+            !.github
+
+      # To avoid giving unknown scripts elevated permissions, download them from the master branch
+      - name: Get CI scripts from master
+        if: ${{ steps.check-build.outputs.enabled == 'true' }}
+        run: |
+          mkdir -p .github
+          cd .github
+          curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github
 
       - name: Get libs cache
         uses: actions/cache@v4

.github/workflows/hw.yml

@@ -12,10 +12,6 @@ on:
         description: 'Chip to run tests for'
         required: true
 
-concurrency:
-  group: tests-hw-${{ github.event.pull_request.number || github.ref }}-${{ inputs.chip }}-${{ inputs.type }}
-  cancel-in-progress: true
-
 jobs:
   hardware-test:
     name: Hardware ${{ inputs.chip }} ${{ inputs.type }} tests
@@ -49,36 +45,48 @@ jobs:
 
           echo "enabled=$enabled" >> $GITHUB_OUTPUT
 
-      - name: Checkout repository
+      - name: Checkout user repository
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          persist-credentials: false
+          sparse-checkout-cone-mode: false
+          sparse-checkout: |
+            /*
+            !.github
 
-      - uses: actions/setup-python@v5
-        if: steps.check-tests.outputs.enabled == 'true'
-        with:
-          cache-dependency-path: tests/requirements.txt
-          cache: 'pip'
-          python-version: '3.10.1'
+      # To avoid giving unknown scripts elevated permissions, download them from the master branch
+      - name: Get CI scripts from master
+        if: ${{ steps.check-tests.outputs.enabled == 'true' }}
+        run: |
+          mkdir -p .github
+          cd .github
+          curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github
+
+      # setup-python currently only works on ubuntu images
+      # - uses: actions/setup-python@v5
+      #   if: ${{ steps.check-tests.outputs.enabled == 'true' }}
+      #   with:
+      #     cache-dependency-path: tests/requirements.txt
+      #     cache: 'pip'
+      #     python-version: '3.10.1'
 
       - name: Install dependencies
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
         run: |
           pip install -U pip
           pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi
+          apt update
+          apt install -y jq
 
       - name: Get binaries
-        id: cache-build-binaries
-        uses: actions/cache/restore@v4
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}
+        uses: actions/download-artifact@v4
         with:
-          fail-on-cache-miss: true
-          key: tests-${{ env.id }}-bin
+          name: tests-bin-${{ inputs.chip }}-${{ inputs.type }}
           path: |
-            ~/.arduino/tests/**/build*.tmp/*.bin
-            ~/.arduino/tests/**/build*.tmp/*.elf
-            ~/.arduino/tests/**/build*.tmp/*.json
+            ~/.arduino/tests
 
       - name: Run Tests
         if: ${{ steps.check-tests.outputs.enabled == 'true' }}

.github/workflows/lib.yml

@@ -120,8 +120,8 @@ jobs:
 
       - name: Push to github repo
         run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git add ${{ env.RESULT_LIBRARY_TEST_FILE }}
           git commit -m "Generated External Libraries Test Results"
           git push origin HEAD:gh-pages

.github/workflows/pre-commit.yml

@@ -28,6 +28,8 @@ jobs:
       - name: Set up Python 3
         uses: actions/setup-python@v5
         with:
+          cache-dependency-path: tools/pre-commit/requirements.txt
+          cache: 'pip'
           python-version: "3.x"
 
       - name: Get Python version hash
@@ -41,11 +43,10 @@ jobs:
         with:
           path: |
             ~/.cache/pre-commit
-            ~/.cache/pip
-          key: pre-commit|${{ env.PY_HASH }}|${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml') }}
+          key: pre-commit-${{ env.PY_HASH }}-${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml', 'tools/pre-commit/requirements.txt') }}
 
       - name: Install python dependencies
-        run: python -m pip install pre-commit docutils
+        run: python -m pip install -r tools/pre-commit/requirements.txt
 
       - name: Get changed files
         id: changed-files
@@ -61,7 +62,6 @@ jobs:
         with:
           path: |
             ~/.cache/pre-commit
-            ~/.cache/pip
           key: ${{ steps.restore-cache.outputs.cache-primary-key }}
 
       - name: Push changes using pre-commit-ci-lite

.github/workflows/publishsizes-2.x.yml

@@ -32,22 +32,22 @@ jobs:
         run: |
           mv master_cli_compile/*.json artifacts/sizes-report/pr/
           mv v2.x_cli_compile/*.json artifacts/sizes-report/master/
-    
+
       - name: Report results
         uses: P-R-O-C-H-Y/report-size-deltas@sizes_v2
         with:
           sketches-reports-source: ${{ env.SKETCHES_REPORTS_PATH }}
           github-token: ${{ env.GITHUB_TOKEN }}
           destination-file: ${{ env.RESULT_SIZES_TEST_FILE }}
-          
+
       - name: Append file with action URL
         run:
           echo "/ [GitHub Action Link](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}})" >> ${{ env.RESULT_SIZES_TEST_FILE }}
 
       - name: Push to github repo
         run: |
-          git config user.name github-actions
-          git config user.email github-actions@github.com
+          git config user.name "github-actions[bot]"
+          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
           git add ${{ env.RESULT_SIZES_TEST_FILE }}
           git commit -m "Generated Sizes Results (master-v2.x)"
           git push origin HEAD:gh-pages