From 5d9cd9ac58471624c07295112095cc3864b9cbaf Mon Sep 17 00:00:00 2001 From: Unexpected Maker Date: Fri, 7 Jun 2024 06:25:43 +1000 Subject: [PATCH 01/24] Added new UM FeatherS3 Neo board. (#9788) * Added new UM FeatherS3 Neo board. * ci(pre-commit): Apply automatic fixes --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- boards.txt | 169 ++++++++++++++++++++++++ variants/um_feathers3neo/pins_arduino.h | 69 ++++++++++ 2 files changed, 238 insertions(+) create mode 100644 variants/um_feathers3neo/pins_arduino.h diff --git a/boards.txt b/boards.txt index 4a64ecc65b0..5163a2d84c9 100644 --- a/boards.txt +++ b/boards.txt @@ -3339,6 +3339,175 @@ um_feathers3.menu.EraseFlash.all.upload.erase_cmd=-e ############################################################## +um_feathers3neo.name=UM FeatherS3 Neo +um_feathers3neo.vid.0=0x303a +um_feathers3neo.pid.0=0x81FB +um_feathers3neo.upload_port.0.vid=0x303a +um_feathers3neo.upload_port.0.pid=0x81FB + +um_feathers3neo.bootloader.tool=esptool_py +um_feathers3neo.bootloader.tool.default=esptool_py + +um_feathers3neo.upload.tool=esptool_py +um_feathers3neo.upload.tool.default=esptool_py +um_feathers3neo.upload.tool.network=esp_ota + +um_feathers3neo.upload.maximum_size=1310720 +um_feathers3neo.upload.maximum_data_size=327680 +um_feathers3neo.upload.flags= +um_feathers3neo.upload.extra_flags= +um_feathers3neo.upload.use_1200bps_touch=false +um_feathers3neo.upload.wait_for_upload_port=false + +um_feathers3neo.serial.disableDTR=false +um_feathers3neo.serial.disableRTS=false + +um_feathers3neo.build.tarch=xtensa +um_feathers3neo.build.bootloader_addr=0x0 +um_feathers3neo.build.target=esp32s3 +um_feathers3neo.build.mcu=esp32s3 +um_feathers3neo.build.core=esp32 +um_feathers3neo.build.variant=um_feathers3neo +um_feathers3neo.build.board=FEATHERS3NEO + +um_feathers3neo.build.usb_mode=1 +um_feathers3neo.build.cdc_on_boot=1 +um_feathers3neo.build.msc_on_boot=0 +um_feathers3neo.build.dfu_on_boot=0 +um_feathers3neo.build.f_cpu=240000000L +um_feathers3neo.build.flash_size=16MB +um_feathers3neo.build.flash_freq=80m +um_feathers3neo.build.flash_mode=dio +um_feathers3neo.build.boot=qio +um_feathers3neo.build.partitions=default +um_feathers3neo.build.defines= +um_feathers3neo.build.loop_core= +um_feathers3neo.build.event_core= +um_feathers3neo.build.flash_type=qio +um_feathers3neo.build.psram_type=qspi +um_feathers3neo.build.memory_type=qio_qspi + +um_feathers3neo.menu.LoopCore.1=Core 1 +um_feathers3neo.menu.LoopCore.1.build.loop_core=-DARDUINO_RUNNING_CORE=1 +um_feathers3neo.menu.LoopCore.0=Core 0 +um_feathers3neo.menu.LoopCore.0.build.loop_core=-DARDUINO_RUNNING_CORE=0 + +um_feathers3neo.menu.EventsCore.1=Core 1 +um_feathers3neo.menu.EventsCore.1.build.event_core=-DARDUINO_EVENT_RUNNING_CORE=1 +um_feathers3neo.menu.EventsCore.0=Core 0 +um_feathers3neo.menu.EventsCore.0.build.event_core=-DARDUINO_EVENT_RUNNING_CORE=0 + +um_feathers3neo.menu.USBMode.hwcdc=Hardware CDC and JTAG +um_feathers3neo.menu.USBMode.hwcdc.build.usb_mode=1 +um_feathers3neo.menu.USBMode.default=USB-OTG (TinyUSB) +um_feathers3neo.menu.USBMode.default.build.usb_mode=0 + +um_feathers3neo.menu.CDCOnBoot.cdc=Enabled +um_feathers3neo.menu.CDCOnBoot.cdc.build.cdc_on_boot=1 +um_feathers3neo.menu.CDCOnBoot.default=Disabled +um_feathers3neo.menu.CDCOnBoot.default.build.cdc_on_boot=0 + +um_feathers3neo.menu.MSCOnBoot.default=Disabled +um_feathers3neo.menu.MSCOnBoot.default.build.msc_on_boot=0 +um_feathers3neo.menu.MSCOnBoot.msc=Enabled (Requires USB-OTG Mode) +um_feathers3neo.menu.MSCOnBoot.msc.build.msc_on_boot=1 + +um_feathers3neo.menu.DFUOnBoot.default=Disabled +um_feathers3neo.menu.DFUOnBoot.default.build.dfu_on_boot=0 +um_feathers3neo.menu.DFUOnBoot.dfu=Enabled (Requires USB-OTG Mode) +um_feathers3neo.menu.DFUOnBoot.dfu.build.dfu_on_boot=1 + +um_feathers3neo.menu.UploadMode.cdc.upload.wait_for_upload_port=true +um_feathers3neo.menu.UploadMode.default=UART0 / Hardware CDC +um_feathers3neo.menu.UploadMode.cdc=USB-OTG CDC (TinyUSB) +um_feathers3neo.menu.UploadMode.cdc.upload.use_1200bps_touch=true +um_feathers3neo.menu.UploadMode.default.upload.use_1200bps_touch=false +um_feathers3neo.menu.UploadMode.default.upload.wait_for_upload_port=false + +um_feathers3neo.menu.PSRAM.enabled=Enabled +um_feathers3neo.menu.PSRAM.enabled.build.defines=-DBOARD_HAS_PSRAM +um_feathers3neo.menu.PSRAM.disabled=Disabled +um_feathers3neo.menu.PSRAM.disabled.build.defines= + +um_feathers3neo.menu.PartitionScheme.default_8MB=8M with spiffs (3MB APP/1.5MB SPIFFS) +um_feathers3neo.menu.PartitionScheme.default_8MB.build.partitions=default_8MB +um_feathers3neo.menu.PartitionScheme.default_8MB.upload.maximum_size=3342336 +um_feathers3neo.menu.PartitionScheme.minimal=Minimal (1.3MB APP/700KB SPIFFS) +um_feathers3neo.menu.PartitionScheme.minimal.build.partitions=minimal +um_feathers3neo.menu.PartitionScheme.no_ota=No OTA (2MB APP/2MB SPIFFS) +um_feathers3neo.menu.PartitionScheme.no_ota.build.partitions=no_ota +um_feathers3neo.menu.PartitionScheme.no_ota.upload.maximum_size=2097152 +um_feathers3neo.menu.PartitionScheme.noota_3g=No OTA (1MB APP/3MB SPIFFS) +um_feathers3neo.menu.PartitionScheme.noota_3g.build.partitions=noota_3g +um_feathers3neo.menu.PartitionScheme.noota_3g.upload.maximum_size=1048576 +um_feathers3neo.menu.PartitionScheme.noota_ffat=No OTA (2MB APP/2MB FATFS) +um_feathers3neo.menu.PartitionScheme.noota_ffat.build.partitions=noota_ffat +um_feathers3neo.menu.PartitionScheme.noota_ffat.upload.maximum_size=2097152 +um_feathers3neo.menu.PartitionScheme.noota_3gffat=No OTA (1MB APP/3MB FATFS) +um_feathers3neo.menu.PartitionScheme.noota_3gffat.build.partitions=noota_3gffat +um_feathers3neo.menu.PartitionScheme.noota_3gffat.upload.maximum_size=1048576 +um_feathers3neo.menu.PartitionScheme.huge_app=Huge APP (3MB No OTA/1MB SPIFFS) +um_feathers3neo.menu.PartitionScheme.huge_app.build.partitions=huge_app +um_feathers3neo.menu.PartitionScheme.huge_app.upload.maximum_size=3145728 +um_feathers3neo.menu.PartitionScheme.min_spiffs=Minimal SPIFFS (1.9MB APP with OTA/190KB SPIFFS) +um_feathers3neo.menu.PartitionScheme.min_spiffs.build.partitions=min_spiffs +um_feathers3neo.menu.PartitionScheme.min_spiffs.upload.maximum_size=1966080 + +um_feathers3neo.menu.CPUFreq.240=240MHz (WiFi) +um_feathers3neo.menu.CPUFreq.240.build.f_cpu=240000000L +um_feathers3neo.menu.CPUFreq.160=160MHz (WiFi) +um_feathers3neo.menu.CPUFreq.160.build.f_cpu=160000000L +um_feathers3neo.menu.CPUFreq.80=80MHz (WiFi) +um_feathers3neo.menu.CPUFreq.80.build.f_cpu=80000000L +um_feathers3neo.menu.CPUFreq.40=40MHz +um_feathers3neo.menu.CPUFreq.40.build.f_cpu=40000000L +um_feathers3neo.menu.CPUFreq.20=20MHz +um_feathers3neo.menu.CPUFreq.20.build.f_cpu=20000000L +um_feathers3neo.menu.CPUFreq.10=10MHz +um_feathers3neo.menu.CPUFreq.10.build.f_cpu=10000000L + +um_feathers3neo.menu.FlashMode.qio=QIO +um_feathers3neo.menu.FlashMode.qio.build.flash_mode=dio +um_feathers3neo.menu.FlashMode.qio.build.boot=qio +um_feathers3neo.menu.FlashMode.dio=DIO +um_feathers3neo.menu.FlashMode.dio.build.flash_mode=dio +um_feathers3neo.menu.FlashMode.dio.build.boot=dio + +um_feathers3neo.menu.UploadSpeed.921600=921600 +um_feathers3neo.menu.UploadSpeed.921600.upload.speed=921600 +um_feathers3neo.menu.UploadSpeed.115200=115200 +um_feathers3neo.menu.UploadSpeed.115200.upload.speed=115200 +um_feathers3neo.menu.UploadSpeed.256000.windows=256000 +um_feathers3neo.menu.UploadSpeed.256000.upload.speed=256000 +um_feathers3neo.menu.UploadSpeed.230400.windows.upload.speed=256000 +um_feathers3neo.menu.UploadSpeed.230400=230400 +um_feathers3neo.menu.UploadSpeed.230400.upload.speed=230400 +um_feathers3neo.menu.UploadSpeed.460800.linux=460800 +um_feathers3neo.menu.UploadSpeed.460800.macosx=460800 +um_feathers3neo.menu.UploadSpeed.460800.upload.speed=460800 +um_feathers3neo.menu.UploadSpeed.512000.windows=512000 +um_feathers3neo.menu.UploadSpeed.512000.upload.speed=512000 + +um_feathers3neo.menu.DebugLevel.none=None +um_feathers3neo.menu.DebugLevel.none.build.code_debug=0 +um_feathers3neo.menu.DebugLevel.error=Error +um_feathers3neo.menu.DebugLevel.error.build.code_debug=1 +um_feathers3neo.menu.DebugLevel.warn=Warn +um_feathers3neo.menu.DebugLevel.warn.build.code_debug=2 +um_feathers3neo.menu.DebugLevel.info=Info +um_feathers3neo.menu.DebugLevel.info.build.code_debug=3 +um_feathers3neo.menu.DebugLevel.debug=Debug +um_feathers3neo.menu.DebugLevel.debug.build.code_debug=4 +um_feathers3neo.menu.DebugLevel.verbose=Verbose +um_feathers3neo.menu.DebugLevel.verbose.build.code_debug=5 + +um_feathers3neo.menu.EraseFlash.none=Disabled +um_feathers3neo.menu.EraseFlash.none.upload.erase_cmd= +um_feathers3neo.menu.EraseFlash.all=Enabled +um_feathers3neo.menu.EraseFlash.all.upload.erase_cmd=-e + +############################################################## + um_nanos3.name=UM NanoS3 um_nanos3.vid.0=0x303a um_nanos3.pid.0=0x8179 diff --git a/variants/um_feathers3neo/pins_arduino.h b/variants/um_feathers3neo/pins_arduino.h new file mode 100644 index 00000000000..c0a1aae867e --- /dev/null +++ b/variants/um_feathers3neo/pins_arduino.h @@ -0,0 +1,69 @@ +#ifndef Pins_Arduino_h +#define Pins_Arduino_h + +#include +#include "soc/soc_caps.h" + +#define USB_VID 0x303A +#define USB_PID 0x81FB +#define USB_MANUFACTURER "Unexpected Maker" +#define USB_PRODUCT "FeatherS3 Neo" +#define USB_SERIAL "" + +static const uint8_t TX = 43; +static const uint8_t RX = 44; + +static const uint8_t SDA = 8; +static const uint8_t SCL = 9; + +static const uint8_t SS = 5; +static const uint8_t MOSI = 35; +static const uint8_t MISO = 37; +static const uint8_t SDO = 35; +static const uint8_t SDI = 37; +static const uint8_t SCK = 36; + +static const uint8_t A0 = 17; +static const uint8_t A1 = 18; +static const uint8_t A2 = 14; +static const uint8_t A3 = 12; +static const uint8_t A4 = 6; +static const uint8_t A5 = 5; +static const uint8_t A6 = 1; +static const uint8_t A7 = 3; +static const uint8_t A8 = 7; +static const uint8_t A9 = 10; +static const uint8_t A10 = 11; +static const uint8_t A11 = 8; +static const uint8_t A12 = 9; + +static const uint8_t T1 = 1; +static const uint8_t T3 = 3; +static const uint8_t T5 = 5; +static const uint8_t T6 = 6; +static const uint8_t T7 = 7; +static const uint8_t T8 = 8; +static const uint8_t T9 = 9; +static const uint8_t T10 = 10; +static const uint8_t T11 = 11; +static const uint8_t T12 = 12; +static const uint8_t T14 = 14; + +static const uint8_t VBAT_SENSE = 2; +static const uint8_t VBUS_SENSE = 15; + +// User LED +#define LED_BUILTIN 13 +#define BUILTIN_LED LED_BUILTIN // backward compatibility + +static const uint8_t RGB_DATA = 40; +// RGB_BUILTIN and RGB_BRIGHTNESS can be used in new Arduino API neopixelWrite() +#define RGB_BUILTIN (RGB_DATA + SOC_GPIO_PIN_COUNT) +#define RGB_BRIGHTNESS 64 + +static const uint8_t RGB_PWR = 39; +static const uint8_t RGB_MATRIX_PWR = 39; +static const uint8_t RGB_MATRIX_DATA = 16; +static const uint8_t LED = 13; + +#endif /* Pins_Arduino_h */ From e3fedc5e8c47427d2fc8c4b45c60dd083243d786 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Fri, 7 Jun 2024 12:29:27 -0300 Subject: [PATCH 02/24] ci(refactor): Fix workflows and small improvements (#9793) * ci(fix): Fix new workflows * ci(pre-commit): Properly cache python modules * fix(get.py): Verify files after extraction and make it more readable * ci(idf): Compile as component when libs change * ci(wokwi): Re-enable cache for wokwi tests * ci(tests): Run scripts only from master * Revert "fix(get.py): Verify files after extraction and make it more readable" This reverts commit cc40edb40766ea8a013c0d76326bc57314418bff. * ci(compilation): Fix changed files check * ci(wokwi): Disable tests that use scenarios --- .github/scripts/tests_run.sh | 3 +- .github/workflows/build_tests.yml | 17 +++++++- .github/workflows/hw.yml | 28 +++++++++---- .github/workflows/pre-commit.yml | 8 ++-- .github/workflows/push.yml | 12 ++++-- .github/workflows/qemu.yml | 33 +++++++++++----- .github/workflows/tests.yml | 15 +++---- .github/workflows/wokwi.yml | 65 ++++++++++++++++++------------- tests/validation/gpio/ci.json | 3 +- tools/pre-commit/requirements.txt | 2 + 10 files changed, 121 insertions(+), 65 deletions(-) create mode 100644 tools/pre-commit/requirements.txt diff --git a/.github/scripts/tests_run.sh b/.github/scripts/tests_run.sh index 43ac41e2d4d..bb90373b56c 100755 --- a/.github/scripts/tests_run.sh +++ b/.github/scripts/tests_run.sh @@ -101,7 +101,7 @@ function run_test() { fi fi done - printf "\n" + printf "Test return code: $error\n" return $error } @@ -250,6 +250,7 @@ else exit_code=0 run_test $target $sketch $options $erase || exit_code=$? + echo "Sketch $sketch exit code: $exit_code" if [ $exit_code -ne 0 ]; then error=$exit_code fi diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml index a142342748a..17cd3b0a1bc 100644 --- a/.github/workflows/build_tests.yml +++ b/.github/workflows/build_tests.yml @@ -47,11 +47,24 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT - - name: Checkout Repository - uses: actions/checkout@v4 + - name: Checkout user repository if: ${{ steps.check-build.outputs.enabled == 'true' }} + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /* + !.github + + # To avoid giving unknown scripts elevated permissions, download them from the master branch + - name: Get CI scripts from master + if: ${{ steps.check-build.outputs.enabled == 'true' }} + run: | + mkdir -p .github + cd .github + curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github - name: Get libs cache uses: actions/cache@v4 diff --git a/.github/workflows/hw.yml b/.github/workflows/hw.yml index 4be0abe6669..376c38a6530 100644 --- a/.github/workflows/hw.yml +++ b/.github/workflows/hw.yml @@ -49,18 +49,32 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT - - name: Checkout repository + - name: Checkout user repository if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /* + !.github - - uses: actions/setup-python@v5 - if: steps.check-tests.outputs.enabled == 'true' - with: - cache-dependency-path: tests/requirements.txt - cache: 'pip' - python-version: '3.10.1' + # To avoid giving unknown scripts elevated permissions, download them from the master branch + - name: Get CI scripts from master + if: ${{ steps.check-tests.outputs.enabled == 'true' }} + run: | + mkdir -p .github + cd .github + curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github + + # setup-python currently only works on ubuntu images + # - uses: actions/setup-python@v5 + # if: ${{ steps.check-tests.outputs.enabled == 'true' }} + # with: + # cache-dependency-path: tests/requirements.txt + # cache: 'pip' + # python-version: '3.10.1' - name: Install dependencies if: ${{ steps.check-tests.outputs.enabled == 'true' }} diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 56393d46202..c4ae017c229 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -28,6 +28,8 @@ jobs: - name: Set up Python 3 uses: actions/setup-python@v5 with: + cache-dependency-path: tools/pre-commit/requirements.txt + cache: 'pip' python-version: "3.x" - name: Get Python version hash @@ -41,11 +43,10 @@ jobs: with: path: | ~/.cache/pre-commit - ~/.cache/pip - key: pre-commit|${{ env.PY_HASH }}|${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml') }} + key: pre-commit-${{ env.PY_HASH }}-${{ hashFiles('.pre-commit-config.yaml', '.github/workflows/pre-commit.yml', 'tools/pre-commit/requirements.txt') }} - name: Install python dependencies - run: python -m pip install pre-commit docutils + run: python -m pip install -r tools/pre-commit/requirements.txt - name: Get changed files id: changed-files @@ -61,7 +62,6 @@ jobs: with: path: | ~/.cache/pre-commit - ~/.cache/pip key: ${{ steps.restore-cache.outputs.cache-primary-key }} - name: Push changes using pre-commit-ci-lite diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index ca3f2de55cf..05fd1a8a2b8 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -48,6 +48,7 @@ jobs: runs-on: ubuntu-latest outputs: build_all: ${{ steps.set-chunks.outputs.build_all }} + build_libraries: ${{ steps.set-chunks.outputs.build_libraries }} build_static_sketches: ${{ steps.set-chunks.outputs.build_static_sketches }} build_idf: ${{ steps.set-chunks.outputs.build_idf }} build_platformio: ${{ steps.set-chunks.outputs.build_platformio }} @@ -103,6 +104,7 @@ jobs: build_platformio=${{ steps.changed-files.outputs.platformio_any_changed == 'true' }} build_idf=${{ steps.changed-files.outputs.idf_any_changed == 'true' }} + build_libraries=${{ steps.changed-files.outputs.libraries_any_changed == 'true' }} build_static_sketches=${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }} core_changed=${{ steps.changed-files.outputs.core_any_changed == 'true' }} @@ -136,9 +138,6 @@ jobs: fi echo "" done - else - echo "Unhandled change triggered the build. This should not happen." - exit 1 fi if [[ -n $sketches ]]; then @@ -164,6 +163,7 @@ jobs: chunks+="]" echo "build_all=$build_all" >> $GITHUB_OUTPUT + echo "build_libraries=$build_libraries" >> $GITHUB_OUTPUT echo "build_static_sketches=$build_static_sketches" >> $GITHUB_OUTPUT echo "build_idf=$build_idf" >> $GITHUB_OUTPUT echo "build_platformio=$build_platformio" >> $GITHUB_OUTPUT @@ -182,6 +182,7 @@ jobs: # Ubuntu build-arduino-linux: name: Arduino ${{ matrix.chunk }} on ubuntu-latest + if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_libraries == 'true' }} needs: gen-chunks runs-on: ubuntu-latest strategy: @@ -274,7 +275,10 @@ jobs: build-esp-idf-component: name: Build with ESP-IDF ${{ matrix.idf_ver }} for ${{ matrix.idf_target }} needs: gen-chunks - if: ${{ needs.gen-chunks.outputs.build_all == 'true' || needs.gen-chunks.outputs.build_idf == 'true' }} + if: | + needs.gen-chunks.outputs.build_all == 'true' || + needs.gen-chunks.outputs.build_libraries == 'true' || + needs.gen-chunks.outputs.build_idf == 'true' runs-on: ubuntu-20.04 strategy: fail-fast: false diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml index 629df904126..1b8d4538901 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/qemu.yml @@ -45,35 +45,48 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT - - name: Checkout repository + - name: Checkout user repository + if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: actions/checkout@v4 - if: steps.check-tests.outputs.enabled == 'true' with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /* + !.github + + # To avoid giving unknown scripts elevated permissions, download them from the master branch + - name: Get CI scripts from master + if: ${{ steps.check-tests.outputs.enabled == 'true' }} + run: | + mkdir -p .github + cd .github + curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github - uses: actions/setup-python@v5 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} with: cache-dependency-path: tests/requirements.txt cache: 'pip' python-version: '3.x' - name: Install Python dependencies - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} run: | pip install -U pip pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi - name: Install APT dependencies uses: awalsh128/cache-apt-pkgs-action@v1.4.2 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} with: packages: libpixman-1-0 libnuma1 libglib2.0-0 libslirp0 libsdl2-2.0-0 version: 1.0 - name: Get QEMU version uses: pozetroninc/github-action-get-latest-release@v0.7.0 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} id: get-qemu-version with: token: ${{secrets.GITHUB_TOKEN}} @@ -84,14 +97,14 @@ jobs: - name: Cache QEMU id: cache-qemu uses: actions/cache@v4 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} with: path: | ~/qemu key: qemu-${{ steps.get-qemu-version.outputs.release }}-${{ hashFiles('.github/workflows/qemu.yml') }} - name: Download QEMU - if: steps.cache-qemu.outputs.cache-hit != 'true' && steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.cache-qemu.outputs.cache-hit != 'true' && steps.check-tests.outputs.enabled == 'true' }} run: | cd ${{ env.QEMU_INSTALL_PATH }} underscore_release=$(echo ${{ steps.get-qemu-version.outputs.release }} | sed 's/\-/_/g') @@ -103,7 +116,7 @@ jobs: echo "QEMU_PATH=${{ env.QEMU_INSTALL_PATH }}/qemu" >> $GITHUB_ENV - name: Get binaries - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} id: cache-build-binaries uses: actions/cache/restore@v4 with: @@ -115,7 +128,7 @@ jobs: ~/.arduino/tests/**/build*.tmp/*.json - name: Run Tests - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} run: QEMU_PATH="${{ env.QEMU_INSTALL_PATH }}" bash .github/scripts/tests_run.sh -c -type ${{inputs.type}} -t ${{inputs.chip}} -i 0 -m 1 -Q - name: Upload ${{ inputs.chip }} ${{ inputs.type }} QEMU results as cache diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 25243cf9440..d5a8e267c5f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -12,14 +12,6 @@ on: - '!libraries/**.txt' - '!libraries/**.properties' - 'package/**' - - '.github/workflows/tests.yml' - - '.github/workflows/build_tests.yml' - - '.github/workflows/hw.yml' - - '.github/workflows/wokwi.yml' - - '.github/workflows/qemu.yml' - - '.github/scripts/install-*.sh' - - '.github/scripts/tests_*.sh' - - '.github/scripts/sketch_utils.sh' schedule: - cron: '0 2 * * *' @@ -167,7 +159,7 @@ jobs: clean: name: Clean objects needs: unit-test-results - if: always() && ${{ github.event_name }} == 'pull_request_target' && ${{ github.event.action }} != 'closed' + if: always() permissions: actions: write runs-on: ubuntu-latest @@ -179,6 +171,11 @@ jobs: const ref = '${{ github.event.pull_request.number || github.ref }}'; const key_prefix = 'tests-' + ref + '-'; + if ('${{ github.event_name }}' == 'pull_request_target' && '${{ github.event.action }}' != 'closed') { + console.log('Skipping cache cleanup for open PR'); + return; + } + await github.paginate(github.rest.actions.getActionsCacheList, { owner: context.repo.owner, repo: context.repo.repo, diff --git a/.github/workflows/wokwi.yml b/.github/workflows/wokwi.yml index ae68daff9e0..8462f47eb12 100644 --- a/.github/workflows/wokwi.yml +++ b/.github/workflows/wokwi.yml @@ -30,16 +30,15 @@ jobs: id: ${{ github.event.pull_request.number || github.ref }}-${{ github.event.pull_request.head.sha || github.sha }}-${{ inputs.chip }}-${{ inputs.type }} runs-on: ubuntu-latest steps: - # Disabled as Wokwi infrastrucutre is not stable (so we can re-trigger the tests manually) - # - name: Check if already run - # if: ${{ github.event.pull_request.number != null }} - # id: get-cache-results - # uses: actions/cache/restore@v4 - # with: - # key: tests-${{ env.id }}-results-wokwi - # path: | - # tests/**/*.xml - # tests/**/result_*.json + - name: Check if already run + if: ${{ github.event.pull_request.number != null }} + id: get-cache-results + uses: actions/cache/restore@v4 + with: + key: tests-${{ env.id }}-results-wokwi + path: | + tests/**/*.xml + tests/**/result_*.json - name: Evaluate if tests should be run id: check-tests @@ -54,35 +53,48 @@ jobs: echo "enabled=$enabled" >> $GITHUB_OUTPUT - - name: Checkout Repository + - name: Checkout user repository + if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: actions/checkout@v4 - if: steps.check-tests.outputs.enabled == 'true' with: ref: ${{ github.event.pull_request.head.sha || github.sha }} + persist-credentials: false + sparse-checkout-cone-mode: false + sparse-checkout: | + /* + !.github + + # To avoid giving unknown scripts elevated permissions, download them from the master branch + - name: Get CI scripts from master + if: ${{ steps.check-tests.outputs.enabled == 'true' }} + run: | + mkdir -p .github + cd .github + curl https://codeload.github.com/${{ github.repository }}/tar.gz/master | tar -xz --strip=2 arduino-esp32-master/.github - uses: actions/setup-python@v5 - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} with: cache-dependency-path: tests/requirements.txt cache: 'pip' python-version: '3.x' - name: Install dependencies - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} run: | pip install -U pip pip install -r tests/requirements.txt --extra-index-url https://dl.espressif.com/pypi - name: Install Wokwi CLI - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} run: curl -L https://wokwi.com/ci/install.sh | sh - name: Wokwi CI Server - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} uses: wokwi/wokwi-ci-server-action@v1 - name: Get binaries - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} id: cache-build-binaries uses: actions/cache/restore@v4 with: @@ -94,21 +106,20 @@ jobs: ~/.arduino/tests/**/build*.tmp/*.json - name: Run Tests - if: steps.check-tests.outputs.enabled == 'true' + if: ${{ steps.check-tests.outputs.enabled == 'true' }} env: WOKWI_CLI_TOKEN: ${{ secrets.WOKWI_CLI_TOKEN }} run: | bash .github/scripts/tests_run.sh -c -type ${{ inputs.type }} -t ${{inputs.chip}} -i 0 -m 1 -W ${{env.WOKWI_TIMEOUT}} - # Disabled as Wokwi infrastrucutre is not stable (so we can re-trigger the tests manually) - # - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as cache - # uses: actions/cache/save@v4 - # if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }} - # with: - # key: tests-${{ env.id }}-results-wokwi - # path: | - # tests/**/*.xml - # tests/**/result_*.json + - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as cache + uses: actions/cache/save@v4 + if: ${{ always() && steps.check-tests.outputs.enabled == 'true' }} + with: + key: tests-${{ env.id }}-results-wokwi + path: | + tests/**/*.xml + tests/**/result_*.json - name: Upload ${{ inputs.chip }} ${{ inputs.type }} Wokwi results as artifacts uses: actions/upload-artifact@v4 diff --git a/tests/validation/gpio/ci.json b/tests/validation/gpio/ci.json index 8e24ae58e5a..13a4b8c1a31 100644 --- a/tests/validation/gpio/ci.json +++ b/tests/validation/gpio/ci.json @@ -1,6 +1,7 @@ { "platforms": { "qemu": false, - "hardware": false + "hardware": false, + "wokwi": false } } diff --git a/tools/pre-commit/requirements.txt b/tools/pre-commit/requirements.txt new file mode 100644 index 00000000000..10db94a1393 --- /dev/null +++ b/tools/pre-commit/requirements.txt @@ -0,0 +1,2 @@ +pre-commit==3.7.1 +docutils==0.21.2 From 29286540278e267b9137fbf6a5a94591d86954db Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:30:18 -0300 Subject: [PATCH 03/24] change(idf): Rename component example and fix compilation warnings (#9801) * change(idf): Rename component example * ci(push): Fix steps conditions * ci(tests): Remove unnecessary concurrency * ci(push): Fix step condition * fix(idf): Fix compilation warnings when as component --- .github/workflows/build_tests.yml | 4 ---- .github/workflows/hw.yml | 4 ---- .github/workflows/push.yml | 6 +++--- .github/workflows/qemu.yml | 4 ---- .github/workflows/wokwi.yml | 4 ---- cores/esp32/MacAddress.cpp | 8 ++++---- cores/esp32/esp32-hal-adc.c | 2 +- idf_component.yml | 2 +- .../{Hello_world => hello_world}/CMakeLists.txt | 0 .../{Hello_world => hello_world}/README.md | 4 ++-- .../{Hello_world => hello_world}/main/CMakeLists.txt | 0 .../{Hello_world => hello_world}/main/idf_component.yml | 0 .../{Hello_world => hello_world}/main/main.cpp | 0 .../{Hello_world => hello_world}/sdkconfig.defaults | 0 libraries/Ethernet/src/ETH.cpp | 8 ++++---- 15 files changed, 15 insertions(+), 31 deletions(-) rename idf_component_examples/{Hello_world => hello_world}/CMakeLists.txt (100%) rename idf_component_examples/{Hello_world => hello_world}/README.md (95%) rename idf_component_examples/{Hello_world => hello_world}/main/CMakeLists.txt (100%) rename idf_component_examples/{Hello_world => hello_world}/main/idf_component.yml (100%) rename idf_component_examples/{Hello_world => hello_world}/main/main.cpp (100%) rename idf_component_examples/{Hello_world => hello_world}/sdkconfig.defaults (100%) diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml index 17cd3b0a1bc..b7c4ec40f1a 100644 --- a/.github/workflows/build_tests.yml +++ b/.github/workflows/build_tests.yml @@ -12,10 +12,6 @@ on: description: 'Chip to build tests for' required: true -concurrency: - group: tests-build-${{ github.event.pull_request.number || github.ref }}-${{ inputs.chip }}-${{ inputs.type }} - cancel-in-progress: true - jobs: build-tests: name: Build ${{ inputs.type }} tests for ${{ inputs.chip }} diff --git a/.github/workflows/hw.yml b/.github/workflows/hw.yml index 376c38a6530..8c7dc0dffb2 100644 --- a/.github/workflows/hw.yml +++ b/.github/workflows/hw.yml @@ -12,10 +12,6 @@ on: description: 'Chip to run tests for' required: true -concurrency: - group: tests-hw-${{ github.event.pull_request.number || github.ref }}-${{ inputs.chip }}-${{ inputs.type }} - cancel-in-progress: true - jobs: hardware-test: name: Hardware ${{ inputs.chip }} ${{ inputs.type }} tests diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 05fd1a8a2b8..3ef15df1566 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -171,7 +171,7 @@ jobs: echo "chunks=$chunks" >> $GITHUB_OUTPUT - name: Upload sketches found - if: ${{ steps.set-chunks.outputs.build_all == 'false' }} + if: ${{ steps.set-chunks.outputs.build_all == 'false' && steps.set-chunks.outputs.build_libraries == 'true' }} uses: actions/upload-artifact@v4 with: name: sketches_found @@ -214,13 +214,13 @@ jobs: run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} ${{ env.MAX_CHUNKS }} 1 - name: Download sketches found - if: ${{ needs.gen-chunks.outputs.build_all == 'false' }} + if: ${{ needs.gen-chunks.outputs.build_all == 'false' && needs.gen-chunks.outputs.build_libraries == 'true' }} uses: actions/download-artifact@v4 with: name: sketches_found - name: Build selected sketches - if: ${{ needs.gen-chunks.outputs.build_all == 'false' }} + if: ${{ needs.gen-chunks.outputs.build_all == 'false' && needs.gen-chunks.outputs.build_libraries == 'true' }} run: bash ./.github/scripts/on-push.sh ${{ matrix.chunk }} ${{ needs.gen-chunks.outputs.chunk_count }} 1 sketches_found.txt #Upload cli compile json as artifact diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml index 1b8d4538901..1aa155129c8 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/qemu.yml @@ -10,10 +10,6 @@ on: required: true type: string -concurrency: - group: qemu-${{ github.event.pull_request.number || github.ref }}-${{ inputs.chip }}-${{ inputs.type }} - cancel-in-progress: true - jobs: qemu-test: name: QEMU ${{ inputs.chip }} ${{ inputs.type }} tests diff --git a/.github/workflows/wokwi.yml b/.github/workflows/wokwi.yml index 8462f47eb12..84b0acf06b4 100644 --- a/.github/workflows/wokwi.yml +++ b/.github/workflows/wokwi.yml @@ -16,10 +16,6 @@ on: description: 'Wokwi CLI API token' required: true -concurrency: - group: tests-wokwi-${{ github.event.pull_request.number || github.ref }}-${{ inputs.chip }}-${{ inputs.type }} - cancel-in-progress: true - env: WOKWI_TIMEOUT: 600000 # Milliseconds diff --git a/cores/esp32/MacAddress.cpp b/cores/esp32/MacAddress.cpp index 718b5cfbaf1..8b4fab1781a 100644 --- a/cores/esp32/MacAddress.cpp +++ b/cores/esp32/MacAddress.cpp @@ -67,12 +67,12 @@ bool MacAddress::fromString(const char *buf) { //Parse user entered string into MAC address bool MacAddress::fromString6(const char *buf) { - char cs[18]; + char cs[18]; // 17 + 1 for null terminator char *token; char *next; //Unused but required int i; - strncpy(cs, buf, sizeof(cs)); //strtok modifies the buffer: copy to working buffer. + strncpy(cs, buf, sizeof(cs) - 1); //strtok modifies the buffer: copy to working buffer. for (i = 0; i < 6; i++) { token = strtok((i == 0) ? cs : NULL, ":"); //Find first or next token @@ -86,12 +86,12 @@ bool MacAddress::fromString6(const char *buf) { } bool MacAddress::fromString8(const char *buf) { - char cs[24]; + char cs[24]; // 23 + 1 for null terminator char *token; char *next; //Unused but required int i; - strncpy(cs, buf, sizeof(cs)); //strtok modifies the buffer: copy to working buffer. + strncpy(cs, buf, sizeof(cs) - 1); //strtok modifies the buffer: copy to working buffer. for (i = 0; i < 8; i++) { token = strtok((i == 0) ? cs : NULL, ":"); //Find first or next token diff --git a/cores/esp32/esp32-hal-adc.c b/cores/esp32/esp32-hal-adc.c index ed9108b6a7f..89ddd726159 100644 --- a/cores/esp32/esp32-hal-adc.c +++ b/cores/esp32/esp32-hal-adc.c @@ -458,7 +458,7 @@ esp_err_t __analogContinuousInit(adc_channel_t *channel, uint8_t channel_num, ad bool analogContinuous(uint8_t pins[], size_t pins_count, uint32_t conversions_per_pin, uint32_t sampling_freq_hz, void (*userFunc)(void)) { adc_channel_t channel[pins_count]; - adc_unit_t adc_unit; + adc_unit_t adc_unit = ADC_UNIT_1; esp_err_t err = ESP_OK; //Convert pins to channels and check if all are ADC1s unit diff --git a/idf_component.yml b/idf_component.yml index bc987f25a93..3accad1fb9d 100644 --- a/idf_component.yml +++ b/idf_component.yml @@ -87,4 +87,4 @@ dependencies: version: "1.0.3" require: public examples: - - path: ./idf_component_examples/Hello_world + - path: ./idf_component_examples/hello_world diff --git a/idf_component_examples/Hello_world/CMakeLists.txt b/idf_component_examples/hello_world/CMakeLists.txt similarity index 100% rename from idf_component_examples/Hello_world/CMakeLists.txt rename to idf_component_examples/hello_world/CMakeLists.txt diff --git a/idf_component_examples/Hello_world/README.md b/idf_component_examples/hello_world/README.md similarity index 95% rename from idf_component_examples/Hello_world/README.md rename to idf_component_examples/hello_world/README.md index f666805dd8a..6e5f1e9acff 100644 --- a/idf_component_examples/Hello_world/README.md +++ b/idf_component_examples/hello_world/README.md @@ -12,13 +12,13 @@ To create a ESP-IDF project from this example with the latest release of Arduino ESP-IDF will download all dependencies needed from the component registry and setup the project for you. If you want to use cloned Arduino-esp32 repository, you can build this example directly. -Go to the example folder `arduino-esp32/idf_component_examples/Hello_world`. +Go to the example folder `arduino-esp32/idf_component_examples/hello_world`. First you need to comment line 6 `pre_release: true` in examples `/main/idf_component.yml`. Then just run command: `idf.py build`. ## Example folder contents -The project **Hello_world** contains one source file in C++ language [main.cpp](main/main.cpp). The file is located in folder [main](main). +The project **hello_world** contains one source file in C++ language [main.cpp](main/main.cpp). The file is located in folder [main](main). ESP-IDF projects are built using CMake. The project build configuration is contained in `CMakeLists.txt` files that provide set of directives and instructions describing the project's source files and targets diff --git a/idf_component_examples/Hello_world/main/CMakeLists.txt b/idf_component_examples/hello_world/main/CMakeLists.txt similarity index 100% rename from idf_component_examples/Hello_world/main/CMakeLists.txt rename to idf_component_examples/hello_world/main/CMakeLists.txt diff --git a/idf_component_examples/Hello_world/main/idf_component.yml b/idf_component_examples/hello_world/main/idf_component.yml similarity index 100% rename from idf_component_examples/Hello_world/main/idf_component.yml rename to idf_component_examples/hello_world/main/idf_component.yml diff --git a/idf_component_examples/Hello_world/main/main.cpp b/idf_component_examples/hello_world/main/main.cpp similarity index 100% rename from idf_component_examples/Hello_world/main/main.cpp rename to idf_component_examples/hello_world/main/main.cpp diff --git a/idf_component_examples/Hello_world/sdkconfig.defaults b/idf_component_examples/hello_world/sdkconfig.defaults similarity index 100% rename from idf_component_examples/Hello_world/sdkconfig.defaults rename to idf_component_examples/hello_world/sdkconfig.defaults diff --git a/libraries/Ethernet/src/ETH.cpp b/libraries/Ethernet/src/ETH.cpp index d7845b9ca3a..2480dd982ec 100644 --- a/libraries/Ethernet/src/ETH.cpp +++ b/libraries/Ethernet/src/ETH.cpp @@ -358,11 +358,11 @@ bool ETHClass::begin(eth_phy_type_t type, int32_t phy_addr, int mdc, int mdio, i #endif /* CONFIG_ETH_USE_ESP32_EMAC */ #if ETH_SPI_SUPPORTS_CUSTOM -static void *_eth_spi_init(const void *ctx) { +__unused static void *_eth_spi_init(const void *ctx) { return (void *)ctx; } -static esp_err_t _eth_spi_deinit(void *ctx) { +__unused static esp_err_t _eth_spi_deinit(void *ctx) { return ESP_OK; } @@ -575,8 +575,8 @@ bool ETHClass::beginSPI( } // Init common MAC and PHY configs to default - eth_mac_config_t eth_mac_config = ETH_MAC_DEFAULT_CONFIG(); - eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG(); + __unused eth_mac_config_t eth_mac_config = ETH_MAC_DEFAULT_CONFIG(); + __unused eth_phy_config_t phy_config = ETH_PHY_DEFAULT_CONFIG(); // Update PHY config based on board specific configuration phy_config.phy_addr = phy_addr; From 034d568d7cb6c7f9e7f52eefbdf99fe86ef0c6c9 Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Mon, 10 Jun 2024 09:30:46 -0300 Subject: [PATCH 04/24] ci(wifi): Add scan to wifi test (#9805) --- tests/validation/wifi/test_wifi.py | 3 +++ tests/validation/wifi/wifi.ino | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/tests/validation/wifi/test_wifi.py b/tests/validation/wifi/test_wifi.py index aa074192341..49dd22797d2 100644 --- a/tests/validation/wifi/test_wifi.py +++ b/tests/validation/wifi/test_wifi.py @@ -1,3 +1,6 @@ def test_wifi(dut): + dut.expect_exact("Scan start") + dut.expect_exact("Scan done") + dut.expect_exact("Wokwi-GUEST") dut.expect_exact("WiFi connected") dut.expect_exact("IP address:") diff --git a/tests/validation/wifi/wifi.ino b/tests/validation/wifi/wifi.ino index dd93ac137f9..696234505cc 100644 --- a/tests/validation/wifi/wifi.ino +++ b/tests/validation/wifi/wifi.ino @@ -112,6 +112,28 @@ void setup() { Serial.println(eventID); // WiFi.removeEvent(eventID); + Serial.println("Scan start"); + + // WiFi.scanNetworks will return the number of networks found. + int n = WiFi.scanNetworks(); + Serial.println("Scan done"); + if (n == 0) { + Serial.println("no networks found"); + } else { + Serial.print(n); + Serial.println(" networks found"); + for (int i = 0; i < n; ++i) { + // Print SSID for each network found + Serial.printf("%s\n", WiFi.SSID(i).c_str()); + Serial.println(); + delay(10); + } + } + Serial.println(""); + + // Delete the scan result to free memory for code below. + WiFi.scanDelete(); + WiFi.begin(ssid, password); Serial.println(); From 575a415719d875f88460ec1461408597744f7053 Mon Sep 17 00:00:00 2001 From: Rodrigo Garcia Date: Mon, 10 Jun 2024 09:31:30 -0300 Subject: [PATCH 05/24] fix (adc): fixes limit test (#9807) Fixes analogContinuousSetWidth() border test. --- cores/esp32/esp32-hal-adc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cores/esp32/esp32-hal-adc.c b/cores/esp32/esp32-hal-adc.c index 89ddd726159..6788f2eb123 100644 --- a/cores/esp32/esp32-hal-adc.c +++ b/cores/esp32/esp32-hal-adc.c @@ -682,7 +682,7 @@ void analogContinuousSetAtten(adc_attenuation_t attenuation) { } void analogContinuousSetWidth(uint8_t bits) { - if ((bits < SOC_ADC_DIGI_MIN_BITWIDTH) && (bits > SOC_ADC_DIGI_MAX_BITWIDTH)) { + if ((bits < SOC_ADC_DIGI_MIN_BITWIDTH) || (bits > SOC_ADC_DIGI_MAX_BITWIDTH)) { log_e("Selected width cannot be set. Range is from %d to %d", SOC_ADC_DIGI_MIN_BITWIDTH, SOC_ADC_DIGI_MAX_BITWIDTH); return; } From e382746b95be9f19a74a10408ae3e250e36e1930 Mon Sep 17 00:00:00 2001 From: lbernstone Date: Tue, 11 Jun 2024 19:53:54 -1000 Subject: [PATCH 06/24] feat(sdmmc): Add RAW disk functions (#9796) * feat(sdmmc): Add RAW disk functions feat(sdmmc): fixed printf mismatches and missing callback feat(sdmmc): added ci.json Removed excess log_i * ci(pre-commit): Apply automatic fixes * Fixed sdmmc host check to pass CI * feat(sdmmc): fixed example USB check --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> Co-authored-by: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> --- .../SD_MMC/examples/SD2USBMSC/SD2USBMSC.ino | 102 ++++++++++++++++++ libraries/SD_MMC/examples/SD2USBMSC/ci.json | 9 ++ libraries/SD_MMC/src/SD_MMC.cpp | 25 +++++ libraries/SD_MMC/src/SD_MMC.h | 11 +- 4 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 libraries/SD_MMC/examples/SD2USBMSC/SD2USBMSC.ino create mode 100644 libraries/SD_MMC/examples/SD2USBMSC/ci.json diff --git a/libraries/SD_MMC/examples/SD2USBMSC/SD2USBMSC.ino b/libraries/SD_MMC/examples/SD2USBMSC/SD2USBMSC.ino new file mode 100644 index 00000000000..d379e409960 --- /dev/null +++ b/libraries/SD_MMC/examples/SD2USBMSC/SD2USBMSC.ino @@ -0,0 +1,102 @@ +#if !SOC_USB_OTG_SUPPORTED || ARDUINO_USB_MODE +#error Device does not support USB_OTG or native USB CDC/JTAG is selected +#endif + +#include +#include +#include + +// USB Mass Storage Class (MSC) object +USBMSC msc; + +int clk = 36; +int cmd = 35; +int d0 = 37; +int d1 = 38; +int d2 = 33; +int d3 = 34; +bool onebit = true; // set to false for 4-bit. 1-bit will ignore the d1-d3 pins (but d3 must be pulled high) + +static int32_t onWrite(uint32_t lba, uint32_t offset, uint8_t *buffer, uint32_t bufsize) { + uint32_t secSize = SD_MMC.sectorSize(); + if (!secSize) { + return false; // disk error + } + log_v("Write lba: %ld\toffset: %ld\tbufsize: %ld", lba, offset, bufsize); + for (int x = 0; x < bufsize / secSize; x++) { + uint8_t blkbuffer[secSize]; + memcpy(blkbuffer, (uint8_t *)buffer + secSize * x, secSize); + if (!SD_MMC.writeRAW(blkbuffer, lba + x)) { + return false; + } + } + return bufsize; +} + +static int32_t onRead(uint32_t lba, uint32_t offset, void *buffer, uint32_t bufsize) { + uint32_t secSize = SD_MMC.sectorSize(); + if (!secSize) { + return false; // disk error + } + log_v("Read lba: %ld\toffset: %ld\tbufsize: %ld\tsector: %lu", lba, offset, bufsize, secSize); + for (int x = 0; x < bufsize / secSize; x++) { + if (!SD_MMC.readRAW((uint8_t *)buffer + (x * secSize), lba + x)) { + return false; // outside of volume boundary + } + } + return bufsize; +} + +static bool onStartStop(uint8_t power_condition, bool start, bool load_eject) { + log_i("Start/Stop power: %u\tstart: %d\teject: %d", power_condition, start, load_eject); + return true; +} + +static void usbEventCallback(void *arg, esp_event_base_t event_base, int32_t event_id, void *event_data) { + if (event_base == ARDUINO_USB_EVENTS) { + arduino_usb_event_data_t *data = (arduino_usb_event_data_t *)event_data; + switch (event_id) { + case ARDUINO_USB_STARTED_EVENT: Serial.println("USB PLUGGED"); break; + case ARDUINO_USB_STOPPED_EVENT: Serial.println("USB UNPLUGGED"); break; + case ARDUINO_USB_SUSPEND_EVENT: Serial.printf("USB SUSPENDED: remote_wakeup_en: %u\n", data->suspend.remote_wakeup_en); break; + case ARDUINO_USB_RESUME_EVENT: Serial.println("USB RESUMED"); break; + + default: break; + } + } +} + +void setup() { + Serial.begin(115200); + Serial.println("Starting Serial"); + + Serial.println("Mounting SDcard"); + SD_MMC.setPins(clk, cmd, d0, d1, d2, d3); + if (!SD_MMC.begin("/sdcard", onebit)) { + Serial.println("Mount Failed"); + return; + } + + Serial.println("Initializing MSC"); + // Initialize USB metadata and callbacks for MSC (Mass Storage Class) + msc.vendorID("ESP32"); + msc.productID("USB_MSC"); + msc.productRevision("1.0"); + msc.onRead(onRead); + msc.onWrite(onWrite); + msc.onStartStop(onStartStop); + msc.mediaPresent(true); + msc.begin(SD_MMC.numSectors(), SD_MMC.sectorSize()); + + Serial.println("Initializing USB"); + + USB.begin(); + USB.onEvent(usbEventCallback); + + Serial.printf("Card Size: %lluMB\n", SD_MMC.totalBytes() / 1024 / 1024); + Serial.printf("Sector: %d\tCount: %d\n", SD_MMC.sectorSize(), SD_MMC.numSectors()); +} + +void loop() { + delay(-1); +} diff --git a/libraries/SD_MMC/examples/SD2USBMSC/ci.json b/libraries/SD_MMC/examples/SD2USBMSC/ci.json new file mode 100644 index 00000000000..2a5ca52e079 --- /dev/null +++ b/libraries/SD_MMC/examples/SD2USBMSC/ci.json @@ -0,0 +1,9 @@ +{ + "targets": { + "esp32": false, + "esp32s2": false, + "esp32c3": false, + "esp32c6": false, + "esp32h2": false + } +} diff --git a/libraries/SD_MMC/src/SD_MMC.cpp b/libraries/SD_MMC/src/SD_MMC.cpp index 18aa9169f59..13e5fcf27fc 100644 --- a/libraries/SD_MMC/src/SD_MMC.cpp +++ b/libraries/SD_MMC/src/SD_MMC.cpp @@ -26,6 +26,8 @@ #include "driver/sdmmc_host.h" #include "driver/sdmmc_defs.h" #include "sdmmc_cmd.h" +#include "diskio_sdmmc.h" +#include "diskio.h" #include "soc/sdmmc_pins.h" #include "ff.h" #include "esp32-hal-periman.h" @@ -191,6 +193,7 @@ bool SDMMCFS::begin(const char *mountpoint, bool mode1bit, bool format_if_mount_ return false; } _impl->mountpoint(mountpoint); + _pdrv = ff_diskio_get_pdrv_card(_card); if (!perimanSetPinBus(_pin_cmd, ESP32_BUS_TYPE_SDMMC_CMD, (void *)(this), -1, -1)) { goto err; @@ -280,5 +283,27 @@ uint64_t SDMMCFS::usedBytes() { return size; } +int SDMMCFS::sectorSize() { + if (!_card) { + return 0; + } + return _card->csd.sector_size; +} + +int SDMMCFS::numSectors() { + if (!_card) { + return 0; + } + return (totalBytes() / _card->csd.sector_size); +} + +bool SDMMCFS::readRAW(uint8_t *buffer, uint32_t sector) { + return (disk_read(_pdrv, buffer, sector, 1) == 0); +} + +bool SDMMCFS::writeRAW(uint8_t *buffer, uint32_t sector) { + return (disk_write(_pdrv, buffer, sector, 1) == 0); +} + SDMMCFS SD_MMC = SDMMCFS(FSImplPtr(new VFSImpl())); #endif /* SOC_SDMMC_HOST_SUPPORTED */ diff --git a/libraries/SD_MMC/src/SD_MMC.h b/libraries/SD_MMC/src/SD_MMC.h index 3a69850470c..a2bc12aed64 100644 --- a/libraries/SD_MMC/src/SD_MMC.h +++ b/libraries/SD_MMC/src/SD_MMC.h @@ -16,7 +16,11 @@ #include "sdkconfig.h" #include "soc/soc_caps.h" -#ifdef SOC_SDMMC_HOST_SUPPORTED +#ifndef SOC_SDMMC_HOST_SUPPORTED +#ifdef ARDUINO +#warning The SDMMC library requires a device with an SDIO Host +#endif +#else #include "FS.h" #include "driver/sdmmc_types.h" @@ -40,6 +44,7 @@ class SDMMCFS : public FS { int8_t _pin_d1 = -1; int8_t _pin_d2 = -1; int8_t _pin_d3 = -1; + uint8_t _pdrv = 0xFF; bool _mode1bit = false; public: @@ -55,6 +60,10 @@ class SDMMCFS : public FS { uint64_t cardSize(); uint64_t totalBytes(); uint64_t usedBytes(); + int sectorSize(); + int numSectors(); + bool readRAW(uint8_t *buffer, uint32_t sector); + bool writeRAW(uint8_t *buffer, uint32_t sector); private: static bool sdmmcDetachBus(void *bus_pointer); From a31a5fca1739993173caba995f7785b8eed6b30e Mon Sep 17 00:00:00 2001 From: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> Date: Thu, 13 Jun 2024 01:53:42 -0300 Subject: [PATCH 07/24] fix(ci): Fix issues found in workflows (#9829) * ci(tests): Swap cache to artifacts to avoid errors between OSes * ci(push): Fix chunk generation for compilation * ci(tests): Fix error code propagation * ci(push): Add shebang to new script * ci(push): Fix sizes upload if there is no changes * ci(bot): Fix GitHub actions bot commit info --- .github/scripts/on-push.sh | 2 +- .github/scripts/set_push_chunks.sh | 83 +++++++++++++++++++++++ .github/scripts/sketch_utils.sh | 7 +- .github/scripts/tests_run.sh | 4 +- .github/workflows/hw.yml | 12 ++-- .github/workflows/lib.yml | 4 +- .github/workflows/publishsizes-2.x.yml | 8 +-- .github/workflows/push.yml | 91 +++++--------------------- .github/workflows/qemu.yml | 10 +-- .github/workflows/wokwi.yml | 10 +-- 10 files changed, 124 insertions(+), 107 deletions(-) create mode 100644 .github/scripts/set_push_chunks.sh diff --git a/.github/scripts/on-push.sh b/.github/scripts/on-push.sh index 7abe3600d80..f925390da13 100755 --- a/.github/scripts/on-push.sh +++ b/.github/scripts/on-push.sh @@ -108,7 +108,7 @@ if [ "$BUILD_PIO" -eq 0 ]; then if [ "$BUILD_LOG" -eq 1 ]; then #remove last comma from the last JSON object - sed -i '$ s/.$//' "$sizes_file" + sed -i '$ s/,$//' "$sizes_file" #echo end of JSON array echo "]}" >> $sizes_file fi diff --git a/.github/scripts/set_push_chunks.sh b/.github/scripts/set_push_chunks.sh new file mode 100644 index 00000000000..472205f0c38 --- /dev/null +++ b/.github/scripts/set_push_chunks.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +build_all=false +chunks_count=0 + +if [[ $CORE_CHANGED == 'true' ]] || [[ $IS_PR != 'true' ]]; then + echo "Core files changed or not a PR. Building all." + build_all=true + chunks_count=$MAX_CHUNKS +elif [[ $LIB_CHANGED == 'true' ]]; then + echo "Libraries changed. Building only affected sketches." + if [[ $NETWORKING_CHANGED == 'true' ]]; then + echo "Networking libraries changed. Building networking related sketches." + networking_sketches="$(find libraries/WiFi -name *.ino) " + networking_sketches+="$(find libraries/Ethernet -name *.ino) " + networking_sketches+="$(find libraries/PPP -name *.ino) " + networking_sketches+="$(find libraries/NetworkClientSecure -name *.ino) " + networking_sketches+="$(find libraries/WebServer -name *.ino) " + fi + if [[ $FS_CHANGED == 'true' ]]; then + echo "FS libraries changed. Building FS related sketches." + fs_sketches="$(find libraries/SD -name *.ino) " + fs_sketches+="$(find libraries/SD_MMC -name *.ino) " + fs_sketches+="$(find libraries/SPIFFS -name *.ino) " + fs_sketches+="$(find libraries/LittleFS -name *.ino) " + fs_sketches+="$(find libraries/FFat -name *.ino) " + fi + sketches="$networking_sketches $fs_sketches" + for file in $LIB_FILES; do + if [[ $file == *.ino ]]; then + # If file ends with .ino, add it to the list of sketches + echo "Sketch found: $file" + sketches+="$file " + elif [[ $(basename $(dirname $file)) == "src" ]]; then + # If file is in a src directory, find all sketches in the parent/examples directory + echo "Library src file found: $file" + lib=$(dirname $(dirname $file)) + if [[ -d $lib/examples ]]; then + lib_sketches=$(find $lib/examples -name *.ino) + sketches+="$lib_sketches " + echo "Library sketches: $lib_sketches" + fi + else + # If file is in a example folder but it is not a sketch, find all sketches in the current directory + echo "File in example folder found: $file" + sketch=$(find $(dirname $file) -name *.ino) + sketches+="$sketch " + echo "Sketch in example folder: $sketch" + fi + echo "" + done +fi + +if [[ -n $sketches ]]; then + # Remove duplicates + sketches=$(echo $sketches | tr ' ' '\n' | sort | uniq) + for sketch in $sketches; do + echo $sketch >> sketches_found.txt + chunks_count=$((chunks_count+1)) + done + echo "Number of sketches found: $chunks_count" + echo "Sketches:" + echo "$sketches" + + if [[ $chunks_count -gt $MAX_CHUNKS ]]; then + echo "More sketches than the allowed number of chunks found. Limiting to $MAX_CHUNKS chunks." + chunks_count=$MAX_CHUNKS + fi +fi + +chunks='["0"' +for i in $(seq 1 $(( $chunks_count - 1 )) ); do + chunks+=",\"$i\"" +done +chunks+="]" + +echo "build_all=$build_all" >> $GITHUB_OUTPUT +echo "build_libraries=$BUILD_LIBRARIES" >> $GITHUB_OUTPUT +echo "build_static_sketches=$BUILD_STATIC_SKETCHES" >> $GITHUB_OUTPUT +echo "build_idf=$BUILD_IDF" >> $GITHUB_OUTPUT +echo "build_platformio=$BUILD_PLATFORMIO" >> $GITHUB_OUTPUT +echo "chunk_count=$chunks_count" >> $GITHUB_OUTPUT +echo "chunks=$chunks" >> $GITHUB_OUTPUT diff --git a/.github/scripts/sketch_utils.sh b/.github/scripts/sketch_utils.sh index 1b43ced1480..cdf334a4fc6 100755 --- a/.github/scripts/sketch_utils.sh +++ b/.github/scripts/sketch_utils.sh @@ -192,7 +192,7 @@ function build_sketch(){ # build_sketch [ex exit_status=$? if [ $exit_status -ne 0 ]; then - echo ""ERROR: Compilation failed with error code $exit_status"" + echo "ERROR: Compilation failed with error code $exit_status" exit $exit_status fi @@ -236,7 +236,7 @@ function build_sketch(){ # build_sketch [ex exit_status=$? if [ $exit_status -ne 0 ]; then - echo ""ERROR: Compilation failed with error code $exit_status"" + echo "ERROR: Compilation failed with error code $exit_status" exit $exit_status fi # $ide_path/arduino-builder -compile -logger=human -core-api-version=10810 \ @@ -398,6 +398,7 @@ function build_sketches(){ # build_sketches > ${{ env.RESULT_SIZES_TEST_FILE }} - name: Push to github repo run: | - git config user.name github-actions - git config user.email github-actions@github.com + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add ${{ env.RESULT_SIZES_TEST_FILE }} git commit -m "Generated Sizes Results (master-v2.x)" git push origin HEAD:gh-pages diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 3ef15df1566..2f14a6fb62f 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -77,6 +77,10 @@ jobs: libraries: - 'libraries/**/examples/**' - 'libraries/**/src/**' + networking: + - 'libraries/Network/src/**' + fs: + - 'libraries/FS/src/**' static_sketeches: - 'libraries/NetworkClientSecure/examples/WiFiClientSecure/WiFiClientSecure.ino' - 'libraries/BLE/examples/Server/Server.ino' @@ -97,78 +101,18 @@ jobs: id: set-chunks env: LIB_FILES: ${{ steps.changed-files.outputs.libraries_all_changed_files }} + IS_PR: ${{ github.event_name == 'pull_request' }} + MAX_CHUNKS: ${{ env.MAX_CHUNKS }} + BUILD_PLATFORMIO: ${{ steps.changed-files.outputs.platformio_any_changed == 'true' }} + BUILD_IDF: ${{ steps.changed-files.outputs.idf_any_changed == 'true' }} + BUILD_LIBRARIES: ${{ steps.changed-files.outputs.libraries_any_changed == 'true' }} + BUILD_STATIC_SKETCHES: ${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }} + FS_CHANGED: ${{ steps.changed-files.outputs.fs_any_changed == 'true' }} + NETWORKING_CHANGED: ${{ steps.changed-files.outputs.networking_any_changed == 'true' }} + CORE_CHANGED: ${{ steps.changed-files.outputs.core_any_changed == 'true' }} + LIB_CHANGED: ${{ steps.changed-files.outputs.libraries_any_changed == 'true' }} run: | - build_all=false - chunks_count=0 - is_pr=${{ github.event_name == 'pull_request' }} - - build_platformio=${{ steps.changed-files.outputs.platformio_any_changed == 'true' }} - build_idf=${{ steps.changed-files.outputs.idf_any_changed == 'true' }} - build_libraries=${{ steps.changed-files.outputs.libraries_any_changed == 'true' }} - build_static_sketches=${{ steps.changed-files.outputs.static_sketeches_any_changed == 'true' }} - - core_changed=${{ steps.changed-files.outputs.core_any_changed == 'true' }} - lib_changed=${{ steps.changed-files.outputs.libraries_any_changed == 'true' }} - - if [[ $core_changed == 'true' ]] || [[ $is_pr != 'true' ]]; then - echo "Core files changed or not a PR. Building all." - build_all=true - chunks_count=${{ env.MAX_CHUNKS }} - elif [[ $lib_changed == 'true' ]]; then - echo "Libraries changed. Building only affected sketches." - sketches="" - for file in $LIB_FILES; do - if [[ $file == *.ino ]]; then - # If file ends with .ino, add it to the list of sketches - echo "Sketch found: $file" - sketches+="$file " - elif [[ $(basename $(dirname $file)) == "src" ]]; then - # If file is in a src directory, find all sketches in the parent/examples directory - echo "Library src file found: $file" - lib=$(dirname $(dirname $file)) - lib_sketches=$(find $lib/examples -name *.ino) - sketches+="$lib_sketches " - echo "Library sketches: $lib_sketches" - else - # If file is in a example folder but it is not a sketch, find all sketches in the current directory - echo "File in example folder found: $file" - sketch=$(find $(dirname $file) -name *.ino) - sketches+="$sketch " - echo "Sketch in example folder: $sketch" - fi - echo "" - done - fi - - if [[ -n $sketches ]]; then - # Remove duplicates - sketches=$(echo $sketches | tr ' ' '\n' | sort | uniq) - for sketch in $sketches; do - echo $sketch >> sketches_found.txt - chunks_count=$((chunks_count+1)) - done - echo "Number of sketches found: $chunks_count" - echo "Sketches: $sketches" - - if [[ $chunks_count -gt ${{ env.MAX_CHUNKS }} ]]; then - echo "More sketches than the allowed number of chunks found. Limiting to ${{ env.MAX_CHUNKS }} chunks." - chunks_count=${{ env.MAX_CHUNKS }} - fi - fi - - chunks='["0"' - for i in $(seq 1 $(( $chunks_count - 1 )) ); do - chunks+=",\"$i\"" - done - chunks+="]" - - echo "build_all=$build_all" >> $GITHUB_OUTPUT - echo "build_libraries=$build_libraries" >> $GITHUB_OUTPUT - echo "build_static_sketches=$build_static_sketches" >> $GITHUB_OUTPUT - echo "build_idf=$build_idf" >> $GITHUB_OUTPUT - echo "build_platformio=$build_platformio" >> $GITHUB_OUTPUT - echo "chunk_count=$chunks_count" >> $GITHUB_OUTPUT - echo "chunks=$chunks" >> $GITHUB_OUTPUT + bash ./.github/scripts/set_push_chunks.sh - name: Upload sketches found if: ${{ steps.set-chunks.outputs.build_all == 'false' && steps.set-chunks.outputs.build_libraries == 'true' }} @@ -336,9 +280,10 @@ jobs: - name: Commit json files to gh-pages if on master if: github.event_name == 'push' && github.ref == 'refs/heads/master' + continue-on-error: true run: | - git config user.name github-actions - git config user.email github-actions@github.com + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" git add --all git commit -m "Updated cli compile json files" git push origin HEAD:gh-pages diff --git a/.github/workflows/qemu.yml b/.github/workflows/qemu.yml index 1aa155129c8..da31054aeef 100644 --- a/.github/workflows/qemu.yml +++ b/.github/workflows/qemu.yml @@ -113,15 +113,11 @@ jobs: - name: Get binaries if: ${{ steps.check-tests.outputs.enabled == 'true' }} - id: cache-build-binaries - uses: actions/cache/restore@v4 + uses: actions/download-artifact@v4 with: - fail-on-cache-miss: true - key: tests-${{ env.id }}-bin + name: tests-bin-${{ inputs.chip }}-${{ inputs.type }} path: | - ~/.arduino/tests/**/build*.tmp/*.bin - ~/.arduino/tests/**/build*.tmp/*.elf - ~/.arduino/tests/**/build*.tmp/*.json + ~/.arduino/tests - name: Run Tests if: ${{ steps.check-tests.outputs.enabled == 'true' }} diff --git a/.github/workflows/wokwi.yml b/.github/workflows/wokwi.yml index 84b0acf06b4..f9eee85f95f 100644 --- a/.github/workflows/wokwi.yml +++ b/.github/workflows/wokwi.yml @@ -91,15 +91,11 @@ jobs: - name: Get binaries if: ${{ steps.check-tests.outputs.enabled == 'true' }} - id: cache-build-binaries - uses: actions/cache/restore@v4 + uses: actions/download-artifact@v4 with: - fail-on-cache-miss: true - key: tests-${{ env.id }}-bin + name: tests-bin-${{ inputs.chip }}-${{ inputs.type }} path: | - ~/.arduino/tests/**/build*.tmp/*.bin - ~/.arduino/tests/**/build*.tmp/*.elf - ~/.arduino/tests/**/build*.tmp/*.json + ~/.arduino/tests - name: Run Tests if: ${{ steps.check-tests.outputs.enabled == 'true' }} From 1ef2208349b24dd6a8c7ae4a76c36af41c29437f Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Thu, 13 Jun 2024 08:38:25 +0300 Subject: [PATCH 08/24] feat(crypto): Add libsodium to the included components (#9821) --- idf_component.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/idf_component.yml b/idf_component.yml index 3accad1fb9d..a9bc90f8788 100644 --- a/idf_component.yml +++ b/idf_component.yml @@ -81,6 +81,9 @@ dependencies: version: "^1.4.2" rules: - if: "target in [esp32s3]" + espressif/libsodium: + version: "^1.0.20~1" + require: public joltwallet/littlefs: version: "^1.10.2" chmorgan/esp-libhelix-mp3: From bc79feb2175d29c12638fc7a9da835d99c1405e9 Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Thu, 13 Jun 2024 08:39:04 +0300 Subject: [PATCH 09/24] fix(client): Implement readBytes in NetworkClient for faster downloads (#9824) * fix(client): Implement readBytes in NetworkClient for faster downloads * fix(client): Implement readBytes to obey the client timeout * fix(clieant): use getTimeout() instead * ci(pre-commit): Apply automatic fixes --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- libraries/Network/src/NetworkClient.cpp | 28 +++++++++++++++++++++++++ libraries/Network/src/NetworkClient.h | 4 ++++ 2 files changed, 32 insertions(+) diff --git a/libraries/Network/src/NetworkClient.cpp b/libraries/Network/src/NetworkClient.cpp index 7e59d2aa3f3..1fe89da4492 100644 --- a/libraries/Network/src/NetworkClient.cpp +++ b/libraries/Network/src/NetworkClient.cpp @@ -479,6 +479,34 @@ int NetworkClient::read(uint8_t *buf, size_t size) { return res; } +size_t NetworkClient::readBytes(char *buffer, size_t length) { + size_t left = length, sofar = 0; + int r = 0, to = millis() + getTimeout(); + while (left) { + r = read((uint8_t *)buffer + sofar, left); + if (r < 0) { + // Error has occurred + break; + } + if (r > 0) { + // We got some data + left -= r; + sofar += r; + to = millis() + getTimeout(); + } else { + // We got no data + if (millis() >= to) { + // We have waited for data enough + log_w("Timeout waiting for data on fd %d", fd()); + break; + } + // Allow other tasks to run + delay(2); + } + } + return sofar; +} + int NetworkClient::peek() { int res = -1; if (fd() >= 0 && _rxBuffer) { diff --git a/libraries/Network/src/NetworkClient.h b/libraries/Network/src/NetworkClient.h index cb92a805900..572292a7a99 100644 --- a/libraries/Network/src/NetworkClient.h +++ b/libraries/Network/src/NetworkClient.h @@ -60,6 +60,10 @@ class NetworkClient : public ESPLwIPClient { int available(); int read(); int read(uint8_t *buf, size_t size); + size_t readBytes(char *buffer, size_t length); + size_t readBytes(uint8_t *buffer, size_t length) { + return readBytes((char *)buffer, length); + } int peek(); void clear(); // clear rx void stop(); From 211520b4ba68e340d3b7d1891c12560f79c6463e Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Thu, 13 Jun 2024 08:40:06 +0300 Subject: [PATCH 10/24] fix(client): Fix NetworkClient::localIP() (#9845) It was returning zero, because it was not able to handle IPv4 mapped address into IPv6 address --- libraries/Network/src/NetworkClient.cpp | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/libraries/Network/src/NetworkClient.cpp b/libraries/Network/src/NetworkClient.cpp index 1fe89da4492..9f4806df18a 100644 --- a/libraries/Network/src/NetworkClient.cpp +++ b/libraries/Network/src/NetworkClient.cpp @@ -617,8 +617,24 @@ IPAddress NetworkClient::localIP(int fd) const { struct sockaddr_storage addr; socklen_t len = sizeof addr; getsockname(fd, (struct sockaddr *)&addr, &len); - struct sockaddr_in *s = (struct sockaddr_in *)&addr; - return IPAddress((uint32_t)(s->sin_addr.s_addr)); + + // IPv4 socket, old way + if (((struct sockaddr *)&addr)->sa_family == AF_INET) { + struct sockaddr_in *s = (struct sockaddr_in *)&addr; + return IPAddress((uint32_t)(s->sin_addr.s_addr)); + } + + // IPv6, but it might be IPv4 mapped address + if (((struct sockaddr *)&addr)->sa_family == AF_INET6) { + struct sockaddr_in6 *saddr6 = (struct sockaddr_in6 *)&addr; + if (IN6_IS_ADDR_V4MAPPED(saddr6->sin6_addr.un.u32_addr)) { + return IPAddress(IPv4, (uint8_t *)saddr6->sin6_addr.s6_addr + IPADDRESS_V4_BYTES_INDEX); + } else { + return IPAddress(IPv6, (uint8_t *)(saddr6->sin6_addr.s6_addr), saddr6->sin6_scope_id); + } + } + log_e("NetworkClient::localIP Not AF_INET or AF_INET6?"); + return (IPAddress(0, 0, 0, 0)); } uint16_t NetworkClient::localPort(int fd) const { From 849ec57ca6b815e55c8b86767d3cb33d5db9d4ce Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Thu, 13 Jun 2024 11:21:32 +0530 Subject: [PATCH 11/24] Added support for removing routes in WebServer library (#9832) * feat: added removeRoutes and removeHandler methods * feat: added removeRoute and removeHandler methods * ci(pre-commit): Apply automatic fixes --------- Co-authored-by: Me No Dev Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- libraries/WebServer/src/WebServer.cpp | 54 +++++++++++++++++++++++++++ libraries/WebServer/src/WebServer.h | 6 +++ 2 files changed, 60 insertions(+) diff --git a/libraries/WebServer/src/WebServer.cpp b/libraries/WebServer/src/WebServer.cpp index 84b4de33518..83c22b7e493 100644 --- a/libraries/WebServer/src/WebServer.cpp +++ b/libraries/WebServer/src/WebServer.cpp @@ -318,10 +318,38 @@ void WebServer::on(const Uri &uri, HTTPMethod method, WebServer::THandlerFunctio _addRequestHandler(new FunctionRequestHandler(fn, ufn, uri, method)); } +bool WebServer::removeRoute(const char *uri) { + return removeRoute(String(uri), HTTP_ANY); +} + +bool WebServer::removeRoute(const char *uri, HTTPMethod method) { + return removeRoute(String(uri), method); +} + +bool WebServer::removeRoute(const String &uri) { + return removeRoute(uri, HTTP_ANY); +} + +bool WebServer::removeRoute(const String &uri, HTTPMethod method) { + // Loop through all request handlers and see if there is a match + RequestHandler *handler = _firstHandler; + while (handler) { + if (handler->canHandle(method, uri)) { + return _removeRequestHandler(handler); + } + handler = handler->next(); + } + return false; +} + void WebServer::addHandler(RequestHandler *handler) { _addRequestHandler(handler); } +bool WebServer::removeHandler(RequestHandler *handler) { + return _removeRequestHandler(handler); +} + void WebServer::_addRequestHandler(RequestHandler *handler) { if (!_lastHandler) { _firstHandler = handler; @@ -332,6 +360,32 @@ void WebServer::_addRequestHandler(RequestHandler *handler) { } } +bool WebServer::_removeRequestHandler(RequestHandler *handler) { + RequestHandler *current = _firstHandler; + RequestHandler *previous = nullptr; + + while (current != nullptr) { + if (current == handler) { + if (previous == nullptr) { + _firstHandler = current->next(); + } else { + previous->next(current->next()); + } + + if (current == _lastHandler) { + _lastHandler = previous; + } + + // Delete 'matching' handler + delete current; + return true; + } + previous = current; + current = current->next(); + } + return false; +} + void WebServer::serveStatic(const char *uri, FS &fs, const char *path, const char *cache_header) { _addRequestHandler(new StaticRequestHandler(fs, path, uri, cache_header)); } diff --git a/libraries/WebServer/src/WebServer.h b/libraries/WebServer/src/WebServer.h index 1382885e1d9..f69c08f22b6 100644 --- a/libraries/WebServer/src/WebServer.h +++ b/libraries/WebServer/src/WebServer.h @@ -147,7 +147,12 @@ class WebServer { void on(const Uri &uri, THandlerFunction fn); void on(const Uri &uri, HTTPMethod method, THandlerFunction fn); void on(const Uri &uri, HTTPMethod method, THandlerFunction fn, THandlerFunction ufn); //ufn handles file uploads + bool removeRoute(const char *uri); + bool removeRoute(const char *uri, HTTPMethod method); + bool removeRoute(const String &uri); + bool removeRoute(const String &uri, HTTPMethod method); void addHandler(RequestHandler *handler); + bool removeHandler(RequestHandler *handler); void serveStatic(const char *uri, fs::FS &fs, const char *path, const char *cache_header = NULL); void onNotFound(THandlerFunction fn); //called when handler is not assigned void onFileUpload(THandlerFunction ufn); //handle file uploads @@ -230,6 +235,7 @@ class WebServer { return _currentClient.write_P(b, l); } void _addRequestHandler(RequestHandler *handler); + bool _removeRequestHandler(RequestHandler *handler); void _handleRequest(); void _finalizeResponse(); bool _parseRequest(NetworkClient &client); From 1d895e58e7b131682cc2ac7b5dcd2ac3ef84dcb3 Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Thu, 13 Jun 2024 10:26:54 +0300 Subject: [PATCH 12/24] fix(xtal): Add a way to change the XTAL frequency for SparkFun ESP32 Thing (#9844) * fix(xtal): Add a way to change the XTAL frequency Add support for boards like SparkFun ESP32 Thing that use 26MHz XTAL * ci(pre-commit): Apply automatic fixes * feat(dbg): Print the XTAL frequency in the debug report --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- cores/esp32/chip-debug-report.cpp | 3 ++- cores/esp32/esp32-hal-misc.c | 3 --- cores/esp32/main.cpp | 10 ++++++++++ variants/esp32thing/pins_arduino.h | 2 ++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cores/esp32/chip-debug-report.cpp b/cores/esp32/chip-debug-report.cpp index 982b01ffa8f..c5b9866e3cc 100644 --- a/cores/esp32/chip-debug-report.cpp +++ b/cores/esp32/chip-debug-report.cpp @@ -96,7 +96,8 @@ static void printChipInfo(void) { chip_report_printf(" Cores : %d\n", info.cores); rtc_cpu_freq_config_t conf; rtc_clk_cpu_freq_get_config(&conf); - chip_report_printf(" Frequency : %lu MHz\n", conf.freq_mhz); + chip_report_printf(" CPU Frequency : %lu MHz\n", conf.freq_mhz); + chip_report_printf(" XTAL Frequency : %d MHz\n", rtc_clk_xtal_freq_get()); chip_report_printf(" Embedded Flash : %s\n", (info.features & CHIP_FEATURE_EMB_FLASH) ? "Yes" : "No"); chip_report_printf(" Embedded PSRAM : %s\n", (info.features & CHIP_FEATURE_EMB_PSRAM) ? "Yes" : "No"); chip_report_printf(" 2.4GHz WiFi : %s\n", (info.features & CHIP_FEATURE_WIFI_BGN) ? "Yes" : "No"); diff --git a/cores/esp32/esp32-hal-misc.c b/cores/esp32/esp32-hal-misc.c index cf8edcf279f..82363b97bd0 100644 --- a/cores/esp32/esp32-hal-misc.c +++ b/cores/esp32/esp32-hal-misc.c @@ -252,9 +252,6 @@ extern bool btInUse(); void initArduino() { //init proper ref tick value for PLL (uncomment if REF_TICK is different than 1MHz) //ESP_REG(APB_CTRL_PLL_TICK_CONF_REG) = APB_CLK_FREQ / REF_CLK_FREQ - 1; -#ifdef F_CPU - setCpuFrequencyMhz(F_CPU / 1000000); -#endif #if CONFIG_SPIRAM_SUPPORT || CONFIG_SPIRAM psramInit(); #endif diff --git a/cores/esp32/main.cpp b/cores/esp32/main.cpp index 21f15083473..6c4d50a9a84 100644 --- a/cores/esp32/main.cpp +++ b/cores/esp32/main.cpp @@ -1,6 +1,7 @@ #include "freertos/FreeRTOS.h" #include "freertos/task.h" #include "esp_task_wdt.h" +#include "soc/rtc.h" #include "Arduino.h" #if (ARDUINO_USB_CDC_ON_BOOT | ARDUINO_USB_MSC_ON_BOOT | ARDUINO_USB_DFU_ON_BOOT) && !ARDUINO_USB_MODE #include "USB.h" @@ -78,6 +79,15 @@ void loopTask(void *pvParameters) { } extern "C" void app_main() { +#ifdef F_XTAL_MHZ +#if !CONFIG_IDF_TARGET_ESP32S2 // ESP32-S2 does not support rtc_clk_xtal_freq_update + rtc_clk_xtal_freq_update((rtc_xtal_freq_t)F_XTAL_MHZ); + rtc_clk_cpu_freq_set_xtal(); +#endif +#endif +#ifdef F_CPU + setCpuFrequencyMhz(F_CPU / 1000000); +#endif #if ARDUINO_USB_CDC_ON_BOOT && !ARDUINO_USB_MODE Serial.begin(); #endif diff --git a/variants/esp32thing/pins_arduino.h b/variants/esp32thing/pins_arduino.h index 66f81860f5a..00abcdfb191 100644 --- a/variants/esp32thing/pins_arduino.h +++ b/variants/esp32thing/pins_arduino.h @@ -3,6 +3,8 @@ #include +#define F_XTAL_MHZ 26 //SparkFun ESP32 Thing has 26MHz Crystal + static const uint8_t LED_BUILTIN = 5; #define BUILTIN_LED LED_BUILTIN // backward compatibility #define LED_BUILTIN LED_BUILTIN // allow testing #ifdef LED_BUILTIN From 3428eb6a9eb8f472c6565f65ac7bab027c8c8cd7 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Fri, 14 Jun 2024 02:55:20 +0530 Subject: [PATCH 13/24] feat: added support for filters in WebServer library (#9842) * feat: added support for filters in webserver * feat: add setFilter function in StaticRequestHandler * fix: ON_STA_FILTER & ON_AP_FILTER * fix: make request handlers backward compatible * fix: ON_STA_FILTER & ON_AP_FILTER * fix: more filters to their own example * chore: grammar * fix: remove filters from header file * fix: use same root route for both interfaces * ci(pre-commit): Apply automatic fixes --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- .../WebServer/examples/Filters/Filters.ino | 110 ++++++++++++++++++ libraries/WebServer/examples/Filters/ci.json | 5 + libraries/WebServer/src/Parsing.cpp | 14 +-- libraries/WebServer/src/WebServer.cpp | 14 ++- libraries/WebServer/src/WebServer.h | 7 +- .../WebServer/src/detail/RequestHandler.h | 31 +++++ .../src/detail/RequestHandlersImpl.h | 68 +++++++++-- 7 files changed, 226 insertions(+), 23 deletions(-) create mode 100644 libraries/WebServer/examples/Filters/Filters.ino create mode 100644 libraries/WebServer/examples/Filters/ci.json diff --git a/libraries/WebServer/examples/Filters/Filters.ino b/libraries/WebServer/examples/Filters/Filters.ino new file mode 100644 index 00000000000..8974e55d322 --- /dev/null +++ b/libraries/WebServer/examples/Filters/Filters.ino @@ -0,0 +1,110 @@ +#include +#include +#include +#include + +// Your STA WiFi Credentials +// ( This is the AP your ESP will connect to ) +const char *ssid = "........"; +const char *password = "........"; + +// Your AP WiFi Credentials +// ( This is the AP your ESP will broadcast ) +const char *ap_ssid = "ESP32_Demo"; +const char *ap_password = ""; + +WebServer server(80); + +const int led = 13; + +// ON_STA_FILTER - Only accept requests coming from STA interface +bool ON_STA_FILTER(WebServer &server) { + return WiFi.STA.hasIP() && WiFi.STA.localIP() == server.client().localIP(); +} + +// ON_AP_FILTER - Only accept requests coming from AP interface +bool ON_AP_FILTER(WebServer &server) { + return WiFi.AP.hasIP() && WiFi.AP.localIP() == server.client().localIP(); +} + +void handleNotFound() { + digitalWrite(led, 1); + String message = "File Not Found\n\n"; + message += "URI: "; + message += server.uri(); + message += "\nMethod: "; + message += (server.method() == HTTP_GET) ? "GET" : "POST"; + message += "\nArguments: "; + message += server.args(); + message += "\n"; + for (uint8_t i = 0; i < server.args(); i++) { + message += " " + server.argName(i) + ": " + server.arg(i) + "\n"; + } + server.send(404, "text/plain", message); + digitalWrite(led, 0); +} + +void setup(void) { + pinMode(led, OUTPUT); + digitalWrite(led, 0); + Serial.begin(115200); + WiFi.mode(WIFI_AP_STA); + // Connect to STA + WiFi.begin(ssid, password); + // Start AP + WiFi.softAP(ap_ssid, ap_password); + Serial.println(""); + + // Wait for connection + while (WiFi.status() != WL_CONNECTED) { + delay(500); + Serial.print("."); + } + Serial.println(""); + Serial.print("Connected to "); + Serial.println(ssid); + Serial.print("IP address: "); + Serial.println(WiFi.localIP()); + + if (MDNS.begin("esp32")) { + Serial.println("MDNS responder started"); + } + + // This route will be accessible by STA clients only + server + .on( + "/", + [&]() { + digitalWrite(led, 1); + server.send(200, "text/plain", "Hi!, This route is accessible for STA clients only"); + digitalWrite(led, 0); + } + ) + .setFilter(ON_STA_FILTER); + + // This route will be accessible by AP clients only + server + .on( + "/", + [&]() { + digitalWrite(led, 1); + server.send(200, "text/plain", "Hi!, This route is accessible for AP clients only"); + digitalWrite(led, 0); + } + ) + .setFilter(ON_AP_FILTER); + + server.on("/inline", []() { + server.send(200, "text/plain", "this works as well"); + }); + + server.onNotFound(handleNotFound); + + server.begin(); + Serial.println("HTTP server started"); +} + +void loop(void) { + server.handleClient(); + delay(2); //allow the cpu to switch to other tasks +} diff --git a/libraries/WebServer/examples/Filters/ci.json b/libraries/WebServer/examples/Filters/ci.json new file mode 100644 index 00000000000..d8b3664bc65 --- /dev/null +++ b/libraries/WebServer/examples/Filters/ci.json @@ -0,0 +1,5 @@ +{ + "targets": { + "esp32h2": false + } +} diff --git a/libraries/WebServer/src/Parsing.cpp b/libraries/WebServer/src/Parsing.cpp index 3d3e7d1c55a..200244e6848 100644 --- a/libraries/WebServer/src/Parsing.cpp +++ b/libraries/WebServer/src/Parsing.cpp @@ -124,7 +124,7 @@ bool WebServer::_parseRequest(NetworkClient &client) { //attach handler RequestHandler *handler; for (handler = _firstHandler; handler; handler = handler->next()) { - if (handler->canHandle(_currentMethod, _currentUri)) { + if (handler->canHandle(*this, _currentMethod, _currentUri)) { break; } } @@ -176,7 +176,7 @@ bool WebServer::_parseRequest(NetworkClient &client) { } } - if (!isForm && _currentHandler && _currentHandler->canRaw(_currentUri)) { + if (!isForm && _currentHandler && _currentHandler->canRaw(*this, _currentUri)) { log_v("Parse raw"); _currentRaw.reset(new HTTPRaw()); _currentRaw->status = RAW_START; @@ -334,7 +334,7 @@ void WebServer::_parseArguments(String data) { void WebServer::_uploadWriteByte(uint8_t b) { if (_currentUpload->currentSize == HTTP_UPLOAD_BUFLEN) { - if (_currentHandler && _currentHandler->canUpload(_currentUri)) { + if (_currentHandler && _currentHandler->canUpload(*this, _currentUri)) { _currentHandler->upload(*this, _currentUri, *_currentUpload); } _currentUpload->totalSize += _currentUpload->currentSize; @@ -449,7 +449,7 @@ bool WebServer::_parseForm(NetworkClient &client, String boundary, uint32_t len) _currentUpload->totalSize = 0; _currentUpload->currentSize = 0; log_v("Start File: %s Type: %s", _currentUpload->filename.c_str(), _currentUpload->type.c_str()); - if (_currentHandler && _currentHandler->canUpload(_currentUri)) { + if (_currentHandler && _currentHandler->canUpload(*this, _currentUri)) { _currentHandler->upload(*this, _currentUri, *_currentUpload); } _currentUpload->status = UPLOAD_FILE_WRITE; @@ -488,12 +488,12 @@ bool WebServer::_parseForm(NetworkClient &client, String boundary, uint32_t len) } } // Found the boundary string, finish processing this file upload - if (_currentHandler && _currentHandler->canUpload(_currentUri)) { + if (_currentHandler && _currentHandler->canUpload(*this, _currentUri)) { _currentHandler->upload(*this, _currentUri, *_currentUpload); } _currentUpload->totalSize += _currentUpload->currentSize; _currentUpload->status = UPLOAD_FILE_END; - if (_currentHandler && _currentHandler->canUpload(_currentUri)) { + if (_currentHandler && _currentHandler->canUpload(*this, _currentUri)) { _currentHandler->upload(*this, _currentUri, *_currentUpload); } log_v("End File: %s Type: %s Size: %d", _currentUpload->filename.c_str(), _currentUpload->type.c_str(), (int)_currentUpload->totalSize); @@ -567,7 +567,7 @@ String WebServer::urlDecode(const String &text) { bool WebServer::_parseFormUploadAborted() { _currentUpload->status = UPLOAD_FILE_ABORTED; - if (_currentHandler && _currentHandler->canUpload(_currentUri)) { + if (_currentHandler && _currentHandler->canUpload(*this, _currentUri)) { _currentHandler->upload(*this, _currentUri, *_currentUpload); } return false; diff --git a/libraries/WebServer/src/WebServer.cpp b/libraries/WebServer/src/WebServer.cpp index 83c22b7e493..048bd529d7b 100644 --- a/libraries/WebServer/src/WebServer.cpp +++ b/libraries/WebServer/src/WebServer.cpp @@ -306,16 +306,18 @@ void WebServer::requestAuthentication(HTTPAuthMethod mode, const char *realm, co send(401, String(FPSTR(mimeTable[html].mimeType)), authFailMsg); } -void WebServer::on(const Uri &uri, WebServer::THandlerFunction handler) { - on(uri, HTTP_ANY, handler); +RequestHandler &WebServer::on(const Uri &uri, WebServer::THandlerFunction handler) { + return on(uri, HTTP_ANY, handler); } -void WebServer::on(const Uri &uri, HTTPMethod method, WebServer::THandlerFunction fn) { - on(uri, method, fn, _fileUploadHandler); +RequestHandler &WebServer::on(const Uri &uri, HTTPMethod method, WebServer::THandlerFunction fn) { + return on(uri, method, fn, _fileUploadHandler); } -void WebServer::on(const Uri &uri, HTTPMethod method, WebServer::THandlerFunction fn, WebServer::THandlerFunction ufn) { - _addRequestHandler(new FunctionRequestHandler(fn, ufn, uri, method)); +RequestHandler &WebServer::on(const Uri &uri, HTTPMethod method, WebServer::THandlerFunction fn, WebServer::THandlerFunction ufn) { + FunctionRequestHandler *handler = new FunctionRequestHandler(fn, ufn, uri, method); + _addRequestHandler(handler); + return *handler; } bool WebServer::removeRoute(const char *uri) { diff --git a/libraries/WebServer/src/WebServer.h b/libraries/WebServer/src/WebServer.h index f69c08f22b6..c43dd4542ea 100644 --- a/libraries/WebServer/src/WebServer.h +++ b/libraries/WebServer/src/WebServer.h @@ -144,9 +144,10 @@ class WebServer { void requestAuthentication(HTTPAuthMethod mode = BASIC_AUTH, const char *realm = NULL, const String &authFailMsg = String("")); typedef std::function THandlerFunction; - void on(const Uri &uri, THandlerFunction fn); - void on(const Uri &uri, HTTPMethod method, THandlerFunction fn); - void on(const Uri &uri, HTTPMethod method, THandlerFunction fn, THandlerFunction ufn); //ufn handles file uploads + typedef std::function FilterFunction; + RequestHandler &on(const Uri &uri, THandlerFunction fn); + RequestHandler &on(const Uri &uri, HTTPMethod method, THandlerFunction fn); + RequestHandler &on(const Uri &uri, HTTPMethod method, THandlerFunction fn, THandlerFunction ufn); //ufn handles file uploads bool removeRoute(const char *uri); bool removeRoute(const char *uri, HTTPMethod method); bool removeRoute(const String &uri); diff --git a/libraries/WebServer/src/detail/RequestHandler.h b/libraries/WebServer/src/detail/RequestHandler.h index 4ef4b1d0075..f19e7ab4613 100644 --- a/libraries/WebServer/src/detail/RequestHandler.h +++ b/libraries/WebServer/src/detail/RequestHandler.h @@ -7,6 +7,11 @@ class RequestHandler { public: virtual ~RequestHandler() {} + + /* + note: old handler API for backward compatibility + */ + virtual bool canHandle(HTTPMethod method, String uri) { (void)method; (void)uri; @@ -20,6 +25,27 @@ class RequestHandler { (void)uri; return false; } + + /* + note: new handler API with support for filters etc. + */ + + virtual bool canHandle(WebServer &server, HTTPMethod method, String uri) { + (void)server; + (void)method; + (void)uri; + return false; + } + virtual bool canUpload(WebServer &server, String uri) { + (void)server; + (void)uri; + return false; + } + virtual bool canRaw(WebServer &server, String uri) { + (void)server; + (void)uri; + return false; + } virtual bool handle(WebServer &server, HTTPMethod requestMethod, String requestUri) { (void)server; (void)requestMethod; @@ -37,6 +63,11 @@ class RequestHandler { (void)raw; } + virtual RequestHandler &setFilter(std::function filter) { + (void)filter; + return *this; + } + RequestHandler *next() { return _next; } diff --git a/libraries/WebServer/src/detail/RequestHandlersImpl.h b/libraries/WebServer/src/detail/RequestHandlersImpl.h index d24d36fd9d4..b6eae6adea0 100644 --- a/libraries/WebServer/src/detail/RequestHandlersImpl.h +++ b/libraries/WebServer/src/detail/RequestHandlersImpl.h @@ -36,6 +36,7 @@ class FunctionRequestHandler : public RequestHandler { return true; } + bool canRaw(String requestUri) override { if (!_ufn || _method == HTTP_GET) { return false; @@ -44,9 +45,32 @@ class FunctionRequestHandler : public RequestHandler { return true; } + bool canHandle(WebServer &server, HTTPMethod requestMethod, String requestUri) override { + if (_method != HTTP_ANY && _method != requestMethod) { + return false; + } + + return _uri->canHandle(requestUri, pathArgs) && (_filter != NULL ? _filter(server) : true); + } + + bool canUpload(WebServer &server, String requestUri) override { + if (!_ufn || !canHandle(server, HTTP_POST, requestUri)) { + return false; + } + + return true; + } + + bool canRaw(WebServer &server, String requestUri) override { + if (!_ufn || _method == HTTP_GET || (_filter != NULL ? _filter(server) == false : false)) { + return false; + } + + return true; + } + bool handle(WebServer &server, HTTPMethod requestMethod, String requestUri) override { - (void)server; - if (!canHandle(requestMethod, requestUri)) { + if (!canHandle(server, requestMethod, requestUri)) { return false; } @@ -55,24 +79,30 @@ class FunctionRequestHandler : public RequestHandler { } void upload(WebServer &server, String requestUri, HTTPUpload &upload) override { - (void)server; (void)upload; - if (canUpload(requestUri)) { + if (canUpload(server, requestUri)) { _ufn(); } } void raw(WebServer &server, String requestUri, HTTPRaw &raw) override { - (void)server; (void)raw; - if (canRaw(requestUri)) { + if (canRaw(server, requestUri)) { _ufn(); } } + FunctionRequestHandler &setFilter(WebServer::FilterFunction filter) { + _filter = filter; + return *this; + } + protected: WebServer::THandlerFunction _fn; WebServer::THandlerFunction _ufn; + // _filter should return 'true' when the request should be handled + // and 'false' when the request should be ignored + WebServer::FilterFunction _filter; Uri *_uri; HTTPMethod _method; }; @@ -100,8 +130,24 @@ class StaticRequestHandler : public RequestHandler { return true; } + bool canHandle(WebServer &server, HTTPMethod requestMethod, String requestUri) override { + if (requestMethod != HTTP_GET) { + return false; + } + + if ((_isFile && requestUri != _uri) || !requestUri.startsWith(_uri)) { + return false; + } + + if (_filter != NULL ? _filter(server) == false : false) { + return false; + } + + return true; + } + bool handle(WebServer &server, HTTPMethod requestMethod, String requestUri) override { - if (!canHandle(requestMethod, requestUri)) { + if (!canHandle(server, requestMethod, requestUri)) { return false; } @@ -197,7 +243,15 @@ class StaticRequestHandler : public RequestHandler { return (result); } // calcETag + StaticRequestHandler &setFilter(WebServer::FilterFunction filter) { + _filter = filter; + return *this; + } + protected: + // _filter should return 'true' when the request should be handled + // and 'false' when the request should be ignored + WebServer::FilterFunction _filter; FS _fs; String _uri; String _path; From 08ef62531dd3afa1b0dc8b45549216b28f6d8e06 Mon Sep 17 00:00:00 2001 From: Ayush Sharma Date: Fri, 14 Jun 2024 02:55:49 +0530 Subject: [PATCH 14/24] fix: removeRoute should remove all matching routes (#9851) --- libraries/WebServer/src/WebServer.cpp | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/libraries/WebServer/src/WebServer.cpp b/libraries/WebServer/src/WebServer.cpp index 048bd529d7b..92623b79c01 100644 --- a/libraries/WebServer/src/WebServer.cpp +++ b/libraries/WebServer/src/WebServer.cpp @@ -333,15 +333,28 @@ bool WebServer::removeRoute(const String &uri) { } bool WebServer::removeRoute(const String &uri, HTTPMethod method) { - // Loop through all request handlers and see if there is a match + bool anyHandlerRemoved = false; RequestHandler *handler = _firstHandler; + RequestHandler *previousHandler = nullptr; + while (handler) { if (handler->canHandle(method, uri)) { - return _removeRequestHandler(handler); + if (_removeRequestHandler(handler)) { + anyHandlerRemoved = true; + // Move to the next handler + if (previousHandler) { + handler = previousHandler->next(); + } else { + handler = _firstHandler; + } + continue; + } } + previousHandler = handler; handler = handler->next(); } - return false; + + return anyHandlerRemoved; } void WebServer::addHandler(RequestHandler *handler) { From a8ce679d8119b4bbe7c897ab8141bc32e069040c Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Fri, 14 Jun 2024 00:26:11 +0300 Subject: [PATCH 15/24] fix(ota): Magic byte check fails with encrypted firmware (#9852) --- libraries/Update/src/Updater.cpp | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/libraries/Update/src/Updater.cpp b/libraries/Update/src/Updater.cpp index f5f6992db5a..4be13b9d1ed 100644 --- a/libraries/Update/src/Updater.cpp +++ b/libraries/Update/src/Updater.cpp @@ -524,9 +524,11 @@ size_t UpdateClass::writeStream(Stream &data) { return 0; } - if (!_verifyHeader(data.peek())) { - _reset(); - return 0; + if (_command == U_FLASH && !_cryptMode) { + if (!_verifyHeader(data.peek())) { + _reset(); + return 0; + } } if (_ledPin != -1) { From f33cc7e94462de2821d61bcbe0ea72b5ed024e48 Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Fri, 14 Jun 2024 00:26:36 +0300 Subject: [PATCH 16/24] fix(ota): Allow password and partition change while idle (#9853) Previously it was allowed only once before begin() was called --- libraries/ArduinoOTA/src/ArduinoOTA.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/libraries/ArduinoOTA/src/ArduinoOTA.cpp b/libraries/ArduinoOTA/src/ArduinoOTA.cpp index 6fa482e1335..769193efb69 100644 --- a/libraries/ArduinoOTA/src/ArduinoOTA.cpp +++ b/libraries/ArduinoOTA/src/ArduinoOTA.cpp @@ -57,25 +57,28 @@ String ArduinoOTAClass::getHostname() { } ArduinoOTAClass &ArduinoOTAClass::setPassword(const char *password) { - if (!_initialized && !_password.length() && password) { + if (_state == OTA_IDLE && password) { MD5Builder passmd5; passmd5.begin(); passmd5.add(password); passmd5.calculate(); + _password.clear(); _password = passmd5.toString(); } return *this; } ArduinoOTAClass &ArduinoOTAClass::setPasswordHash(const char *password) { - if (!_initialized && !_password.length() && password) { + if (_state == OTA_IDLE && password) { + _password.clear(); _password = password; } return *this; } ArduinoOTAClass &ArduinoOTAClass::setPartitionLabel(const char *partition_label) { - if (!_initialized && !_partition_label.length() && partition_label) { + if (_state == OTA_IDLE && partition_label) { + _partition_label.clear(); _partition_label = partition_label; } return *this; From f22ddb30b7d46845825f968d2367d4ddc0eccedf Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Fri, 14 Jun 2024 00:26:59 +0300 Subject: [PATCH 17/24] feat(uart): Add esp-modbus to the included components (#9855) It used to come with IDF 4.x --- idf_component.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/idf_component.yml b/idf_component.yml index a9bc90f8788..81d79180dbb 100644 --- a/idf_component.yml +++ b/idf_component.yml @@ -84,6 +84,9 @@ dependencies: espressif/libsodium: version: "^1.0.20~1" require: public + espressif/esp-modbus: + version: "^1.0.15" + require: public joltwallet/littlefs: version: "^1.10.2" chmorgan/esp-libhelix-mp3: From cbf1e94dd5f9ac8bdabaae9603611a3b81d8fbe4 Mon Sep 17 00:00:00 2001 From: Lesords <98899771+Lesords@users.noreply.github.com> Date: Fri, 14 Jun 2024 05:27:49 +0800 Subject: [PATCH 18/24] chore: delete the definition of pin A3 (#9798) Co-authored-by: Lucas Saavedra Vaz <32426024+lucasssvaz@users.noreply.github.com> --- variants/XIAO_ESP32C3/pins_arduino.h | 1 - 1 file changed, 1 deletion(-) diff --git a/variants/XIAO_ESP32C3/pins_arduino.h b/variants/XIAO_ESP32C3/pins_arduino.h index ca89c5f557e..061e743f523 100644 --- a/variants/XIAO_ESP32C3/pins_arduino.h +++ b/variants/XIAO_ESP32C3/pins_arduino.h @@ -17,7 +17,6 @@ static const uint8_t SCK = 8; static const uint8_t A0 = 2; static const uint8_t A1 = 3; static const uint8_t A2 = 4; -static const uint8_t A3 = 5; static const uint8_t D0 = 2; static const uint8_t D1 = 3; From 7d73a74d65ea517c7426877d15537b32ba20a385 Mon Sep 17 00:00:00 2001 From: Thibo Verheyde <33724539+vThibo@users.noreply.github.com> Date: Mon, 17 Jun 2024 21:31:04 +0200 Subject: [PATCH 19/24] Add Walter board support (#9817) --- boards.txt | 173 +++++++++++++++++++++++++++++++++ variants/walter/pins_arduino.h | 64 ++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 variants/walter/pins_arduino.h diff --git a/boards.txt b/boards.txt index 5163a2d84c9..59e3f8311d0 100644 --- a/boards.txt +++ b/boards.txt @@ -36821,3 +36821,176 @@ aslcanx2.menu.EraseFlash.all=Enabled aslcanx2.menu.EraseFlash.all.upload.erase_cmd=-e ############################################################## + +walter.name=DPTechnics Walter + +walter.bootloader.tool=esptool_py +walter.bootloader.tool.default=esptool_py + +walter.upload.tool=esptool_py +walter.upload.tool.default=esptool_py +walter.upload.tool.network=esp_ota + +walter.upload.maximum_size=1310720 +walter.upload.maximum_data_size=327680 +walter.upload.flags= +walter.upload.extra_flags= +walter.upload.use_1200bps_touch=false +walter.upload.wait_for_upload_port=false + +walter.serial.disableDTR=false +walter.serial.disableRTS=false + +walter.build.tarch=xtensa +walter.build.bootloader_addr=0x0 +walter.build.target=esp32s3 +walter.build.mcu=esp32s3 +walter.build.core=esp32 +walter.build.variant=walter +walter.build.board=DPTECHNICS_WALTER + +walter.build.usb_mode=1 +walter.build.cdc_on_boot=1 +walter.build.msc_on_boot=0 +walter.build.dfu_on_boot=0 +walter.build.f_cpu=240000000L +walter.build.flash_size=16MB +walter.build.flash_freq=80m +walter.build.flash_mode=dio +walter.build.boot=qio +walter.build.boot_freq=80m +walter.build.partitions=default +walter.build.defines= +walter.build.loop_core= +walter.build.event_core= +walter.build.psram_type=qspi +walter.build.memory_type={build.boot}_{build.psram_type} + +## IDE 2.0 Seems to not update the value +walter.menu.JTAGAdapter.default=Disabled +walter.menu.JTAGAdapter.default.build.copy_jtag_files=0 +walter.menu.JTAGAdapter.builtin=Integrated USB JTAG +walter.menu.JTAGAdapter.builtin.build.openocdscript=esp32s3-builtin.cfg +walter.menu.JTAGAdapter.builtin.build.copy_jtag_files=1 +walter.menu.JTAGAdapter.external=FTDI Adapter +walter.menu.JTAGAdapter.external.build.openocdscript=esp32s3-ftdi.cfg +walter.menu.JTAGAdapter.external.build.copy_jtag_files=1 +walter.menu.JTAGAdapter.bridge=ESP USB Bridge +walter.menu.JTAGAdapter.bridge.build.openocdscript=esp32s3-bridge.cfg +walter.menu.JTAGAdapter.bridge.build.copy_jtag_files=1 + +walter.menu.PSRAM.enabled=QSPI PSRAM +walter.menu.PSRAM.enabled.build.defines=-DBOARD_HAS_PSRAM +walter.menu.PSRAM.enabled.build.psram_type=qspi +walter.menu.PSRAM.disabled=Disabled +walter.menu.PSRAM.disabled.build.defines= +walter.menu.PSRAM.disabled.build.psram_type=qspi + +walter.menu.FlashMode.qio=QIO 80MHz +walter.menu.FlashMode.qio.build.flash_mode=dio +walter.menu.FlashMode.qio.build.boot=qio +walter.menu.FlashMode.qio.build.boot_freq=80m +walter.menu.FlashMode.qio.build.flash_freq=80m +walter.menu.FlashMode.dio=DIO 80MHz +walter.menu.FlashMode.dio.build.flash_mode=dio +walter.menu.FlashMode.dio.build.boot=dio +walter.menu.FlashMode.dio.build.boot_freq=80m +walter.menu.FlashMode.dio.build.flash_freq=80m + +walter.menu.FlashSize.16M=16MB (128Mb) +walter.menu.FlashSize.16M.build.flash_size=16MB + +walter.menu.LoopCore.1=Core 1 +walter.menu.LoopCore.1.build.loop_core=-DARDUINO_RUNNING_CORE=1 +walter.menu.LoopCore.0=Core 0 +walter.menu.LoopCore.0.build.loop_core=-DARDUINO_RUNNING_CORE=0 + +walter.menu.EventsCore.1=Core 1 +walter.menu.EventsCore.1.build.event_core=-DARDUINO_EVENT_RUNNING_CORE=1 +walter.menu.EventsCore.0=Core 0 +walter.menu.EventsCore.0.build.event_core=-DARDUINO_EVENT_RUNNING_CORE=0 + +walter.menu.USBMode.hwcdc=Hardware CDC and JTAG +walter.menu.USBMode.hwcdc.build.usb_mode=1 +walter.menu.USBMode.default=USB-OTG (TinyUSB) +walter.menu.USBMode.default.build.usb_mode=0 + +walter.menu.CDCOnBoot.cdc=Enabled +walter.menu.CDCOnBoot.cdc.build.cdc_on_boot=1 +walter.menu.CDCOnBoot.default=Disabled +walter.menu.CDCOnBoot.default.build.cdc_on_boot=0 + +walter.menu.MSCOnBoot.default=Disabled +walter.menu.MSCOnBoot.default.build.msc_on_boot=0 +walter.menu.MSCOnBoot.msc=Enabled (Requires USB-OTG Mode) +walter.menu.MSCOnBoot.msc.build.msc_on_boot=1 + +walter.menu.DFUOnBoot.default=Disabled +walter.menu.DFUOnBoot.default.build.dfu_on_boot=0 +walter.menu.DFUOnBoot.dfu=Enabled (Requires USB-OTG Mode) +walter.menu.DFUOnBoot.dfu.build.dfu_on_boot=1 + +walter.menu.UploadMode.default=UART0 / Hardware CDC +walter.menu.UploadMode.default.upload.use_1200bps_touch=false +walter.menu.UploadMode.default.upload.wait_for_upload_port=false +walter.menu.UploadMode.cdc=USB-OTG CDC (TinyUSB) +walter.menu.UploadMode.cdc.upload.use_1200bps_touch=true +walter.menu.UploadMode.cdc.upload.wait_for_upload_port=true + +walter.menu.PartitionScheme.fatflash=16M Flash (2MB APP/12.5MB FATFS) +walter.menu.PartitionScheme.fatflash.build.partitions=ffat +walter.menu.PartitionScheme.fatflash.upload.maximum_size=2097152 +walter.menu.PartitionScheme.app3M_fat9M_16MB=16M Flash (3MB APP/9.9MB FATFS) +walter.menu.PartitionScheme.app3M_fat9M_16MB.build.partitions=app3M_fat9M_16MB +walter.menu.PartitionScheme.app3M_fat9M_16MB.upload.maximum_size=3145728 +walter.menu.PartitionScheme.default_16MB=Default (6.25MB APP/3.43MB SPIFFS) +walter.menu.PartitionScheme.default_16MB.build.partitions=default_16MB +walter.menu.PartitionScheme.default_16MB.upload.maximum_size=6553600 + +walter.menu.CPUFreq.240=240MHz (WiFi) +walter.menu.CPUFreq.240.build.f_cpu=240000000L +walter.menu.CPUFreq.160=160MHz (WiFi) +walter.menu.CPUFreq.160.build.f_cpu=160000000L +walter.menu.CPUFreq.80=80MHz (WiFi) +walter.menu.CPUFreq.80.build.f_cpu=80000000L +walter.menu.CPUFreq.40=40MHz +walter.menu.CPUFreq.40.build.f_cpu=40000000L +walter.menu.CPUFreq.20=20MHz +walter.menu.CPUFreq.20.build.f_cpu=20000000L +walter.menu.CPUFreq.10=10MHz +walter.menu.CPUFreq.10.build.f_cpu=10000000L + +walter.menu.UploadSpeed.921600=921600 +walter.menu.UploadSpeed.921600.upload.speed=921600 +walter.menu.UploadSpeed.115200=115200 +walter.menu.UploadSpeed.115200.upload.speed=115200 +walter.menu.UploadSpeed.256000.windows=256000 +walter.menu.UploadSpeed.256000.upload.speed=256000 +walter.menu.UploadSpeed.230400.windows.upload.speed=256000 +walter.menu.UploadSpeed.230400=230400 +walter.menu.UploadSpeed.230400.upload.speed=230400 +walter.menu.UploadSpeed.460800.linux=460800 +walter.menu.UploadSpeed.460800.macosx=460800 +walter.menu.UploadSpeed.460800.upload.speed=460800 +walter.menu.UploadSpeed.512000.windows=512000 +walter.menu.UploadSpeed.512000.upload.speed=512000 + +walter.menu.DebugLevel.none=None +walter.menu.DebugLevel.none.build.code_debug=0 +walter.menu.DebugLevel.error=Error +walter.menu.DebugLevel.error.build.code_debug=1 +walter.menu.DebugLevel.warn=Warn +walter.menu.DebugLevel.warn.build.code_debug=2 +walter.menu.DebugLevel.info=Info +walter.menu.DebugLevel.info.build.code_debug=3 +walter.menu.DebugLevel.debug=Debug +walter.menu.DebugLevel.debug.build.code_debug=4 +walter.menu.DebugLevel.verbose=Verbose +walter.menu.DebugLevel.verbose.build.code_debug=5 + +walter.menu.EraseFlash.none=Disabled +walter.menu.EraseFlash.none.upload.erase_cmd= +walter.menu.EraseFlash.all=Enabled +walter.menu.EraseFlash.all.upload.erase_cmd=-e + +############################################################## diff --git a/variants/walter/pins_arduino.h b/variants/walter/pins_arduino.h new file mode 100644 index 00000000000..53076e7b48e --- /dev/null +++ b/variants/walter/pins_arduino.h @@ -0,0 +1,64 @@ +#ifndef Pins_Arduino_h +#define Pins_Arduino_h + +#include + +#define USB_VID 0x303a +#define USB_PID 0x1001 +#define USB_MANUFACTURER "DPTechnics" +#define USB_PRODUCT "Walter" +#define USB_SERIAL "" + +#define MODEM_TX 48 // Sequans modem UART0 TX +#define MODEM_RX 14 // Sequans modem UART0 RX +#define MODEM_CTS 47 // Sequans modem UART0 CTS +#define MODEM_RTS 19 // Sequans modem UART0 RTS +#define MODEM_RESET 45 // Sequans modem reset signal +#define MODEM_WAKE 46 // Sequans modem wake signal + +static const uint8_t TX = 43; +static const uint8_t RX = 44; + +static const uint8_t SDA = 8; +static const uint8_t SCL = 9; + +static const uint8_t SS = 10; +static const uint8_t MOSI = 11; +static const uint8_t MISO = 13; +static const uint8_t SCK = 12; + +static const uint8_t A0 = 1; +static const uint8_t A1 = 2; +static const uint8_t A2 = 3; +static const uint8_t A3 = 4; +static const uint8_t A4 = 5; +static const uint8_t A5 = 6; +static const uint8_t A6 = 7; +static const uint8_t A7 = 8; +static const uint8_t A8 = 9; +static const uint8_t A9 = 10; +static const uint8_t A10 = 11; +static const uint8_t A11 = 12; +static const uint8_t A12 = 13; +static const uint8_t A14 = 15; +static const uint8_t A15 = 16; +static const uint8_t A16 = 17; +static const uint8_t A17 = 18; +static const uint8_t A18 = 19; +static const uint8_t A19 = 20; + +static const uint8_t T1 = 1; +static const uint8_t T2 = 2; +static const uint8_t T3 = 3; +static const uint8_t T4 = 4; +static const uint8_t T5 = 5; +static const uint8_t T6 = 6; +static const uint8_t T7 = 7; +static const uint8_t T8 = 8; +static const uint8_t T9 = 9; +static const uint8_t T10 = 10; +static const uint8_t T11 = 11; +static const uint8_t T12 = 12; +static const uint8_t T13 = 13; + +#endif /* Pins_Arduino_h */ From 5b7c615a0ad25e0ba685595e56c305c5f59d1c2e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Proch=C3=A1zka?= <90197375+P-R-O-C-H-Y@users.noreply.github.com> Date: Mon, 17 Jun 2024 21:32:05 +0200 Subject: [PATCH 20/24] ci(boards): Add cache for libs (#9877) --- .github/workflows/boards.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.github/workflows/boards.yml b/.github/workflows/boards.yml index 30175c3e8d2..8d5868b083b 100644 --- a/.github/workflows/boards.yml +++ b/.github/workflows/boards.yml @@ -59,6 +59,19 @@ jobs: exit 1; fi + - name: Get libs cache + uses: actions/cache@v4 + with: + key: libs-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('package/package_esp32_index.template.json', 'tools/get.py') }} + path: | + ./tools/dist + ./tools/esp32-arduino-libs + ./tools/esptool + ./tools/mk* + ./tools/openocd-esp32 + ./tools/riscv32-* + ./tools/xtensa-* + - name: Compile sketch uses: P-R-O-C-H-Y/compile-sketches@main with: @@ -73,3 +86,4 @@ jobs: exit-on-fail: true sketch-paths: "- ./libraries/ESP32/examples/CI/CIBoardsTest/CIBoardsTest.ino" + verbose: true From 2c7f722e70fdac3ee83ec1bd626fb034d625400c Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Mon, 17 Jun 2024 22:32:30 +0300 Subject: [PATCH 21/24] add(board): Add LILYGO T-ETH-Lite (#9865) * add(board): Add LILYGO T-ETH-Lite Adds board support for Lilygo T-ETH Lite * ci(pre-commit): Apply automatic fixes --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- boards.txt | 116 ++++++++++++++++++++++ variants/lilygo_t_eth_lite/pins_arduino.h | 50 ++++++++++ 2 files changed, 166 insertions(+) create mode 100644 variants/lilygo_t_eth_lite/pins_arduino.h diff --git a/boards.txt b/boards.txt index 59e3f8311d0..d42e1b0571b 100644 --- a/boards.txt +++ b/boards.txt @@ -4697,6 +4697,122 @@ lilygo_t_display_s3.menu.EraseFlash.all.upload.erase_cmd=-e ############################################################## +lilygo_t_eth_lite.name=LilyGo T-ETH-Lite + +lilygo_t_eth_lite.bootloader.tool=esptool_py +lilygo_t_eth_lite.bootloader.tool.default=esptool_py + +lilygo_t_eth_lite.upload.tool=esptool_py +lilygo_t_eth_lite.upload.tool.default=esptool_py +lilygo_t_eth_lite.upload.tool.network=esp_ota + +lilygo_t_eth_lite.upload.maximum_size=3145728 +lilygo_t_eth_lite.upload.maximum_data_size=327680 +lilygo_t_eth_lite.upload.speed=921600 +lilygo_t_eth_lite.upload.flags= +lilygo_t_eth_lite.upload.extra_flags= +lilygo_t_eth_lite.upload.use_1200bps_touch=false +lilygo_t_eth_lite.upload.wait_for_upload_port=false + +lilygo_t_eth_lite.serial.disableDTR=false +lilygo_t_eth_lite.serial.disableRTS=false + +lilygo_t_eth_lite.build.tarch=xtensa +lilygo_t_eth_lite.build.bootloader_addr=0x0 +lilygo_t_eth_lite.build.target=esp32s3 +lilygo_t_eth_lite.build.mcu=esp32s3 +lilygo_t_eth_lite.build.core=esp32 +lilygo_t_eth_lite.build.variant=lilygo_t_eth_lite +lilygo_t_eth_lite.build.board=LILYGO_T_ETH_LITE + +lilygo_t_eth_lite.build.usb_mode=1 +lilygo_t_eth_lite.build.cdc_on_boot=0 +lilygo_t_eth_lite.build.msc_on_boot=0 +lilygo_t_eth_lite.build.dfu_on_boot=0 +lilygo_t_eth_lite.build.f_cpu=240000000L +lilygo_t_eth_lite.build.flash_size=16MB +lilygo_t_eth_lite.build.flash_freq=80m +lilygo_t_eth_lite.build.flash_mode=dio +lilygo_t_eth_lite.build.boot=qio +lilygo_t_eth_lite.build.boot_freq=80m +lilygo_t_eth_lite.build.partitions=app3M_fat9M_16MB +lilygo_t_eth_lite.build.defines= +lilygo_t_eth_lite.build.loop_core= +lilygo_t_eth_lite.build.event_core= +lilygo_t_eth_lite.build.psram_type=opi +lilygo_t_eth_lite.build.memory_type={build.boot}_{build.psram_type} + +## IDE 2.0 Seems to not update the value +lilygo_t_eth_lite.menu.JTAGAdapter.default=Disabled +lilygo_t_eth_lite.menu.JTAGAdapter.default.build.copy_jtag_files=0 +lilygo_t_eth_lite.menu.JTAGAdapter.builtin=Integrated USB JTAG +lilygo_t_eth_lite.menu.JTAGAdapter.builtin.build.openocdscript=esp32s3-builtin.cfg +lilygo_t_eth_lite.menu.JTAGAdapter.builtin.build.copy_jtag_files=1 + +lilygo_t_eth_lite.menu.LoopCore.1=Core 1 +lilygo_t_eth_lite.menu.LoopCore.1.build.loop_core=-DARDUINO_RUNNING_CORE=1 +lilygo_t_eth_lite.menu.LoopCore.0=Core 0 +lilygo_t_eth_lite.menu.LoopCore.0.build.loop_core=-DARDUINO_RUNNING_CORE=0 + +lilygo_t_eth_lite.menu.EventsCore.1=Core 1 +lilygo_t_eth_lite.menu.EventsCore.1.build.event_core=-DARDUINO_EVENT_RUNNING_CORE=1 +lilygo_t_eth_lite.menu.EventsCore.0=Core 0 +lilygo_t_eth_lite.menu.EventsCore.0.build.event_core=-DARDUINO_EVENT_RUNNING_CORE=0 + +lilygo_t_eth_lite.menu.USBMode.hwcdc=Hardware CDC and JTAG +lilygo_t_eth_lite.menu.USBMode.hwcdc.build.usb_mode=1 +lilygo_t_eth_lite.menu.USBMode.default=USB-OTG (TinyUSB) +lilygo_t_eth_lite.menu.USBMode.default.build.usb_mode=0 + +lilygo_t_eth_lite.menu.CDCOnBoot.default=Disabled +lilygo_t_eth_lite.menu.CDCOnBoot.default.build.cdc_on_boot=0 +lilygo_t_eth_lite.menu.CDCOnBoot.cdc=Enabled +lilygo_t_eth_lite.menu.CDCOnBoot.cdc.build.cdc_on_boot=1 + +lilygo_t_eth_lite.menu.MSCOnBoot.default=Disabled +lilygo_t_eth_lite.menu.MSCOnBoot.default.build.msc_on_boot=0 +lilygo_t_eth_lite.menu.MSCOnBoot.msc=Enabled (Requires USB-OTG Mode) +lilygo_t_eth_lite.menu.MSCOnBoot.msc.build.msc_on_boot=1 + +lilygo_t_eth_lite.menu.DFUOnBoot.default=Disabled +lilygo_t_eth_lite.menu.DFUOnBoot.default.build.dfu_on_boot=0 +lilygo_t_eth_lite.menu.DFUOnBoot.dfu=Enabled (Requires USB-OTG Mode) +lilygo_t_eth_lite.menu.DFUOnBoot.dfu.build.dfu_on_boot=1 + +lilygo_t_eth_lite.menu.UploadMode.default=UART0 / Hardware CDC +lilygo_t_eth_lite.menu.UploadMode.default.upload.use_1200bps_touch=false +lilygo_t_eth_lite.menu.UploadMode.default.upload.wait_for_upload_port=false +lilygo_t_eth_lite.menu.UploadMode.cdc=USB-OTG CDC (TinyUSB) +lilygo_t_eth_lite.menu.UploadMode.cdc.upload.use_1200bps_touch=true +lilygo_t_eth_lite.menu.UploadMode.cdc.upload.wait_for_upload_port=true + +lilygo_t_eth_lite.menu.PartitionScheme.app3M_fat9M_16MB=16M Flash (3MB APP/9.9MB FATFS) +lilygo_t_eth_lite.menu.PartitionScheme.app3M_fat9M_16MB.build.partitions=app3M_fat9M_16MB +lilygo_t_eth_lite.menu.PartitionScheme.app3M_fat9M_16MB.upload.maximum_size=3145728 +lilygo_t_eth_lite.menu.PartitionScheme.rainmaker=RainMaker +lilygo_t_eth_lite.menu.PartitionScheme.rainmaker.build.partitions=rainmaker +lilygo_t_eth_lite.menu.PartitionScheme.rainmaker.upload.maximum_size=3145728 + +lilygo_t_eth_lite.menu.DebugLevel.none=None +lilygo_t_eth_lite.menu.DebugLevel.none.build.code_debug=0 +lilygo_t_eth_lite.menu.DebugLevel.error=Error +lilygo_t_eth_lite.menu.DebugLevel.error.build.code_debug=1 +lilygo_t_eth_lite.menu.DebugLevel.warn=Warn +lilygo_t_eth_lite.menu.DebugLevel.warn.build.code_debug=2 +lilygo_t_eth_lite.menu.DebugLevel.info=Info +lilygo_t_eth_lite.menu.DebugLevel.info.build.code_debug=3 +lilygo_t_eth_lite.menu.DebugLevel.debug=Debug +lilygo_t_eth_lite.menu.DebugLevel.debug.build.code_debug=4 +lilygo_t_eth_lite.menu.DebugLevel.verbose=Verbose +lilygo_t_eth_lite.menu.DebugLevel.verbose.build.code_debug=5 + +lilygo_t_eth_lite.menu.EraseFlash.none=Disabled +lilygo_t_eth_lite.menu.EraseFlash.none.upload.erase_cmd= +lilygo_t_eth_lite.menu.EraseFlash.all=Enabled +lilygo_t_eth_lite.menu.EraseFlash.all.upload.erase_cmd=-e + +############################################################## + micros2.name=microS2 micros2.vid.0=0x239A micros2.pid.0=0x80C5 diff --git a/variants/lilygo_t_eth_lite/pins_arduino.h b/variants/lilygo_t_eth_lite/pins_arduino.h new file mode 100644 index 00000000000..cb8fed779d9 --- /dev/null +++ b/variants/lilygo_t_eth_lite/pins_arduino.h @@ -0,0 +1,50 @@ +#ifndef Pins_Arduino_h +#define Pins_Arduino_h + +#include + +#define USB_VID 0x303a +#define USB_PID 0x1001 + +static const uint8_t TX = 43; +static const uint8_t RX = 44; + +static const uint8_t SDA = 15; +static const uint8_t SCL = 16; + +static const uint8_t SS = 4; +static const uint8_t MISO = 5; +static const uint8_t MOSI = 6; +static const uint8_t SCK = 7; +static const uint8_t SD_SS = 42; + +// Analog +static const uint8_t A2 = 3; +static const uint8_t A3 = 4; +static const uint8_t A4 = 5; +static const uint8_t A5 = 6; +static const uint8_t A6 = 7; +static const uint8_t A7 = 8; + +// Touch +static const uint8_t T1 = 1; +static const uint8_t T2 = 2; +static const uint8_t T3 = 3; +static const uint8_t T4 = 4; +static const uint8_t T5 = 5; +static const uint8_t T6 = 6; +static const uint8_t T7 = 7; +static const uint8_t T8 = 8; + +// Ethernet +#define ETH_PHY_TYPE ETH_PHY_W5500 +#define ETH_PHY_ADDR 1 +#define ETH_PHY_CS 9 +#define ETH_PHY_IRQ 13 +#define ETH_PHY_RST 14 +#define ETH_PHY_SPI_HOST SPI2_HOST +#define ETH_PHY_SPI_SCK 10 +#define ETH_PHY_SPI_MISO 11 +#define ETH_PHY_SPI_MOSI 12 + +#endif /* Pins_Arduino_h */ From 1efab83432ee8e5b4c0b9dadceb476aed7dfcf36 Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Mon, 17 Jun 2024 22:32:52 +0300 Subject: [PATCH 22/24] feat(http): Allow to set Accept-Encoding header (#9863) Similar to setUserAgent --- libraries/HTTPClient/src/HTTPClient.cpp | 14 +++++++++++--- libraries/HTTPClient/src/HTTPClient.h | 2 ++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/libraries/HTTPClient/src/HTTPClient.cpp b/libraries/HTTPClient/src/HTTPClient.cpp index 54eedf0ac2e..64dfe3a7db0 100644 --- a/libraries/HTTPClient/src/HTTPClient.cpp +++ b/libraries/HTTPClient/src/HTTPClient.cpp @@ -408,6 +408,14 @@ void HTTPClient::setUserAgent(const String &userAgent) { _userAgent = userAgent; } +/** + * set Accept Encoding Header + * @param acceptEncoding const char * + */ +void HTTPClient::setAcceptEncoding(const String &acceptEncoding) { + _acceptEncoding = acceptEncoding; +} + /** * set the Authorizatio for the http request * @param user const char * @@ -969,8 +977,8 @@ String HTTPClient::errorToString(int error) { */ void HTTPClient::addHeader(const String &name, const String &value, bool first, bool replace) { // not allow set of Header handled by code - if (!name.equalsIgnoreCase(F("Connection")) && !name.equalsIgnoreCase(F("User-Agent")) && !name.equalsIgnoreCase(F("Host")) - && !(name.equalsIgnoreCase(F("Authorization")) && _base64Authorization.length())) { + if (!name.equalsIgnoreCase(F("Connection")) && !name.equalsIgnoreCase(F("User-Agent")) && !name.equalsIgnoreCase(F("Accept-Encoding")) + && !name.equalsIgnoreCase(F("Host")) && !(name.equalsIgnoreCase(F("Authorization")) && _base64Authorization.length())) { String headerLine = name; headerLine += ": "; @@ -1130,7 +1138,7 @@ bool HTTPClient::sendHeader(const char *type) { header += "\r\n"; if (!_useHTTP10) { - header += F("Accept-Encoding: identity;q=1,chunked;q=0.1,*;q=0\r\n"); + header += String(F("Accept-Encoding: ")) + _acceptEncoding + F("\r\n"); } if (_base64Authorization.length()) { diff --git a/libraries/HTTPClient/src/HTTPClient.h b/libraries/HTTPClient/src/HTTPClient.h index 7841f355640..edc050ab0dd 100644 --- a/libraries/HTTPClient/src/HTTPClient.h +++ b/libraries/HTTPClient/src/HTTPClient.h @@ -194,6 +194,7 @@ class HTTPClient { void setReuse(bool reuse); /// keep-alive void setUserAgent(const String &userAgent); + void setAcceptEncoding(const String &acceptEncoding); void setAuthorization(const char *user, const char *password); void setAuthorization(const char *auth); void setAuthorizationType(const char *authType); @@ -285,6 +286,7 @@ class HTTPClient { String _userAgent = "ESP32HTTPClient"; String _base64Authorization; String _authorizationType = "Basic"; + String _acceptEncoding = "identity;q=1,chunked;q=0.1,*;q=0"; /// Response handling RequestArgument *_currentHeaders = nullptr; From 5c22402025cc166bfe3f9eb19e521f4ec64225c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juraj=20Andr=C3=A1ssy?= <10706773+JAndrassy@users.noreply.github.com> Date: Mon, 17 Jun 2024 21:40:10 +0200 Subject: [PATCH 23/24] fix: ClientSecure.available() fix for connection closed by remote socket (#9869) --- libraries/NetworkClientSecure/src/NetworkClientSecure.cpp | 6 ++++-- libraries/NetworkClientSecure/src/ssl_client.cpp | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/libraries/NetworkClientSecure/src/NetworkClientSecure.cpp b/libraries/NetworkClientSecure/src/NetworkClientSecure.cpp index d5e3b63c5be..73ff6da5dcc 100644 --- a/libraries/NetworkClientSecure/src/NetworkClientSecure.cpp +++ b/libraries/NetworkClientSecure/src/NetworkClientSecure.cpp @@ -305,9 +305,11 @@ int NetworkClientSecure::available() { res = data_to_read(sslclient.get()); if (res < 0 && !_stillinPlainStart) { - log_e("Closing connection on failed available check"); + if (res != MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) { + log_e("Closing connection on failed available check"); + } stop(); - return peeked ? peeked : res; + return peeked; } return res + peeked; } diff --git a/libraries/NetworkClientSecure/src/ssl_client.cpp b/libraries/NetworkClientSecure/src/ssl_client.cpp index c8d5bbd21ea..0f93f5cd7fe 100644 --- a/libraries/NetworkClientSecure/src/ssl_client.cpp +++ b/libraries/NetworkClientSecure/src/ssl_client.cpp @@ -27,7 +27,7 @@ const char *pers = "esp32-tls"; static int _handle_error(int err, const char *function, int line) { - if (err == -30848) { + if (err == MBEDTLS_ERR_SSL_PEER_CLOSE_NOTIFY) { return err; } #ifdef MBEDTLS_ERROR_C From c43187a3864bf81be632afb2c6eea42e6df63a01 Mon Sep 17 00:00:00 2001 From: Me No Dev Date: Mon, 17 Jun 2024 22:47:12 +0300 Subject: [PATCH 24/24] fix(wifi): Fix WiFi setTxPower and getTxPower (#9862) * fix(wifi): Fix WiFi setTxPower and getTxPower We need to wait for the interface to be started in order to be able to set/get TX Power. Code was returning too early, so calling the functions after `begin()` resulted in failure. * ci(pre-commit): Apply automatic fixes --------- Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com> --- libraries/WiFi/src/AP.cpp | 4 ++++ libraries/WiFi/src/STA.cpp | 4 ++++ libraries/WiFi/src/WiFiGeneric.cpp | 10 ++++++++-- libraries/WiFi/src/WiFiGeneric.h | 3 +++ 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/libraries/WiFi/src/AP.cpp b/libraries/WiFi/src/AP.cpp index db61c2f0ff1..a61be662495 100644 --- a/libraries/WiFi/src/AP.cpp +++ b/libraries/WiFi/src/AP.cpp @@ -187,6 +187,10 @@ bool APClass::begin() { log_e("AP enable failed!"); return false; } + if (!waitStatusBits(ESP_NETIF_STARTED_BIT, 1000)) { + log_e("Failed to start AP!"); + return false; + } return true; } diff --git a/libraries/WiFi/src/STA.cpp b/libraries/WiFi/src/STA.cpp index dacb0cae2d4..443d2621957 100644 --- a/libraries/WiFi/src/STA.cpp +++ b/libraries/WiFi/src/STA.cpp @@ -298,6 +298,10 @@ bool STAClass::begin(bool tryConnect) { log_e("STA enable failed!"); return false; } + if (!waitStatusBits(ESP_NETIF_STARTED_BIT, 1000)) { + log_e("Failed to start STA!"); + return false; + } if (tryConnect) { return connect(); } diff --git a/libraries/WiFi/src/WiFiGeneric.cpp b/libraries/WiFi/src/WiFiGeneric.cpp index 1903a5c9c41..75a01d1362f 100644 --- a/libraries/WiFi/src/WiFiGeneric.cpp +++ b/libraries/WiFi/src/WiFiGeneric.cpp @@ -639,7 +639,11 @@ bool WiFiGenericClass::setTxPower(wifi_power_t power) { log_w("Neither AP or STA has been started"); return false; } - return esp_wifi_set_max_tx_power(power) == ESP_OK; + esp_err_t err = esp_wifi_set_max_tx_power(power); + if (err != ESP_OK) { + log_e("Failed to set TX Power: 0x%x: %s", err, esp_err_to_name(err)); + } + return err == ESP_OK; } wifi_power_t WiFiGenericClass::getTxPower() { @@ -648,7 +652,9 @@ wifi_power_t WiFiGenericClass::getTxPower() { log_w("Neither AP or STA has been started"); return WIFI_POWER_19_5dBm; } - if (esp_wifi_get_max_tx_power(&power)) { + esp_err_t err = esp_wifi_get_max_tx_power(&power); + if (err != ESP_OK) { + log_e("Failed to get TX Power: 0x%x: %s", err, esp_err_to_name(err)); return WIFI_POWER_19_5dBm; } return (wifi_power_t)power; diff --git a/libraries/WiFi/src/WiFiGeneric.h b/libraries/WiFi/src/WiFiGeneric.h index 6ed74dfed84..3cb1515b324 100644 --- a/libraries/WiFi/src/WiFiGeneric.h +++ b/libraries/WiFi/src/WiFiGeneric.h @@ -44,6 +44,9 @@ #define wifi_event_id_t network_event_handle_t typedef enum { + WIFI_POWER_21dBm = 84, // 21dBm + WIFI_POWER_20_5dBm = 82, // 20.5dBm + WIFI_POWER_20dBm = 80, // 20dBm WIFI_POWER_19_5dBm = 78, // 19.5dBm WIFI_POWER_19dBm = 76, // 19dBm WIFI_POWER_18_5dBm = 74, // 18.5dBm