Test release dev

.git-blame-ignore-revs

@@ -10,3 +10,6 @@ b57f69bd13222b1753446a0f7c17386eda1dc2c9
 
 # Formating with Black
 45f1da954eeab4897fb852894fae0c1b901b3926
+
+# Refactoring documentation
+6282f98dbfca58add4992c259c0aac3c3ec64d3f

.github/ISSUE_TEMPLATE/bug-report-no-hw.yml

@@ -14,7 +14,7 @@ body:
     id: version
     attributes:
       label: Esptool Version
-      description: The output of `git describe` if working with the sources, output of `esptool.py version` otherwise. If possible, consider updating esptool.
+      description: The output of `git describe` if working with the sources, output of `esptool.py version` otherwise. If possible, consider [updating esptool](https://docs.espressif.com/projects/esptool/en/latest/installation.html#how-to-update).
       placeholder: ex. v4.0.1, commit v4.1-10-g2059335aa
     validations:
       required: true

.github/ISSUE_TEMPLATE/issue-with-hw.yml

@@ -7,7 +7,7 @@ body:
       value: |
         * Most failures to connect, flash, etc. are problems with the hardware.
         * Please check any guide that came with your hardware, and also check [the esptool troubleshooting guide](https://docs.espressif.com/projects/esptool/en/latest/troubleshooting.html).
-        * If your board is a custom design, consider using our [free-of-charge schematic and PCB review service](https://www.espressif.com/en/contact-us/circuit-schematic-pcb-design-review).
+        * If your board is a custom design, check the [ESP Hardware Design Guidelines](https://docs.espressif.com/projects/esp-hardware-design-guidelines/) and consider using our [free-of-charge schematic and PCB review service](https://www.espressif.com/en/contact-us/circuit-schematic-pcb-design-review).
         * If still experiencing the issue, please provide as many details as possible below about your hardware and computer setup.
   - type: input
     id: os
@@ -21,7 +21,7 @@ body:
     id: version
     attributes:
       label: Esptool Version
-      description: The output of `git describe` if working with the sources, output of `esptool.py version` otherwise. If possible, consider updating esptool.
+      description: The output of `git describe` if working with the sources, output of `esptool.py version` otherwise. If possible, consider [updating esptool](https://docs.espressif.com/projects/esptool/en/latest/installation.html#how-to-update).
       placeholder: ex. v4.0.1, commit v4.1-10-g2059335aa
     validations:
       required: true

.github/pull_request_template.md

@@ -1,15 +1,20 @@
-(Please delete any lines which don't apply)
-
-# Description of change
+<!-- Fill in a description of the change here, at least 100 characters.
+Make sure other people will be able to understand what your pull request is about.
+Delete any sections which don't apply, including the section header. -->
 
 # This change fixes the following bug(s):
+<!-- If your change fixes any bugs, list them in this section.
 
-(Please put issue URLs or #number-of-issue here.)
+Please include the issue URL or the # issue number here. -->
 
 # I have tested this change with the following hardware & software combinations:
+<!-- In this section, describe the hardware and software combinations with which you tested the PR change - operating system(s), development board name(s), ESP8266 and/or ESP32 series.
+
+If you did not perform any testing, write "NO TESTING" in this section. -->
 
-(Operating system(s), development board name(s), ESP8266 and/or ESP32.)
+# I have run the esptool.py automated integration tests with this change and the above hardware:
+<!-- In this section, post the results of running the automatic integration tests of esptool.py with this change and the above hardware.
 
-# I have run the esptool.py automated integration tests with this change and the above hardware. The results were:
+Details here: https://docs.espressif.com/projects/esptool/en/latest/contributing.html#automated-integration-tests
 
-(Details here: https://docs.espressif.com/projects/esptool/en/latest/contributing.html#automated-integration-tests)
+If you did not perform any testing, write "NO TESTING" in this section. -->

.github/workflows/build_esptool.yml

@@ -4,65 +4,73 @@ on: [push, pull_request]
 
 jobs:
   build-esptool-binaries:
-    name: Build esptool binaries for ${{ matrix.os }}
-    runs-on: ${{ matrix.os }}
+    name: Build esptool binaries for ${{ matrix.platform }}
+    runs-on: ${{ matrix.RUN_ON }}
     strategy:
       matrix:
-        os: [macos-latest, ubuntu-20.04, windows-latest, ARM, ARM64]
+        platform: [macos, windows, linux-amd64]
         include:
-        - os: macos-latest
+        - platform: macos
           TARGET: macos
           SEPARATOR: ':'
-        - os: ubuntu-20.04
-          TARGET: linux-amd64
-          SEPARATOR: ':'
-        - os: windows-latest
+          RUN_ON: macos-latest
+        - platform: windows
           TARGET: win64
           EXTEN: .exe
           SEPARATOR: ';'
-        - os: ARM
-          CONTAINER: python:3.8-bullseye
-          TARGET: arm
-          SEPARATOR: ':'
-        - os: ARM64
-          CONTAINER: python:3.8-bullseye
-          TARGET: arm64
+          RUN_ON: windows-latest
+        - platform: linux-amd64
+          TARGET: linux-amd64
           SEPARATOR: ':'
+          RUN_ON: ubuntu-20.04
+        # - platform: linux-arm32
+        #   CONTAINER: python:3.8-bullseye
+        #   TARGET: linux-arm32
+        #   SEPARATOR: ':'
+        #   RUN_ON: [ARM, self-hosted, linux]
+        # - platform: linux-arm64
+        #   CONTAINER: python:3.8-bullseye
+        #   TARGET: linux-arm64
+        #   SEPARATOR: ':'
+        #   RUN_ON: [ARM64, self-hosted, linux]
     container: ${{ matrix.CONTAINER }} # use python container on ARM
     env:
       DISTPATH: esptool-${{ matrix.TARGET }}
-      STUBS_DIR: /esptool/targets/stub_flasher/
+      STUBS_DIR: ./esptool/targets/stub_flasher/
+      EFUSE_DIR: ./espefuse/efuse_defs/
+      PIP_EXTRA_INDEX_URL: "https://dl.espressif.com/pypi"
     steps:
       - name: Checkout repository
         uses: actions/checkout@master
       - name: Set up Python 3.8
         # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108
-        if: matrix.os != 'ARM' && matrix.os != 'ARM64'
+        if: matrix.platform != 'linux-arm32' && matrix.platform != 'linux-arm64'
         uses: actions/setup-python@master
         with:
           python-version: 3.8
       - name: Install dependencies
+        # PyInstaller >=6.0 results in significantly more antivirus false positives
         run: |
           python -m pip install --upgrade pip
-          pip install pyinstaller
+          pip install pyinstaller==5.13.2
           pip install --user -e .
       - name: Build with PyInstaller
         run: |
-          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico --add-data=".${{ env.STUBS_DIR }}*.json${{ matrix.SEPARATOR }}${{ env.STUBS_DIR }}" esptool.py
-          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico espefuse.py
+          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico --add-data="${{ env.STUBS_DIR }}*.json${{ matrix.SEPARATOR }}${{ env.STUBS_DIR }}" esptool.py
+          pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico --add-data="${{ env.EFUSE_DIR }}*.yaml${{ matrix.SEPARATOR }}${{ env.EFUSE_DIR }}" espefuse.py
           pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico espsecure.py
           pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=ci/espressif.ico esp_rfc2217_server.py
-      - name: Sign binaries
-        if: matrix.os == 'windows-latest' && github.event_name != 'pull_request'
-        env:
-          CERTIFICATE: ${{ secrets.CERTIFICATE }}
-          CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}
-        shell: pwsh
-        run: |
-          ./ci/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/esptool.exe
-          ./ci/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/espefuse.exe
-          ./ci/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/espsecure.exe
-          ./ci/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/esp_rfc2217_server.exe
+      # - name: Sign binaries
+      #   if: matrix.platform == 'windows' && github.event_name != 'pull_request'
+      #   env:
+      #     CERTIFICATE: ${{ secrets.CERTIFICATE }}
+      #     CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}
+      #   shell: pwsh
+      #   run: |
+      #     ./ci/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/esptool.exe
+      #     ./ci/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/espefuse.exe
+      #     ./ci/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/espsecure.exe
+      #     ./ci/Sign-File.ps1 -Path ./${{ env.DISTPATH }}/esp_rfc2217_server.exe
       - name: Test binaries
         shell: bash
         run: |
@@ -84,49 +92,46 @@ jobs:
     if: startsWith(github.ref, 'refs/tags/')
     needs: build-esptool-binaries
     runs-on: ubuntu-latest
-    outputs:
-      upload_url: ${{ steps.create_release.outputs.upload_url }}
-      VERSION: ${{ steps.get_version.outputs.VERSION }}
+    permissions:
+      contents: write
     steps:
-      - name: Create release
-        id: create_release
-        uses: actions/create-release@v1
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        with:
-          tag_name: ${{ github.ref }}
-          release_name: Version ${{ github.ref }}
-          draft: true
-          prerelease: false
       - name: Get version
         id: get_version
-        run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//}
+        run: echo "VERSION=${GITHUB_REF#refs/tags/v}" >> $GITHUB_OUTPUT
         shell: bash
-
-  upload_assets:
-    name: Upload release assets
-    if: startsWith(github.ref, 'refs/tags/')
-    needs: create_release
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        TARGET: [macos, linux-amd64, win64, arm, arm64]
-    env:
-      DISTPATH: esptool-${{ needs.create_release.outputs.VERSION }}-${{ matrix.TARGET }}
-    steps:
+      - name: Get previous version
+        id: get_previous_version
+        env:
+          FILTER: "${{ contains(github.ref_name, 'dev') && 'first' || 'map(select(.prerelease == false)) | first' }} | .tag_name"
+        run: echo "VERSION=$(curl https://api.github.com/repos/peterdragun/esptool/releases | jq -r '${{env.FILTER}}')" >> $GITHUB_OUTPUT
+        shell: bash
+      - name: Checkout
+        uses: actions/checkout@master
+        with:
+          fetch-depth: 0
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install --user -e ".[dev]"
+      - name: Generate changelog
+        run: |
+          cz changelog ${{ steps.get_previous_version.outputs.VERSION }}..${{ steps.get_version.outputs.VERSION }} --template ci/gh_changelog_template.md.j2 --file-name changelog_body.md
+          cat changelog_body.md
       - name: Download built binaries
         uses: actions/download-artifact@master
-      - name: Rename and package binaries
+      - name: Compress and rename binaries
         run: |
-          mv esptool-${{ matrix.TARGET }} ${{ env.DISTPATH }}
-          zip -r ${{ env.DISTPATH }}.zip ./${{ env.DISTPATH }}/*
-      - name: Upload release assets
-        id: upload-release-asset
-        uses: actions/upload-release-asset@v1
+          for dir in esptool-*; do
+            zip -r "esptool-v${{ steps.get_version.outputs.VERSION }}-${dir#esptool-}.zip" "$dir"
+          done
+      - name: Create release
+        id: create_release
+        uses: softprops/action-gh-release@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         with:
-          upload_url: ${{ needs.create_release.outputs.upload_url }}
-          asset_path: "${{ env.DISTPATH }}.zip"
-          asset_name: "${{ env.DISTPATH }}.zip"
-          asset_content_type: application/zip
+          body_path: changelog_body.md
+          name: Version ${{ steps.get_version.outputs.VERSION }}
+          draft: true
+          prerelease: ${{ contains(github.ref_name, 'dev') }}
+          files: esptool-v${{ steps.get_version.outputs.VERSION }}-*.zip

.github/workflows/dangerjs.yml

@@ -0,0 +1,25 @@
+name: DangerJS Pull Request linter
+on:
+  pull_request_target:
+    types: [opened, edited, reopened, synchronize]
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  pull-request-style-linter:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Check out PR head
+      uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: DangerJS pull request linter
+      uses: espressif/shared-github-dangerjs@v1
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        instructions-contributions-file: 'CONTRIBUTING.rst'
+        instructions-gitlab-mirror: 'true'

.github/workflows/issue_comment.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@master
       - name: Sync issue comments to Jira
-        uses: espressif/github-actions/sync_issues_to_jira@master
+        uses: espressif/sync-jira-actions@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           JIRA_PASS: ${{ secrets.JIRA_PASS }}

.github/workflows/new_issues.yml

@@ -10,7 +10,7 @@ jobs:
     steps:
       - uses: actions/checkout@master
       - name: Sync GitHub issues to Jira project
-        uses: espressif/github-actions/sync_issues_to_jira@master
+        uses: espressif/sync-jira-actions@v1
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           JIRA_PASS: ${{ secrets.JIRA_PASS }}

.github/workflows/new_prs.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - uses: actions/checkout@master
       - name: Sync PRs to Jira project
-        uses: espressif/github-actions/sync_issues_to_jira@master
+        uses: espressif/sync-jira-actions@v1
         with:
           cron_job: true
         env:

.github/workflows/release_esptool_pypi.yml

@@ -4,7 +4,7 @@ name: PyPI release
 
 on:
   release:
-    types: [released]
+    types: [published]
 
 jobs:
   build_and_upload:

.github/workflows/test_esptool.yml

@@ -8,17 +8,29 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ['3.7', '3.8', '3.9', '3.10']
+        python-version: ['3.7', '3.8', '3.9', '3.10', '3.11', '3.12']
 
     steps:
     - name: Checkout ref commit
       uses: actions/checkout@master
 
+    - name: Get previous version
+      id: get_previous_version
+      env:
+        FILTER: "${{ contains(github.ref_name, 'dev') && 'first' || 'map(select(.prerelease == false)) | first' }} | .tag_name"
+      run: |
+        echo "VERSION=$(curl https://api.github.com/repos/peterdragun/esptool/releases | jq -r '${{env.FILTER}}')"
+
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@master
       with:
         python-version: ${{ matrix.python-version }}
 
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install setuptools
+
     - name: SoftHSM2 setup
       run: |
         sudo apt-get update
@@ -28,23 +40,19 @@ jobs:
         sudo chown -R $(whoami) /var/lib/softhsm
         ./ci/setup_softhsm2.sh || exit 1
 
-    - name: Test python components (fast)
+    - name: Install esptool and check if the installed versions can run
       run: |
         python setup.py build
         pip install --extra-index-url https://dl.espressif.com/pypi -e .[dev,hsm]
-        pytest test/test_imagegen.py
-        pytest test/test_espsecure.py
-        pytest test/test_espsecure_hsm.py
-        pytest test/test_merge_bin.py
-        pytest test/test_image_info.py
-        pytest test/test_modules.py
-
-    - name: Check the installed versions can run
-      run: |
         esptool.py --help
         espefuse.py --help
         espsecure.py --help
 
+    - name: Test esptool and components
+      run: |
+        pytest -m host_test
+        pytest test/test_espsecure_hsm.py
+
     - name: Build stub (Python 3.7 only)
       if: matrix.python-version == 3.7
       run: |
@@ -68,10 +76,12 @@ jobs:
         - name: Checkout
           uses: actions/checkout@master
 
-        - name: Lint with flake8
+        - name: Set up Python 3.7
+          uses: actions/setup-python@master
+          with:
+            python-version: 3.7
+
+        - name: Run pre-commit hooks
           run: |
             pip install --extra-index-url https://dl.espressif.com/pypi -e .[dev]
-            python -m flake8
-
-        - name: Check formatting with Black
-          uses: psf/black@stable
+            pre-commit run --all-files