ci(tools): Fix tools workflows (#9846)

* ci(tools): Remove ARM64 runner and use get.exe

* ci(tools): Optimize get.py and verify extraction

* change(tools): Push generated binaries to PR

---------

Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

.github/scripts/install-arduino-core-esp32.sh

@@ -28,7 +28,11 @@ if [ ! -d "$ARDUINO_ESP32_PATH" ]; then
     #git submodule update --init --recursive > /dev/null 2>&1
 
     echo "Installing Platform Tools ..."
-    cd tools && python get.py
+    if [ "$OS_IS_WINDOWS" == "1" ]; then
+      cd tools && ./get.exe
+    else
+      cd tools && python get.py
+    fi
     cd $script_init_path
 
     echo "ESP32 Arduino has been installed in '$ARDUINO_ESP32_PATH'"

.github/workflows/build_py_tools.yml

@@ -3,10 +3,11 @@ name: Build Python Tools
 on:
   pull_request:
     paths:
-    - 'tools/get.py'
-    - 'tools/espota.py'
-    - 'tools/gen_esp32part.py'
-    - 'tools/gen_insights_package.py'
+      - '.github/workflows/build_py_tools.yml'
+      - 'tools/get.py'
+      - 'tools/espota.py'
+      - 'tools/gen_esp32part.py'
+      - 'tools/gen_insights_package.py'
 
 jobs:
   find-changed-tools:
@@ -21,6 +22,13 @@ jobs:
         with:
           fetch-depth: 2
           ref: ${{ github.event.pull_request.head.ref }}
+
+      - name: Check if checkout failed
+        if: failure()
+        run: |
+          echo "Checkout failed."
+          echo "Make sure you are using a branch inside the repository and not a fork."
+
       - name: Verify Python Tools Changed
         uses: tj-actions/changed-files@v41
         id: verify-changed-files
@@ -47,7 +55,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [windows-latest, macos-latest, ubuntu-20.04, ARM, ARM64]
+        os: [windows-latest, macos-latest, ubuntu-20.04, ARM]
         include:
         - os: windows-latest
           TARGET: win64
@@ -63,10 +71,6 @@ jobs:
           CONTAINER: python:3.8-bullseye
           TARGET: arm
           SEPARATOR: ':'
-        - os: ARM64
-          CONTAINER: python:3.8-bullseye
-          TARGET: arm64
-          SEPARATOR: ':'
     container: ${{ matrix.CONTAINER }} # use python container on ARM
     env:
       DISTPATH: pytools-${{ matrix.TARGET }}
@@ -93,7 +97,7 @@ jobs:
           ref: ${{ github.event.pull_request.head.ref }}
       - name: Set up Python 3.8
         # Skip setting python on ARM because of missing compatibility: https://github.com/actions/setup-python/issues/108
-        if: matrix.os != 'ARM' && matrix.os != 'ARM64'
+        if: matrix.os != 'ARM'
         uses: actions/setup-python@master
         with:
           python-version: 3.8
@@ -108,7 +112,7 @@ jobs:
             pyinstaller --distpath ./${{ env.DISTPATH }} -F --icon=.github/pytools/espressif.ico tools/$tool.py
           done
       - name: Sign binaries
-        if: matrix.os == 'windows-latest' && env.CERTIFICATE != '' && env.CERTIFICATE_PASSWORD != ''
+        if: matrix.os == 'windows-latest'
         env:
           CERTIFICATE: ${{ secrets.CERTIFICATE }}
           CERTIFICATE_PASSWORD: ${{ secrets.CERTIFICATE_PASSWORD }}

tools/get.py

@@ -101,52 +101,46 @@ def verify_files(filename, destination, rename_to):
     t1 = time.time()
     if filename.endswith(".zip"):
         try:
-            with zipfile.ZipFile(filename, "r") as archive:
-                first_dir = archive.namelist()[0].split("/")[0]
-                total_files = len(archive.namelist())
-                for i, zipped_file in enumerate(archive.namelist(), 1):
-                    local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
-                    if not os.path.exists(local_path):
-                        print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
-                        print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
-                        return False
-                    print_verification_progress(total_files, i, t1)
+            archive = zipfile.ZipFile(filename, "r")
+            file_list = archive.namelist()
         except zipfile.BadZipFile:
-            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+            if verbose:
+                print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
             return False
     elif filename.endswith(".tar.gz"):
         try:
-            with tarfile.open(filename, "r:gz") as archive:
-                first_dir = archive.getnames()[0].split("/")[0]
-                total_files = len(archive.getnames())
-                for i, zipped_file in enumerate(archive.getnames(), 1):
-                    local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
-                    if not os.path.exists(local_path):
-                        print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
-                        print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
-                        return False
-                    print_verification_progress(total_files, i, t1)
+            archive = tarfile.open(filename, "r:gz")
+            file_list = archive.getnames()
         except tarfile.ReadError:
-            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+            if verbose:
+                print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
             return False
     elif filename.endswith(".tar.xz"):
         try:
-            with tarfile.open(filename, "r:xz") as archive:
-                first_dir = archive.getnames()[0].split("/")[0]
-                total_files = len(archive.getnames())
-                for i, zipped_file in enumerate(archive.getnames(), 1):
-                    local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
-                    if not os.path.exists(local_path):
-                        print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
-                        print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
-                        return False
-                    print_verification_progress(total_files, i, t1)
+            archive = tarfile.open(filename, "r:xz")
+            file_list = archive.getnames()
         except tarfile.ReadError:
-            print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+            if verbose:
+                print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
             return False
     else:
         raise NotImplementedError("Unsupported archive type")
 
+    try:
+        first_dir = file_list[0].split("/")[0]
+        total_files = len(file_list)
+        for i, zipped_file in enumerate(file_list, 1):
+            local_path = os.path.join(extracted_dir_path, zipped_file.replace(first_dir, rename_to, 1))
+            if not os.path.exists(local_path):
+                if verbose:
+                    print(f"\nMissing {zipped_file} on location: {extracted_dir_path}")
+                    print(f"Verification failed; aborted in {format_time(time.time() - t1)}")
+                return False
+            print_verification_progress(total_files, i, t1)
+    except Exception as e:
+        print(f"\nError: {e}")
+        return False
+
     if verbose:
         print(f"\nVerification passed; completed in {format_time(time.time() - t1)}")
 
@@ -231,7 +225,12 @@ def unpack(filename, destination, force_extract):  # noqa: C901
             shutil.rmtree(rename_to)
         shutil.move(dirname, rename_to)
 
-    return True
+    if verify_files(filename, destination, rename_to):
+        print(" Files extracted successfully.")
+        return True
+    else:
+        print(" Failed to extract files.")
+        return False
 
 
 def download_file_with_progress(url, filename, start_time):
@@ -291,6 +290,7 @@ def get_tool(tool, force_download, force_extract):
     local_path = dist_dir + archive_name
     url = tool["url"]
     start_time = time.time()
+    print("")
     if not os.path.isfile(local_path) or force_download:
         if verbose:
             print("Downloading '" + archive_name + "' to '" + local_path + "'")
@@ -421,6 +421,9 @@ def identify_platform():
         current_dir + "/../package/package_esp32_index.template.json", identified_platform
     )
     mkdir_p(dist_dir)
+
+    print("\nDownloading and extracting tools...")
+
     for tool in tools_to_download:
         if is_test:
             print("Would install: {0}".format(tool["archiveFileName"]))
@@ -432,4 +435,4 @@ def identify_platform():
                     print(f"Tool {tool['archiveFileName']} was corrupted, but re-downloading did not help!\n")
                     sys.exit(1)
 
-    print("Platform Tools Installed")
+    print("\nPlatform Tools Installed")