From ca31f7ff5daeabee6471b6533a041661d1df385d Mon Sep 17 00:00:00 2001 From: puddly <32534428+puddly@users.noreply.github.com> Date: Mon, 11 Nov 2024 16:44:48 -0500 Subject: [PATCH] Release manifests (#91) * Create a tool to generate release manifests * Support `requirements.txt` * Add `metadata.created_at` * Bump pre-commit and re-run * Add manifest generation to CI * Properly support manifest in CI * Inject the current Python `bin` path into `make` invocation * Adjust `needs` for manfiest generation and checkout the current repo --- .github/workflows/build.yaml | 44 +++++++++++++++++++++-- .pre-commit-config.yaml | 2 +- Dockerfile | 10 +++++- README.md | 2 +- requirements.txt | 2 ++ tools/build_project.py | 10 ++++-- tools/create_manifest.py | 67 ++++++++++++++++++++++++++++++++++++ 7 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 requirements.txt create mode 100755 tools/create_manifest.py diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index aa706f1e..736375e1 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -159,7 +159,7 @@ jobs: done # Build it - python3 tools/build_project.py \ + /opt/venv/bin/python3 tools/build_project.py \ $sdk_args \ $toolchain_args \ --manifest "${{ matrix.manifest }}" \ @@ -188,14 +188,50 @@ jobs: compression-level: 9 if-no-files-found: error + generate-manifest: + name: Generate manifest + needs: [build-container, build-firmwares] + runs-on: ubuntu-latest + container: + image: ${{ needs.build-container.outputs.container_name }} + options: --user root + steps: + - uses: actions/checkout@v4 + + - name: Download all workflow artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + pattern: firmware-build-* + + - name: Generate manifest + run: | + /opt/venv/bin/python3 tools/create_manifest.py artifacts > artifacts/manifest.json + + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: manifest + path: artifacts/manifest.json + compression-level: 9 + if-no-files-found: error + release-assets: name: Upload release assets - needs: [build-firmwares] + needs: [generate-manifest] if: github.event_name == 'release' runs-on: ubuntu-latest permissions: contents: write steps: + - name: Download manifest + uses: actions/download-artifact@v4 + with: + path: artifacts + merge-multiple: true + pattern: manifest + - name: Download all workflow artifacts uses: actions/download-artifact@v4 with: @@ -206,4 +242,6 @@ jobs: - name: Upload artifacts uses: softprops/action-gh-release@v2 with: - files: artifacts/*.gbl + files: | + artifacts/*.gbl + artifacts/manifest.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4b9b1710..a4a3fba8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.3 + rev: v0.7.3 hooks: - id: ruff args: [--fix] diff --git a/Dockerfile b/Dockerfile index c8cedaa4..68ecd16c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,10 +16,18 @@ RUN \ default-jre-headless \ patch \ python3 \ - python3-ruamel.yaml \ + python3-pip \ + python3-virtualenv \ unzip \ xz-utils +COPY requirements.txt /tmp/ + +RUN \ + virtualenv /opt/venv \ + && /opt/venv/bin/pip install -r /tmp/requirements.txt \ + && rm /tmp/requirements.txt + # Install Simplicity Commander (unfortunately no stable URL available, this # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz). RUN \ diff --git a/README.md b/README.md index e3aaf22a..dfc212c5 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ tool will automatically determine which SDK and toolchain to use. > automatically found so these flags can be omitted. ```bash -pip install ruamel.yaml # Only dependency +pip install -r requirements.txt python tools/build_project.py \ # The following SDK and toolchain flags can be omitted on macOS diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..e4dad3c7 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +ruamel.yaml +universal-silabs-flasher>=0.0.25 \ No newline at end of file diff --git a/tools/build_project.py b/tools/build_project.py index 385766c1..6ff69d84 100755 --- a/tools/build_project.py +++ b/tools/build_project.py @@ -3,6 +3,7 @@ from __future__ import annotations +import os import re import ast import sys @@ -169,9 +170,9 @@ def load_toolchains(paths: list[pathlib.Path]) -> dict[pathlib.Path, str]: return toolchains -def subprocess_run_verbose(command: list[str], prefix: str) -> None: +def subprocess_run_verbose(command: list[str], prefix: str, **kwargs) -> None: with subprocess.Popen( - command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT + command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs ) as proc: for line in proc.stdout: LOGGER.info("[%s] %r", prefix, line.decode("utf-8").strip()) @@ -543,7 +544,10 @@ def main(): f"POST_BUILD_EXE={args.postbuild}", "VERBOSE=1", ], - "make" + "make", + env={ + "PATH": f"{pathlib.Path(sys.executable).parent}:{os.environ['PATH']}" + } ) # fmt: on diff --git a/tools/create_manifest.py b/tools/create_manifest.py new file mode 100755 index 00000000..58497071 --- /dev/null +++ b/tools/create_manifest.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Tool to create a JSON manifest file for a collection of firmwares.""" + +from __future__ import annotations + +import json +import logging +import hashlib +import pathlib +import argparse + +from datetime import datetime, timezone + +from universal_silabs_flasher.firmware import parse_firmware_image + +_LOGGER = logging.getLogger(__name__) + + +def main(): + parser = argparse.ArgumentParser( + formatter_class=argparse.ArgumentDefaultsHelpFormatter + ) + parser.add_argument( + "firmware_dir", + type=pathlib.Path, + help="Directory containing firmware images", + ) + + args = parser.parse_args() + manifest = { + "metadata": { + "created_at": datetime.now(timezone.utc).isoformat(), + }, + "firmwares": [], + } + + for firmware_file in args.firmware_dir.glob("*.gbl"): + data = firmware_file.read_bytes() + + try: + firmware = parse_firmware_image(data) + except ValueError: + _LOGGER.warning("Ignoring invalid firmware file: %s", firmware_file) + continue + + try: + gbl_metadata = firmware.get_nabucasa_metadata() + except (KeyError, ValueError): + metadata = None + else: + metadata = gbl_metadata.original_json + + manifest["firmwares"].append( + { + "filename": firmware_file.name, + "checksum": f"sha3-256:{hashlib.sha3_256(data).hexdigest()}", + "size": len(data), + "metadata": metadata, + "release_notes": None, + } + ) + + print(json.dumps(manifest, indent=2)) + + +if __name__ == "__main__": + main()