Release manifests (#91)

* Create a tool to generate release manifests

* Support `requirements.txt`

* Add `metadata.created_at`

* Bump pre-commit and re-run

* Add manifest generation to CI

* Properly support manifest in CI

* Inject the current Python `bin` path into `make` invocation

* Adjust `needs` for manfiest generation and checkout the current repo

.github/workflows/build.yaml

@@ -159,7 +159,7 @@ jobs:
           done
 
           # Build it
-          python3 tools/build_project.py \
+          /opt/venv/bin/python3 tools/build_project.py \
             $sdk_args \
             $toolchain_args \
             --manifest "${{ matrix.manifest }}" \
@@ -188,14 +188,50 @@ jobs:
           compression-level: 9
           if-no-files-found: error
 
+  generate-manifest:
+    name: Generate manifest
+    needs: [build-container, build-firmwares]
+    runs-on: ubuntu-latest
+    container:
+      image: ${{ needs.build-container.outputs.container_name }}
+      options: --user root
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download all workflow artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          merge-multiple: true
+          pattern: firmware-build-*
+
+      - name: Generate manifest
+        run: |
+          /opt/venv/bin/python3 tools/create_manifest.py artifacts > artifacts/manifest.json
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: manifest
+          path: artifacts/manifest.json
+          compression-level: 9
+          if-no-files-found: error
+
   release-assets:
     name: Upload release assets
-    needs: [build-firmwares]
+    needs: [generate-manifest]
     if: github.event_name == 'release'
     runs-on: ubuntu-latest
     permissions:
       contents: write
     steps:
+      - name: Download manifest
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          merge-multiple: true
+          pattern: manifest
+
       - name: Download all workflow artifacts
         uses: actions/download-artifact@v4
         with:
@@ -206,4 +242,6 @@ jobs:
       - name: Upload artifacts
         uses: softprops/action-gh-release@v2
         with:
-          files: artifacts/*.gbl
+          files: |
+            artifacts/*.gbl
+            artifacts/manifest.json

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.4.3
+    rev: v0.7.3
     hooks:
       - id: ruff
         args: [--fix]

Dockerfile

@@ -16,10 +16,18 @@ RUN \
        default-jre-headless \
        patch \
        python3 \
-       python3-ruamel.yaml \
+       python3-pip \
+       python3-virtualenv \
        unzip \
        xz-utils
 
+COPY requirements.txt /tmp/
+
+RUN \
+    virtualenv /opt/venv \
+    && /opt/venv/bin/pip install -r /tmp/requirements.txt \
+    && rm /tmp/requirements.txt
+
 # Install Simplicity Commander (unfortunately no stable URL available, this
 # is known to be working with Commander_linux_x86_64_1v15p0b1306.tar.bz).
 RUN \

README.md

@@ -70,7 +70,7 @@ tool will automatically determine which SDK and toolchain to use.
 > automatically found so these flags can be omitted.
 
 ```bash
-pip install ruamel.yaml  # Only dependency
+pip install -r requirements.txt
 
 python tools/build_project.py \
     # The following SDK and toolchain flags can be omitted on macOS

requirements.txt

@@ -0,0 +1,2 @@
+ruamel.yaml
+universal-silabs-flasher>=0.0.25

tools/build_project.py

@@ -3,6 +3,7 @@
 
 from __future__ import annotations
 
+import os
 import re
 import ast
 import sys
@@ -169,9 +170,9 @@ def load_toolchains(paths: list[pathlib.Path]) -> dict[pathlib.Path, str]:
     return toolchains
 
 
-def subprocess_run_verbose(command: list[str], prefix: str) -> None:
+def subprocess_run_verbose(command: list[str], prefix: str, **kwargs) -> None:
     with subprocess.Popen(
-        command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT
+        command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs
     ) as proc:
         for line in proc.stdout:
             LOGGER.info("[%s] %r", prefix, line.decode("utf-8").strip())
@@ -543,7 +544,10 @@ def main():
             f"POST_BUILD_EXE={args.postbuild}",
             "VERBOSE=1",
         ],
-        "make"
+        "make",
+        env={
+            "PATH": f"{pathlib.Path(sys.executable).parent}:{os.environ['PATH']}"
+        }
     )
     # fmt: on
 

tools/create_manifest.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Tool to create a JSON manifest file for a collection of firmwares."""
+
+from __future__ import annotations
+
+import json
+import logging
+import hashlib
+import pathlib
+import argparse
+
+from datetime import datetime, timezone
+
+from universal_silabs_flasher.firmware import parse_firmware_image
+
+_LOGGER = logging.getLogger(__name__)
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter
+    )
+    parser.add_argument(
+        "firmware_dir",
+        type=pathlib.Path,
+        help="Directory containing firmware images",
+    )
+
+    args = parser.parse_args()
+    manifest = {
+        "metadata": {
+            "created_at": datetime.now(timezone.utc).isoformat(),
+        },
+        "firmwares": [],
+    }
+
+    for firmware_file in args.firmware_dir.glob("*.gbl"):
+        data = firmware_file.read_bytes()
+
+        try:
+            firmware = parse_firmware_image(data)
+        except ValueError:
+            _LOGGER.warning("Ignoring invalid firmware file: %s", firmware_file)
+            continue
+
+        try:
+            gbl_metadata = firmware.get_nabucasa_metadata()
+        except (KeyError, ValueError):
+            metadata = None
+        else:
+            metadata = gbl_metadata.original_json
+
+        manifest["firmwares"].append(
+            {
+                "filename": firmware_file.name,
+                "checksum": f"sha3-256:{hashlib.sha3_256(data).hexdigest()}",
+                "size": len(data),
+                "metadata": metadata,
+                "release_notes": None,
+            }
+        )
+
+    print(json.dumps(manifest, indent=2))
+
+
+if __name__ == "__main__":
+    main()