CI: tests and builds on every push, a signed release on a tag (#5)

The workflow runs on the runner's host and builds in the project's Docker
image through scripts/ci.sh, as on a developer's machine. A tag v*, or a
run by hand for an older tag, builds that tag's sources, signs the Update
File with the key held in the repository's secrets, checks the signature
against the public key in the sources, and publishes a Gitea release with
the .ota, the factory image, the ELF and checksums.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT

.gitea/workflows/ci.yml

@@ -1,14 +1,69 @@
+# CI and releases (docs/milestones/R1.md).
+#   Any push:          host tests, then the release firmware and the Debug Build.
+#   A tag v*:          the same, then a Gitea release with the signed Update File.
+#   Run by hand:       the release of a tag that exists already (the ones from before CI).
+#
+# The runner executes jobs on its own host, where Docker is: the steps are plain shell and the build
+# runs in the project's image, exactly as scripts/ci.sh does on a developer's machine. No JavaScript
+# actions (the host has no Node), so the checkout is done with git.
 name: CI
-on: [push]
+on:
+  push:
+    branches: ['**']
+    tags: ['v*']
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: An existing tag to build and publish as a release
+        required: true
 
 jobs:
-  probe:
+  build:
     runs-on: "ubuntu://docker:ubuntu:resolute"
     steps:
-      - name: What the runner gives a job
+      - name: Check out
         run: |
-          set +e
+          find . -mindepth 1 -maxdepth 1 -exec rm -rf {} +
-          echo "shell: $0"; id; uname -a; cat /etc/os-release | head -3
+          git init -q .
-          nproc; free -m | head -2; df -h / | tail -1
+          git remote add origin "${{ github.server_url }}/${{ github.repository }}.git"
-          for t in git python3 pip node docker curl openssl gcc make; do printf '%s: ' $t; command -v $t || echo none; done
+          git fetch -q --tags origin '+refs/heads/*:refs/remotes/origin/*'
-          env | grep -E '^(GITHUB|GITEA|RUNNER|ACTIONS|CI)' | grep -vi token | sort
+          git checkout -q --detach "${{ github.sha }}"
+          git describe --tags --always
+
+      - name: Host tests and both builds
+        if: github.event_name == 'push'
+        run: scripts/ci.sh
+
+      - name: Which release
+        id: release
+        run: |
+          if [ "${{ github.event_name }}" = workflow_dispatch ]; then
+            echo "tag=${{ inputs.tag }}" >> "$GITHUB_OUTPUT"
+          elif [ "${{ github.ref_type }}" = tag ]; then
+            echo "tag=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
+          fi
+
+      - name: Build and sign the release
+        if: steps.release.outputs.tag != ''
+        env:
+          OTA_SIGNING_KEY: ${{ secrets.OTA_SIGNING_KEY }}
+        run: |
+          # The sources of the tag in a clone of their own; the tools are this commit's.
+          rm -rf ../release-src dist
+          git clone -q . ../release-src
+          git -C ../release-src checkout -q --detach "refs/tags/${{ steps.release.outputs.tag }}"
+          # The key exists as a file only while this step runs.
+          umask 077
+          export RORO_OTA_KEY="$(mktemp)"
+          trap 'rm -f "$RORO_OTA_KEY"' EXIT
+          printf '%s\n' "$OTA_SIGNING_KEY" > "$RORO_OTA_KEY"
+          umask 022
+          scripts/release_build.sh ../release-src dist
+
+      - name: Publish the release
+        if: steps.release.outputs.tag != ''
+        env:
+          GITEA_API: ${{ github.server_url }}/api/v1
+          GITEA_REPO: ${{ github.repository }}
+          GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }}
+        run: scripts/release_publish.py dist

scripts/ota_verify.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python3
+"""Checks an Update File (.ota) the way the device does, on a PC: header, signature, image hash.
+
+Usage: scripts/ota_verify.py <file.ota> [public key, default keys/ota-public.pem]
+Exits 0 and prints the version if a device would accept the file.
+"""
+import hashlib
+import os
+import struct
+import subprocess
+import sys
+import tempfile
+
+HEADER_SIZE = 160
+SIGNED_BYTES = 80
+
+
+def main():
+    if len(sys.argv) < 2:
+        sys.exit(__doc__)
+    root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    key = sys.argv[2] if len(sys.argv) > 2 else os.path.join(root, "keys", "ota-public.pem")
+    data = open(sys.argv[1], "rb").read()
+    if len(data) < HEADER_SIZE or data[:8] != b"RORO-OTA":
+        sys.exit("not an update file")
+    fmt, header_size, image_size = struct.unpack("<HHI", data[8:16])
+    if fmt != 1 or header_size != HEADER_SIZE:
+        sys.exit("unsupported update format")
+    image = data[HEADER_SIZE:]
+    if len(image) != image_size:
+        sys.exit(f"the header announces {image_size} bytes of image, the file has {len(image)}")
+    if hashlib.sha256(image).digest() != data[16:48]:
+        sys.exit("image corrupted (hash mismatch)")
+    version = data[48:80].split(b"\0")[0].decode()
+    (sig_len,) = struct.unpack("<H", data[80:82])
+    with tempfile.NamedTemporaryFile() as signed, tempfile.NamedTemporaryFile() as sig:
+        signed.write(data[:SIGNED_BYTES])
+        signed.flush()
+        sig.write(data[82:82 + sig_len])
+        sig.flush()
+        ok = subprocess.run(["openssl", "dgst", "-sha256", "-verify", key, "-signature", sig.name, signed.name],
+                            capture_output=True).returncode == 0
+    if not ok:
+        sys.exit("bad signature (wrong key)")
+    print(f"{sys.argv[1]}: {version}, {image_size} bytes, signature good")
+
+
+if __name__ == "__main__":
+    main()

scripts/release_build.sh

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+# Builds what a release publishes, from a checkout of the repository at a tag (docs/milestones/R1.md).
+# Usage: scripts/release_build.sh <checkout> <out folder>
+#   <checkout>   a clone with its tags, at the commit to release: its own sources are built, with
+#                this copy's build image and signing tools, so an old tag can be released today.
+# The signing key is read from $RORO_OTA_KEY (a file), as scripts/make_ota.py does.
+# Out: roro9stack-<version>.ota (signed, checked), -factory.bin (USB), .elf.gz (to decode crashes),
+#      SHA256SUMS, and notes.md for the release's text.
+set -euo pipefail
+SRC="$(cd "$1" && pwd)"
+mkdir -p "$2"
+OUT="$(cd "$2" && pwd)"
+TOOLS="$(cd "$(dirname "$0")" && pwd)"
+
+VERSION="$(git -C "$SRC" describe --tags --always --dirty)"
+case "$VERSION" in
+    *-dirty) echo "release: $SRC has uncommitted changes ($VERSION)" >&2; exit 1 ;;
+esac
+git -C "$SRC" describe --tags --exact-match >/dev/null 2>&1 || { echo "release: $VERSION is not a tag" >&2; exit 1; }
+
+source "$TOOLS/_docker.sh"
+ROOT="$SRC"  # _docker.sh mounts $ROOT as /work: the checkout to build, not necessarily this copy
+DOCKER_EXTRA=()
+run_in_container bash -c 'git config --global --add safe.directory /work && pio run -e cardputer-adv'
+
+BUILD="$SRC/.pio/build/cardputer-adv"
+NAME="roro9stack-$VERSION"
+"$TOOLS/make_ota.py" "$BUILD/firmware.bin" "$VERSION" "$OUT/$NAME.ota"
+# A wrong key must stop the release here, not on a device: checked against the public key the
+# sources being built carry.
+"$TOOLS/ota_verify.py" "$OUT/$NAME.ota" "$SRC/keys/ota-public.pem"
+cp "$BUILD/firmware.factory.bin" "$OUT/$NAME-factory.bin"
+gzip -9 -c "$BUILD/firmware.elf" > "$OUT/$NAME.elf.gz"
+(cd "$OUT" && sha256sum "$NAME.ota" "$NAME-factory.bin" "$NAME.elf.gz" > SHA256SUMS)
+
+# The release's text: what the tag says, then what went in since the tag before.
+PREVIOUS="$(git -C "$SRC" describe --tags --abbrev=0 "$VERSION^" 2>/dev/null || true)"
+{
+    git -C "$SRC" tag -l --format='%(contents)' "$VERSION" | sed -e '/^-----BEGIN PGP/,$d'
+    echo
+    echo "## Files"
+    echo
+    echo "- \`$NAME.ota\`: the signed Update File. Copy it to \`/updates\` on the SD card and install it from Settings > Firmware or the Storage App, or push it over Wi-Fi with \`scripts/ota_push.py\`."
+    echo "- \`$NAME-factory.bin\`: the whole flash image, for a first install over USB at offset 0."
+    echo "- \`$NAME.elf.gz\`: the symbols, to decode a crash report from this build."
+    echo "- \`SHA256SUMS\`: checksums of the three."
+    if [ -n "$PREVIOUS" ]; then
+        echo
+        echo "## Changes since $PREVIOUS"
+        echo
+        git -C "$SRC" log --no-merges --format='- %s' "$PREVIOUS..$VERSION"
+    fi
+} > "$OUT/notes.md"
+echo "$VERSION" > "$OUT/version"
+ls -l "$OUT"

scripts/release_publish.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Creates or completes a Gitea release from what scripts/release_build.sh made.
+
+Usage: scripts/release_publish.py <folder>
+Environment: GITEA_API (https://host/api/v1), GITEA_REPO (owner/name), GITEA_TOKEN.
+Run again for the same version, it replaces the files and the text instead of failing.
+"""
+import json
+import os
+import sys
+import urllib.error
+import urllib.parse
+import urllib.request
+
+API = os.environ.get("GITEA_API", "").rstrip("/")
+REPO = os.environ.get("GITEA_REPO", "")
+TOKEN = os.environ.get("GITEA_TOKEN", "")
+
+
+def call(method, path, body=None, raw=None, content_type="application/json"):
+    data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None)
+    request = urllib.request.Request(f"{API}/repos/{REPO}{path}", data=data, method=method)
+    request.add_header("Authorization", f"token {TOKEN}")
+    if data is not None:
+        request.add_header("Content-Type", content_type)
+    try:
+        with urllib.request.urlopen(request, timeout=300) as response:
+            text = response.read()
+            return response.status, json.loads(text) if text else None
+    except urllib.error.HTTPError as e:
+        return e.code, e.read().decode(errors="replace")[:300]
+
+
+def main():
+    if len(sys.argv) != 2 or not (API and REPO and TOKEN):
+        sys.exit(__doc__)
+    folder = sys.argv[1]
+    version = open(os.path.join(folder, "version")).read().strip()
+    notes = open(os.path.join(folder, "notes.md")).read()
+    files = sorted(f for f in os.listdir(folder) if f not in ("version", "notes.md"))
+
+    status, release = call("GET", f"/releases/tags/{urllib.parse.quote(version)}")
+    fields = {"tag_name": version, "name": f"roro9stack {version}", "body": notes, "draft": False, "prerelease": False}
+    if status == 200:
+        status, release = call("PATCH", f"/releases/{release['id']}", fields)
+    else:
+        status, release = call("POST", "/releases", fields)
+    if status not in (200, 201):
+        sys.exit(f"release: Gitea answered {status}: {release}")
+
+    for asset in release.get("assets") or []:  # a second run replaces what the first uploaded
+        if asset["name"] in files:
+            call("DELETE", f"/releases/{release['id']}/assets/{asset['id']}")
+    for name in files:
+        with open(os.path.join(folder, name), "rb") as f:
+            status, answer = call("POST", f"/releases/{release['id']}/assets?name={urllib.parse.quote(name)}", raw=f.read(),
+                                  content_type="application/octet-stream")
+        if status != 201:
+            sys.exit(f"release: uploading {name}: Gitea answered {status}: {answer}")
+        print(f"uploaded {name} ({answer['size']} bytes)")
+    print(f"published {release['html_url']}")
+
+
+if __name__ == "__main__":
+    main()