diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 9d14203..b2e41c6 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -1,14 +1,69 @@ +# CI and releases (docs/milestones/R1.md). +# Any push: host tests, then the release firmware and the Debug Build. +# A tag v*: the same, then a Gitea release with the signed Update File. +# Run by hand: the release of a tag that exists already (the ones from before CI). +# +# The runner executes jobs on its own host, where Docker is: the steps are plain shell and the build +# runs in the project's image, exactly as scripts/ci.sh does on a developer's machine. No JavaScript +# actions (the host has no Node), so the checkout is done with git. name: CI -on: [push] +on: + push: + branches: ['**'] + tags: ['v*'] + workflow_dispatch: + inputs: + tag: + description: An existing tag to build and publish as a release + required: true jobs: - probe: + build: runs-on: "ubuntu://docker:ubuntu:resolute" steps: - - name: What the runner gives a job + - name: Check out run: | - set +e - echo "shell: $0"; id; uname -a; cat /etc/os-release | head -3 - nproc; free -m | head -2; df -h / | tail -1 - for t in git python3 pip node docker curl openssl gcc make; do printf '%s: ' $t; command -v $t || echo none; done - env | grep -E '^(GITHUB|GITEA|RUNNER|ACTIONS|CI)' | grep -vi token | sort + find . -mindepth 1 -maxdepth 1 -exec rm -rf {} + + git init -q . + git remote add origin "${{ github.server_url }}/${{ github.repository }}.git" + git fetch -q --tags origin '+refs/heads/*:refs/remotes/origin/*' + git checkout -q --detach "${{ github.sha }}" + git describe --tags --always + + - name: Host tests and both builds + if: github.event_name == 'push' + run: scripts/ci.sh + + - name: Which release + id: release + run: | + if [ "${{ github.event_name }}" = workflow_dispatch ]; then + echo "tag=${{ inputs.tag }}" >> "$GITHUB_OUTPUT" + elif [ "${{ github.ref_type }}" = tag ]; then + echo "tag=${{ github.ref_name }}" >> "$GITHUB_OUTPUT" + fi + + - name: Build and sign the release + if: steps.release.outputs.tag != '' + env: + OTA_SIGNING_KEY: ${{ secrets.OTA_SIGNING_KEY }} + run: | + # The sources of the tag in a clone of their own; the tools are this commit's. + rm -rf ../release-src dist + git clone -q . ../release-src + git -C ../release-src checkout -q --detach "refs/tags/${{ steps.release.outputs.tag }}" + # The key exists as a file only while this step runs. + umask 077 + export RORO_OTA_KEY="$(mktemp)" + trap 'rm -f "$RORO_OTA_KEY"' EXIT + printf '%s\n' "$OTA_SIGNING_KEY" > "$RORO_OTA_KEY" + umask 022 + scripts/release_build.sh ../release-src dist + + - name: Publish the release + if: steps.release.outputs.tag != '' + env: + GITEA_API: ${{ github.server_url }}/api/v1 + GITEA_REPO: ${{ github.repository }} + GITEA_TOKEN: ${{ secrets.GITEA_TOKEN }} + run: scripts/release_publish.py dist diff --git a/scripts/ota_verify.py b/scripts/ota_verify.py new file mode 100755 index 0000000..7e07171 --- /dev/null +++ b/scripts/ota_verify.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Checks an Update File (.ota) the way the device does, on a PC: header, signature, image hash. + +Usage: scripts/ota_verify.py [public key, default keys/ota-public.pem] +Exits 0 and prints the version if a device would accept the file. +""" +import hashlib +import os +import struct +import subprocess +import sys +import tempfile + +HEADER_SIZE = 160 +SIGNED_BYTES = 80 + + +def main(): + if len(sys.argv) < 2: + sys.exit(__doc__) + root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + key = sys.argv[2] if len(sys.argv) > 2 else os.path.join(root, "keys", "ota-public.pem") + data = open(sys.argv[1], "rb").read() + if len(data) < HEADER_SIZE or data[:8] != b"RORO-OTA": + sys.exit("not an update file") + fmt, header_size, image_size = struct.unpack(" +# a clone with its tags, at the commit to release: its own sources are built, with +# this copy's build image and signing tools, so an old tag can be released today. +# The signing key is read from $RORO_OTA_KEY (a file), as scripts/make_ota.py does. +# Out: roro9stack-.ota (signed, checked), -factory.bin (USB), .elf.gz (to decode crashes), +# SHA256SUMS, and notes.md for the release's text. +set -euo pipefail +SRC="$(cd "$1" && pwd)" +mkdir -p "$2" +OUT="$(cd "$2" && pwd)" +TOOLS="$(cd "$(dirname "$0")" && pwd)" + +VERSION="$(git -C "$SRC" describe --tags --always --dirty)" +case "$VERSION" in + *-dirty) echo "release: $SRC has uncommitted changes ($VERSION)" >&2; exit 1 ;; +esac +git -C "$SRC" describe --tags --exact-match >/dev/null 2>&1 || { echo "release: $VERSION is not a tag" >&2; exit 1; } + +source "$TOOLS/_docker.sh" +ROOT="$SRC" # _docker.sh mounts $ROOT as /work: the checkout to build, not necessarily this copy +DOCKER_EXTRA=() +run_in_container bash -c 'git config --global --add safe.directory /work && pio run -e cardputer-adv' + +BUILD="$SRC/.pio/build/cardputer-adv" +NAME="roro9stack-$VERSION" +"$TOOLS/make_ota.py" "$BUILD/firmware.bin" "$VERSION" "$OUT/$NAME.ota" +# A wrong key must stop the release here, not on a device: checked against the public key the +# sources being built carry. +"$TOOLS/ota_verify.py" "$OUT/$NAME.ota" "$SRC/keys/ota-public.pem" +cp "$BUILD/firmware.factory.bin" "$OUT/$NAME-factory.bin" +gzip -9 -c "$BUILD/firmware.elf" > "$OUT/$NAME.elf.gz" +(cd "$OUT" && sha256sum "$NAME.ota" "$NAME-factory.bin" "$NAME.elf.gz" > SHA256SUMS) + +# The release's text: what the tag says, then what went in since the tag before. +PREVIOUS="$(git -C "$SRC" describe --tags --abbrev=0 "$VERSION^" 2>/dev/null || true)" +{ + git -C "$SRC" tag -l --format='%(contents)' "$VERSION" | sed -e '/^-----BEGIN PGP/,$d' + echo + echo "## Files" + echo + echo "- \`$NAME.ota\`: the signed Update File. Copy it to \`/updates\` on the SD card and install it from Settings > Firmware or the Storage App, or push it over Wi-Fi with \`scripts/ota_push.py\`." + echo "- \`$NAME-factory.bin\`: the whole flash image, for a first install over USB at offset 0." + echo "- \`$NAME.elf.gz\`: the symbols, to decode a crash report from this build." + echo "- \`SHA256SUMS\`: checksums of the three." + if [ -n "$PREVIOUS" ]; then + echo + echo "## Changes since $PREVIOUS" + echo + git -C "$SRC" log --no-merges --format='- %s' "$PREVIOUS..$VERSION" + fi +} > "$OUT/notes.md" +echo "$VERSION" > "$OUT/version" +ls -l "$OUT" diff --git a/scripts/release_publish.py b/scripts/release_publish.py new file mode 100755 index 0000000..7ab0f26 --- /dev/null +++ b/scripts/release_publish.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Creates or completes a Gitea release from what scripts/release_build.sh made. + +Usage: scripts/release_publish.py +Environment: GITEA_API (https://host/api/v1), GITEA_REPO (owner/name), GITEA_TOKEN. +Run again for the same version, it replaces the files and the text instead of failing. +""" +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request + +API = os.environ.get("GITEA_API", "").rstrip("/") +REPO = os.environ.get("GITEA_REPO", "") +TOKEN = os.environ.get("GITEA_TOKEN", "") + + +def call(method, path, body=None, raw=None, content_type="application/json"): + data = raw if raw is not None else (json.dumps(body).encode() if body is not None else None) + request = urllib.request.Request(f"{API}/repos/{REPO}{path}", data=data, method=method) + request.add_header("Authorization", f"token {TOKEN}") + if data is not None: + request.add_header("Content-Type", content_type) + try: + with urllib.request.urlopen(request, timeout=300) as response: + text = response.read() + return response.status, json.loads(text) if text else None + except urllib.error.HTTPError as e: + return e.code, e.read().decode(errors="replace")[:300] + + +def main(): + if len(sys.argv) != 2 or not (API and REPO and TOKEN): + sys.exit(__doc__) + folder = sys.argv[1] + version = open(os.path.join(folder, "version")).read().strip() + notes = open(os.path.join(folder, "notes.md")).read() + files = sorted(f for f in os.listdir(folder) if f not in ("version", "notes.md")) + + status, release = call("GET", f"/releases/tags/{urllib.parse.quote(version)}") + fields = {"tag_name": version, "name": f"roro9stack {version}", "body": notes, "draft": False, "prerelease": False} + if status == 200: + status, release = call("PATCH", f"/releases/{release['id']}", fields) + else: + status, release = call("POST", "/releases", fields) + if status not in (200, 201): + sys.exit(f"release: Gitea answered {status}: {release}") + + for asset in release.get("assets") or []: # a second run replaces what the first uploaded + if asset["name"] in files: + call("DELETE", f"/releases/{release['id']}/assets/{asset['id']}") + for name in files: + with open(os.path.join(folder, name), "rb") as f: + status, answer = call("POST", f"/releases/{release['id']}/assets?name={urllib.parse.quote(name)}", raw=f.read(), + content_type="application/octet-stream") + if status != 201: + sys.exit(f"release: uploading {name}: Gitea answered {status}: {answer}") + print(f"uploaded {name} ({answer['size']} bytes)") + print(f"published {release['html_url']}") + + +if __name__ == "__main__": + main()