Safe Mode, crash reports, and a watched main loop

Every build now records at boot which version runs and, after a crash restart, which one crashed (even across a Rollback). The core dump summary (task, PC, reason, backtrace) is printed and raised as a Notification; `crash` shows it later. After 3 crash restarts in a row the firmware starts in Safe Mode: clock, Wi-Fi, Update Service and Debug Console only (SafeMode, 2 host tests). A normal restart or a minute up resets the count. The main loop is now on the task watchdog (enableLoopWDT): Arduino only watched core 0's idle task, so a stuck loop hung the device for good. The Update Service restarts into an installed update by itself if the main loop hasn't after 90 s. Debug Builds: `coredump get` and `reset` are answered by the console's own task; rdbg.py crash decodes the backtrace and rdbg.py coredump runs esp-coredump, against ELFs archived by version and digest in .pio/elves. The StorageService mutex is now made in the constructor: Safe Mode never starts that Service, and `info` crashed on the null mutex, 29 times in a row before the fix was pushed into Safe Mode over Wi-Fi. Verified on the device: crash report and full core dump decoded over Wi-Fi; Safe Mode at exactly 3 crashes, left by `reboot`; a hung loop caught by the watchdog in 5 s; `reset` from the console task. ADR 0005. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT
2026-10-04 03:07:56 +02:00
parent 0fb7f4e9d5
commit 14ff13f634
16 changed files with 452 additions and 21 deletions
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# Turns crash addresses into functions and source lines, using the archived ELF of the build that
+# crashed (.pio/elves/, kept by scripts/version.py).
+# Usage: scripts/decode_backtrace.sh <version or ELF sha256 prefix> <address>...
+set -euo pipefail
+source "$(dirname "$0")/_docker.sh"
+[ $# -ge 2 ] || { echo "Usage: scripts/decode_backtrace.sh <version|sha> <address>..." >&2; exit 1; }
+ELF="$(ls -t "$ROOT"/.pio/elves/*"$1"*.elf 2>/dev/null | head -1 || true)"
+[ -n "$ELF" ] || { echo "No archived ELF matches '$1' in .pio/elves/" >&2; exit 1; }
+echo "using ${ELF#$ROOT/}" >&2
+DOCKER_EXTRA=()
+run_in_container /pio/tools/toolchain-xtensa-esp-elf/bin/xtensa-esp32s3-elf-addr2line -pfiaC -e "/work/${ELF#$ROOT/}" "${@:2}"
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Full post-mortem of a core dump fetched with `scripts/rdbg.py coredump`: every task's backtrace,
+# registers and the crashed task's stack, via esp-coredump and GDB in the container.
+# Usage: scripts/decode_coredump.sh <core.bin> <version or ELF sha256 prefix>
+set -euo pipefail
+source "$(dirname "$0")/_docker.sh"
+[ $# -eq 2 ] || { echo "Usage: scripts/decode_coredump.sh <core.bin> <version|sha>" >&2; exit 1; }
+CORE="$(realpath "$1")"
+ELF="$(ls -t "$ROOT"/.pio/elves/*"$2"*.elf 2>/dev/null | head -1 || true)"
+[ -n "$ELF" ] || { echo "No archived ELF matches '$2' in .pio/elves/" >&2; exit 1; }
+echo "using ${ELF#$ROOT/}" >&2
+DOCKER_EXTRA=(-v "$(dirname "$CORE"):/core:ro")
+run_in_container /pio/penv/bin/esp-coredump --chip esp32s3 info_corefile -t raw \
+    -g /pio/tools/tool-xtensa-esp-elf-gdb/bin/xtensa-esp32s3-elf-gdb \
+    -c "/core/$(basename "$CORE")" "/work/${ELF#$ROOT/}"
@@ -6,10 +6,16 @@ Usage: scripts/rdbg.py [-H host] [-b] [command ...]
  command      runs it and prints what follows, until the console has been quiet for a moment
  -H host      the device's IP (Settings > Firmware), default $RORO_OTA_HOST
  -b           also print the backlog the device sends on connecting (boot messages and so on)
+
+Commands handled here as well as on the device:
+  crash             the last crash, with its backtrace decoded (scripts/decode_backtrace.sh)
+  coredump [file]   fetch the core dump (default core-<date>.bin) and decode it with esp-coredump
 The token is read from ~/.config/roro9stack/debug-token (made by the first build).
 """
 import os
+import re
 import select
+import subprocess
 import socket
 import sys
 import time
@@ -48,6 +54,86 @@ def read_until_quiet(sock, quiet, out):
            out.flush()


+SCRIPTS = os.path.dirname(os.path.abspath(__file__))
+
+
+def run(sock, command, out=sys.stdout):
+    """Sends one command and returns its reply (also copied to `out`)."""
+    sock.sendall((command + "\n").encode())
+    # The main loop echoes "> command" when it runs it; the reply follows.
+    echoed = read_until(sock, f"> {command}\n".encode(), 15)
+    if echoed is None:
+        sys.exit("device: the command never ran")
+    reply = echoed.decode(errors="replace").split(f"> {command}\n", 1)[1]
+
+    class Tee:
+        def write(self, text):
+            nonlocal reply
+            reply += text
+            if out:
+                out.write(text)
+
+        def flush(self):
+            if out:
+                out.flush()
+
+    if out:
+        out.write(reply)
+    try:
+        read_until_quiet(sock, 1.5, Tee())
+    except BrokenPipeError:  # e.g. piped into head
+        pass
+    return reply
+
+
+def crash_firmware(reply):
+    """The archived-ELF key for the crashed firmware: its ELF digest, or else its version."""
+    sha = re.search(r"elf sha256 ([0-9a-f]{8,})", reply)
+    if sha:
+        return sha.group(1)
+    version = re.search(r"last one in (\S+)", reply)
+    return version.group(1) if version else None
+
+
+def crash(sock):
+    reply = run(sock, "crash")
+    trace = re.search(r"backtrace((?: 0x[0-9a-f]+)+)", reply)
+    key = crash_firmware(reply)
+    if trace and key:
+        print()
+        subprocess.call([os.path.join(SCRIPTS, "decode_backtrace.sh"), key] + trace.group(1).split())
+
+
+def coredump(sock, path):
+    info = run(sock, "crash", out=None)
+    sock.sendall(b"coredump get\n")
+    # One buffer throughout: the header, the size and the first bytes often share a packet.
+    buf = b""
+    sock.settimeout(15)
+    while b"coredump: none" not in buf and not re.search(rb"coredump: data (\d+)\n", buf):
+        chunk = sock.recv(65536)
+        if not chunk:
+            sys.exit("device: closed the connection")
+        buf += chunk
+    header = re.search(rb"coredump: data (\d+)\n", buf)
+    if not header:
+        sys.exit("device: no core dump in flash")
+    size = int(header.group(1))
+    data = buf[header.end():]
+    while len(data) < size:
+        chunk = sock.recv(65536)
+        if not chunk:
+            sys.exit(f"device: the connection closed after {len(data)} of {size} bytes")
+        data += chunk
+    data = data[:size]
+    path = path or time.strftime("core-%Y%m%d-%H%M%S.bin")
+    open(path, "wb").write(data)
+    print(f"saved {path} ({size} bytes)")
+    key = crash_firmware(info)
+    if key:
+        subprocess.call([os.path.join(SCRIPTS, "decode_coredump.sh"), path, key])
+
+
 def interactive(sock):
    sock.setblocking(False)
    while True:
@@ -93,18 +179,19 @@ def main():
        read_until_quiet(sock, 0.5, show)  # the backlog
        if not args:
            return interactive(sock)
-        command = " ".join(args)
-        sock.sendall((command + "\n").encode())
-        # The main loop echoes "> command" when it runs it; the reply follows.
-        echoed = read_until(sock, f"> {command}\n".encode(), 15)
-        if echoed is None:
-            sys.exit("device: the command never ran")
-        sys.stdout.write(echoed.decode(errors="replace").split(f"> {command}\n", 1)[1])
-        try:
-            read_until_quiet(sock, 1.5, sys.stdout)
-        except BrokenPipeError:  # e.g. piped into head
-            pass
+        if args == ["crash"]:
+            return crash(sock)
+        if args == ["reset"]:  # answered by the console's own task, not the main loop
+            sock.sendall(b"reset\n")
+            print((read_until(sock, b"restarting now\n", 10) or b"device: no answer").decode().strip().splitlines()[-1])
+            return
+        if args[0] == "coredump" and args[1:2] != ["erase"]:
+            return coredump(sock, args[1] if len(args) > 1 else None)
+        run(sock, " ".join(args))


 if __name__ == "__main__":
-    main()
+    try:
+        main()
+    except (ConnectionResetError, BrokenPipeError):
+        sys.exit("device: the connection dropped (restarting?)")
@@ -14,3 +14,20 @@ if env["PIOENV"].endswith("-debug"):  # noqa: F821
    version += "+debug"  # a Debug Build says so wherever the version shows

 env.Append(CPPDEFINES=[("RORO_VERSION", '\\"%s\\"' % version)])  # noqa: F821
+
+
+# Keep every build's ELF, named by version and the first 16 hex digits of its SHA-256 (the core dump
+# names the crashed firmware by the same digest), so a crash can be decoded after later builds.
+def archive_elf(source, target, env):
+    import hashlib
+    import os
+    import shutil
+
+    elf = str(target[0])
+    digest = hashlib.sha256(open(elf, "rb").read()).hexdigest()[:16]
+    folder = os.path.join(env.subst("$PROJECT_DIR"), ".pio", "elves")
+    os.makedirs(folder, exist_ok=True)
+    shutil.copy(elf, os.path.join(folder, "%s.%s.elf" % (version, digest)))
+
+
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.elf", archive_elf)  # noqa: F821