Safe Mode, crash reports, and a watched main loop

Every build now records at boot which version runs and, after a crash
restart, which one crashed (even across a Rollback). The core dump
summary (task, PC, reason, backtrace) is printed and raised as a
Notification; `crash` shows it later. After 3 crash restarts in a row
the firmware starts in Safe Mode: clock, Wi-Fi, Update Service and Debug
Console only (SafeMode, 2 host tests). A normal restart or a minute up
resets the count.

The main loop is now on the task watchdog (enableLoopWDT): Arduino only
watched core 0's idle task, so a stuck loop hung the device for good.
The Update Service restarts into an installed update by itself if the
main loop hasn't after 90 s.

Debug Builds: `coredump get` and `reset` are answered by the console's
own task; rdbg.py crash decodes the backtrace and rdbg.py coredump runs
esp-coredump, against ELFs archived by version and digest in .pio/elves.

The StorageService mutex is now made in the constructor: Safe Mode never
starts that Service, and `info` crashed on the null mutex, 29 times in a
row before the fix was pushed into Safe Mode over Wi-Fi.

Verified on the device: crash report and full core dump decoded over
Wi-Fi; Safe Mode at exactly 3 crashes, left by `reboot`; a hung loop
caught by the watchdog in 5 s; `reset` from the console task. ADR 0005.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT

src/main.cpp

@@ -16,6 +16,7 @@
 #include "file_receiver.h"
 #include "key_mapper.h"
 #include "platform/console.h"
+#include "platform/crash_report.h"
 #include "platform/nvs_store.h"
 #include "platform/system_info.h"
 #include "service_manager.h"
@@ -28,6 +29,7 @@
 #include "services/storage_service.h"
 #include "services/update_service.h"
 #include "services/wifi_service.h"
+#include "safe_mode.h"
 #include "settings.h"
 #include "storage_paths.h"
 #include "ui/notifier.h"
@@ -58,6 +60,7 @@ static DebugConsole* debugConsole;
 static Notifier* notifier;
 static LauncherApp launcher;
 static AppManager* apps;
+static bool safeMode = false;  // see SafeMode: only what it takes to be fixed over the air
 
 static RawKeys readKeys() {
     auto state = M5Cardputer.Keyboard.keysState();
@@ -104,9 +107,13 @@ static StatusInfo currentStatus() {
 // (UpdateService::tick) decides instead, and an unconfirmed image stays PENDING_VERIFY.
 extern "C" bool verifyRollbackLater() { return true; }  // C linkage, or the weak default wins
 
+static void setupSafeMode(int crashes);
+
 void setup() {
     nvs.begin();
     UpdateService::bootGuard(nvs);  // first: before anything that could crash on new firmware
+    int crashes = crash_report::noteBoot(nvs);
+    safeMode = SafeMode::active(crashes);
     Serial.setRxBufferSize(2 * FileReceiver::kChunk);  // before the port opens; sd put sends 1 chunk at a time
     Serial.setTxBufferSize(2048);  // the console skips Serial when it's full: room for a burst like `tasks`
 
@@ -116,6 +123,7 @@ void setup() {
     Serial.begin(115200);
 
     settings.load();
+    if (safeMode) return setupSafeMode(crashes);
 
     battery = new BatteryService(bus);
     storageService = new StorageService(bus);
@@ -157,6 +165,39 @@ void setup() {
     if (!settings.getBool(Setting::SetupDone)) apps->openModal("setup");
     console.printf("%s %s ready, free heap %u, last start: %s\n", kProductName, versionString(), ESP.getFreeHeap(),
                    system_info::resetReason());
+    if (system_info::resetWasCrash()) {
+        crash_report::print(console, nvs);
+        bus.publish(Event::withText(EventType::Notification, crash_report::headline().c_str(),
+                                    static_cast<int32_t>(NotificationLevel::Warning)));
+    }
+    // A main loop stuck for 5 s panics (and leaves a core dump) instead of hanging forever: Arduino
+    // only watches the idle task on core 0, and the loop runs on core 1.
+    enableLoopWDT();
+}
+
+// Safe Mode: Wi-Fi, the clock, Firmware Updates and (in a Debug Build) the Debug Console. No Apps,
+// no IRC, no SD card: whatever crashed three times in a row is most likely among them.
+static void setupSafeMode(int crashes) {
+    clockService = new ClockService(settings, bus);
+    savedNetworks = new SavedNetworks(nvs);
+    savedNetworks->load();
+    wifi = new WifiService(settings, *savedNetworks, *clockService);
+    storageService = new StorageService(bus);  // never started: no card access
+    update = new UpdateService(nvs, *wifi, *savedNetworks, *storageService, bus, settings);
+    services.add(*clockService);
+    services.add(*wifi);
+    services.add(*update);
+#ifdef RORO_DEBUG
+    debugConsole = new DebugConsole(*wifi);
+    services.add(*debugConsole);
+#endif
+    bus.subscribe(EventType::Notification, [](const Event& e) { console.printf("notification: %s\n", e.text); });
+    screen.begin();
+    services.startAll(millis());
+    console.printf("%s %s in SAFE MODE: %d crash restarts in a row. Only Wi-Fi and Firmware Updates run.\n",
+                   kProductName, versionString(), crashes);
+    crash_report::print(console, nvs);
+    enableLoopWDT();
 }
 
 // Dev aid: commands to drive the UI without the keyboard, from the serial port or the Debug Console.
@@ -276,19 +317,31 @@ static const char* const kHelp =
     "reboot            restart\n"
     "boot other        restart into the other app slot (manual Rollback)\n"
     "log level <0-5>   ESP-IDF log level (0 none ... 5 verbose)\n"
+    "crash             the last crash: firmware, reason, task, backtrace\n"
+    "coredump erase    forget the core dump in flash\n"
     "key <name|char>   press a key: up down left right select back home, or one character\n"
     "wifi status | wifi add <ssid><TAB><password>\n"
     "irc start | irc dump | irc say <buffer> <text>\n"
     "sd list | cat <path> | log <text> | burst | sound on|off | short | normal\n"
 #ifdef RORO_DEBUG
     "crash abort|wdt   crash on purpose (to test crash reports and Safe Mode)\n"
+    "coredump get      (Debug Console only) send the raw core dump: use scripts/rdbg.py coredump\n"
+    "reset             (Debug Console only) restart at once, even if the main loop is stuck\n"
     "quit              close the Debug Console connection\n"
 #endif
     ;
 
+// Commands that only touch what Safe Mode starts.
+static bool safeModeCommand(const String& line) {
+    return line == "help" || line == "info" || line == "tasks" || line == "reboot" || line == "boot other" ||
+           line.startsWith("log level ") || line.startsWith("crash") || line.startsWith("coredump") ||
+           line == "wifi status" || line.startsWith("wifi add ");
+}
+
 static void runCommand(String line) {
     line.trim();
     if (line.isEmpty()) return;
+    if (safeMode && !safeModeCommand(line)) return (void)console.println("not available in Safe Mode");
     if (line == "help") console.print(kHelp);
     if (line == "info") {
         system_info::printSystem(console);
@@ -309,6 +362,8 @@ static void runCommand(String line) {
         esp_log_level_set("*", static_cast<esp_log_level_t>(constrain(level, 0, 5)));
         console.printf("log level: %d\n", constrain(level, 0, 5));
     }
+    if (line == "crash") crash_report::print(console, nvs);
+    if (line == "coredump erase") console.println(crash_report::erase() ? "coredump: erased" : "coredump: nothing to erase");
 #ifdef RORO_DEBUG
     if (line == "crash abort") abort();
     if (line == "crash wdt")
@@ -410,7 +465,48 @@ static void serialCommands() {
     }
 }
 
+static void remoteCommands() {
+#ifdef RORO_DEBUG
+    for (std::string remote; debugConsole->takeCommand(remote);) {
+        console.printf("> %s\n", remote.c_str());  // so the transcript reads the same on both ends
+        runCommand(remote.c_str());
+    }
+#endif
+}
+
+// After a minute up, the crash streak is over (SafeMode counts starts that crash in a row).
+static void noteStableOnce(uint32_t now) {
+    static bool noted = false;
+    if (noted || now < SafeMode::kStableAfterMs) return;
+    noted = true;
+    crash_report::noteStable(nvs);
+}
+
+static void loopSafeMode() {
+    uint32_t now = millis();
+    serialCommands();
+    remoteCommands();
+    services.tick(now);
+    bus.dispatch();
+    noteStableOnce(now);
+    if (update->phase() == UpdateService::Phase::Installed) {
+        screen.renderUpdate("Restarting", "into " + update->incomingVersion(), -1);
+        delay(800);
+        ESP.restart();
+    }
+    static uint32_t lastDraw = 0;
+    if (update->phase() == UpdateService::Phase::Receiving) {
+        screen.renderUpdate("Firmware update", "Receiving " + update->incomingVersion(), update->percent());
+    } else if (now - lastDraw > 2000 || !lastDraw) {
+        lastDraw = now;
+        bool up = wifi->state() == WifiController::State::Connected;
+        screen.renderUpdate("Safe Mode", up ? "Updates: " + wifi->ip() + ":3232" : "Waiting for Wi-Fi", -1);
+    }
+    delay(10);
+}
+
 void loop() {
+    if (safeMode) return loopSafeMode();
 #ifdef RORO_TEST_CRASH
     // Test builds only (never in a release): crash during Probation to exercise Rollback.
     if (millis() > 5000) abort();
@@ -420,12 +516,8 @@ void loop() {
     uint32_t now = millis();
 
     serialCommands();
-#ifdef RORO_DEBUG
-    for (std::string remote; debugConsole->takeCommand(remote);) {
-        console.printf("> %s\n", remote.c_str());  // so the transcript reads the same on both ends
-        runCommand(remote.c_str());
-    }
-#endif
+    remoteCommands();
+    noteStableOnce(now);
     uploadStep();
     printListingWhenReady();
     M5Cardputer.update();

src/platform/crash_report.cpp

@@ -0,0 +1,66 @@
+#include "platform/crash_report.h"
+
+#include <esp_core_dump.h>
+
+#include "platform/system_info.h"
+#include "safe_mode.h"
+#include "version.h"
+
+namespace roro::crash_report {
+
+int noteBoot(KeyValueStore& store) {
+    bool crashed = system_info::resetWasCrash();
+    std::string last;
+    store.getString("run_ver", last);
+    if (crashed) {
+        store.putString("crash_ver", last.empty() ? versionString() : last);
+        store.putString("crash_why", system_info::resetReason());
+    }
+    if (last != versionString()) store.putString("run_ver", versionString());
+
+    int32_t before = 0;
+    store.getInt("crash_boots", before);
+    int now = SafeMode::countAtBoot(crashed, before);
+    if (now != before) store.putInt("crash_boots", now);
+    return now;
+}
+
+void noteStable(KeyValueStore& store) {
+    int32_t n = 0;
+    if (store.getInt("crash_boots", n) && n != 0) store.putInt("crash_boots", 0);
+}
+
+void print(Print& out, KeyValueStore& store) {
+    std::string version, why;
+    store.getString("crash_ver", version);
+    store.getString("crash_why", why);
+    if (version.empty()) out.println("crash: none recorded");
+    else out.printf("crash: last one in %s (%s)\n", version.c_str(), why.c_str());
+
+    esp_core_dump_summary_t sum;
+    if (esp_core_dump_image_check() != ESP_OK || esp_core_dump_get_summary(&sum) != ESP_OK) {
+        out.println("crash: no core dump in flash");
+        return;
+    }
+    char reason[160] = "";
+    esp_core_dump_get_panic_reason(reason, sizeof reason);
+    out.printf("crash: task %s, pc 0x%08lx, cause %lu, address 0x%08lx\n", sum.exc_task, (unsigned long)sum.exc_pc,
+               (unsigned long)sum.ex_info.exc_cause, (unsigned long)sum.ex_info.exc_vaddr);
+    if (reason[0]) out.printf("crash: reason: %s\n", reason);
+    out.print("crash: backtrace");
+    for (uint32_t i = 0; i < sum.exc_bt_info.depth && i < 16; i++) out.printf(" 0x%08lx", (unsigned long)sum.exc_bt_info.bt[i]);
+    out.println(sum.exc_bt_info.corrupted ? " (corrupted)" : "");
+    out.printf("crash: elf sha256 %s\n", reinterpret_cast<const char*>(sum.app_elf_sha256));
+}
+
+std::string headline() {
+    esp_core_dump_summary_t sum;
+    std::string where = "";
+    if (esp_core_dump_image_check() == ESP_OK && esp_core_dump_get_summary(&sum) == ESP_OK)
+        where = std::string(" in ") + sum.exc_task;
+    return std::string("Restarted after a ") + system_info::resetReason() + where;
+}
+
+bool erase() { return esp_core_dump_image_erase() == ESP_OK; }
+
+}  // namespace roro::crash_report

src/platform/crash_report.h

@@ -0,0 +1,26 @@
+#pragma once
+
+#include <Print.h>
+
+#include <string>
+
+#include "key_value_store.h"
+
+namespace roro::crash_report {
+
+// First thing at boot: remembers which version runs, and after a crash restart, which version
+// crashed (the one that ran last time, even if a Rollback has switched slots since). Returns how
+// many starts in a row ended in a crash, for Safe Mode.
+int noteBoot(KeyValueStore& store);
+// The device has been up long enough: the crash streak is over.
+void noteStable(KeyValueStore& store);
+
+// The last crash: which firmware, why, and the task, PC and backtrace from the core dump in flash.
+// Decode the backtrace on the PC with scripts/decode_backtrace.sh <version> <addresses>.
+void print(Print& out, KeyValueStore& store);
+// One line for a Notification after a crash restart, e.g. "Restarted after a crash in loopTask".
+std::string headline();
+// Forgets the core dump, so the next crash's is the one kept.
+bool erase();
+
+}  // namespace roro::crash_report

src/services/debug_console.cpp

@@ -4,6 +4,10 @@
 
 #include <WiFi.h>
 
+#include <algorithm>
+#include <esp_core_dump.h>
+#include <esp_flash.h>
+
 #include "platform/console.h"
 #include "version.h"
 
@@ -43,6 +47,24 @@ bool readLine(NetworkClient& c, std::string& line, uint32_t timeoutMs) {
     return false;
 }
 
+// The raw core dump partition contents, as esp-coredump reads them ("-t raw").
+void sendCoreDump(NetworkClient& client) {
+    size_t addr = 0, size = 0;
+    if (esp_core_dump_image_check() != ESP_OK || esp_core_dump_image_get(&addr, &size) != ESP_OK) {
+        client.print("coredump: none\n");
+        return;
+    }
+    client.printf("coredump: data %u\n", (unsigned)size);
+    uint8_t buf[1024];
+    for (size_t off = 0; off < size;) {
+        size_t n = std::min(sizeof buf, size - off);
+        if (esp_flash_read(nullptr, buf, addr + off, n) != ESP_OK) break;  // the host sees a short file
+        if (client.write(buf, n) != n) return;
+        off += n;
+    }
+    client.print("coredump: end\n");
+}
+
 }  // namespace
 
 DebugConsole::DebugConsole(WifiService& wifi) : wifi_(wifi), lock_(xSemaphoreCreateMutex()) {}
@@ -125,6 +147,17 @@ void DebugConsole::serve(NetworkClient& client) {
                 client.stop();
                 break;
             }
+            if (line == "reset") {  // answered here: works even when the main loop is stuck
+                client.print("debug: restarting now\n");
+                client.flush();
+                delay(200);
+                esp_restart();
+            }
+            if (line == "coredump get") {  // binary: answered here, not by the main loop
+                sendCoreDump(client);
+                line.clear();
+                continue;
+            }
             xSemaphoreTake(lock_, portMAX_DELAY);
             bool full = commands_.size() >= kMaxQueued;
             if (!full && !line.empty()) commands_.push_back(line);

src/services/storage_service.cpp

@@ -14,7 +14,6 @@ namespace roro {
 
 void StorageService::start() {
     if (task_) return;
-    lock_ = xSemaphoreCreateMutex();
     xTaskCreate(taskEntry, "storage", 10240, this, 1, &task_);  // room for a signature check
 }
 

src/services/storage_service.h

@@ -20,7 +20,8 @@ namespace roro {
 // queued Log lines in batches, lists files for Storage Clean-up, deletes, and formats.
 class StorageService : public Service {
    public:
-    explicit StorageService(EventBus& bus) : monitor_(bus), bus_(bus) {}
+    // The lock exists from the start, so a Service that's never started (Safe Mode) can still be asked.
+    explicit StorageService(EventBus& bus) : monitor_(bus), bus_(bus), lock_(xSemaphoreCreateMutex()) {}
     const char* name() const override { return "storage"; }
     void start() override;
     void stop() override;

src/services/update_service.cpp

@@ -28,6 +28,7 @@ class UpdateSource {
 namespace {
 
 constexpr uint32_t kStallMs = 10000;
+constexpr uint32_t kForceRestartMs = 90000;  // the main loop waits up to 60 s for someone typing
 constexpr size_t kChunk = 4096;
 
 class NetSource : public UpdateSource {
@@ -190,6 +191,7 @@ void UpdateService::taskEntry(void* self) { static_cast<UpdateService*>(self)->l
 void UpdateService::listen() {
     NetworkServer server(kPort);
     bool listening = false;
+    uint32_t installedAt = 0;
     for (;;) {
         bool connected = wifi_.state() == WifiController::State::Connected;
         if (connected && !listening) {
@@ -201,6 +203,15 @@ void UpdateService::listen() {
             MDNS.end();
             listening = false;
         }
+        // The main loop restarts into an installed update when it's safe. If it never does (stuck,
+        // or waiting on someone typing for too long), restart from here: the update must not wait.
+        if (phase_ == Phase::Installed) {
+            if (!installedAt) installedAt = millis();
+            if (millis() - installedAt > kForceRestartMs) {
+                ESP_LOGW("update", "the main loop never restarted into the update: restarting");
+                esp_restart();
+            }
+        }
         if (listening && phase_ == Phase::Idle) {
             NetworkClient client = server.accept();
             if (client) {