Safe Mode, crash reports, and a watched main loop

Every build now records at boot which version runs and, after a crash
restart, which one crashed (even across a Rollback). The core dump
summary (task, PC, reason, backtrace) is printed and raised as a
Notification; `crash` shows it later. After 3 crash restarts in a row
the firmware starts in Safe Mode: clock, Wi-Fi, Update Service and Debug
Console only (SafeMode, 2 host tests). A normal restart or a minute up
resets the count.

The main loop is now on the task watchdog (enableLoopWDT): Arduino only
watched core 0's idle task, so a stuck loop hung the device for good.
The Update Service restarts into an installed update by itself if the
main loop hasn't after 90 s.

Debug Builds: `coredump get` and `reset` are answered by the console's
own task; rdbg.py crash decodes the backtrace and rdbg.py coredump runs
esp-coredump, against ELFs archived by version and digest in .pio/elves.

The StorageService mutex is now made in the constructor: Safe Mode never
starts that Service, and `info` crashed on the null mutex, 29 times in a
row before the fix was pushed into Safe Mode over Wi-Fi.

Verified on the device: crash report and full core dump decoded over
Wi-Fi; Safe Mode at exactly 3 crashes, left by `reboot`; a hung loop
caught by the watchdog in 5 s; `reset` from the console task. ADR 0005.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT

src/services/debug_console.cpp

@@ -4,6 +4,10 @@
 
 #include <WiFi.h>
 
+#include <algorithm>
+#include <esp_core_dump.h>
+#include <esp_flash.h>
+
 #include "platform/console.h"
 #include "version.h"
 
@@ -43,6 +47,24 @@ bool readLine(NetworkClient& c, std::string& line, uint32_t timeoutMs) {
     return false;
 }
 
+// The raw core dump partition contents, as esp-coredump reads them ("-t raw").
+void sendCoreDump(NetworkClient& client) {
+    size_t addr = 0, size = 0;
+    if (esp_core_dump_image_check() != ESP_OK || esp_core_dump_image_get(&addr, &size) != ESP_OK) {
+        client.print("coredump: none\n");
+        return;
+    }
+    client.printf("coredump: data %u\n", (unsigned)size);
+    uint8_t buf[1024];
+    for (size_t off = 0; off < size;) {
+        size_t n = std::min(sizeof buf, size - off);
+        if (esp_flash_read(nullptr, buf, addr + off, n) != ESP_OK) break;  // the host sees a short file
+        if (client.write(buf, n) != n) return;
+        off += n;
+    }
+    client.print("coredump: end\n");
+}
+
 }  // namespace
 
 DebugConsole::DebugConsole(WifiService& wifi) : wifi_(wifi), lock_(xSemaphoreCreateMutex()) {}
@@ -125,6 +147,17 @@ void DebugConsole::serve(NetworkClient& client) {
                 client.stop();
                 break;
             }
+            if (line == "reset") {  // answered here: works even when the main loop is stuck
+                client.print("debug: restarting now\n");
+                client.flush();
+                delay(200);
+                esp_restart();
+            }
+            if (line == "coredump get") {  // binary: answered here, not by the main loop
+                sendCoreDump(client);
+                line.clear();
+                continue;
+            }
             xSemaphoreTake(lock_, portMAX_DELAY);
             bool full = commands_.size() >= kMaxQueued;
             if (!full && !line.empty()) commands_.push_back(line);

src/services/storage_service.cpp

@@ -14,7 +14,6 @@ namespace roro {
 
 void StorageService::start() {
     if (task_) return;
-    lock_ = xSemaphoreCreateMutex();
     xTaskCreate(taskEntry, "storage", 10240, this, 1, &task_);  // room for a signature check
 }
 

src/services/storage_service.h

@@ -20,7 +20,8 @@ namespace roro {
 // queued Log lines in batches, lists files for Storage Clean-up, deletes, and formats.
 class StorageService : public Service {
    public:
-    explicit StorageService(EventBus& bus) : monitor_(bus), bus_(bus) {}
+    // The lock exists from the start, so a Service that's never started (Safe Mode) can still be asked.
+    explicit StorageService(EventBus& bus) : monitor_(bus), bus_(bus), lock_(xSemaphoreCreateMutex()) {}
     const char* name() const override { return "storage"; }
     void start() override;
     void stop() override;

src/services/update_service.cpp

@@ -28,6 +28,7 @@ class UpdateSource {
 namespace {
 
 constexpr uint32_t kStallMs = 10000;
+constexpr uint32_t kForceRestartMs = 90000;  // the main loop waits up to 60 s for someone typing
 constexpr size_t kChunk = 4096;
 
 class NetSource : public UpdateSource {
@@ -190,6 +191,7 @@ void UpdateService::taskEntry(void* self) { static_cast<UpdateService*>(self)->l
 void UpdateService::listen() {
     NetworkServer server(kPort);
     bool listening = false;
+    uint32_t installedAt = 0;
     for (;;) {
         bool connected = wifi_.state() == WifiController::State::Connected;
         if (connected && !listening) {
@@ -201,6 +203,15 @@ void UpdateService::listen() {
             MDNS.end();
             listening = false;
         }
+        // The main loop restarts into an installed update when it's safe. If it never does (stuck,
+        // or waiting on someone typing for too long), restart from here: the update must not wait.
+        if (phase_ == Phase::Installed) {
+            if (!installedAt) installedAt = millis();
+            if (millis() - installedAt > kForceRestartMs) {
+                ESP_LOGW("update", "the main loop never restarted into the update: restarting");
+                esp_restart();
+            }
+        }
         if (listening && phase_ == Phase::Idle) {
             NetworkClient client = server.accept();
             if (client) {