twisla/roro9stack
Public Access
1
0
Fork 0
Code Issues 48 Pull Requests Actions Packages Releases 16 Activity

Safe Mode, crash reports, and a watched main loop

Browse Source 
Every build now records at boot which version runs and, after a crash
restart, which one crashed (even across a Rollback). The core dump
summary (task, PC, reason, backtrace) is printed and raised as a
Notification; `crash` shows it later. After 3 crash restarts in a row
the firmware starts in Safe Mode: clock, Wi-Fi, Update Service and Debug
Console only (SafeMode, 2 host tests). A normal restart or a minute up
resets the count.

The main loop is now on the task watchdog (enableLoopWDT): Arduino only
watched core 0's idle task, so a stuck loop hung the device for good.
The Update Service restarts into an installed update by itself if the
main loop hasn't after 90 s.

Debug Builds: `coredump get` and `reset` are answered by the console's
own task; rdbg.py crash decodes the backtrace and rdbg.py coredump runs
esp-coredump, against ELFs archived by version and digest in .pio/elves.

The StorageService mutex is now made in the constructor: Safe Mode never
starts that Service, and `info` crashed on the null mutex, 29 times in a
row before the fix was pushed into Safe Mode over Wi-Fi.

Verified on the device: crash report and full core dump decoded over
Wi-Fi; Safe Mode at exactly 3 crashes, left by `reboot`; a hung loop
caught by the watchdog in 5 s; `reset` from the console task. ADR 0005.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT
This commit is contained in:
twisla
2026-10-04 03:07:56 +02:00
co-authored by Claude Opus 5.5
parent 0fb7f4e9d5
commit 14ff13f634
16 changed files with 452 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/main.cpp
+98 -6
View File
@@ -16,6 +16,7 @@
#include "file_receiver.h"
#include "key_mapper.h"
#include "platform/console.h"
#include "platform/crash_report.h"
#include "platform/nvs_store.h"
#include "platform/system_info.h"
#include "service_manager.h"
@@ -28,6 +29,7 @@
#include "services/storage_service.h"
#include "services/update_service.h"
#include "services/wifi_service.h"
#include "safe_mode.h"
#include "settings.h"
#include "storage_paths.h"
#include "ui/notifier.h"
@@ -58,6 +60,7 @@ static DebugConsole* debugConsole;
static Notifier* notifier;
static LauncherApp launcher;
static AppManager* apps;
static bool safeMode = false; // see SafeMode: only what it takes to be fixed over the air
static RawKeys readKeys() {
auto state = M5Cardputer.Keyboard.keysState();
@@ -104,9 +107,13 @@ static StatusInfo currentStatus() {
// (UpdateService::tick) decides instead, and an unconfirmed image stays PENDING_VERIFY.
extern "C" bool verifyRollbackLater() { return true; } // C linkage, or the weak default wins
static void setupSafeMode(int crashes);
void setup() {
nvs.begin();
UpdateService::bootGuard(nvs); // first: before anything that could crash on new firmware
int crashes = crash_report::noteBoot(nvs);
safeMode = SafeMode::active(crashes);
Serial.setRxBufferSize(2 * FileReceiver::kChunk); // before the port opens; sd put sends 1 chunk at a time
Serial.setTxBufferSize(2048); // the console skips Serial when it's full: room for a burst like `tasks`
@@ -116,6 +123,7 @@ void setup() {
Serial.begin(115200);
settings.load();
if (safeMode) return setupSafeMode(crashes);
battery = new BatteryService(bus);
storageService = new StorageService(bus);
@@ -157,6 +165,39 @@ void setup() {
if (!settings.getBool(Setting::SetupDone)) apps->openModal("setup");
console.printf("%s %s ready, free heap %u, last start: %s\n", kProductName, versionString(), ESP.getFreeHeap(),
system_info::resetReason());
if (system_info::resetWasCrash()) {
crash_report::print(console, nvs);
bus.publish(Event::withText(EventType::Notification, crash_report::headline().c_str(),
static_cast<int32_t>(NotificationLevel::Warning)));
}
// A main loop stuck for 5 s panics (and leaves a core dump) instead of hanging forever: Arduino
// only watches the idle task on core 0, and the loop runs on core 1.
enableLoopWDT();
}
// Safe Mode: Wi-Fi, the clock, Firmware Updates and (in a Debug Build) the Debug Console. No Apps,
// no IRC, no SD card: whatever crashed three times in a row is most likely among them.
static void setupSafeMode(int crashes) {
clockService = new ClockService(settings, bus);
savedNetworks = new SavedNetworks(nvs);
savedNetworks->load();
wifi = new WifiService(settings, *savedNetworks, *clockService);
storageService = new StorageService(bus); // never started: no card access
update = new UpdateService(nvs, *wifi, *savedNetworks, *storageService, bus, settings);
services.add(*clockService);
services.add(*wifi);
services.add(*update);
#ifdef RORO_DEBUG
debugConsole = new DebugConsole(*wifi);
services.add(*debugConsole);
#endif
bus.subscribe(EventType::Notification, [](const Event& e) { console.printf("notification: %s\n", e.text); });
screen.begin();
services.startAll(millis());
console.printf("%s %s in SAFE MODE: %d crash restarts in a row. Only Wi-Fi and Firmware Updates run.\n",
kProductName, versionString(), crashes);
crash_report::print(console, nvs);
enableLoopWDT();
}
// Dev aid: commands to drive the UI without the keyboard, from the serial port or the Debug Console.
@@ -276,19 +317,31 @@ static const char* const kHelp =
"reboot restart\n"
"boot other restart into the other app slot (manual Rollback)\n"
"log level <0-5> ESP-IDF log level (0 none ... 5 verbose)\n"
"crash the last crash: firmware, reason, task, backtrace\n"
"coredump erase forget the core dump in flash\n"
"key <name|char> press a key: up down left right select back home, or one character\n"
"wifi status | wifi add <ssid><TAB><password>\n"
"irc start | irc dump | irc say <buffer> <text>\n"
"sd list | cat <path> | log <text> | burst | sound on|off | short | normal\n"
#ifdef RORO_DEBUG
"crash abort|wdt crash on purpose (to test crash reports and Safe Mode)\n"
"coredump get (Debug Console only) send the raw core dump: use scripts/rdbg.py coredump\n"
"reset (Debug Console only) restart at once, even if the main loop is stuck\n"
"quit close the Debug Console connection\n"
#endif
;
// Commands that only touch what Safe Mode starts.
static bool safeModeCommand(const String& line) {
return line == "help" || line == "info" || line == "tasks" || line == "reboot" || line == "boot other" ||
line.startsWith("log level ") || line.startsWith("crash") || line.startsWith("coredump") ||
line == "wifi status" || line.startsWith("wifi add ");
}
static void runCommand(String line) {
line.trim();
if (line.isEmpty()) return;
if (safeMode && !safeModeCommand(line)) return (void)console.println("not available in Safe Mode");
if (line == "help") console.print(kHelp);
if (line == "info") {
system_info::printSystem(console);
@@ -309,6 +362,8 @@ static void runCommand(String line) {
esp_log_level_set("*", static_cast<esp_log_level_t>(constrain(level, 0, 5)));
console.printf("log level: %d\n", constrain(level, 0, 5));
}
if (line == "crash") crash_report::print(console, nvs);
if (line == "coredump erase") console.println(crash_report::erase() ? "coredump: erased" : "coredump: nothing to erase");
#ifdef RORO_DEBUG
if (line == "crash abort") abort();
if (line == "crash wdt")
@@ -410,7 +465,48 @@ static void serialCommands() {
}
}
static void remoteCommands() {
#ifdef RORO_DEBUG
for (std::string remote; debugConsole->takeCommand(remote);) {
console.printf("> %s\n", remote.c_str()); // so the transcript reads the same on both ends
runCommand(remote.c_str());
}
#endif
}
// After a minute up, the crash streak is over (SafeMode counts starts that crash in a row).
static void noteStableOnce(uint32_t now) {
static bool noted = false;
if (noted || now < SafeMode::kStableAfterMs) return;
noted = true;
crash_report::noteStable(nvs);
}
static void loopSafeMode() {
uint32_t now = millis();
serialCommands();
remoteCommands();
services.tick(now);
bus.dispatch();
noteStableOnce(now);
if (update->phase() == UpdateService::Phase::Installed) {
screen.renderUpdate("Restarting", "into " + update->incomingVersion(), -1);
delay(800);
ESP.restart();
}
static uint32_t lastDraw = 0;
if (update->phase() == UpdateService::Phase::Receiving) {
screen.renderUpdate("Firmware update", "Receiving " + update->incomingVersion(), update->percent());
} else if (now - lastDraw > 2000 || !lastDraw) {
lastDraw = now;
bool up = wifi->state() == WifiController::State::Connected;
screen.renderUpdate("Safe Mode", up ? "Updates: " + wifi->ip() + ":3232" : "Waiting for Wi-Fi", -1);
}
delay(10);
}
void loop() {
if (safeMode) return loopSafeMode();
#ifdef RORO_TEST_CRASH
// Test builds only (never in a release): crash during Probation to exercise Rollback.
if (millis() > 5000) abort();
@@ -420,12 +516,8 @@ void loop() {
uint32_t now = millis();
serialCommands();
#ifdef RORO_DEBUG
for (std::string remote; debugConsole->takeCommand(remote);) {
console.printf("> %s\n", remote.c_str()); // so the transcript reads the same on both ends
runCommand(remote.c_str());
}
#endif
remoteCommands();
noteStableOnce(now);
uploadStep();
printListingWhenReady();
M5Cardputer.update();