From d3948ccc3a17b9f22c2f96644f5b406e736fe6d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Martin?= Date: Fri, 2 Oct 2026 14:56:06 +0200 Subject: [PATCH] M1 step 3: Log writing and Storage Clean-up - lib/storage_model (host-tested): FAT-safe names, daily Log paths, dates from Log/Capture file names, CleanupPlan by category and age, byte formatting - StorageService does all card I/O on its task: queued Log lines are written in batches each second (dropped while Logs are paused), plus file listing and deletion jobs - Settings > Storage moves into StoragePage: usage, Clean up (category, age with size preview, confirmation), Erase SD card - Clock: local date for Log names - Serial: log , sd list Verified on the device: lines land in /irc/dev/#test/2026-10-02.log with folders created as needed. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT --- README.md | 2 + lib/services/src/clock_model.cpp | 9 + lib/services/src/clock_model.h | 1 + lib/storage_model/src/cleanup_plan.cpp | 53 ++++++ lib/storage_model/src/cleanup_plan.h | 40 ++++ lib/storage_model/src/storage_paths.cpp | 67 +++++++ lib/storage_model/src/storage_paths.h | 22 +++ src/apps/settings_app.cpp | 51 +----- src/apps/settings_app.h | 10 +- src/apps/storage_page.cpp | 171 ++++++++++++++++++ src/apps/storage_page.h | 49 +++++ src/main.cpp | 36 +++- src/services/clock_service.h | 5 + src/services/storage_service.cpp | 159 ++++++++++++++-- src/services/storage_service.h | 49 ++++- test/test_cleanup_plan/test_cleanup_plan.cpp | 74 ++++++++ test/test_clock/test_clock.cpp | 7 + .../test_storage_paths/test_storage_paths.cpp | 54 ++++++ 18 files changed, 785 insertions(+), 74 deletions(-) create mode 100644 lib/storage_model/src/cleanup_plan.cpp create mode 100644 lib/storage_model/src/cleanup_plan.h create mode 100644 lib/storage_model/src/storage_paths.cpp create mode 100644 lib/storage_model/src/storage_paths.h create mode 100644 src/apps/storage_page.cpp create mode 100644 src/apps/storage_page.h create mode 100644 test/test_cleanup_plan/test_cleanup_plan.cpp create mode 100644 test/test_storage_paths/test_storage_paths.cpp diff --git a/README.md b/README.md index 5383907..580ac2a 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,8 @@ sudo usermod -aG dialout "$USER" | `sound on` / `sound off` | Toggles the Sound setting (beep + LED) | | `short` / `normal` | Screen timeouts 5 s / 10 s, or 30 s / 60 s | | `wifi add ` | Adds a Saved Network (so credentials stay out of the repo) | +| `log ` | Appends a line to a test IRC Log (`/irc/dev/#test/.log`) | +| `sd list` | Lists the files of each Storage Clean-up category | | `wifi status` | Prints Wi-Fi state, network, signal, clock and free heap | `scripts/flash.sh` stops a running serial log first, since it would hold the port. diff --git a/lib/services/src/clock_model.cpp b/lib/services/src/clock_model.cpp index 4ef4544..61d315a 100644 --- a/lib/services/src/clock_model.cpp +++ b/lib/services/src/clock_model.cpp @@ -34,6 +34,15 @@ std::string ClockModel::formatLocalTime(int64_t utcSeconds) { return buf; } +std::string ClockModel::formatLocalDate(int64_t utcSeconds) { + time_t t = static_cast(utcSeconds); + struct tm local; + localtime_r(&t, &local); + char buf[12]; + std::snprintf(buf, sizeof(buf), "%04d-%02d-%02d", local.tm_year + 1900, local.tm_mon + 1, local.tm_mday); + return buf; +} + std::string ClockModel::formatAge(int64_t seconds) { char buf[24]; if (seconds < 60) return "now"; diff --git a/lib/services/src/clock_model.h b/lib/services/src/clock_model.h index adb4cbb..cb3abfe 100644 --- a/lib/services/src/clock_model.h +++ b/lib/services/src/clock_model.h @@ -21,6 +21,7 @@ class ClockModel { static void applyTimezone(const char* posixTz); static std::string formatLocalTime(int64_t utcSeconds); // "HH:MM" + static std::string formatLocalDate(int64_t utcSeconds); // "YYYY-MM-DD" static std::string formatAge(int64_t seconds); // "now", "5 min ago", "3 h ago", "2 d ago" private: diff --git a/lib/storage_model/src/cleanup_plan.cpp b/lib/storage_model/src/cleanup_plan.cpp new file mode 100644 index 0000000..045fe98 --- /dev/null +++ b/lib/storage_model/src/cleanup_plan.cpp @@ -0,0 +1,53 @@ +#include "cleanup_plan.h" + +#include + +#include "storage_paths.h" + +namespace roro { + +namespace { +int maxAgeDays(CleanupAge age) { + switch (age) { + case CleanupAge::OneMonth: return 30; + case CleanupAge::ThreeMonths: return 91; + case CleanupAge::SixMonths: return 182; + case CleanupAge::OneYear: return 365; + default: return -1; + } +} +} // namespace + +CleanupPlan CleanupPlan::make(const std::vector& files, CleanupAge age, int todayDay) { + CleanupPlan plan; + for (auto& f : files) { + if (f.path.rfind("/notes", 0) == 0) continue; // Notes are never cleaned up + bool selected = age == CleanupAge::Everything; + if (!selected) { + size_t slash = f.path.find_last_of('/'); + int day = storage::fileDay(slash == std::string::npos ? f.path : f.path.substr(slash + 1)); + selected = day >= 0 && todayDay - day > maxAgeDays(age); + } + if (selected) { + plan.paths.push_back(f.path); + plan.bytes += f.bytes; + } + } + return plan; +} + +std::string formatBytes(uint64_t bytes) { + const char* units[] = {"B", "KB", "MB", "GB"}; + double v = static_cast(bytes); + int u = 0; + while (v >= 1024 && u < 3) { + v /= 1024; + u++; + } + char buf[16]; + if (u == 0 || v >= 10) std::snprintf(buf, sizeof(buf), "%.0f %s", v, units[u]); + else std::snprintf(buf, sizeof(buf), "%.1f %s", v, units[u]); + return buf; +} + +} // namespace roro diff --git a/lib/storage_model/src/cleanup_plan.h b/lib/storage_model/src/cleanup_plan.h new file mode 100644 index 0000000..68c2605 --- /dev/null +++ b/lib/storage_model/src/cleanup_plan.h @@ -0,0 +1,40 @@ +#pragma once + +#include +#include +#include + +namespace roro { + +struct StoredFile { + std::string path; + uint64_t bytes; +}; + +// What Storage Clean-up offers (see CONTEXT.md). Notes are deliberately absent. +struct CleanupCategory { + const char* label; + const char* folder; +}; +inline const CleanupCategory kCleanupCategories[] = { + {"IRC logs", "/irc"}, + {"Probe-request logs", "/wifi/probes"}, + {"Wi-Fi captures", "/captures/wifi"}, +}; + +enum class CleanupAge { OneMonth, ThreeMonths, SixMonths, OneYear, Everything }; +inline const char* const kCleanupAgeLabels[] = {"Older than 1 month", "Older than 3 months", "Older than 6 months", + "Older than 1 year", "Everything"}; + +// The files a clean-up would delete, and how much space that frees. Dates come from file names; +// undated files are only removed by Everything. +struct CleanupPlan { + std::vector paths; + uint64_t bytes = 0; + + static CleanupPlan make(const std::vector& files, CleanupAge age, int todayDay); +}; + +std::string formatBytes(uint64_t bytes); + +} // namespace roro diff --git a/lib/storage_model/src/storage_paths.cpp b/lib/storage_model/src/storage_paths.cpp new file mode 100644 index 0000000..399ddb3 --- /dev/null +++ b/lib/storage_model/src/storage_paths.cpp @@ -0,0 +1,67 @@ +#include "storage_paths.h" + +#include + +namespace roro::storage { + +namespace { + +// Howard Hinnant's days_from_civil. +int daysFromCivil(int y, int m, int d) { + y -= m <= 2; + int era = (y >= 0 ? y : y - 399) / 400; + int yoe = y - era * 400; + int doy = (153 * (m + (m > 2 ? -3 : 9)) + 2) / 5 + d - 1; + int doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + return era * 146097 + doe - 719468; +} + +bool digits(const std::string& s, size_t pos, size_t n, int& out) { + if (pos + n > s.size()) return false; + out = 0; + for (size_t i = pos; i < pos + n; i++) { + if (!std::isdigit(static_cast(s[i]))) return false; + out = out * 10 + (s[i] - '0'); + } + return true; +} + +int validDay(int y, int m, int d) { + static const int kDays[] = {31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}; + if (y < 1970 || m < 1 || m > 12 || d < 1 || d > kDays[m - 1]) return -1; + return daysFromCivil(y, m, d); +} + +} // namespace + +std::string sanitize(const std::string& name) { + std::string out; + for (char c : name) { + bool forbidden = c == '\\' || c == '/' || c == ':' || c == '*' || c == '?' || c == '"' || c == '<' || + c == '>' || c == '|' || static_cast(c) < 0x20; + out += forbidden ? '_' : c; + } + if (out.empty() || out == ".") return "_"; + if (out == "..") return "_."; + return out; +} + +std::string dailyLogPath(std::initializer_list folders, const std::string& date) { + std::string path; + for (auto& f : folders) path += "/" + sanitize(f); + return path + "/" + (date.empty() ? "undated" : date) + ".log"; +} + +int fileDay(const std::string& name) { + int y, m, d; + if (digits(name, 0, 4, y) && name.size() > 10 && name[4] == '-' && digits(name, 5, 2, m) && name[7] == '-' && + digits(name, 8, 2, d)) + return validDay(y, m, d); + if (digits(name, 0, 4, y) && digits(name, 4, 2, m) && digits(name, 6, 2, d) && name.size() > 8 && name[8] == '-') + return validDay(y, m, d); + return -1; +} + +int dayFromUtc(int64_t utcSeconds) { return static_cast(utcSeconds / 86400); } + +} // namespace roro::storage diff --git a/lib/storage_model/src/storage_paths.h b/lib/storage_model/src/storage_paths.h new file mode 100644 index 0000000..061d070 --- /dev/null +++ b/lib/storage_model/src/storage_paths.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include +#include + +namespace roro::storage { + +// Makes one path component safe for FAT: forbidden characters become '_', never empty or a dot name. +std::string sanitize(const std::string& name); + +// "/irc/libera/#roro/2026-10-02.log" from {"irc", "libera", "#roro"} and a local date +// ("YYYY-MM-DD"); an empty date (clock not set yet) gives "undated.log". +std::string dailyLogPath(std::initializer_list folders, const std::string& date); + +// Day number (days since 1970-01-01) encoded in a Log ("2026-10-02.log") or Capture +// ("20261002-144500.pcap") file name, or -1 if the name carries no valid date. +int fileDay(const std::string& fileName); + +int dayFromUtc(int64_t utcSeconds); + +} // namespace roro::storage diff --git a/src/apps/settings_app.cpp b/src/apps/settings_app.cpp index 7ee76b8..062c5a8 100644 --- a/src/apps/settings_app.cpp +++ b/src/apps/settings_app.cpp @@ -14,7 +14,6 @@ using Row = SettingsMenu::Row; void SettingsApp::onEnter() { page_ = Page::Menu; - eraseDialog_.reset(); list_.setCount(menu_.count()); } @@ -28,7 +27,9 @@ bool SettingsApp::onKey(const KeyEvent& e) { case Page::Menu: return onMenuKey(e); case Page::Text: return onTextKey(e); case Page::Choice: return onChoiceKey(e); - case Page::Storage: return onStorageKey(e); + case Page::Storage: + if (!storagePage_.onKey(e)) page_ = Page::Menu; + return true; case Page::About: return onAboutKey(e); case Page::Wifi: if (!wifiPage_.onKey(e)) page_ = Page::Menu; @@ -66,7 +67,10 @@ bool SettingsApp::onMenuKey(const KeyEvent& e) { case Kind::Slider: break; case Kind::Page: switch (menu_.row(i)) { - case Row::Storage: page_ = Page::Storage; break; + case Row::Storage: + page_ = Page::Storage; + storagePage_.enter(); + break; case Row::Wifi: page_ = Page::Wifi; wifiPage_.enter(); @@ -112,21 +116,6 @@ bool SettingsApp::onChoiceKey(const KeyEvent& e) { return true; } -bool SettingsApp::onStorageKey(const KeyEvent& e) { - if (eraseDialog_) { - eraseDialog_->onKey(e); - if (eraseDialog_->result() == 1 && !d_.storage.requestFormat()) warn("A format is already running"); - if (eraseDialog_->result() != DialogModel::kPending) eraseDialog_.reset(); - return true; - } - if (e.key == Key::Select) { - if (d_.storage.state().present) eraseDialog_.reset(new DialogModel({"Cancel", "Erase"})); - else warn("No SD card"); - } - if (e.key == Key::Back) page_ = Page::Menu; - return true; -} - bool SettingsApp::onAboutKey(const KeyEvent& e) { if (e.key == Key::Back) page_ = Page::Menu; if (e.key == Key::Char && e.ch == 'w') d_.apps.open("demo"); // hidden widget demo @@ -142,25 +131,6 @@ void SettingsApp::update(uint32_t nowMs) { } } -std::vector SettingsApp::storageLines() const { - auto s = d_.storage.state(); - std::vector lines; - char buf[64]; - if (d_.storage.formatting()) { - lines.push_back("Erasing the SD card..."); - } else if (!s.present) { - lines.push_back("No SD card inserted."); - } else { - snprintf(buf, sizeof(buf), "SD card: %u MB, %d%% used", static_cast(s.totalBytes >> 20), s.usedPercent); - lines.push_back(buf); - lines.push_back(s.logsAllowed ? "Logs: recording" : "Logs: paused (over 90%)"); - lines.push_back(s.capturesAllowed ? "Captures: allowed" : "Captures: stopped (card full)"); - } - lines.push_back(""); - lines.push_back("Enter: erase SD card"); - return lines; -} - std::vector SettingsApp::aboutLines() const { std::vector lines; char buf[64]; @@ -210,12 +180,7 @@ void SettingsApp::draw(Canvas& c) { widgets::list(c, choices_, area, [&](int i) { return (i == current ? "* " : " ") + options[i]; }); break; } - case Page::Storage: - widgets::textLines(c, storageLines(), 0, area); - if (eraseDialog_) - widgets::dialog(c, "Erase SD card?", "Everything on the card is deleted. It becomes one FAT32 partition.", - *eraseDialog_); - break; + case Page::Storage: storagePage_.draw(c); break; case Page::About: widgets::textLines(c, aboutLines(), 0, area); break; case Page::Wifi: wifiPage_.draw(c); break; } diff --git a/src/apps/settings_app.h b/src/apps/settings_app.h index 8ad6e08..f822850 100644 --- a/src/apps/settings_app.h +++ b/src/apps/settings_app.h @@ -13,6 +13,7 @@ #include "services/battery_service.h" #include "services/clock_service.h" #include "services/storage_service.h" +#include "apps/storage_page.h" #include "apps/wifi_settings_page.h" #include "settings_menu.h" #include "ui/theme.h" @@ -34,7 +35,10 @@ struct SettingsAppDeps { class SettingsApp : public App { public: explicit SettingsApp(const SettingsAppDeps& deps) - : d_(deps), menu_(deps.settings), wifiPage_(deps.settings, deps.savedNetworks, deps.wifi, deps.bus) {} + : d_(deps), + menu_(deps.settings), + wifiPage_(deps.settings, deps.savedNetworks, deps.wifi, deps.bus), + storagePage_(deps.storage, deps.clock, deps.bus) {} void onEnter() override; bool onKey(const KeyEvent& e) override; void update(uint32_t nowMs) override; @@ -49,20 +53,18 @@ class SettingsApp : public App { bool onMenuKey(const KeyEvent& e); bool onTextKey(const KeyEvent& e); bool onChoiceKey(const KeyEvent& e); - bool onStorageKey(const KeyEvent& e); bool onAboutKey(const KeyEvent& e); void warn(const std::string& text); - std::vector storageLines() const; std::vector aboutLines() const; SettingsAppDeps d_; SettingsMenu menu_; WifiSettingsPage wifiPage_; + StoragePage storagePage_; Page page_ = Page::Menu; ListModel list_{theme::kContent.h / theme::kLineHeight}; ListModel choices_{theme::kContent.h / theme::kLineHeight}; LineEditor editor_{39}; - std::unique_ptr eraseDialog_; int editingRow_ = 0; uint32_t lastRefreshMs_ = 0; }; diff --git a/src/apps/storage_page.cpp b/src/apps/storage_page.cpp new file mode 100644 index 0000000..bd87d8f --- /dev/null +++ b/src/apps/storage_page.cpp @@ -0,0 +1,171 @@ +#include "storage_page.h" + +#include "ui/widgets.h" + +namespace roro { + +namespace { +constexpr int kAges = sizeof(kCleanupAgeLabels) / sizeof(kCleanupAgeLabels[0]); +constexpr int kCategories = sizeof(kCleanupCategories) / sizeof(kCleanupCategories[0]); +} // namespace + +void StoragePage::enter() { + view_ = View::Main; + dialog_.reset(); + main_.setCount(kRows); + main_.select(kCleanUp); +} + +void StoragePage::warn(const char* text) { + bus_.publish(Event::withText(EventType::Notification, text, static_cast(NotificationLevel::Warning))); +} + +uint64_t StoragePage::categoryBytes(int category) const { + uint64_t sum = 0; + if (category < static_cast(listing_.size())) + for (auto& f : listing_[category]) sum += f.bytes; + return sum; +} + +CleanupPlan StoragePage::planFor(int age) const { + int today = clock_.today(); + auto cutoff = static_cast(age); + // Without a clock only "Everything" can be judged. + if (today < 0 && cutoff != CleanupAge::Everything) return {}; + return CleanupPlan::make(listing_[categories_.selected()], cutoff, today); +} + +bool StoragePage::onKey(const KeyEvent& e) { + if (dialog_) { + dialog_->onKey(e); + int result = dialog_->result(); + if (result == 1) { + if (view_ == View::Main) { + storage_.requestFormat(); + } else { + storage_.requestDelete(pending_.paths); + enter(); + } + } + if (result != DialogModel::kPending) dialog_.reset(); + return true; + } + + switch (view_) { + case View::Main: + switch (e.key) { + case Key::Up: main_.up(); break; + case Key::Down: main_.down(); break; + case Key::Back: return false; + case Key::Select: + if (!storage_.state().present) { + warn("No SD card"); + } else if (main_.selected() == kCleanUp) { + storage_.requestListing(); + haveListing_ = false; + categories_.setCount(kCategories); + view_ = View::Categories; + } else if (main_.selected() == kErase) { + dialog_.reset(new DialogModel({"Cancel", "Erase"})); + } + break; + default: break; + } + return true; + + case View::Categories: + switch (e.key) { + case Key::Up: categories_.up(); break; + case Key::Down: categories_.down(); break; + case Key::Back: view_ = View::Main; break; + case Key::Select: + if (!haveListing_) break; + if (listing_[categories_.selected()].empty()) { + warn("Nothing to clean up here"); + break; + } + ages_.setCount(kAges); + view_ = View::Ages; + break; + default: break; + } + return true; + + case View::Ages: + switch (e.key) { + case Key::Up: ages_.up(); break; + case Key::Down: ages_.down(); break; + case Key::Back: view_ = View::Categories; break; + case Key::Select: + pending_ = planFor(ages_.selected()); + if (pending_.paths.empty()) warn(clock_.today() < 0 ? "Clock not set: only Everything works" + : "No files that old"); + else dialog_.reset(new DialogModel({"Cancel", "Delete"})); + break; + default: break; + } + return true; + } + return true; +} + +void StoragePage::draw(Canvas& c) { + const auto& area = theme::kContent; + if (!haveListing_ && storage_.listingReady()) { + listing_ = storage_.listing(); + haveListing_ = true; + } + + switch (view_) { + case View::Main: { + auto s = storage_.state(); + widgets::list( + c, main_, area, + [](int i) -> std::string { + const char* labels[] = {"SD card", "Logs", "Captures", "Clean up", "Erase SD card"}; + return labels[i]; + }, + [&](int i) -> std::string { + if (storage_.formatting() && i == kCard) return "erasing..."; + if (!s.present) return i == kCard ? "none" : ""; + switch (i) { + case kCard: return formatBytes(s.totalBytes) + ", " + std::to_string(s.usedPercent) + "% used"; + case kLogs: return s.logsAllowed ? "recording" : "paused (over 90%)"; + case kCaptures: return s.capturesAllowed ? "allowed" : "stopped (full)"; + default: return ">"; + } + }); + if (dialog_) + widgets::dialog(c, "Erase SD card?", "Everything on the card is deleted. It becomes one FAT32 partition.", + *dialog_); + break; + } + case View::Categories: + if (!haveListing_) { + widgets::textLines(c, {"Listing files..."}, 0, area); + break; + } + widgets::list( + c, categories_, area, [](int i) { return std::string(kCleanupCategories[i].label); }, + [this](int i) { + return std::to_string(listing_[i].size()) + " files, " + formatBytes(categoryBytes(i)); + }); + break; + case View::Ages: + widgets::list( + c, ages_, area, [](int i) { return std::string(kCleanupAgeLabels[i]); }, + [this](int i) { + auto plan = planFor(i); + return plan.paths.empty() ? std::string("-") : formatBytes(plan.bytes); + }); + if (dialog_) { + std::string message = "Delete " + std::to_string(pending_.paths.size()) + " files from " + + kCleanupCategories[categories_.selected()].label + ", freeing " + + formatBytes(pending_.bytes) + "?"; + widgets::dialog(c, "Clean up", message, *dialog_); + } + break; + } +} + +} // namespace roro diff --git a/src/apps/storage_page.h b/src/apps/storage_page.h new file mode 100644 index 0000000..8de205c --- /dev/null +++ b/src/apps/storage_page.h @@ -0,0 +1,49 @@ +#pragma once + +#include +#include + +#include "cleanup_plan.h" +#include "dialog_model.h" +#include "event_bus.h" +#include "key_event.h" +#include "list_model.h" +#include "services/clock_service.h" +#include "services/storage_service.h" +#include "ui/canvas.h" +#include "ui/theme.h" + +namespace roro { + +// Settings → Storage: card usage, Storage Clean-up (category → age → confirm) and erasing the card. +class StoragePage { + public: + StoragePage(StorageService& storage, ClockService& clock, EventBus& bus) + : storage_(storage), clock_(clock), bus_(bus) {} + + void enter(); + bool onKey(const KeyEvent& e); // false: leave the page + void draw(Canvas& c); + + private: + enum class View { Main, Categories, Ages }; + enum Row { kCard, kLogs, kCaptures, kCleanUp, kErase, kRows }; + + CleanupPlan planFor(int age) const; + uint64_t categoryBytes(int category) const; + void warn(const char* text); + + StorageService& storage_; + ClockService& clock_; + EventBus& bus_; + View view_ = View::Main; + ListModel main_{theme::kContent.h / theme::kLineHeight}; + ListModel categories_{theme::kContent.h / theme::kLineHeight}; + ListModel ages_{theme::kContent.h / theme::kLineHeight}; + std::vector> listing_; + bool haveListing_ = false; + std::unique_ptr dialog_; + CleanupPlan pending_; +}; + +} // namespace roro diff --git a/src/main.cpp b/src/main.cpp index bc06c5b..48bedf8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -16,6 +16,7 @@ #include "services/storage_service.h" #include "services/wifi_service.h" #include "settings.h" +#include "storage_paths.h" #include "ui/notifier.h" #include "ui/screen.h" #include "version.h" @@ -31,7 +32,7 @@ static Screen screen; // Constructed in setup(), after the hardware and Settings are ready. static BatteryService* battery; -static StorageService* storage; +static StorageService* storageService; static PowerService* power; static ClockService* clockService; static SavedNetworks* savedNetworks; @@ -61,7 +62,7 @@ static StatusInfo currentStatus() { if (apps->foregroundTitle()) s.title = apps->foregroundTitle(); if (battery->estimator().hasReading()) s.batteryPercent = battery->estimator().percent(); s.clock = clockService->displayTime(); - auto storageState = storage->state(); + auto storageState = storageService->state(); s.sdPresent = storageState.present; s.sdLevel = storageState.level; s.compose = keyMapper.pendingCompose(); @@ -90,7 +91,7 @@ void setup() { settings.load(); battery = new BatteryService(bus); - storage = new StorageService(bus); + storageService = new StorageService(bus); power = new PowerService(settings); clockService = new ClockService(settings, bus); savedNetworks = new SavedNetworks(nvs); @@ -101,13 +102,13 @@ void setup() { services.add(*power); services.add(*clockService); services.add(*battery); - services.add(*storage); + services.add(*storageService); services.add(*wifi); apps = new AppManager(launcher); launcher.setManager(*apps); apps->registerApp({"settings", "Settings", false, - new SettingsApp({settings, bus, *apps, *battery, *storage, *clockService, *wifi, *savedNetworks})}); + new SettingsApp({settings, bus, *apps, *battery, *storageService, *clockService, *wifi, *savedNetworks})}); apps->registerApp({"demo", "Widget demo", true, new DemoApp(bus)}); apps->registerApp({"setup", "Setup", true, new SetupApp(settings, *apps)}); @@ -121,6 +122,21 @@ void setup() { } // Dev aid: serial commands to drive the UI without the keyboard. +static bool listingWanted = false; + +static void printListingWhenReady() { + if (!listingWanted || !storageService->listingReady()) return; + listingWanted = false; + auto listing = storageService->listing(); + for (size_t i = 0; i < listing.size(); i++) { + uint64_t bytes = 0; + for (auto& f : listing[i]) bytes += f.bytes; + Serial.printf("sd list: %-20s %u files, %s\n", kCleanupCategories[i].label, (unsigned)listing[i].size(), + formatBytes(bytes).c_str()); + for (auto& f : listing[i]) Serial.printf(" %s (%u B)\n", f.path.c_str(), (unsigned)f.bytes); + } +} + static void serialCommands() { static String line; while (Serial.available()) { @@ -147,6 +163,15 @@ static void serialCommands() { Serial.printf("wifi add: %s\n", error.empty() ? "ok" : error.c_str()); wifi->savedNetworksChanged(); } + if (line.startsWith("log ")) { // appends to a test IRC Log + std::string path = storage::dailyLogPath({"irc", "dev", "#test"}, clockService->localDate()); + storageService->appendLine(path, clockService->displayTime() + " " + line.substring(4).c_str()); + Serial.printf("log: queued to %s\n", path.c_str()); + } + if (line == "sd list") { + storageService->requestListing(); + listingWanted = true; + } if (line == "wifi status") Serial.printf("wifi: state %d ssid '%s' rssi %d clock %s heap %u min %u\n", (int)wifi->state(), wifi->ssid().c_str(), wifi->rssi(), clockService->displayTime().c_str(), ESP.getFreeHeap(), @@ -171,6 +196,7 @@ void loop() { uint32_t now = millis(); serialCommands(); + printListingWhenReady(); M5Cardputer.update(); if (M5Cardputer.Keyboard.isChange()) { bool swallow = M5Cardputer.Keyboard.isPressed() && power->onKey(now); // only woke the screen diff --git a/src/services/clock_service.h b/src/services/clock_service.h index a9d6222..f830862 100644 --- a/src/services/clock_service.h +++ b/src/services/clock_service.h @@ -23,6 +23,11 @@ class ClockService : public Service { bool set(int64_t utcSeconds, TimeSource source) { return model_.set(utcSeconds, source, millis()); } const ClockModel& model() const { return model_; } + // Local "YYYY-MM-DD" once set; otherwise empty (Logs then go to undated files). + std::string localDate() const { return model_.isSet() ? ClockModel::formatLocalDate(model_.utcNow(millis())) : ""; } + // Days since 1970-01-01 (UTC), or -1 if the clock isn't set. + int today() const { return model_.isSet() ? static_cast(model_.utcNow(millis()) / 86400) : -1; } + // "14:05" once set; otherwise "--:--". std::string displayTime() const { return model_.isSet() ? ClockModel::formatLocalTime(model_.utcNow(millis())) : "--:--"; diff --git a/src/services/storage_service.cpp b/src/services/storage_service.cpp index 08ed21c..cfacfc7 100644 --- a/src/services/storage_service.cpp +++ b/src/services/storage_service.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include "platform/pins.h" @@ -14,7 +15,7 @@ namespace roro { void StorageService::start() { if (task_) return; lock_ = xSemaphoreCreateMutex(); - xTaskCreate(taskEntry, "storage", 4096, this, 1, &task_); + xTaskCreate(taskEntry, "storage", 6144, this, 1, &task_); } void StorageService::stop() { @@ -26,23 +27,52 @@ void StorageService::stop() { } StorageState StorageService::state() const { - xSemaphoreTake(lock_, portMAX_DELAY); + lock(); StorageState copy = monitor_.state(); - xSemaphoreGive(lock_); + unlock(); return copy; } -void StorageService::taskEntry(void* self) { - auto* service = static_cast(self); - for (;;) { - if (service->formatRequested_) { - service->format(); - service->formatRequested_ = false; - } - service->poll(); - // Sleep until the next poll, or until a format request wakes us early. - ulTaskNotifyTake(pdTRUE, pdMS_TO_TICKS(kPollMs)); +void StorageService::appendLine(const std::string& path, const std::string& line) { + lock(); + size_t bytes = path.size() + line.size(); + bool accept = monitor_.state().logsAllowed && pendingBytes_ + bytes <= kMaxPendingBytes; + if (accept) { + pending_.emplace_back(path, line); + pendingBytes_ += bytes; + } else { + dropped_++; } + unlock(); +} + +void StorageService::requestListing() { + lock(); + listingRequested_ = true; + listingReady_ = false; + unlock(); + if (task_) xTaskNotifyGive(task_); +} + +bool StorageService::listingReady() const { + lock(); + bool ready = listingReady_; + unlock(); + return ready; +} + +std::vector> StorageService::listing() const { + lock(); + auto copy = listing_; + unlock(); + return copy; +} + +void StorageService::requestDelete(std::vector paths) { + lock(); + toDelete_.insert(toDelete_.end(), paths.begin(), paths.end()); + unlock(); + if (task_) xTaskNotifyGive(task_); } bool StorageService::requestFormat() { @@ -52,6 +82,38 @@ bool StorageService::requestFormat() { return true; } +void StorageService::taskEntry(void* self) { static_cast(self)->loop(); } + +void StorageService::loop() { + uint32_t wakes = kPollEvery; // poll right away + for (;;) { + if (formatRequested_) { + format(); + formatRequested_ = false; + wakes = kPollEvery; + } + if (wakes++ >= kPollEvery) { + wakes = 1; + poll(); + } + writePending(); + + lock(); + bool wantListing = listingRequested_; + std::vector deleting; + deleting.swap(toDelete_); + unlock(); + if (!deleting.empty()) { + remove(deleting); + poll(); + } + if (wantListing) list(); + + // Sleep until the next batch, or until a request wakes us early. + ulTaskNotifyTake(pdTRUE, pdMS_TO_TICKS(kWakeMs)); + } +} + void StorageService::format() { if (mounted_) SD.end(); mounted_ = false; @@ -91,9 +153,76 @@ void StorageService::poll() { uint64_t total = present ? SD.totalBytes() : 0; uint64_t used = present ? SD.usedBytes() : 0; - xSemaphoreTake(lock_, portMAX_DELAY); + lock(); monitor_.update(present, total, used); - xSemaphoreGive(lock_); + unlock(); +} + +void StorageService::writePending() { + lock(); + std::deque> batch; + batch.swap(pending_); + pendingBytes_ = 0; + unlock(); + if (batch.empty() || !mounted_) return; + + // Keep each file's lines in order while opening each file once. + std::stable_sort(batch.begin(), batch.end(), [](const auto& a, const auto& b) { return a.first < b.first; }); + File file; + std::string openPath; + for (auto& [path, line] : batch) { + if (path != openPath) { + if (file) file.close(); + file = SD.open(path.c_str(), FILE_APPEND, true); // true: create missing folders + openPath = path; + } + if (file) { + file.write(reinterpret_cast(line.data()), line.size()); + file.write('\n'); + } + } + if (file) file.close(); +} + +namespace { +void walk(File dir, std::vector& out) { + for (File f = dir.openNextFile(); f; f = dir.openNextFile()) { + if (f.isDirectory()) walk(f, out); + else out.push_back({f.path(), f.size()}); + } +} +} // namespace + +void StorageService::list() { + std::vector> result; + for (auto& category : kCleanupCategories) { + std::vector files; + if (mounted_) { + File dir = SD.open(category.folder); + if (dir && dir.isDirectory()) walk(dir, files); + } + result.push_back(std::move(files)); + } + lock(); + listing_ = std::move(result); + listingRequested_ = false; + listingReady_ = true; + unlock(); +} + +void StorageService::remove(const std::vector& paths) { + uint64_t freed = 0; + int failed = 0; + for (auto& p : paths) { + File f = SD.open(p.c_str()); + uint64_t size = f ? f.size() : 0; + if (f) f.close(); + if (SD.remove(p.c_str())) freed += size; + else failed++; + } + std::string text = "Freed " + formatBytes(freed) + (failed ? " (" + std::to_string(failed) + " failed)" : ""); + bus_.publish(Event::withText(EventType::Notification, text.c_str(), + static_cast(failed ? NotificationLevel::Warning : NotificationLevel::Info))); } } // namespace roro diff --git a/src/services/storage_service.h b/src/services/storage_service.h index b3caa9a..c280f1a 100644 --- a/src/services/storage_service.h +++ b/src/services/storage_service.h @@ -3,13 +3,20 @@ #include #include +#include +#include +#include +#include + +#include "cleanup_plan.h" #include "service.h" #include "storage_monitor.h" namespace roro { -// Mounts the microSD card (retrying while absent) and feeds its usage to the StorageMonitor. -// Runs on its own task: a failed mount blocks for seconds and must never stall the UI. +// Owns the microSD card. All card access happens on this Service's own task (the SD driver isn't +// safe across tasks, and a failed mount blocks for seconds): it mounts and watches usage, writes +// queued Log lines in batches, lists files for Storage Clean-up, deletes, and formats. class StorageService : public Service { public: explicit StorageService(EventBus& bus) : monitor_(bus), bus_(bus) {} @@ -19,24 +26,52 @@ class StorageService : public Service { StorageState state() const; - // Erases the whole card: one partition spanning the card, formatted FAT32. Runs on the storage - // task; the result arrives as a Notification. Returns false if a format is already pending. + // Queues one line for a Log file (created with its folders as needed). Dropped while Logs are + // paused (over 90 % full, or no card) or if too much is already waiting. + void appendLine(const std::string& path, const std::string& line); + uint32_t droppedLines() const { return dropped_; } + + // Storage Clean-up: list every file of every category, then delete a selection. + void requestListing(); + bool listingReady() const; + std::vector> listing() const; // one vector per kCleanupCategories entry + void requestDelete(std::vector paths); + + // Erases the whole card: one partition spanning the card, formatted FAT32. bool requestFormat(); bool formatting() const { return formatRequested_; } private: - static constexpr uint32_t kPollMs = 15000; + static constexpr uint32_t kWakeMs = 1000; + static constexpr uint32_t kPollEvery = 15; // wake-ups between usage checks + static constexpr size_t kMaxPendingBytes = 16 * 1024; + static void taskEntry(void* self); + void loop(); void poll(); bool mount(); void format(); + void writePending(); + void list(); + void remove(const std::vector& paths); + void lock() const { xSemaphoreTake(lock_, portMAX_DELAY); } + void unlock() const { xSemaphoreGive(lock_); } StorageMonitor monitor_; - bool mounted_ = false; EventBus& bus_; - volatile bool formatRequested_ = false; + bool mounted_ = false; TaskHandle_t task_ = nullptr; mutable SemaphoreHandle_t lock_ = nullptr; + + // Shared with other tasks, guarded by lock_. + std::deque> pending_; + size_t pendingBytes_ = 0; + uint32_t dropped_ = 0; + bool listingRequested_ = false; + bool listingReady_ = false; + std::vector> listing_; + std::vector toDelete_; + volatile bool formatRequested_ = false; }; } // namespace roro diff --git a/test/test_cleanup_plan/test_cleanup_plan.cpp b/test/test_cleanup_plan/test_cleanup_plan.cpp new file mode 100644 index 0000000..eadae6d --- /dev/null +++ b/test/test_cleanup_plan/test_cleanup_plan.cpp @@ -0,0 +1,74 @@ +#include + +#include "cleanup_plan.h" + +using namespace roro; + +void setUp() {} +void tearDown() {} + +static const int kToday = 20728; // 2026-10-02 + +static std::vector files() { + return { + {"/irc/libera/#roro/2026-10-01.log", 1000}, // 1 day old + {"/irc/libera/#roro/2026-08-01.log", 2000}, // 62 days + {"/irc/libera/#roro/2025-09-01.log", 4000}, // 396 days + {"/irc/libera/#roro/undated.log", 8000}, + }; +} + +void test_age_cutoffs_select_strictly_older_files() { + auto plan = CleanupPlan::make(files(), CleanupAge::OneMonth, kToday); + TEST_ASSERT_EQUAL(2, plan.paths.size()); + TEST_ASSERT_EQUAL(6000, plan.bytes); +} + +void test_a_year_keeps_everything_younger() { + auto plan = CleanupPlan::make(files(), CleanupAge::OneYear, kToday); + TEST_ASSERT_EQUAL(1, plan.paths.size()); + TEST_ASSERT_EQUAL_STRING("/irc/libera/#roro/2025-09-01.log", plan.paths[0].c_str()); +} + +void test_undated_files_only_go_with_everything() { + auto plan = CleanupPlan::make(files(), CleanupAge::Everything, kToday); + TEST_ASSERT_EQUAL(4, plan.paths.size()); + TEST_ASSERT_EQUAL(15000, plan.bytes); +} + +void test_nothing_to_delete_is_an_empty_plan() { + auto plan = CleanupPlan::make({{"/irc/a/b/2026-10-02.log", 10}}, CleanupAge::OneMonth, kToday); + TEST_ASSERT_TRUE(plan.paths.empty()); + TEST_ASSERT_EQUAL(0, plan.bytes); +} + +void test_notes_are_never_selected() { + auto plan = CleanupPlan::make({{"/notes/2020-01-01.log", 10}}, CleanupAge::Everything, kToday); + TEST_ASSERT_TRUE(plan.paths.empty()); +} + +void test_categories_cover_logs_and_captures_but_not_notes() { + bool notes = false; + for (auto& c : kCleanupCategories) notes |= std::string(c.folder) == "/notes"; + TEST_ASSERT_FALSE(notes); + TEST_ASSERT_TRUE(sizeof(kCleanupCategories) / sizeof(kCleanupCategories[0]) >= 3); +} + +void test_size_formatting() { + TEST_ASSERT_EQUAL_STRING("512 B", formatBytes(512).c_str()); + TEST_ASSERT_EQUAL_STRING("1.5 KB", formatBytes(1536).c_str()); + TEST_ASSERT_EQUAL_STRING("340 MB", formatBytes(340ull * 1024 * 1024).c_str()); + TEST_ASSERT_EQUAL_STRING("2.1 GB", formatBytes(2254857830ull).c_str()); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(test_age_cutoffs_select_strictly_older_files); + RUN_TEST(test_a_year_keeps_everything_younger); + RUN_TEST(test_undated_files_only_go_with_everything); + RUN_TEST(test_nothing_to_delete_is_an_empty_plan); + RUN_TEST(test_notes_are_never_selected); + RUN_TEST(test_categories_cover_logs_and_captures_but_not_notes); + RUN_TEST(test_size_formatting); + return UNITY_END(); +} diff --git a/test/test_clock/test_clock.cpp b/test/test_clock/test_clock.cpp index 197f391..4152297 100644 --- a/test/test_clock/test_clock.cpp +++ b/test/test_clock/test_clock.cpp @@ -61,6 +61,12 @@ void test_local_time_in_brussels_summer_and_winter() { TEST_ASSERT_EQUAL_STRING("13:00", ClockModel::formatLocalTime(k2026_01_15_1200Z).c_str()); } +void test_local_date_in_brussels() { + ClockModel::applyTimezone(kBrussels); + // 2026-07-01 23:30Z is already July 2nd in Brussels (UTC+2) + TEST_ASSERT_EQUAL_STRING("2026-07-02", ClockModel::formatLocalDate(k2026_07_01_1200Z + 11 * 3600 + 1800).c_str()); +} + int main() { UNITY_BEGIN(); RUN_TEST(test_clock_is_unset_until_a_source_sets_it); @@ -70,5 +76,6 @@ int main() { RUN_TEST(test_utc_now_survives_millis_wraparound); RUN_TEST(test_relative_age_formatting); RUN_TEST(test_local_time_in_brussels_summer_and_winter); + RUN_TEST(test_local_date_in_brussels); return UNITY_END(); } diff --git a/test/test_storage_paths/test_storage_paths.cpp b/test/test_storage_paths/test_storage_paths.cpp new file mode 100644 index 0000000..c98e998 --- /dev/null +++ b/test/test_storage_paths/test_storage_paths.cpp @@ -0,0 +1,54 @@ +#include + +#include "storage_paths.h" + +using namespace roro; + +void setUp() {} +void tearDown() {} + +void test_sanitize_replaces_characters_fat_forbids() { + TEST_ASSERT_EQUAL_STRING("a_b_c_d_e_f_g_h_i", storage::sanitize("a\\b/c:d*e?f\"gi").c_str()); + TEST_ASSERT_EQUAL_STRING("#roro", storage::sanitize("#roro").c_str()); + TEST_ASSERT_EQUAL_STRING("Cl\xC3\xA9ment", storage::sanitize("Cl\xC3\xA9ment").c_str()); +} + +void test_sanitize_never_returns_empty_or_dot_names() { + TEST_ASSERT_EQUAL_STRING("_", storage::sanitize("").c_str()); + TEST_ASSERT_EQUAL_STRING("_", storage::sanitize(".").c_str()); + TEST_ASSERT_EQUAL_STRING("_.", storage::sanitize("..").c_str()); +} + +void test_daily_log_path() { + TEST_ASSERT_EQUAL_STRING("/irc/libera/#roro/2026-10-02.log", + storage::dailyLogPath({"irc", "libera", "#roro"}, "2026-10-02").c_str()); +} + +void test_daily_log_path_sanitizes_each_part() { + TEST_ASSERT_EQUAL_STRING("/irc/irc.libera.chat_6697/nick_name/undated.log", + storage::dailyLogPath({"irc", "irc.libera.chat:6697", "nick/name"}, "").c_str()); +} + +void test_parse_date_from_log_and_capture_names() { + TEST_ASSERT_EQUAL(20728, storage::fileDay("2026-10-02.log")); // days since 1970-01-01 + TEST_ASSERT_EQUAL(20728, storage::fileDay("20261002-144500.pcap")); + TEST_ASSERT_EQUAL(-1, storage::fileDay("undated.log")); + TEST_ASSERT_EQUAL(-1, storage::fileDay("notes.txt")); + TEST_ASSERT_EQUAL(-1, storage::fileDay("2026-13-02.log")); +} + +void test_day_from_utc_seconds() { + TEST_ASSERT_EQUAL(20728, storage::dayFromUtc(1790942400)); // 2026-10-02 12:00Z + TEST_ASSERT_EQUAL(0, storage::dayFromUtc(0)); +} + +int main() { + UNITY_BEGIN(); + RUN_TEST(test_sanitize_replaces_characters_fat_forbids); + RUN_TEST(test_sanitize_never_returns_empty_or_dot_names); + RUN_TEST(test_daily_log_path); + RUN_TEST(test_daily_log_path_sanitizes_each_part); + RUN_TEST(test_parse_date_from_log_and_capture_names); + RUN_TEST(test_day_from_utc_seconds); + return UNITY_END(); +}