twisla/roro9stack
Public Access
1
0
Fork 0
Code Issues 48 Pull Requests Actions Packages Releases 16 Activity

Gemini: read big pages from the card as you scroll (Q88)

Browse Source 
A page larger than memory allows is now windowed instead of cut: one
pass over its file (cache or Saved Page) counts lines, indexes every
64th (offset, and the preformatted state there in bit 31: about 1 KB
for a 1 MB page) and loads the first window. Scrolling near either end
reads the next or previous window on the Gemini task; the line on top
of the screen stays put, the scrollbar follows the whole page, and Tab
at a window's edge pages on instead of wrapping. Window budgets count
the memory the old window gives back.

Display pages alternate between two cache files so the one on screen
is never overwritten by the next fetch; jobs use a third.

On the device, Cosmos with IRC connected: 226 of 419 lines at first,
then lines 192-419, back to 64 and 0 while scrolling. `key space` added
to the console's key command.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT
This commit is contained in:
twisla
2026-10-05 01:44:01 +02:00
co-authored by Claude Opus 5.5
parent fd306d5013
commit 9ca9a16de8
6 changed files with 259 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/gemini_service.cpp
+145 -34
View File
@@ -82,6 +82,17 @@ void listSaved(const std::string& dir, std::vector<SavedEntry>& out) {
} // namespace
const char* GeminiService::slotPath(int slot) {
static const char* const kPaths[] = {"/gemini/cache/page0.gmi", "/gemini/cache/page1.gmi", "/gemini/cache/work.gmi"};
return kPaths[slot];
}
const char* GeminiService::cacheOf(const std::string& url) const {
for (int s = 0; s < 3; s++)
if (!url.empty() && slotUrl_[s] == url) return slotPath(s);
return nullptr;
}
GeminiService::GeminiService(KeyValueStore& store, StorageService& storage, ClockService& clock, EventBus& bus)
: store_(store), storage_(storage), clock_(clock), bus_(bus), lock_(xSemaphoreCreateMutex()) {}
@@ -173,6 +184,18 @@ bool GeminiService::download(const std::string& url) {
return start(Job::Download);
}
bool GeminiService::loadWindow(const GeminiPage& page, size_t firstLine) {
if (busy_ || page.backing.empty()) return false;
windowPage_ = GeminiPage();
windowPage_.backing = page.backing;
windowPage_.totalLines = page.totalLines;
windowPage_.lineIndex = page.lineIndex;
windowLine_ = firstLine;
// The App drops its current window once the new one is in: count that memory as coming back.
windowReleasing_ = page.text.bytes() + page.text.lineCount() * 8;
return start(Job::Window);
}
bool GeminiService::takeResult(GeminiPage& out) {
if (!ready_) return false;
xSemaphoreTake(lock_, portMAX_DELAY);
@@ -234,7 +257,7 @@ void GeminiService::run() {
for (const std::string& link : links_) {
if (cancel_) break;
GeminiPage page;
fetchFollowing(link, page, false);
fetchFollowing(link, page, false, kWorkSlot);
bool text = page.outcome == GeminiPage::Outcome::Ok &&
page.header.category() == gemini::Category::Success && page.header.mimeType() == "text/gemini";
if (text && saveCached(link).empty()) saved++;
@@ -256,13 +279,36 @@ void GeminiService::run() {
break;
}
GeminiPage page;
fetchFollowing(from, page, false);
fetchFollowing(from, page, false, kWorkSlot);
std::string why = page.outcome != GeminiPage::Outcome::Ok ? page.error
: page.header.category() != gemini::Category::Success ? page.header.meta
: saveCached(from);
finishWithMessage(why.empty() ? "Refreshed" : "Not refreshed: " + why);
break;
}
case Job::Window: {
// Q86 both ways: old and new together above the transient floor for a moment, and the
// new one alone (once the old is dropped) leaving the steady floor.
size_t free = esp_get_free_heap_size();
size_t transient = free > kTransientFloor + TextBuffer::kChunk ? free - kTransientFloor - TextBuffer::kChunk : 0;
size_t steady = free + windowReleasing_ > kSteadyFloor + TextBuffer::kChunk
? free + windowReleasing_ - kSteadyFloor - TextBuffer::kChunk
: 0;
size_t budget = std::min(transient, steady);
GeminiPage page = std::move(windowPage_);
page.windowStart = windowLine_ - windowLine_ % kIndexEvery;
page.windowed = true;
page.windowOf = page.backing;
page.outcome = GeminiPage::Outcome::Ok;
readWindow(page, page.windowStart, std::min(budget, kMaxBody));
console.printf("gemini: window from line %u: %u lines\n", (unsigned)page.windowStart,
(unsigned)page.text.lineCount());
xSemaphoreTake(lock_, portMAX_DELAY);
result_ = std::move(page);
ready_ = true;
xSemaphoreGive(lock_);
break;
}
case Job::Delete: {
bool removed = false;
bool ran = storage_.runAndWait([&]() { removed = SD.remove(arg_.c_str()); });
@@ -295,7 +341,7 @@ void GeminiService::run() {
if (name.empty()) name = "download";
std::string to = "/gemini/downloads/" + storage::sanitize(name);
bool copied = false;
if (cacheHolds(arg_)) storage_.runAndWait([&]() { copied = copyFile(kCachePath, to, ""); });
if (const char* from = cacheOf(arg_)) storage_.runAndWait([&]() { copied = copyFile(from, to, ""); });
finishWithMessage(copied ? "Saved to " + to : "Not saved (no card, or the page is gone)");
break;
}
@@ -324,7 +370,12 @@ void GeminiService::runFetch(GeminiPage& page) {
page.url = url;
page.error = "Not saved, and offline";
} else
fetchFollowing(url, page, true);
{
int slot = nextDisplay_;
fetchFollowing(url, page, true, slot);
if (page.outcome == GeminiPage::Outcome::Ok && !page.cachePath.empty())
nextDisplay_ = 1 - slot; // keep this one: it backs the page on screen
}
}
}
page.ms = millis() - started;
@@ -332,7 +383,7 @@ void GeminiService::runFetch(GeminiPage& page) {
}
// One request, following up to kMaxRedirects redirects.
void GeminiService::fetchFollowing(const std::string& first, GeminiPage& page, bool loadIntoRam) {
void GeminiService::fetchFollowing(const std::string& first, GeminiPage& page, bool loadIntoRam, int slot) {
std::string url = first;
size_t free = esp_get_free_heap_size();
if (free < kStartFloor) { // Q86, for every fetch: pages, saves, refreshes
@@ -342,7 +393,7 @@ void GeminiService::fetchFollowing(const std::string& first, GeminiPage& page, b
return;
}
for (int hop = 0;; hop++) {
fetchOne(url, page, loadIntoRam);
fetchOne(url, page, loadIntoRam, slot);
if (page.outcome != GeminiPage::Outcome::Ok || page.header.category() != gemini::Category::Redirect) break;
std::string next = gemini::resolve(url, page.header.meta);
if (!gemini::isGemini(next)) {
@@ -401,7 +452,7 @@ void GeminiService::openSaved(const std::string& path, GeminiPage& page) {
page.savedFrom = rest.substr(0, on);
page.savedOn = on == std::string::npos ? "" : rest.substr(on + 4);
}
loadFromCard(page, esp_get_free_heap_size(), path.c_str());
loadFromCard(page, esp_get_free_heap_size(), path);
page.outcome = GeminiPage::Outcome::Ok;
}
@@ -459,12 +510,14 @@ void GeminiService::composeStart(GeminiPage& page) {
std::string GeminiService::saveCached(const std::string& url) {
if (!gemini::isGemini(url)) return "Only Gemini pages can be saved";
if (!storage_.state().present) return "No SD card";
if (!cacheHolds(url)) {
const char* from = cacheOf(url);
if (!from) {
GeminiPage page;
fetchFollowing(url, page, false);
fetchFollowing(url, page, false, kWorkSlot);
if (page.outcome != GeminiPage::Outcome::Ok) return page.error;
if (page.header.category() != gemini::Category::Success) return page.header.meta;
if (!cacheHolds(page.url)) return "Not saved";
from = cacheOf(page.url);
if (!from) return "Not saved";
}
std::string to = gemini::savedPath(url);
bool existed = false, ok = false;
@@ -472,14 +525,14 @@ std::string GeminiService::saveCached(const std::string& url) {
std::string header = std::string(kSavedMark) + url + " on " + now();
storage_.runAndWait([&]() {
existed = SD.exists(to.c_str());
ok = copyFile(kCachePath, to, header);
ok = copyFile(from, to, header);
});
if (!ok) return "Writing to the SD card failed";
replaced_ = existed; // Q82: saving again replaces it, and says so
return "";
}
void GeminiService::fetchOne(const std::string& url, GeminiPage& page, bool loadIntoRam) {
void GeminiService::fetchOne(const std::string& url, GeminiPage& page, bool loadIntoRam, int slot) {
page.outcome = GeminiPage::Outcome::Failed;
page.text.clear();
page.truncated = false;
@@ -553,11 +606,11 @@ void GeminiService::fetchOne(const std::string& url, GeminiPage& page, bool load
// With a card, the body goes there while the connection is open (only 4 KB in RAM),
// and comes back into RAM once the connection's memory is free again.
if (storage_.state().present) {
cachedUrl_.clear();
if (!receiveToCard(tls, buf + i, n - i, page)) return;
slotUrl_[slot].clear();
if (!receiveToCard(tls, buf + i, n - i, page, slot)) return;
tls.stop();
cachedUrl_ = url;
if (loadIntoRam) loadFromCard(page, freeBefore, kCachePath);
slotUrl_[slot] = url;
if (loadIntoRam) loadFromCard(page, freeBefore, slotPath(slot));
page.outcome = GeminiPage::Outcome::Ok;
return;
}
@@ -589,18 +642,18 @@ void GeminiService::fetchOne(const std::string& url, GeminiPage& page, bool load
page.outcome = GeminiPage::Outcome::Ok;
}
// Streams the rest of the body to kCachePath in 1 KB pieces, each written by the storage task:
// Streams the rest of the body to a cache file in 1 KB pieces, each written by the storage task:
// with IRC's TLS connection and this one both open, every KB of RAM counts (Q86).
bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page) {
bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page, int slot) {
struct Cache {
File file;
};
auto cache = std::make_shared<Cache>();
bool opened = storage_.runAndWait([cache]() {
bool opened = storage_.runAndWait([cache, slot]() {
SD.mkdir("/gemini");
SD.mkdir("/gemini/cache");
cache->file = SD.open(kCachePath, FILE_WRITE);
cache->file = SD.open(slotPath(slot), FILE_WRITE);
});
if (!opened || !cache->file) {
page.error = "Can't write to the SD card";
@@ -642,37 +695,95 @@ bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, s
storage_.runAndWait([cache]() { cache->file.close(); });
if (cancel_) page.error = "Cancelled";
else if (!ok) page.error = "Writing to the SD card failed";
else page.cachePath = kCachePath;
else page.cachePath = slotPath(slot);
return !cancel_ && ok;
}
// Back into RAM, now that the connection's memory is free again: as much as the steady floor
// allows (Q86); the rest stays on the card.
void GeminiService::loadFromCard(GeminiPage& page, size_t freeBefore, const char* path) {
// Back into RAM, now that the connection's memory is free again: one pass over the file counts
// its lines and indexes every kIndexEvery-th, and loads lines while the steady floor allows (Q86).
// If they don't all fit, the page is windowed: the App asks for other windows as you scroll.
void GeminiService::loadFromCard(GeminiPage& page, size_t freeBefore, const std::string& path) {
size_t budget = freeBefore > kSteadyFloor + TextBuffer::kChunk ? freeBefore - kSteadyFloor - TextBuffer::kChunk : 0;
budget = std::min(budget, kMaxBody);
page.backing = path;
page.lineIndex.clear();
page.totalLines = 0;
page.windowStart = 0;
bool loading = true;
storage_.runAndWait([&]() {
File f = SD.open(path);
File f = SD.open(path.c_str());
if (!f) return;
std::string line;
uint32_t offset = 0, lineStart = 0;
bool pre = false;
auto endLine = [&]() {
if (page.totalLines % kIndexEvery == 0) page.lineIndex.push_back(lineStart | (pre ? 0x80000000u : 0));
if (line.compare(0, 3, "```") == 0) pre = !pre;
if (loading) {
if (page.text.bytes() + page.text.lineCount() * 8 + line.size() + 1 > budget) loading = false;
else {
line += '\n';
page.text.append(line.data(), line.size());
}
}
page.totalLines++;
line.clear();
};
char buf[512];
int n;
while ((n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
size_t cost = page.text.bytes() + page.text.lineCount() * 8 + n;
if (cost > budget) {
page.truncated = true;
if (page.truncatedWhy.empty())
page.truncatedWhy = "only part of it fits in memory; the whole page is on the card";
break;
for (int i = 0; i < n; i++, offset++) {
if (buf[i] == '\n') {
endLine();
lineStart = offset + 1;
} else if (line.size() < kMaxLineBytes) line += buf[i];
}
page.text.append(buf, n);
}
if (!line.empty()) endLine();
f.close();
});
page.text.finish();
page.windowed = !loading;
if (page.windowed)
console.printf("gemini: %s is windowed: %u of %u lines in memory\n", path.c_str(),
(unsigned)page.text.lineCount(), (unsigned)page.totalLines);
}
// Lines from `firstLine` on, from the page's file, as many as `budget` allows.
void GeminiService::readWindow(GeminiPage& page, size_t firstLine, size_t budget) {
size_t entry = firstLine / kIndexEvery;
if (entry >= page.lineIndex.size()) return;
storage_.runAndWait([&]() {
File f = SD.open(page.backing.c_str());
if (!f || !f.seek(page.lineIndex[entry] & 0x7FFFFFFFu)) return;
size_t lineNo = entry * kIndexEvery;
std::string line;
char buf[512];
int n;
bool done = false;
while (!done && (n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
for (int i = 0; i < n && !done; i++) {
if (buf[i] != '\n') {
if (line.size() < kMaxLineBytes) line += buf[i];
continue;
}
if (lineNo >= firstLine) {
if (page.text.bytes() + page.text.lineCount() * 8 + line.size() + 1 > budget) done = true;
else {
line += '\n';
page.text.append(line.data(), line.size());
}
}
lineNo++;
line.clear();
}
}
if (!done && !line.empty() && lineNo >= firstLine) page.text.append(line.data(), line.size());
f.close();
});
page.text.finish();
}
void GeminiService::report(const GeminiPage& page, size_t heapBefore, size_t heapLowest) {
console.printf("gemini: %s -> %s, %lu ms\n", page.requested.c_str(), page.url.c_str(), (unsigned long)page.ms);
if (page.outcome != GeminiPage::Outcome::Ok) console.printf("gemini: %s\n", page.error.c_str());
src/services/gemini_service.h
+27 -8
View File
@@ -36,6 +36,14 @@ struct GeminiPage {
std::string savedFrom; // a Saved Page: the URL it was saved from (links resolve against it)
std::string savedOn; // and when, "2026-10-05 01:20"
uint32_t ms = 0;
// A page bigger than memory allows is read from its file as you scroll: `text` then holds
// lines [windowStart, windowStart + text.lineCount()) of totalLines. lineIndex has where every
// 64th line starts in the file (bit 31: inside a preformatted block there).
std::string backing; // the file the page is in (a cache file or a Saved Page)
bool windowed = false;
size_t totalLines = 0, windowStart = 0;
std::vector<uint32_t> lineIndex;
std::string windowOf; // a window result: the backing file it belongs to
// What relative links resolve against.
const std::string& base() const { return savedFrom.empty() ? url : savedFrom; }
@@ -54,7 +62,8 @@ class GeminiService {
static constexpr int kMaxRedirects = 5; // Q72
static constexpr size_t kMaxOnCard = 1024 * 1024; // a text page larger than this is cut
static constexpr int kMaxLinkedPages = 30; // Q83
static constexpr const char* kCachePath = "/gemini/cache/page.gmi";
static constexpr size_t kIndexEvery = 64; // lines between lineIndex entries
static constexpr size_t kMaxLineBytes = 8192; // longer lines are cut
static constexpr const char* kBookmarks = "/gemini/bookmarks.gmi";
static constexpr const char* kStartUrl = "about:start";
@@ -70,6 +79,8 @@ class GeminiService {
bool deleteSaved(const std::string& savedPath); // d on a Saved Page (Q85)
bool addBookmark(const std::string& url, const std::string& title); // b (Q77)
bool download(const std::string& url); // s on a non-text page (Q73)
// The window of a big page starting at `firstLine` (a multiple of kIndexEvery), from its file.
bool loadWindow(const GeminiPage& page, size_t firstLine);
bool busy() const { return busy_; }
bool takeResult(GeminiPage& out);
@@ -82,22 +93,27 @@ class GeminiService {
bool fetchToConsole(const std::string& url);
private:
enum class Job { Fetch, Save, SaveWithLinks, Refresh, Delete, Bookmark, Download };
enum class Job { Fetch, Save, SaveWithLinks, Refresh, Delete, Bookmark, Download, Window };
// Cache files: two take turns for pages on screen (the one shown is never overwritten by the
// next), one for jobs (saving linked pages, refreshing).
static constexpr int kWorkSlot = 2;
static const char* slotPath(int slot);
bool start(Job job);
static void taskEntry(void* self);
void run();
void runFetch(GeminiPage& page);
void fetchFollowing(const std::string& url, GeminiPage& page, bool loadIntoRam);
void fetchOne(const std::string& url, GeminiPage& page, bool loadIntoRam);
bool receiveToCard(::NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page);
void loadFromCard(GeminiPage& page, size_t freeBefore, const char* path);
void fetchFollowing(const std::string& url, GeminiPage& page, bool loadIntoRam, int slot);
void fetchOne(const std::string& url, GeminiPage& page, bool loadIntoRam, int slot);
bool receiveToCard(::NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page, int slot);
void loadFromCard(GeminiPage& page, size_t freeBefore, const std::string& path);
void readWindow(GeminiPage& page, size_t firstLine, size_t budget);
void openSaved(const std::string& path, GeminiPage& page);
std::string savedFirstLine(const std::string& path);
std::string savedFrom(const std::string& path);
void composeStart(GeminiPage& page);
std::string saveCached(const std::string& url); // "" or what went wrong
bool cacheHolds(const std::string& url) const { return cachedUrl_ == url; }
const char* cacheOf(const std::string& url) const; // the cache file holding url's body, or nullptr
void finishWithMessage(const std::string& text);
void notify(const std::string& text);
void report(const GeminiPage& page, size_t heapBefore, size_t heapLowest);
@@ -115,7 +131,10 @@ class GeminiService {
bool preferSaved_ = false;
GeminiPage result_;
std::string message_;
std::string cachedUrl_; // the URL whose body is in kCachePath
std::string slotUrl_[3]; // the URL whose body each cache file holds
int nextDisplay_ = 0;
size_t windowLine_ = 0, windowReleasing_ = 0;
GeminiPage windowPage_;
bool replaced_ = false; // the last save replaced an older copy
volatile bool busy_ = false, ready_ = false, messageReady_ = false, cancel_ = false, toConsole_ = false;
size_t lowest_ = 0;