twisla/roro9stack
Public Access
1
0
Fork 0
Code Issues 51 Pull Requests 1 Actions 2 Packages Releases 15 Activity
Files
dafbdaddf6dc2b8b748d9f2fb8c6134f92389ed8
roro9stack/src/services/gemini_service.cpp
T
twislaandClaude Opus 5.5 9ca9a16de8 Gemini: read big pages from the card as you scroll (Q88) 
A page larger than memory allows is now windowed instead of cut: one
pass over its file (cache or Saved Page) counts lines, indexes every
64th (offset, and the preformatted state there in bit 31: about 1 KB
for a 1 MB page) and loads the first window. Scrolling near either end
reads the next or previous window on the Gemini task; the line on top
of the screen stays put, the scrollbar follows the whole page, and Tab
at a window's edge pages on instead of wrapping. Window budgets count
the memory the old window gives back.

Display pages alternate between two cache files so the one on screen
is never overwritten by the next fetch; jobs use a third.

On the device, Cosmos with IRC connected: 226 of 419 lines at first,
then lines 192-419, back to 64 and 0 while scrolling. `key space` added
to the console's key command.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT
2026-10-05 01:44:01 +02:00

804 lines
31 KiB
C++
Raw Blame History

#include "services/gemini_service.h"
#include <NetworkClientSecure.h>
#include <SD.h>
#include <WiFi.h>
#include <esp_heap_caps.h>
#include <algorithm>
#include <map>
#include "gemini_url.h"
#include "gemtext.h"
#include "platform/console.h"
#include "sha256.h"
#include "storage_paths.h"
namespace roro {
using gemini::TextBuffer;
namespace {
constexpr uint32_t kIdleTimeoutMs = 15000;
constexpr const char* kSavedRoot = "/gemini/saved";
constexpr const char* kSavedMark = "> Saved from "; // first line of a Saved Page (Q82)
constexpr const char* kDefaults =
"## Start here\n"
"=> gemini://geminiprotocol.net/ Project Gemini\n"
"=> gemini://kennedy.gemi.dev/ Kennedy: search Geminispace\n"
"=> gemini://skyjake.fi/~Cosmos/ Cosmos: what's new on Geminispace\n"
"=> gemini://bbs.geminispace.org/ Bubble: a bulletin board\n";
std::string hex(const uint8_t* data, size_t len) {
static const char* digits = "0123456789abcdef";
std::string out;
for (size_t i = 0; i < len; i++) {
out += digits[data[i] >> 4];
out += digits[data[i] & 15];
}
return out;
}
// Folders for a file path, one at a time (SD.mkdir makes one level).
void makeParents(const std::string& path) {
for (size_t slash = path.find('/', 1); slash != std::string::npos; slash = path.find('/', slash + 1)) {
std::string dir = path.substr(0, slash);
if (!SD.exists(dir.c_str())) SD.mkdir(dir.c_str());
}
}
// Copies `from` to `to`, optionally after a first line. On the storage task.
bool copyFile(const char* from, const std::string& to, const std::string& firstLine) {
File in = SD.open(from);
if (!in) return false;
makeParents(to);
File out = SD.open(to.c_str(), FILE_WRITE);
if (!out) return false;
bool ok = firstLine.empty() || out.print((firstLine + "\n").c_str()) > 0;
uint8_t buf[512];
int n;
while (ok && (n = in.read(buf, sizeof buf)) > 0) ok = out.write(buf, n) == static_cast<size_t>(n);
out.close();
in.close();
if (!ok) SD.remove(to.c_str());
return ok;
}
struct SavedEntry {
std::string path; // /gemini/saved/host/…/name.gmi
time_t when;
};
void listSaved(const std::string& dir, std::vector<SavedEntry>& out) {
File d = SD.open(dir.c_str());
if (!d || !d.isDirectory()) return;
for (File f = d.openNextFile(); f && out.size() < 200; f = d.openNextFile()) {
std::string path = dir + "/" + f.name();
if (f.isDirectory()) listSaved(path, out);
else out.push_back({path, f.getLastWrite()});
}
}
} // namespace
const char* GeminiService::slotPath(int slot) {
static const char* const kPaths[] = {"/gemini/cache/page0.gmi", "/gemini/cache/page1.gmi", "/gemini/cache/work.gmi"};
return kPaths[slot];
}
const char* GeminiService::cacheOf(const std::string& url) const {
for (int s = 0; s < 3; s++)
if (!url.empty() && slotUrl_[s] == url) return slotPath(s);
return nullptr;
}
GeminiService::GeminiService(KeyValueStore& store, StorageService& storage, ClockService& clock, EventBus& bus)
: store_(store), storage_(storage), clock_(clock), bus_(bus), lock_(xSemaphoreCreateMutex()) {}
// NVS keys are 15 characters at most: "gm" and 12 hex digits of SHA-256("host:port").
std::string GeminiService::pinKey(const std::string& host, int port) const {
std::string id = host + ":" + std::to_string(port);
uint8_t digest[32];
Sha256::hash(reinterpret_cast<const uint8_t*>(id.data()), id.size(), digest);
return "gm" + hex(digest, 6);
}
void GeminiService::trust(const std::string& host, int port, const std::string& fingerprint) {
store_.putString(pinKey(host, port).c_str(), fingerprint);
}
std::string GeminiService::now() const {
int64_t t = clock_.utcNow();
return t < 0 ? "an unknown date" : ClockModel::formatLocalDate(t) + " " + ClockModel::formatLocalTime(t);
}
void GeminiService::notify(const std::string& text) {
bus_.publish(Event::withText(EventType::Notification, text.c_str(), static_cast<int32_t>(NotificationLevel::Info)));
}
bool GeminiService::start(Job job) {
if (busy_) return false;
busy_ = true;
ready_ = messageReady_ = cancel_ = false;
job_ = job;
if (xTaskCreate(taskEntry, "gemini", 6144, this, 1, nullptr) != pdPASS) { // peak 3.6 KB (step 1)
busy_ = false;
return false;
}
return true;
}
bool GeminiService::fetch(const std::string& url, bool preferSaved) {
if (busy_) return false;
arg_ = url;
preferSaved_ = preferSaved;
return start(Job::Fetch);
}
bool GeminiService::fetchToConsole(const std::string& url) {
if (busy_) return false;
toConsole_ = true;
if (!fetch(url)) {
toConsole_ = false;
return false;
}
return true;
}
bool GeminiService::save(const std::string& url) {
if (busy_) return false;
arg_ = url;
return start(Job::Save);
}
bool GeminiService::saveWithLinks(const std::string& url, std::vector<std::string> links) {
if (busy_) return false;
arg_ = url;
links_ = std::move(links);
return start(Job::SaveWithLinks);
}
bool GeminiService::refresh(const std::string& savedPath) {
if (busy_) return false;
arg_ = savedPath;
return start(Job::Refresh);
}
bool GeminiService::deleteSaved(const std::string& savedPath) {
if (busy_) return false;
arg_ = savedPath;
return start(Job::Delete);
}
bool GeminiService::addBookmark(const std::string& url, const std::string& title) {
if (busy_) return false;
arg_ = url;
arg2_ = title;
return start(Job::Bookmark);
}
bool GeminiService::download(const std::string& url) {
if (busy_) return false;
arg_ = url;
return start(Job::Download);
}
bool GeminiService::loadWindow(const GeminiPage& page, size_t firstLine) {
if (busy_ || page.backing.empty()) return false;
windowPage_ = GeminiPage();
windowPage_.backing = page.backing;
windowPage_.totalLines = page.totalLines;
windowPage_.lineIndex = page.lineIndex;
windowLine_ = firstLine;
// The App drops its current window once the new one is in: count that memory as coming back.
windowReleasing_ = page.text.bytes() + page.text.lineCount() * 8;
return start(Job::Window);
}
bool GeminiService::takeResult(GeminiPage& out) {
if (!ready_) return false;
xSemaphoreTake(lock_, portMAX_DELAY);
out = std::move(result_);
result_ = GeminiPage();
ready_ = false;
xSemaphoreGive(lock_);
return true;
}
bool GeminiService::takeMessage(std::string& out) {
if (!messageReady_) return false;
xSemaphoreTake(lock_, portMAX_DELAY);
out = std::move(message_);
messageReady_ = false;
xSemaphoreGive(lock_);
return true;
}
void GeminiService::finishWithMessage(const std::string& text) {
xSemaphoreTake(lock_, portMAX_DELAY);
message_ = text;
messageReady_ = true;
xSemaphoreGive(lock_);
}
void GeminiService::taskEntry(void* self) {
static_cast<GeminiService*>(self)->run();
vTaskDelete(nullptr);
}
void GeminiService::run() {
switch (job_) {
case Job::Fetch: {
GeminiPage page;
runFetch(page);
if (!toConsole_) {
xSemaphoreTake(lock_, portMAX_DELAY);
result_ = std::move(page);
ready_ = true;
xSemaphoreGive(lock_);
}
toConsole_ = false;
break;
}
case Job::Save: {
std::string why = saveCached(arg_);
finishWithMessage(!why.empty() ? why : replaced_ ? "Saved again (replaced the older copy)" : "Saved for offline reading");
break;
}
case Job::SaveWithLinks: {
std::string why = saveCached(arg_);
if (!why.empty()) {
finishWithMessage(why);
break;
}
int saved = 1, failed = 0, total = static_cast<int>(links_.size()) + 1;
notify("Saving " + std::to_string(total) + " pages for offline reading");
for (const std::string& link : links_) {
if (cancel_) break;
GeminiPage page;
fetchFollowing(link, page, false, kWorkSlot);
bool text = page.outcome == GeminiPage::Outcome::Ok &&
page.header.category() == gemini::Category::Success && page.header.mimeType() == "text/gemini";
if (text && saveCached(link).empty()) saved++;
else failed++;
if ((saved + failed) % 5 == 0) notify("Saved " + std::to_string(saved) + " of " + std::to_string(total));
}
std::string done = "Saved " + std::to_string(saved) + " of " + std::to_string(total) + " pages" +
(failed ? " (" + std::to_string(failed) + " not gemtext or failed)" : "");
notify(done);
finishWithMessage(done);
break;
}
case Job::Refresh: {
// Only the first line: where it came from. (Loading the whole page here, next to the
// App's copy and a TLS connection, once took the heap down to 436 bytes.)
std::string from = savedFrom(arg_);
if (from.empty()) {
finishWithMessage("Not a Saved Page");
break;
}
GeminiPage page;
fetchFollowing(from, page, false, kWorkSlot);
std::string why = page.outcome != GeminiPage::Outcome::Ok ? page.error
: page.header.category() != gemini::Category::Success ? page.header.meta
: saveCached(from);
finishWithMessage(why.empty() ? "Refreshed" : "Not refreshed: " + why);
break;
}
case Job::Window: {
// Q86 both ways: old and new together above the transient floor for a moment, and the
// new one alone (once the old is dropped) leaving the steady floor.
size_t free = esp_get_free_heap_size();
size_t transient = free > kTransientFloor + TextBuffer::kChunk ? free - kTransientFloor - TextBuffer::kChunk : 0;
size_t steady = free + windowReleasing_ > kSteadyFloor + TextBuffer::kChunk
? free + windowReleasing_ - kSteadyFloor - TextBuffer::kChunk
: 0;
size_t budget = std::min(transient, steady);
GeminiPage page = std::move(windowPage_);
page.windowStart = windowLine_ - windowLine_ % kIndexEvery;
page.windowed = true;
page.windowOf = page.backing;
page.outcome = GeminiPage::Outcome::Ok;
readWindow(page, page.windowStart, std::min(budget, kMaxBody));
console.printf("gemini: window from line %u: %u lines\n", (unsigned)page.windowStart,
(unsigned)page.text.lineCount());
xSemaphoreTake(lock_, portMAX_DELAY);
result_ = std::move(page);
ready_ = true;
xSemaphoreGive(lock_);
break;
}
case Job::Delete: {
bool removed = false;
bool ran = storage_.runAndWait([&]() { removed = SD.remove(arg_.c_str()); });
finishWithMessage(!ran ? "No SD card" : removed ? "Deleted" : "Not deleted");
break;
}
case Job::Bookmark: {
std::string line = "=> " + arg_ + (arg2_.empty() ? "" : " " + arg2_), already;
bool ran = storage_.runAndWait([&]() {
File f = SD.open(kBookmarks);
while (f && f.available()) {
String l = f.readStringUntil('\n');
if (l.startsWith(("=> " + arg_ + " ").c_str()) || l == ("=> " + arg_).c_str()) already = "yes";
}
if (f) f.close();
if (!already.empty()) return;
makeParents(kBookmarks);
File out = SD.open(kBookmarks, FILE_APPEND);
if (out) {
out.print((line + "\n").c_str());
out.close();
}
});
finishWithMessage(!ran ? "No SD card" : !already.empty() ? "Already bookmarked" : "Bookmarked");
break;
}
case Job::Download: {
gemini::Url u;
std::string name = gemini::parseUrl(arg_, u) ? u.path.substr(u.path.rfind('/') + 1) : "";
if (name.empty()) name = "download";
std::string to = "/gemini/downloads/" + storage::sanitize(name);
bool copied = false;
if (const char* from = cacheOf(arg_)) storage_.runAndWait([&]() { copied = copyFile(from, to, ""); });
finishWithMessage(copied ? "Saved to " + to : "Not saved (no card, or the page is gone)");
break;
}
}
busy_ = false;
}
void GeminiService::runFetch(GeminiPage& page) {
page.requested = arg_;
size_t before = esp_get_free_heap_size();
lowest_ = before;
uint32_t started = millis();
const std::string& url = arg_;
if (url == kStartUrl) composeStart(page);
else if (url.rfind("file://", 0) == 0) openSaved(url.substr(7), page);
else {
// Inside a Saved Page, a link to another Saved Page opens the saved copy (Q84).
bool savedCopy = false;
if (preferSaved_) {
std::string path = gemini::savedPath(url);
storage_.runAndWait([&]() { savedCopy = SD.exists(path.c_str()); });
if (savedCopy) openSaved(path, page);
}
if (!savedCopy) {
if (preferSaved_ && !WiFi.isConnected()) {
page.url = url;
page.error = "Not saved, and offline";
} else
{
int slot = nextDisplay_;
fetchFollowing(url, page, true, slot);
if (page.outcome == GeminiPage::Outcome::Ok && !page.cachePath.empty())
nextDisplay_ = 1 - slot; // keep this one: it backs the page on screen
}
}
}
page.ms = millis() - started;
if (toConsole_) report(page, before, lowest_);
}
// One request, following up to kMaxRedirects redirects.
void GeminiService::fetchFollowing(const std::string& first, GeminiPage& page, bool loadIntoRam, int slot) {
std::string url = first;
size_t free = esp_get_free_heap_size();
if (free < kStartFloor) { // Q86, for every fetch: pages, saves, refreshes
page.url = url;
page.outcome = GeminiPage::Outcome::Failed;
page.error = "Not enough memory (" + std::to_string(free / 1024) + " KB free): stop IRC or retry";
return;
}
for (int hop = 0;; hop++) {
fetchOne(url, page, loadIntoRam, slot);
if (page.outcome != GeminiPage::Outcome::Ok || page.header.category() != gemini::Category::Redirect) break;
std::string next = gemini::resolve(url, page.header.meta);
if (!gemini::isGemini(next)) {
page.outcome = GeminiPage::Outcome::Failed;
page.error = "Redirect to another protocol: " + next;
break;
}
if (hop + 1 > kMaxRedirects) {
page.outcome = GeminiPage::Outcome::Failed;
page.error = "Too many redirects";
break;
}
url = next;
}
page.url = url;
}
// A Saved Page's first line: "> Saved from <url> on <date>" (Q82). Empty if it isn't one.
std::string GeminiService::savedFirstLine(const std::string& path) {
std::string first;
storage_.runAndWait([&]() {
File f = SD.open(path.c_str());
if (!f) return;
char buf[600];
size_t n = f.readBytesUntil('\n', buf, sizeof buf - 1);
buf[n] = 0;
first = buf;
f.close();
});
if (!first.empty() && first.back() == '\r') first.pop_back();
return first.rfind(kSavedMark, 0) == 0 ? first : "";
}
std::string GeminiService::savedFrom(const std::string& path) {
std::string first = savedFirstLine(path);
if (first.empty()) return "";
std::string rest = first.substr(strlen(kSavedMark));
return rest.substr(0, rest.find(" on "));
}
// A Saved Page from the card, as far as memory allows; its first line says where it came from.
void GeminiService::openSaved(const std::string& path, GeminiPage& page) {
page.url = "file://" + path;
page.header.status = 20;
page.header.meta = "text/gemini";
bool found = false;
storage_.runAndWait([&]() { found = SD.exists(path.c_str()); });
std::string first = found ? savedFirstLine(path) : "";
if (!found) {
page.error = "This Saved Page is gone";
return;
}
if (first.rfind(kSavedMark, 0) == 0) {
std::string rest = first.substr(strlen(kSavedMark));
size_t on = rest.find(" on ");
page.savedFrom = rest.substr(0, on);
page.savedOn = on == std::string::npos ? "" : rest.substr(on + 4);
}
loadFromCard(page, esp_get_free_heap_size(), path);
page.outcome = GeminiPage::Outcome::Ok;
}
// about:start (Q78, Q84): bookmarks, then Saved Pages by capsule, newest first, then defaults.
void GeminiService::composeStart(GeminiPage& page) {
std::string doc = "# Gemini\nTab picks a link, Enter follows it, Back returns, g types an address. "
"On a page: b bookmarks it, s saves it to read offline, S with the pages it links to.\n";
std::string bookmarks;
std::vector<SavedEntry> saved;
bool card = storage_.runAndWait([&]() {
File f = SD.open(kBookmarks);
while (f && f.available() && bookmarks.size() < 8192) {
String l = f.readStringUntil('\n');
if (l.startsWith("=>")) bookmarks += std::string(l.c_str()) + "\n";
}
if (f) f.close();
listSaved(kSavedRoot, saved);
});
doc += "## Bookmarks\n";
doc += !card ? "No SD card: bookmarks and Saved Pages live on it.\n"
: bookmarks.empty() ? "None yet: b on a page adds it here.\n" : bookmarks;
if (!saved.empty()) {
// Capsules ordered by their newest Saved Page, pages newest first within each.
std::sort(saved.begin(), saved.end(), [](const SavedEntry& a, const SavedEntry& b) { return a.when > b.when; });
std::vector<std::string> order;
std::map<std::string, std::string> byHost;
size_t root = strlen(kSavedRoot) + 1;
for (auto& e : saved) {
std::string rel = e.path.substr(root);
std::string host = rel.substr(0, rel.find('/'));
if (!byHost.count(host)) order.push_back(host);
std::string label = rel.substr(host.size() + 1);
if (label.size() > 4 && label.compare(label.size() - 4, 4, ".gmi") == 0) label.resize(label.size() - 4);
char day[24] = "";
if (e.when > 1000000000) {
struct tm t;
localtime_r(&e.when, &t);
strftime(day, sizeof day, " (%d %b)", &t);
}
byHost[host] += "=> file://" + e.path + " " + label + day + "\n";
}
doc += "## Saved pages\n";
for (auto& host : order) doc += "### " + host + "\n" + byHost[host];
}
doc += kDefaults;
page.url = kStartUrl;
page.header.status = 20;
page.header.meta = "text/gemini";
page.text.append(doc.data(), doc.size());
page.text.finish();
page.outcome = GeminiPage::Outcome::Ok;
}
// The cached body of `url` (fetched again if the cache holds something else) to its Saved Page.
std::string GeminiService::saveCached(const std::string& url) {
if (!gemini::isGemini(url)) return "Only Gemini pages can be saved";
if (!storage_.state().present) return "No SD card";
const char* from = cacheOf(url);
if (!from) {
GeminiPage page;
fetchFollowing(url, page, false, kWorkSlot);
if (page.outcome != GeminiPage::Outcome::Ok) return page.error;
if (page.header.category() != gemini::Category::Success) return page.header.meta;
from = cacheOf(page.url);
if (!from) return "Not saved";
}
std::string to = gemini::savedPath(url);
bool existed = false, ok = false;
replaced_ = false;
std::string header = std::string(kSavedMark) + url + " on " + now();
storage_.runAndWait([&]() {
existed = SD.exists(to.c_str());
ok = copyFile(from, to, header);
});
if (!ok) return "Writing to the SD card failed";
replaced_ = existed; // Q82: saving again replaces it, and says so
return "";
}
void GeminiService::fetchOne(const std::string& url, GeminiPage& page, bool loadIntoRam, int slot) {
page.outcome = GeminiPage::Outcome::Failed;
page.text.clear();
page.truncated = false;
gemini::Url u;
if (!gemini::parseUrl(url, u) || u.scheme != "gemini") {
page.error = "Not a Gemini URL: " + url;
return;
}
auto track = [this]() { lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size()); };
size_t freeBefore = esp_get_free_heap_size(); // what's left once the connection closes again
NetworkClientSecure tls;
tls.setInsecure(); // trust on first use: the pinned fingerprint is what counts (Q71)
tls.setTimeout(15);
if (!tls.connect(u.host.c_str(), u.portOrDefault())) {
page.error = "Cannot connect to " + u.host;
return;
}
track();
uint8_t sha[32];
if (!tls.getFingerprintSHA256(sha)) {
page.error = "No certificate from " + u.host;
return;
}
std::string presented = hex(sha, sizeof sha), pinned;
std::string key = pinKey(u.host, u.portOrDefault());
if (!store_.getString(key.c_str(), pinned)) store_.putString(key.c_str(), presented); // first use
else if (pinned != presented) {
page.outcome = GeminiPage::Outcome::CertificateChanged;
page.host = u.host;
page.port = u.portOrDefault();
page.pinned = pinned;
page.presented = presented;
page.error = "The certificate of " + u.host + " changed";
return;
}
tls.print((gemini::requestUrl(url) + "\r\n").c_str());
std::string header;
bool inBody = false;
uint32_t last = millis();
char buf[512];
while (!cancel_) {
int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
if (n <= 0) {
if (!tls.connected()) break;
if (millis() - last > kIdleTimeoutMs) {
page.error = "Timed out";
return;
}
delay(5);
continue;
}
last = millis();
int i = 0;
while (!inBody && i < n) {
char c = buf[i++];
if (c == '\n') inBody = true;
else if (c != '\r') header += c;
if (header.size() > 1100) {
page.error = "Malformed response header";
return;
}
}
if (!inBody) continue;
if (!gemini::parseHeader(header, page.header)) {
page.error = "Malformed response header";
return;
}
if (page.header.category() != gemini::Category::Success) break; // no body to read
// With a card, the body goes there while the connection is open (only 4 KB in RAM),
// and comes back into RAM once the connection's memory is free again.
if (storage_.state().present) {
slotUrl_[slot].clear();
if (!receiveToCard(tls, buf + i, n - i, page, slot)) return;
tls.stop();
slotUrl_[slot] = url;
if (loadIntoRam) loadFromCard(page, freeBefore, slotPath(slot));
page.outcome = GeminiPage::Outcome::Ok;
return;
}
// Q86, two floors: once the connection closes, its ~50 KB comes back, so the page may use
// what was free before minus the steady floor; meanwhile the heap must stay above the
// transient floor. Both counted with the next 4 KB chunk and the line index.
size_t take = std::min<size_t>(n - i, kMaxBody - page.text.bytes());
size_t pageCost = page.text.bytes() + page.text.lineCount() * 8 + TextBuffer::kChunk + take;
if (take < static_cast<size_t>(n - i)) page.truncatedWhy = "longer than 64 KB";
else if (freeBefore < kSteadyFloor + pageCost) page.truncatedWhy = "not enough memory to keep it";
else if (esp_get_free_heap_size() < kTransientFloor + TextBuffer::kChunk + take)
page.truncatedWhy = "not enough memory while receiving";
if (!page.truncatedWhy.empty()) {
page.truncated = true;
break;
}
page.text.append(buf + i, take);
track();
}
if (cancel_) {
page.error = "Cancelled";
return;
}
if (!inBody && !gemini::parseHeader(header, page.header)) { // a header with no line end
page.error = header.empty() ? "Empty response" : "Malformed response header";
return;
}
page.text.finish();
page.outcome = GeminiPage::Outcome::Ok;
}
// Streams the rest of the body to a cache file in 1 KB pieces, each written by the storage task:
// with IRC's TLS connection and this one both open, every KB of RAM counts (Q86).
bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page, int slot) {
struct Cache {
File file;
};
auto cache = std::make_shared<Cache>();
bool opened = storage_.runAndWait([cache, slot]() {
SD.mkdir("/gemini");
SD.mkdir("/gemini/cache");
cache->file = SD.open(slotPath(slot), FILE_WRITE);
});
if (!opened || !cache->file) {
page.error = "Can't write to the SD card";
return false;
}
std::string pending(first, len);
constexpr size_t kPiece = 1024;
pending.reserve(kPiece + 512);
bool ok = true;
auto flush = [&]() {
if (pending.empty() || !ok) return;
ok = storage_.runAndWait([cache, &pending, &ok]() {
ok = cache->file.write(reinterpret_cast<const uint8_t*>(pending.data()), pending.size()) == pending.size();
}) && ok;
page.receivedBytes += pending.size();
pending.clear();
};
uint32_t last = millis();
char buf[512];
while (!cancel_ && ok) {
if (pending.size() >= kPiece) flush();
if (page.receivedBytes + pending.size() >= kMaxOnCard) {
page.truncated = true;
page.truncatedWhy = "longer than 1 MB";
break;
}
int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
if (n <= 0) {
if (!tls.connected()) break;
if (millis() - last > kIdleTimeoutMs) break; // keep what came
delay(5);
continue;
}
last = millis();
pending.append(buf, n);
lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size());
}
flush();
storage_.runAndWait([cache]() { cache->file.close(); });
if (cancel_) page.error = "Cancelled";
else if (!ok) page.error = "Writing to the SD card failed";
else page.cachePath = slotPath(slot);
return !cancel_ && ok;
}
// Back into RAM, now that the connection's memory is free again: one pass over the file counts
// its lines and indexes every kIndexEvery-th, and loads lines while the steady floor allows (Q86).
// If they don't all fit, the page is windowed: the App asks for other windows as you scroll.
void GeminiService::loadFromCard(GeminiPage& page, size_t freeBefore, const std::string& path) {
size_t budget = freeBefore > kSteadyFloor + TextBuffer::kChunk ? freeBefore - kSteadyFloor - TextBuffer::kChunk : 0;
budget = std::min(budget, kMaxBody);
page.backing = path;
page.lineIndex.clear();
page.totalLines = 0;
page.windowStart = 0;
bool loading = true;
storage_.runAndWait([&]() {
File f = SD.open(path.c_str());
if (!f) return;
std::string line;
uint32_t offset = 0, lineStart = 0;
bool pre = false;
auto endLine = [&]() {
if (page.totalLines % kIndexEvery == 0) page.lineIndex.push_back(lineStart | (pre ? 0x80000000u : 0));
if (line.compare(0, 3, "```") == 0) pre = !pre;
if (loading) {
if (page.text.bytes() + page.text.lineCount() * 8 + line.size() + 1 > budget) loading = false;
else {
line += '\n';
page.text.append(line.data(), line.size());
}
}
page.totalLines++;
line.clear();
};
char buf[512];
int n;
while ((n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
for (int i = 0; i < n; i++, offset++) {
if (buf[i] == '\n') {
endLine();
lineStart = offset + 1;
} else if (line.size() < kMaxLineBytes) line += buf[i];
}
}
if (!line.empty()) endLine();
f.close();
});
page.text.finish();
page.windowed = !loading;
if (page.windowed)
console.printf("gemini: %s is windowed: %u of %u lines in memory\n", path.c_str(),
(unsigned)page.text.lineCount(), (unsigned)page.totalLines);
}
// Lines from `firstLine` on, from the page's file, as many as `budget` allows.
void GeminiService::readWindow(GeminiPage& page, size_t firstLine, size_t budget) {
size_t entry = firstLine / kIndexEvery;
if (entry >= page.lineIndex.size()) return;
storage_.runAndWait([&]() {
File f = SD.open(page.backing.c_str());
if (!f || !f.seek(page.lineIndex[entry] & 0x7FFFFFFFu)) return;
size_t lineNo = entry * kIndexEvery;
std::string line;
char buf[512];
int n;
bool done = false;
while (!done && (n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
for (int i = 0; i < n && !done; i++) {
if (buf[i] != '\n') {
if (line.size() < kMaxLineBytes) line += buf[i];
continue;
}
if (lineNo >= firstLine) {
if (page.text.bytes() + page.text.lineCount() * 8 + line.size() + 1 > budget) done = true;
else {
line += '\n';
page.text.append(line.data(), line.size());
}
}
lineNo++;
line.clear();
}
}
if (!done && !line.empty() && lineNo >= firstLine) page.text.append(line.data(), line.size());
f.close();
});
page.text.finish();
}
void GeminiService::report(const GeminiPage& page, size_t heapBefore, size_t heapLowest) {
console.printf("gemini: %s -> %s, %lu ms\n", page.requested.c_str(), page.url.c_str(), (unsigned long)page.ms);
if (page.outcome != GeminiPage::Outcome::Ok) console.printf("gemini: %s\n", page.error.c_str());
else console.printf("gemini: %d %s, %u bytes, %u lines%s\n", page.header.status, page.header.meta.c_str(),
(unsigned)page.text.bytes(), (unsigned)page.text.lineCount(),
page.truncated ? (" (truncated: " + page.truncatedWhy + ")").c_str() : "");
if (page.outcome == GeminiPage::Outcome::CertificateChanged)
console.printf("gemini: pinned %s\ngemini: now %s\n", page.pinned.c_str(), page.presented.c_str());
for (size_t i = 0; i < page.text.lineCount() && i < 6; i++) console.printf(" %s\n", page.text.line(i).substr(0, 100).c_str());
if (!page.cachePath.empty())
console.printf("gemini: %u bytes on the card in %s\n", (unsigned)page.receivedBytes, page.cachePath.c_str());
console.printf("gemini: heap %u before, %u lowest seen, %u after\n", (unsigned)heapBefore, (unsigned)heapLowest,
(unsigned)esp_get_free_heap_size());
}
} // namespace roro
Reference in New Issue View Git Blame Copy Permalink