twisla/roro9stack
Public Access
1
0
Fork 0
Code Issues 51 Pull Requests 1 Actions 2 Packages Releases 14 Activity
Files
50fcf6b7a3b6700577b7ad6d6a831c9433016558
roro9stack/src/services/gemini_service.cpp
T
twislaandClaude Opus 5.5 70bb2a4137 S1 #11: bytes read and written, counted per network service 
A Counted<> wrapper around the network clients adds what goes through
their buffer read and write to a per-service counter (IRC, Gemini, Debug
Console, Updates); the single-byte calls and print() end up there, so
each byte counts once. `net` prints the totals. For TLS it's the plain
text the service sees.

Checked on the device: a Gemini fetch counts 164,986 in (a 164,970-byte
page and its 16-byte header) and 42 out (the URL and CRLF).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT
2026-10-06 01:10:55 +02:00

805 lines
31 KiB
C++
Raw Blame History

#include "services/gemini_service.h"
#include <NetworkClientSecure.h>
#include <SD.h>
#include <WiFi.h>
#include <esp_heap_caps.h>
#include <algorithm>
#include <map>
#include "gemini_url.h"
#include "gemtext.h"
#include "platform/console.h"
#include "sha256.h"
#include "platform/counted_client.h"
#include "storage_paths.h"
namespace roro {
using gemini::TextBuffer;
namespace {
constexpr uint32_t kIdleTimeoutMs = 15000;
constexpr const char* kSavedRoot = "/gemini/saved";
constexpr const char* kSavedMark = "> Saved from "; // first line of a Saved Page (Q82)
constexpr const char* kDefaults =
"## Start here\n"
"=> gemini://geminiprotocol.net/ Project Gemini\n"
"=> gemini://kennedy.gemi.dev/ Kennedy: search Geminispace\n"
"=> gemini://skyjake.fi/~Cosmos/ Cosmos: what's new on Geminispace\n"
"=> gemini://bbs.geminispace.org/ Bubble: a bulletin board\n";
std::string hex(const uint8_t* data, size_t len) {
static const char* digits = "0123456789abcdef";
std::string out;
for (size_t i = 0; i < len; i++) {
out += digits[data[i] >> 4];
out += digits[data[i] & 15];
}
return out;
}
// Folders for a file path, one at a time (SD.mkdir makes one level).
void makeParents(const std::string& path) {
for (size_t slash = path.find('/', 1); slash != std::string::npos; slash = path.find('/', slash + 1)) {
std::string dir = path.substr(0, slash);
if (!SD.exists(dir.c_str())) SD.mkdir(dir.c_str());
}
}
// Copies `from` to `to`, optionally after a first line. On the storage task.
bool copyFile(const char* from, const std::string& to, const std::string& firstLine) {
File in = SD.open(from);
if (!in) return false;
makeParents(to);
File out = SD.open(to.c_str(), FILE_WRITE);
if (!out) return false;
bool ok = firstLine.empty() || out.print((firstLine + "\n").c_str()) > 0;
uint8_t buf[512];
int n;
while (ok && (n = in.read(buf, sizeof buf)) > 0) ok = out.write(buf, n) == static_cast<size_t>(n);
out.close();
in.close();
if (!ok) SD.remove(to.c_str());
return ok;
}
struct SavedEntry {
std::string path; // /gemini/saved/host/…/name.gmi
time_t when;
};
void listSaved(const std::string& dir, std::vector<SavedEntry>& out) {
File d = SD.open(dir.c_str());
if (!d || !d.isDirectory()) return;
for (File f = d.openNextFile(); f && out.size() < 200; f = d.openNextFile()) {
std::string path = dir + "/" + f.name();
if (f.isDirectory()) listSaved(path, out);
else out.push_back({path, f.getLastWrite()});
}
}
} // namespace
const char* GeminiService::slotPath(int slot) {
static const char* const kPaths[] = {"/gemini/cache/page0.gmi", "/gemini/cache/page1.gmi", "/gemini/cache/work.gmi"};
return kPaths[slot];
}
const char* GeminiService::cacheOf(const std::string& url) const {
for (int s = 0; s < 3; s++)
if (!url.empty() && slotUrl_[s] == url) return slotPath(s);
return nullptr;
}
GeminiService::GeminiService(KeyValueStore& store, StorageService& storage, ClockService& clock, EventBus& bus)
: store_(store), storage_(storage), clock_(clock), bus_(bus), lock_(xSemaphoreCreateMutex()) {}
// NVS keys are 15 characters at most: "gm" and 12 hex digits of SHA-256("host:port").
std::string GeminiService::pinKey(const std::string& host, int port) const {
std::string id = host + ":" + std::to_string(port);
uint8_t digest[32];
Sha256::hash(reinterpret_cast<const uint8_t*>(id.data()), id.size(), digest);
return "gm" + hex(digest, 6);
}
void GeminiService::trust(const std::string& host, int port, const std::string& fingerprint) {
store_.putString(pinKey(host, port).c_str(), fingerprint);
}
std::string GeminiService::now() const {
int64_t t = clock_.utcNow();
return t < 0 ? "an unknown date" : ClockModel::formatLocalDate(t) + " " + ClockModel::formatLocalTime(t);
}
void GeminiService::notify(const std::string& text) {
bus_.publish(Event::withText(EventType::Notification, text.c_str(), static_cast<int32_t>(NotificationLevel::Info)));
}
bool GeminiService::start(Job job) {
if (busy_) return false;
busy_ = true;
ready_ = messageReady_ = cancel_ = false;
job_ = job;
if (xTaskCreate(taskEntry, "gemini", 6144, this, 1, nullptr) != pdPASS) { // peak 3.6 KB (step 1)
busy_ = false;
return false;
}
return true;
}
bool GeminiService::fetch(const std::string& url, bool preferSaved) {
if (busy_) return false;
arg_ = url;
preferSaved_ = preferSaved;
return start(Job::Fetch);
}
bool GeminiService::fetchToConsole(const std::string& url) {
if (busy_) return false;
toConsole_ = true;
if (!fetch(url)) {
toConsole_ = false;
return false;
}
return true;
}
bool GeminiService::save(const std::string& url) {
if (busy_) return false;
arg_ = url;
return start(Job::Save);
}
bool GeminiService::saveWithLinks(const std::string& url, std::vector<std::string> links) {
if (busy_) return false;
arg_ = url;
links_ = std::move(links);
return start(Job::SaveWithLinks);
}
bool GeminiService::refresh(const std::string& savedPath) {
if (busy_) return false;
arg_ = savedPath;
return start(Job::Refresh);
}
bool GeminiService::deleteSaved(const std::string& savedPath) {
if (busy_) return false;
arg_ = savedPath;
return start(Job::Delete);
}
bool GeminiService::addBookmark(const std::string& url, const std::string& title) {
if (busy_) return false;
arg_ = url;
arg2_ = title;
return start(Job::Bookmark);
}
bool GeminiService::download(const std::string& url) {
if (busy_) return false;
arg_ = url;
return start(Job::Download);
}
bool GeminiService::loadWindow(const GeminiPage& page, size_t firstLine) {
if (busy_ || page.backing.empty()) return false;
windowPage_ = GeminiPage();
windowPage_.backing = page.backing;
windowPage_.totalLines = page.totalLines;
windowPage_.lineIndex = page.lineIndex;
windowLine_ = firstLine;
// The App drops its current window once the new one is in: count that memory as coming back.
windowReleasing_ = page.text.bytes() + page.text.lineCount() * kBytesPerLine;
return start(Job::Window);
}
bool GeminiService::takeResult(GeminiPage& out) {
if (!ready_) return false;
xSemaphoreTake(lock_, portMAX_DELAY);
out = std::move(result_);
result_ = GeminiPage();
ready_ = false;
xSemaphoreGive(lock_);
return true;
}
bool GeminiService::takeMessage(std::string& out) {
if (!messageReady_) return false;
xSemaphoreTake(lock_, portMAX_DELAY);
out = std::move(message_);
messageReady_ = false;
xSemaphoreGive(lock_);
return true;
}
void GeminiService::finishWithMessage(const std::string& text) {
xSemaphoreTake(lock_, portMAX_DELAY);
message_ = text;
messageReady_ = true;
xSemaphoreGive(lock_);
}
void GeminiService::taskEntry(void* self) {
static_cast<GeminiService*>(self)->run();
vTaskDelete(nullptr);
}
void GeminiService::run() {
switch (job_) {
case Job::Fetch: {
GeminiPage page;
runFetch(page);
if (!toConsole_) {
xSemaphoreTake(lock_, portMAX_DELAY);
result_ = std::move(page);
ready_ = true;
xSemaphoreGive(lock_);
}
toConsole_ = false;
break;
}
case Job::Save: {
std::string why = saveCached(arg_);
finishWithMessage(!why.empty() ? why : replaced_ ? "Saved again (replaced the older copy)" : "Saved for offline reading");
break;
}
case Job::SaveWithLinks: {
std::string why = saveCached(arg_);
if (!why.empty()) {
finishWithMessage(why);
break;
}
int saved = 1, failed = 0, total = static_cast<int>(links_.size()) + 1;
notify("Saving " + std::to_string(total) + " pages for offline reading");
for (const std::string& link : links_) {
if (cancel_) break;
GeminiPage page;
fetchFollowing(link, page, false, kWorkSlot);
bool text = page.outcome == GeminiPage::Outcome::Ok &&
page.header.category() == gemini::Category::Success && page.header.mimeType() == "text/gemini";
if (text && saveCached(link).empty()) saved++;
else failed++;
if ((saved + failed) % 5 == 0) notify("Saved " + std::to_string(saved) + " of " + std::to_string(total));
}
std::string done = "Saved " + std::to_string(saved) + " of " + std::to_string(total) + " pages" +
(failed ? " (" + std::to_string(failed) + " not gemtext or failed)" : "");
notify(done);
finishWithMessage(done);
break;
}
case Job::Refresh: {
// Only the first line: where it came from. (Loading the whole page here, next to the
// App's copy and a TLS connection, once took the heap down to 436 bytes.)
std::string from = savedFrom(arg_);
if (from.empty()) {
finishWithMessage("Not a Saved Page");
break;
}
GeminiPage page;
fetchFollowing(from, page, false, kWorkSlot);
std::string why = page.outcome != GeminiPage::Outcome::Ok ? page.error
: page.header.category() != gemini::Category::Success ? page.header.meta
: saveCached(from);
finishWithMessage(why.empty() ? "Refreshed" : "Not refreshed: " + why);
break;
}
case Job::Window: {
// Q86 both ways: old and new together above the transient floor for a moment, and the
// new one alone (once the old is dropped) leaving the steady floor.
size_t free = esp_get_free_heap_size();
size_t transient = free > kTransientFloor + TextBuffer::kChunk ? free - kTransientFloor - TextBuffer::kChunk : 0;
size_t steady = free + windowReleasing_ > kSteadyFloor + TextBuffer::kChunk
? free + windowReleasing_ - kSteadyFloor - TextBuffer::kChunk
: 0;
size_t budget = std::min(transient, steady);
GeminiPage page = std::move(windowPage_);
page.windowStart = windowLine_ - windowLine_ % kIndexEvery;
page.windowed = true;
page.windowOf = page.backing;
page.outcome = GeminiPage::Outcome::Ok;
readWindow(page, page.windowStart, std::min(budget, kMaxBody));
console.printf("gemini: window from line %u: %u lines\n", (unsigned)page.windowStart,
(unsigned)page.text.lineCount());
xSemaphoreTake(lock_, portMAX_DELAY);
result_ = std::move(page);
ready_ = true;
xSemaphoreGive(lock_);
break;
}
case Job::Delete: {
bool removed = false;
bool ran = storage_.runAndWait([&]() { removed = SD.remove(arg_.c_str()); });
finishWithMessage(!ran ? "No SD card" : removed ? "Deleted" : "Not deleted");
break;
}
case Job::Bookmark: {
std::string line = "=> " + arg_ + (arg2_.empty() ? "" : " " + arg2_), already;
bool ran = storage_.runAndWait([&]() {
File f = SD.open(kBookmarks);
while (f && f.available()) {
String l = f.readStringUntil('\n');
if (l.startsWith(("=> " + arg_ + " ").c_str()) || l == ("=> " + arg_).c_str()) already = "yes";
}
if (f) f.close();
if (!already.empty()) return;
makeParents(kBookmarks);
File out = SD.open(kBookmarks, FILE_APPEND);
if (out) {
out.print((line + "\n").c_str());
out.close();
}
});
finishWithMessage(!ran ? "No SD card" : !already.empty() ? "Already bookmarked" : "Bookmarked");
break;
}
case Job::Download: {
gemini::Url u;
std::string name = gemini::parseUrl(arg_, u) ? u.path.substr(u.path.rfind('/') + 1) : "";
if (name.empty()) name = "download";
std::string to = "/gemini/downloads/" + storage::sanitize(name);
bool copied = false;
if (const char* from = cacheOf(arg_)) storage_.runAndWait([&]() { copied = copyFile(from, to, ""); });
finishWithMessage(copied ? "Saved to " + to : "Not saved (no card, or the page is gone)");
break;
}
}
busy_ = false;
}
void GeminiService::runFetch(GeminiPage& page) {
page.requested = arg_;
size_t before = esp_get_free_heap_size();
lowest_ = before;
uint32_t started = millis();
const std::string& url = arg_;
if (url == kStartUrl) composeStart(page);
else if (url.rfind("file://", 0) == 0) openSaved(url.substr(7), page);
else {
// Inside a Saved Page, a link to another Saved Page opens the saved copy (Q84).
bool savedCopy = false;
if (preferSaved_) {
std::string path = gemini::savedPath(url);
storage_.runAndWait([&]() { savedCopy = SD.exists(path.c_str()); });
if (savedCopy) openSaved(path, page);
}
if (!savedCopy) {
if (preferSaved_ && !WiFi.isConnected()) {
page.url = url;
page.error = "Not saved, and offline";
} else
{
int slot = nextDisplay_;
fetchFollowing(url, page, true, slot);
if (page.outcome == GeminiPage::Outcome::Ok && !page.cachePath.empty())
nextDisplay_ = 1 - slot; // keep this one: it backs the page on screen
}
}
}
page.ms = millis() - started;
if (toConsole_) report(page, before, lowest_);
}
// One request, following up to kMaxRedirects redirects.
void GeminiService::fetchFollowing(const std::string& first, GeminiPage& page, bool loadIntoRam, int slot) {
std::string url = first;
size_t free = esp_get_free_heap_size();
if (free < kStartFloor) { // Q86, for every fetch: pages, saves, refreshes
page.url = url;
page.outcome = GeminiPage::Outcome::Failed;
page.error = "Not enough memory (" + std::to_string(free / 1024) + " KB free): stop IRC or retry";
return;
}
for (int hop = 0;; hop++) {
fetchOne(url, page, loadIntoRam, slot);
if (page.outcome != GeminiPage::Outcome::Ok || page.header.category() != gemini::Category::Redirect) break;
std::string next = gemini::resolve(url, page.header.meta);
if (!gemini::isGemini(next)) {
page.outcome = GeminiPage::Outcome::Failed;
page.error = "Redirect to another protocol: " + next;
break;
}
if (hop + 1 > kMaxRedirects) {
page.outcome = GeminiPage::Outcome::Failed;
page.error = "Too many redirects";
break;
}
url = next;
}
page.url = url;
}
// A Saved Page's first line: "> Saved from <url> on <date>" (Q82). Empty if it isn't one.
std::string GeminiService::savedFirstLine(const std::string& path) {
std::string first;
storage_.runAndWait([&]() {
File f = SD.open(path.c_str());
if (!f) return;
char buf[600];
size_t n = f.readBytesUntil('\n', buf, sizeof buf - 1);
buf[n] = 0;
first = buf;
f.close();
});
if (!first.empty() && first.back() == '\r') first.pop_back();
return first.rfind(kSavedMark, 0) == 0 ? first : "";
}
std::string GeminiService::savedFrom(const std::string& path) {
std::string first = savedFirstLine(path);
if (first.empty()) return "";
std::string rest = first.substr(strlen(kSavedMark));
return rest.substr(0, rest.find(" on "));
}
// A Saved Page from the card, as far as memory allows; its first line says where it came from.
void GeminiService::openSaved(const std::string& path, GeminiPage& page) {
page.url = "file://" + path;
page.header.status = 20;
page.header.meta = "text/gemini";
bool found = false;
storage_.runAndWait([&]() { found = SD.exists(path.c_str()); });
std::string first = found ? savedFirstLine(path) : "";
if (!found) {
page.error = "This Saved Page is gone";
return;
}
if (first.rfind(kSavedMark, 0) == 0) {
std::string rest = first.substr(strlen(kSavedMark));
size_t on = rest.find(" on ");
page.savedFrom = rest.substr(0, on);
page.savedOn = on == std::string::npos ? "" : rest.substr(on + 4);
}
loadFromCard(page, esp_get_free_heap_size(), path);
page.outcome = GeminiPage::Outcome::Ok;
}
// about:start (Q78, Q84): bookmarks, then Saved Pages by capsule, newest first, then defaults.
void GeminiService::composeStart(GeminiPage& page) {
std::string doc = "# Gemini\nTab picks a link, Enter follows it, Back returns, g types an address. "
"On a page: b bookmarks it, s saves it to read offline, S with the pages it links to.\n";
std::string bookmarks;
std::vector<SavedEntry> saved;
bool card = storage_.runAndWait([&]() {
File f = SD.open(kBookmarks);
while (f && f.available() && bookmarks.size() < 8192) {
String l = f.readStringUntil('\n');
if (l.startsWith("=>")) bookmarks += std::string(l.c_str()) + "\n";
}
if (f) f.close();
listSaved(kSavedRoot, saved);
});
doc += "## Bookmarks\n";
doc += !card ? "No SD card: bookmarks and Saved Pages live on it.\n"
: bookmarks.empty() ? "None yet: b on a page adds it here.\n" : bookmarks;
if (!saved.empty()) {
// Capsules ordered by their newest Saved Page, pages newest first within each.
std::sort(saved.begin(), saved.end(), [](const SavedEntry& a, const SavedEntry& b) { return a.when > b.when; });
std::vector<std::string> order;
std::map<std::string, std::string> byHost;
size_t root = strlen(kSavedRoot) + 1;
for (auto& e : saved) {
std::string rel = e.path.substr(root);
std::string host = rel.substr(0, rel.find('/'));
if (!byHost.count(host)) order.push_back(host);
std::string label = rel.substr(host.size() + 1);
if (label.size() > 4 && label.compare(label.size() - 4, 4, ".gmi") == 0) label.resize(label.size() - 4);
char day[24] = "";
if (e.when > 1000000000) {
struct tm t;
localtime_r(&e.when, &t);
strftime(day, sizeof day, " (%d %b)", &t);
}
byHost[host] += "=> file://" + e.path + " " + label + day + "\n";
}
doc += "## Saved pages\n";
for (auto& host : order) doc += "### " + host + "\n" + byHost[host];
}
doc += kDefaults;
page.url = kStartUrl;
page.header.status = 20;
page.header.meta = "text/gemini";
page.text.append(doc.data(), doc.size());
page.text.finish();
page.outcome = GeminiPage::Outcome::Ok;
}
// The cached body of `url` (fetched again if the cache holds something else) to its Saved Page.
std::string GeminiService::saveCached(const std::string& url) {
if (!gemini::isGemini(url)) return "Only Gemini pages can be saved";
if (!storage_.state().present) return "No SD card";
const char* from = cacheOf(url);
if (!from) {
GeminiPage page;
fetchFollowing(url, page, false, kWorkSlot);
if (page.outcome != GeminiPage::Outcome::Ok) return page.error;
if (page.header.category() != gemini::Category::Success) return page.header.meta;
from = cacheOf(page.url);
if (!from) return "Not saved";
}
std::string to = gemini::savedPath(url);
bool existed = false, ok = false;
replaced_ = false;
std::string header = std::string(kSavedMark) + url + " on " + now();
storage_.runAndWait([&]() {
existed = SD.exists(to.c_str());
ok = copyFile(from, to, header);
});
if (!ok) return "Writing to the SD card failed";
replaced_ = existed; // Q82: saving again replaces it, and says so
return "";
}
void GeminiService::fetchOne(const std::string& url, GeminiPage& page, bool loadIntoRam, int slot) {
page.outcome = GeminiPage::Outcome::Failed;
page.text.clear();
page.truncated = false;
gemini::Url u;
if (!gemini::parseUrl(url, u) || u.scheme != "gemini") {
page.error = "Not a Gemini URL: " + url;
return;
}
auto track = [this]() { lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size()); };
size_t freeBefore = esp_get_free_heap_size(); // what's left once the connection closes again
Counted<NetworkClientSecure> tls(net::User::Gemini);
tls.setInsecure(); // trust on first use: the pinned fingerprint is what counts (Q71)
tls.setTimeout(15);
if (!tls.connect(u.host.c_str(), u.portOrDefault())) {
page.error = "Cannot connect to " + u.host;
return;
}
track();
uint8_t sha[32];
if (!tls.getFingerprintSHA256(sha)) {
page.error = "No certificate from " + u.host;
return;
}
std::string presented = hex(sha, sizeof sha), pinned;
std::string key = pinKey(u.host, u.portOrDefault());
if (!store_.getString(key.c_str(), pinned)) store_.putString(key.c_str(), presented); // first use
else if (pinned != presented) {
page.outcome = GeminiPage::Outcome::CertificateChanged;
page.host = u.host;
page.port = u.portOrDefault();
page.pinned = pinned;
page.presented = presented;
page.error = "The certificate of " + u.host + " changed";
return;
}
tls.print((gemini::requestUrl(url) + "\r\n").c_str());
std::string header;
bool inBody = false;
uint32_t last = millis();
char buf[512];
while (!cancel_) {
int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
if (n <= 0) {
if (!tls.connected()) break;
if (millis() - last > kIdleTimeoutMs) {
page.error = "Timed out";
return;
}
delay(5);
continue;
}
last = millis();
int i = 0;
while (!inBody && i < n) {
char c = buf[i++];
if (c == '\n') inBody = true;
else if (c != '\r') header += c;
if (header.size() > 1100) {
page.error = "Malformed response header";
return;
}
}
if (!inBody) continue;
if (!gemini::parseHeader(header, page.header)) {
page.error = "Malformed response header";
return;
}
if (page.header.category() != gemini::Category::Success) break; // no body to read
// With a card, the body goes there while the connection is open (only 4 KB in RAM),
// and comes back into RAM once the connection's memory is free again.
if (storage_.state().present) {
slotUrl_[slot].clear();
if (!receiveToCard(tls, buf + i, n - i, page, slot)) return;
tls.stop();
slotUrl_[slot] = url;
if (loadIntoRam) loadFromCard(page, freeBefore, slotPath(slot));
page.outcome = GeminiPage::Outcome::Ok;
return;
}
// Q86, two floors: once the connection closes, its ~50 KB comes back, so the page may use
// what was free before minus the steady floor; meanwhile the heap must stay above the
// transient floor. Both counted with the next 4 KB chunk and the line index.
size_t take = std::min<size_t>(n - i, kMaxBody - page.text.bytes());
size_t pageCost = page.text.bytes() + page.text.lineCount() * kBytesPerLine + TextBuffer::kChunk + take;
if (take < static_cast<size_t>(n - i)) page.truncatedWhy = "longer than 64 KB";
else if (freeBefore < kSteadyFloor + pageCost) page.truncatedWhy = "not enough memory to keep it";
else if (esp_get_free_heap_size() < kTransientFloor + TextBuffer::kChunk + take)
page.truncatedWhy = "not enough memory while receiving";
if (!page.truncatedWhy.empty()) {
page.truncated = true;
break;
}
page.text.append(buf + i, take);
track();
}
if (cancel_) {
page.error = "Cancelled";
return;
}
if (!inBody && !gemini::parseHeader(header, page.header)) { // a header with no line end
page.error = header.empty() ? "Empty response" : "Malformed response header";
return;
}
page.text.finish();
page.outcome = GeminiPage::Outcome::Ok;
}
// Streams the rest of the body to a cache file in 1 KB pieces, each written by the storage task:
// with IRC's TLS connection and this one both open, every KB of RAM counts (Q86).
bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page, int slot) {
struct Cache {
File file;
};
auto cache = std::make_shared<Cache>();
bool opened = storage_.runAndWait([cache, slot]() {
SD.mkdir("/gemini");
SD.mkdir("/gemini/cache");
cache->file = SD.open(slotPath(slot), FILE_WRITE);
});
if (!opened || !cache->file) {
page.error = "Can't write to the SD card";
return false;
}
std::string pending(first, len);
constexpr size_t kPiece = 1024;
pending.reserve(kPiece + 512);
bool ok = true;
auto flush = [&]() {
if (pending.empty() || !ok) return;
ok = storage_.runAndWait([cache, &pending, &ok]() {
ok = cache->file.write(reinterpret_cast<const uint8_t*>(pending.data()), pending.size()) == pending.size();
}) && ok;
page.receivedBytes += pending.size();
pending.clear();
};
uint32_t last = millis();
char buf[512];
while (!cancel_ && ok) {
if (pending.size() >= kPiece) flush();
if (page.receivedBytes + pending.size() >= kMaxOnCard) {
page.truncated = true;
page.truncatedWhy = "longer than 1 MB";
break;
}
int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
if (n <= 0) {
if (!tls.connected()) break;
if (millis() - last > kIdleTimeoutMs) break; // keep what came
delay(5);
continue;
}
last = millis();
pending.append(buf, n);
lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size());
}
flush();
storage_.runAndWait([cache]() { cache->file.close(); });
if (cancel_) page.error = "Cancelled";
else if (!ok) page.error = "Writing to the SD card failed";
else page.cachePath = slotPath(slot);
return !cancel_ && ok;
}
// Back into RAM, now that the connection's memory is free again: one pass over the file counts
// its lines and indexes every kIndexEvery-th, and loads lines while the steady floor allows (Q86).
// If they don't all fit, the page is windowed: the App asks for other windows as you scroll.
void GeminiService::loadFromCard(GeminiPage& page, size_t freeBefore, const std::string& path) {
size_t budget = freeBefore > kSteadyFloor + TextBuffer::kChunk ? freeBefore - kSteadyFloor - TextBuffer::kChunk : 0;
budget = std::min(budget, kMaxBody);
page.backing = path;
page.lineIndex.clear();
page.totalLines = 0;
page.windowStart = 0;
bool loading = true;
storage_.runAndWait([&]() {
File f = SD.open(path.c_str());
if (!f) return;
std::string line;
uint32_t offset = 0, lineStart = 0;
bool pre = false;
auto endLine = [&]() {
if (page.totalLines % kIndexEvery == 0) page.lineIndex.push_back(lineStart | (pre ? 0x80000000u : 0));
if (line.compare(0, 3, "```") == 0) pre = !pre;
if (loading) {
if (page.text.bytes() + page.text.lineCount() * kBytesPerLine + line.size() + 1 > budget) loading = false;
else {
line += '\n';
page.text.append(line.data(), line.size());
}
}
page.totalLines++;
line.clear();
};
char buf[512];
int n;
while ((n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
for (int i = 0; i < n; i++, offset++) {
if (buf[i] == '\n') {
endLine();
lineStart = offset + 1;
} else if (line.size() < kMaxLineBytes) line += buf[i];
}
}
if (!line.empty()) endLine();
f.close();
});
page.text.finish();
page.windowed = !loading;
if (page.windowed)
console.printf("gemini: %s is windowed: %u of %u lines in memory\n", path.c_str(),
(unsigned)page.text.lineCount(), (unsigned)page.totalLines);
}
// Lines from `firstLine` on, from the page's file, as many as `budget` allows.
void GeminiService::readWindow(GeminiPage& page, size_t firstLine, size_t budget) {
size_t entry = firstLine / kIndexEvery;
if (entry >= page.lineIndex.size()) return;
storage_.runAndWait([&]() {
File f = SD.open(page.backing.c_str());
if (!f || !f.seek(page.lineIndex[entry] & 0x7FFFFFFFu)) return;
size_t lineNo = entry * kIndexEvery;
std::string line;
char buf[512];
int n;
bool done = false;
while (!done && (n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
for (int i = 0; i < n && !done; i++) {
if (buf[i] != '\n') {
if (line.size() < kMaxLineBytes) line += buf[i];
continue;
}
if (lineNo >= firstLine) {
if (page.text.bytes() + page.text.lineCount() * kBytesPerLine + line.size() + 1 > budget) done = true;
else {
line += '\n';
page.text.append(line.data(), line.size());
}
}
lineNo++;
line.clear();
}
}
if (!done && !line.empty() && lineNo >= firstLine) page.text.append(line.data(), line.size());
f.close();
});
page.text.finish();
}
void GeminiService::report(const GeminiPage& page, size_t heapBefore, size_t heapLowest) {
console.printf("gemini: %s -> %s, %lu ms\n", page.requested.c_str(), page.url.c_str(), (unsigned long)page.ms);
if (page.outcome != GeminiPage::Outcome::Ok) console.printf("gemini: %s\n", page.error.c_str());
else console.printf("gemini: %d %s, %u bytes, %u lines%s\n", page.header.status, page.header.meta.c_str(),
(unsigned)page.text.bytes(), (unsigned)page.text.lineCount(),
page.truncated ? (" (truncated: " + page.truncatedWhy + ")").c_str() : "");
if (page.outcome == GeminiPage::Outcome::CertificateChanged)
console.printf("gemini: pinned %s\ngemini: now %s\n", page.pinned.c_str(), page.presented.c_str());
for (size_t i = 0; i < page.text.lineCount() && i < 6; i++) console.printf(" %s\n", page.text.line(i).substr(0, 100).c_str());
if (!page.cachePath.empty())
console.printf("gemini: %u bytes on the card in %s\n", (unsigned)page.receivedBytes, page.cachePath.c_str());
console.printf("gemini: heap %u before, %u lowest seen, %u after\n", (unsigned)heapBefore, (unsigned)heapLowest,
(unsigned)esp_get_free_heap_size());
}
} // namespace roro
Reference in New Issue View Git Blame Copy Permalink