twisla/roro9stack
Public Access
1
0
Fork 0
Code Issues 49 Pull Requests 1 Actions Packages Releases 15 Activity

G1 step 3: the Gemini fetcher (TOFU, redirects, floors, pages via the card)

Browse Source 
GeminiService fetches on a short-lived task and hands the App a
GeminiPage: header, the body as lines in 4 KB chunks (TextBuffer: no
large block, no doubling copies), the final URL after up to 5
redirects. Certificates are pinned on first use per host and port; a
change comes back as its own outcome with both fingerprints. No fetch
starts below 55 KB free (Q86).

With a card, the body streams to /gemini/cache/page.gmi in 1 KB pieces
while the connection is open, then loads into RAM once its memory is
back (Q87); StorageService::runAndWait (moved from the Debug Console)
keeps every card access on the storage task. Without a card: RAM, with
the steady and transient floors.

Measured with IRC connected: Cosmos (31.6 KB) went from 4.6 KB to the
whole page on the card and 20 KB on screen; lowest free heap 19.5 KB
in transfer, 43 KB once loaded. `gemini get` and `gemini trust` on the
console. 14 Gemini tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT
This commit is contained in:
twisla
2026-10-05 00:58:01 +02:00
co-authored by Claude Opus 5.5
parent babb1d114e
commit 7b8d9391a4
12 changed files with 552 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/services/gemini_service.cpp
+291 -65
View File
@@ -1,22 +1,22 @@
#include "services/gemini_service.h"
#include <NetworkClientSecure.h>
#include <SD.h>
#include <esp_heap_caps.h>
#include <algorithm>
#include <memory>
#include "gemini_url.h"
#include "platform/console.h"
#include "sha256.h"
using roro::gemini::TextBuffer;
namespace roro {
namespace {
struct ConsoleFetch {
GeminiService* service;
std::string url;
volatile bool* busy;
};
constexpr uint32_t kIdleTimeoutMs = 15000;
std::string hex(const uint8_t* data, size_t len) {
static const char* digits = "0123456789abcdef";
@@ -30,75 +30,301 @@ std::string hex(const uint8_t* data, size_t len) {
} // namespace
bool GeminiService::fetchToConsole(const std::string& url) {
GeminiService::GeminiService(KeyValueStore& store, StorageService& storage)
: store_(store), storage_(storage), lock_(xSemaphoreCreateMutex()) {}
// NVS keys are 15 characters at most: "gm" and 12 hex digits of SHA-256("host:port").
std::string GeminiService::pinKey(const std::string& host, int port) const {
std::string id = host + ":" + std::to_string(port);
uint8_t digest[32];
Sha256::hash(reinterpret_cast<const uint8_t*>(id.data()), id.size(), digest);
return "gm" + hex(digest, 6);
}
void GeminiService::trust(const std::string& host, int port, const std::string& fingerprint) {
store_.putString(pinKey(host, port).c_str(), fingerprint);
}
bool GeminiService::fetch(const std::string& url) {
if (busy_) return false;
busy_ = true;
auto* job = new ConsoleFetch{this, url, &busy_};
if (xTaskCreate(consoleTask, "gemini", 6144, job, 1, nullptr) != pdPASS) {
delete job;
ready_ = cancel_ = false;
pendingUrl_ = url;
if (xTaskCreate(taskEntry, "gemini", 6144, this, 1, nullptr) != pdPASS) { // peak 3.6 KB (step 1)
busy_ = false;
return false;
}
return true;
}
void GeminiService::consoleTask(void* arg) {
std::unique_ptr<ConsoleFetch> job(static_cast<ConsoleFetch*>(arg));
// Until step 2's URL parser: gemini://host[:port]/path
std::string rest = job->url.substr(job->url.find("://") == std::string::npos ? 0 : job->url.find("://") + 3);
std::string authority = rest.substr(0, rest.find('/'));
std::string host = authority.substr(0, authority.find(':'));
uint16_t port = authority.find(':') == std::string::npos ? 1965 : atoi(authority.substr(authority.find(':') + 1).c_str());
size_t before = esp_get_free_heap_size(), lowest = before;
auto track = [&lowest]() { lowest = std::min<size_t>(lowest, esp_get_free_heap_size()); };
uint32_t start = millis();
{
NetworkClientSecure tls;
tls.setInsecure(); // trust on first use: the fingerprint is what counts (Q71)
tls.setTimeout(15);
if (!tls.connect(host.c_str(), port)) {
console.printf("gemini: cannot connect to %s:%u\n", host.c_str(), port);
} else {
uint32_t connected = millis();
track(); // the handshake's buffers are allocated now
uint8_t sha[32];
std::string fingerprint = tls.getFingerprintSHA256(sha) ? hex(sha, 32) : "?";
tls.print((job->url + "\r\n").c_str());
std::string header, sample;
size_t body = 0;
bool inBody = false;
uint32_t last = millis();
while (tls.connected() || tls.available()) {
int c = tls.read();
if (c < 0) {
if (millis() - last > 15000) break;
delay(5);
continue;
}
last = millis();
if ((body & 255) == 0) track();
if (!inBody) {
if (c == '\n') inBody = true;
else if (c != '\r' && header.size() < 1100) header += static_cast<char>(c);
} else {
body++;
if (sample.size() < 400) sample += static_cast<char>(c);
}
}
console.printf("gemini: %s\ngemini: header '%s', %u bytes, TLS %lu ms, total %lu ms\n", job->url.c_str(),
header.c_str(), (unsigned)body, (unsigned long)(connected - start),
(unsigned long)(millis() - start));
console.printf("gemini: certificate sha256 %s\n", fingerprint.c_str());
console.printf("--- first bytes ---\n%s\n--- end ---\n", sample.c_str());
}
tls.stop();
bool GeminiService::fetchToConsole(const std::string& url) {
if (busy_) return false;
toConsole_ = true;
if (!fetch(url)) {
toConsole_ = false;
return false;
}
console.printf("gemini: heap %u before, %u lowest during, %u after; stack left %u\n", (unsigned)before,
(unsigned)lowest, (unsigned)esp_get_free_heap_size(), (unsigned)uxTaskGetStackHighWaterMark(nullptr));
*job->busy = false;
job.reset();
return true;
}
bool GeminiService::takeResult(GeminiPage& out) {
if (!ready_) return false;
xSemaphoreTake(lock_, portMAX_DELAY);
out = std::move(result_);
result_ = GeminiPage();
ready_ = false;
xSemaphoreGive(lock_);
return true;
}
void GeminiService::taskEntry(void* self) {
static_cast<GeminiService*>(self)->run();
vTaskDelete(nullptr);
}
void GeminiService::run() {
GeminiPage page;
page.requested = pendingUrl_;
size_t before = esp_get_free_heap_size();
lowest_ = before;
uint32_t start = millis();
std::string url = pendingUrl_;
if (before < kStartFloor) {
page.error = "Not enough memory (" + std::to_string(before / 1024) + " KB free): stop IRC or retry";
} else {
for (int hop = 0;; hop++) {
fetchOne(url, page);
if (page.outcome != GeminiPage::Outcome::Ok || page.header.category() != gemini::Category::Redirect) break;
std::string next = gemini::resolve(url, page.header.meta);
if (!gemini::isGemini(next)) {
page.outcome = GeminiPage::Outcome::Failed;
page.error = "Redirect to another protocol: " + next;
break;
}
if (hop + 1 > kMaxRedirects) {
page.outcome = GeminiPage::Outcome::Failed;
page.error = "Too many redirects";
break;
}
url = next;
}
}
page.url = url;
page.ms = millis() - start;
if (toConsole_) {
report(page, before, lowest_);
toConsole_ = false;
} else {
xSemaphoreTake(lock_, portMAX_DELAY);
result_ = std::move(page);
ready_ = true;
xSemaphoreGive(lock_);
}
busy_ = false;
}
void GeminiService::fetchOne(const std::string& url, GeminiPage& page) {
page.outcome = GeminiPage::Outcome::Failed;
page.text.clear();
page.truncated = false;
gemini::Url u;
if (!gemini::parseUrl(url, u) || u.scheme != "gemini") {
page.error = "Not a Gemini URL: " + url;
return;
}
auto track = [this]() { lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size()); };
size_t freeBefore = esp_get_free_heap_size(); // what's left once the connection closes again
NetworkClientSecure tls;
tls.setInsecure(); // trust on first use: the pinned fingerprint is what counts (Q71)
tls.setTimeout(15);
if (!tls.connect(u.host.c_str(), u.portOrDefault())) {
page.error = "Cannot connect to " + u.host;
return;
}
track();
uint8_t sha[32];
if (!tls.getFingerprintSHA256(sha)) {
page.error = "No certificate from " + u.host;
return;
}
std::string presented = hex(sha, sizeof sha), pinned;
std::string key = pinKey(u.host, u.portOrDefault());
if (!store_.getString(key.c_str(), pinned)) store_.putString(key.c_str(), presented); // first use
else if (pinned != presented) {
page.outcome = GeminiPage::Outcome::CertificateChanged;
page.host = u.host;
page.port = u.portOrDefault();
page.pinned = pinned;
page.presented = presented;
page.error = "The certificate of " + u.host + " changed";
return;
}
tls.print((gemini::requestUrl(url) + "\r\n").c_str());
std::string header;
bool inBody = false;
uint32_t last = millis();
char buf[512];
while (!cancel_) {
int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
if (n <= 0) {
if (!tls.connected()) break;
if (millis() - last > kIdleTimeoutMs) {
page.error = "Timed out";
return;
}
delay(5);
continue;
}
last = millis();
int i = 0;
while (!inBody && i < n) {
char c = buf[i++];
if (c == '\n') inBody = true;
else if (c != '\r') header += c;
if (header.size() > 1100) {
page.error = "Malformed response header";
return;
}
}
if (!inBody) continue;
if (!gemini::parseHeader(header, page.header)) {
page.error = "Malformed response header";
return;
}
if (page.header.category() != gemini::Category::Success) break; // no body to read
// With a card, the body goes there while the connection is open (only 4 KB in RAM),
// and comes back into RAM once the connection's memory is free again.
if (storage_.state().present) {
if (!receiveToCard(tls, buf + i, n - i, page)) return;
tls.stop();
loadFromCard(page, freeBefore);
page.outcome = GeminiPage::Outcome::Ok;
return;
}
// Q86, two floors: once the connection closes, its ~50 KB comes back, so the page may use
// what was free before minus the steady floor; meanwhile the heap must stay above the
// transient floor. Both counted with the next 4 KB chunk and the line index.
size_t take = std::min<size_t>(n - i, kMaxBody - page.text.bytes());
size_t pageCost = page.text.bytes() + page.text.lineCount() * 8 + TextBuffer::kChunk + take;
if (take < static_cast<size_t>(n - i)) page.truncatedWhy = "longer than 64 KB";
else if (freeBefore < kSteadyFloor + pageCost) page.truncatedWhy = "not enough memory to keep it";
else if (esp_get_free_heap_size() < kTransientFloor + TextBuffer::kChunk + take)
page.truncatedWhy = "not enough memory while receiving";
if (!page.truncatedWhy.empty()) {
page.truncated = true;
break;
}
page.text.append(buf + i, take);
track();
}
if (cancel_) {
page.error = "Cancelled";
return;
}
if (!inBody && !gemini::parseHeader(header, page.header)) { // a header with no line end
page.error = header.empty() ? "Empty response" : "Malformed response header";
return;
}
page.text.finish();
page.outcome = GeminiPage::Outcome::Ok;
}
// Streams the rest of the body to kCachePath in 1 KB pieces, each written by the storage task:
// with IRC's TLS connection and this one both open, every KB of RAM counts (Q86).
bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page) {
struct Cache {
File file;
};
auto cache = std::make_shared<Cache>();
bool opened = storage_.runAndWait([cache]() {
SD.mkdir("/gemini");
SD.mkdir("/gemini/cache");
cache->file = SD.open(kCachePath, FILE_WRITE);
});
if (!opened || !cache->file) {
page.error = "Can't write to the SD card";
return false;
}
std::string pending(first, len);
constexpr size_t kPiece = 1024;
pending.reserve(kPiece + 512);
bool ok = true;
auto flush = [&]() {
if (pending.empty() || !ok) return;
ok = storage_.runAndWait([cache, &pending, &ok]() {
ok = cache->file.write(reinterpret_cast<const uint8_t*>(pending.data()), pending.size()) == pending.size();
}) && ok;
page.receivedBytes += pending.size();
pending.clear();
};
uint32_t last = millis();
char buf[512];
while (!cancel_ && ok) {
if (pending.size() >= kPiece) flush();
if (page.receivedBytes + pending.size() >= kMaxOnCard) {
page.truncated = true;
page.truncatedWhy = "longer than 1 MB";
break;
}
int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
if (n <= 0) {
if (!tls.connected()) break;
if (millis() - last > kIdleTimeoutMs) break; // keep what came
delay(5);
continue;
}
last = millis();
pending.append(buf, n);
lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size());
}
flush();
storage_.runAndWait([cache]() { cache->file.close(); });
if (cancel_) page.error = "Cancelled";
else if (!ok) page.error = "Writing to the SD card failed";
else page.cachePath = kCachePath;
return !cancel_ && ok;
}
// Back into RAM, now that the connection's memory is free again: as much as the steady floor
// allows (Q86); the rest stays on the card.
void GeminiService::loadFromCard(GeminiPage& page, size_t freeBefore) {
size_t budget = freeBefore > kSteadyFloor + TextBuffer::kChunk ? freeBefore - kSteadyFloor - TextBuffer::kChunk : 0;
budget = std::min(budget, kMaxBody);
storage_.runAndWait([&]() {
File f = SD.open(kCachePath);
if (!f) return;
char buf[512];
int n;
while ((n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
size_t cost = page.text.bytes() + page.text.lineCount() * 8 + n;
if (cost > budget) {
page.truncated = true;
if (page.truncatedWhy.empty())
page.truncatedWhy = "only part of it fits in memory; the whole page is on the card";
break;
}
page.text.append(buf, n);
}
f.close();
});
page.text.finish();
}
void GeminiService::report(const GeminiPage& page, size_t heapBefore, size_t heapLowest) {
console.printf("gemini: %s -> %s, %lu ms\n", page.requested.c_str(), page.url.c_str(), (unsigned long)page.ms);
if (page.outcome != GeminiPage::Outcome::Ok) console.printf("gemini: %s\n", page.error.c_str());
else console.printf("gemini: %d %s, %u bytes, %u lines%s\n", page.header.status, page.header.meta.c_str(),
(unsigned)page.text.bytes(), (unsigned)page.text.lineCount(),
page.truncated ? (" (truncated: " + page.truncatedWhy + ")").c_str() : "");
if (page.outcome == GeminiPage::Outcome::CertificateChanged)
console.printf("gemini: pinned %s\ngemini: now %s\n", page.pinned.c_str(), page.presented.c_str());
for (size_t i = 0; i < page.text.lineCount() && i < 6; i++) console.printf(" %s\n", page.text.line(i).substr(0, 100).c_str());
if (!page.cachePath.empty())
console.printf("gemini: %u bytes on the card in %s\n", (unsigned)page.receivedBytes, page.cachePath.c_str());
console.printf("gemini: heap %u before, %u lowest seen, %u after\n", (unsigned)heapBefore, (unsigned)heapLowest,
(unsigned)esp_get_free_heap_size());
}
} // namespace roro