G1 step 3: the Gemini fetcher (TOFU, redirects, floors, pages via the card)

GeminiService fetches on a short-lived task and hands the App a
GeminiPage: header, the body as lines in 4 KB chunks (TextBuffer: no
large block, no doubling copies), the final URL after up to 5
redirects. Certificates are pinned on first use per host and port; a
change comes back as its own outcome with both fingerprints. No fetch
starts below 55 KB free (Q86).

With a card, the body streams to /gemini/cache/page.gmi in 1 KB pieces
while the connection is open, then loads into RAM once its memory is
back (Q87); StorageService::runAndWait (moved from the Debug Console)
keeps every card access on the storage task. Without a card: RAM, with
the steady and transient floors.

Measured with IRC connected: Cosmos (31.6 KB) went from 4.6 KB to the
whole page on the card and 20 KB on screen; lowest free heap 19.5 KB
in transfer, 43 KB once loaded. `gemini get` and `gemini trust` on the
console. 14 Gemini tests.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT

src/services/debug_console.cpp

@@ -187,29 +187,7 @@ bool DebugConsole::binaryCommand(NetworkClient& client, const std::string& line)
     return true;
 }
 
-bool DebugConsole::onStorage(std::function<void()> job) {
-    // Shared with the job, which outlives this wait if the card is missing and it never starts.
-    // Exactly one side wins the state change: the job (Queued -> Running) or the wait giving up
-    // (Queued -> Abandoned), so an abandoned job can never touch this stack frame.
-    enum : int { Queued, Running, Abandoned };
-    struct Run {
-        std::atomic<int> state{Queued};
-        SemaphoreHandle_t done = xSemaphoreCreateBinary();
-        ~Run() { vSemaphoreDelete(done); }
-    };
-    auto run = std::make_shared<Run>();
-    storage_.runJob([run, job]() {
-        int expected = Queued;
-        if (!run->state.compare_exchange_strong(expected, Running)) return;
-        job();
-        xSemaphoreGive(run->done);
-    });
-    for (int waited = 0; xSemaphoreTake(run->done, pdMS_TO_TICKS(500)) != pdTRUE; waited += 500) {
-        int expected = Queued;
-        if (waited >= 5000 && run->state.compare_exchange_strong(expected, Abandoned)) return false;
-    }
-    return true;
-}
+bool DebugConsole::onStorage(std::function<void()> job) { return storage_.runAndWait(std::move(job)); }
 
 void DebugConsole::get(NetworkClient& client, const std::string& path) {
     bool ran = onStorage([&]() {

src/services/gemini_service.cpp

@@ -1,22 +1,22 @@
 #include "services/gemini_service.h"
 
 #include <NetworkClientSecure.h>
+#include <SD.h>
 #include <esp_heap_caps.h>
 
 #include <algorithm>
-#include <memory>
 
+#include "gemini_url.h"
 #include "platform/console.h"
+#include "sha256.h"
+
+using roro::gemini::TextBuffer;
 
 namespace roro {
 
 namespace {
 
-struct ConsoleFetch {
-    GeminiService* service;
-    std::string url;
-    volatile bool* busy;
-};
+constexpr uint32_t kIdleTimeoutMs = 15000;
 
 std::string hex(const uint8_t* data, size_t len) {
     static const char* digits = "0123456789abcdef";
@@ -30,75 +30,301 @@ std::string hex(const uint8_t* data, size_t len) {
 
 }  // namespace
 
-bool GeminiService::fetchToConsole(const std::string& url) {
+GeminiService::GeminiService(KeyValueStore& store, StorageService& storage)
+    : store_(store), storage_(storage), lock_(xSemaphoreCreateMutex()) {}
+
+// NVS keys are 15 characters at most: "gm" and 12 hex digits of SHA-256("host:port").
+std::string GeminiService::pinKey(const std::string& host, int port) const {
+    std::string id = host + ":" + std::to_string(port);
+    uint8_t digest[32];
+    Sha256::hash(reinterpret_cast<const uint8_t*>(id.data()), id.size(), digest);
+    return "gm" + hex(digest, 6);
+}
+
+void GeminiService::trust(const std::string& host, int port, const std::string& fingerprint) {
+    store_.putString(pinKey(host, port).c_str(), fingerprint);
+}
+
+bool GeminiService::fetch(const std::string& url) {
     if (busy_) return false;
     busy_ = true;
-    auto* job = new ConsoleFetch{this, url, &busy_};
-    if (xTaskCreate(consoleTask, "gemini", 6144, job, 1, nullptr) != pdPASS) {
-        delete job;
+    ready_ = cancel_ = false;
+    pendingUrl_ = url;
+    if (xTaskCreate(taskEntry, "gemini", 6144, this, 1, nullptr) != pdPASS) {  // peak 3.6 KB (step 1)
         busy_ = false;
         return false;
     }
     return true;
 }
 
-void GeminiService::consoleTask(void* arg) {
-    std::unique_ptr<ConsoleFetch> job(static_cast<ConsoleFetch*>(arg));
-    // Until step 2's URL parser: gemini://host[:port]/path
-    std::string rest = job->url.substr(job->url.find("://") == std::string::npos ? 0 : job->url.find("://") + 3);
-    std::string authority = rest.substr(0, rest.find('/'));
-    std::string host = authority.substr(0, authority.find(':'));
-    uint16_t port = authority.find(':') == std::string::npos ? 1965 : atoi(authority.substr(authority.find(':') + 1).c_str());
-
-    size_t before = esp_get_free_heap_size(), lowest = before;
-    auto track = [&lowest]() { lowest = std::min<size_t>(lowest, esp_get_free_heap_size()); };
-    uint32_t start = millis();
-    {
-        NetworkClientSecure tls;
-        tls.setInsecure();  // trust on first use: the fingerprint is what counts (Q71)
-        tls.setTimeout(15);
-        if (!tls.connect(host.c_str(), port)) {
-            console.printf("gemini: cannot connect to %s:%u\n", host.c_str(), port);
-        } else {
-            uint32_t connected = millis();
-            track();  // the handshake's buffers are allocated now
-            uint8_t sha[32];
-            std::string fingerprint = tls.getFingerprintSHA256(sha) ? hex(sha, 32) : "?";
-            tls.print((job->url + "\r\n").c_str());
-            std::string header, sample;
-            size_t body = 0;
-            bool inBody = false;
-            uint32_t last = millis();
-            while (tls.connected() || tls.available()) {
-                int c = tls.read();
-                if (c < 0) {
-                    if (millis() - last > 15000) break;
-                    delay(5);
-                    continue;
-                }
-                last = millis();
-                if ((body & 255) == 0) track();
-                if (!inBody) {
-                    if (c == '\n') inBody = true;
-                    else if (c != '\r' && header.size() < 1100) header += static_cast<char>(c);
-                } else {
-                    body++;
-                    if (sample.size() < 400) sample += static_cast<char>(c);
-                }
-            }
-            console.printf("gemini: %s\ngemini: header '%s', %u bytes, TLS %lu ms, total %lu ms\n", job->url.c_str(),
-                           header.c_str(), (unsigned)body, (unsigned long)(connected - start),
-                           (unsigned long)(millis() - start));
-            console.printf("gemini: certificate sha256 %s\n", fingerprint.c_str());
-            console.printf("--- first bytes ---\n%s\n--- end ---\n", sample.c_str());
-        }
-        tls.stop();
+bool GeminiService::fetchToConsole(const std::string& url) {
+    if (busy_) return false;
+    toConsole_ = true;
+    if (!fetch(url)) {
+        toConsole_ = false;
+        return false;
     }
-    console.printf("gemini: heap %u before, %u lowest during, %u after; stack left %u\n", (unsigned)before,
-                   (unsigned)lowest, (unsigned)esp_get_free_heap_size(), (unsigned)uxTaskGetStackHighWaterMark(nullptr));
-    *job->busy = false;
-    job.reset();
+    return true;
+}
+
+bool GeminiService::takeResult(GeminiPage& out) {
+    if (!ready_) return false;
+    xSemaphoreTake(lock_, portMAX_DELAY);
+    out = std::move(result_);
+    result_ = GeminiPage();
+    ready_ = false;
+    xSemaphoreGive(lock_);
+    return true;
+}
+
+void GeminiService::taskEntry(void* self) {
+    static_cast<GeminiService*>(self)->run();
     vTaskDelete(nullptr);
 }
 
+void GeminiService::run() {
+    GeminiPage page;
+    page.requested = pendingUrl_;
+    size_t before = esp_get_free_heap_size();
+    lowest_ = before;
+    uint32_t start = millis();
+    std::string url = pendingUrl_;
+    if (before < kStartFloor) {
+        page.error = "Not enough memory (" + std::to_string(before / 1024) + " KB free): stop IRC or retry";
+    } else {
+        for (int hop = 0;; hop++) {
+            fetchOne(url, page);
+            if (page.outcome != GeminiPage::Outcome::Ok || page.header.category() != gemini::Category::Redirect) break;
+            std::string next = gemini::resolve(url, page.header.meta);
+            if (!gemini::isGemini(next)) {
+                page.outcome = GeminiPage::Outcome::Failed;
+                page.error = "Redirect to another protocol: " + next;
+                break;
+            }
+            if (hop + 1 > kMaxRedirects) {
+                page.outcome = GeminiPage::Outcome::Failed;
+                page.error = "Too many redirects";
+                break;
+            }
+            url = next;
+        }
+    }
+    page.url = url;
+    page.ms = millis() - start;
+    if (toConsole_) {
+        report(page, before, lowest_);
+        toConsole_ = false;
+    } else {
+        xSemaphoreTake(lock_, portMAX_DELAY);
+        result_ = std::move(page);
+        ready_ = true;
+        xSemaphoreGive(lock_);
+    }
+    busy_ = false;
+}
+
+void GeminiService::fetchOne(const std::string& url, GeminiPage& page) {
+    page.outcome = GeminiPage::Outcome::Failed;
+    page.text.clear();
+    page.truncated = false;
+    gemini::Url u;
+    if (!gemini::parseUrl(url, u) || u.scheme != "gemini") {
+        page.error = "Not a Gemini URL: " + url;
+        return;
+    }
+    auto track = [this]() { lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size()); };
+
+    size_t freeBefore = esp_get_free_heap_size();  // what's left once the connection closes again
+    NetworkClientSecure tls;
+    tls.setInsecure();  // trust on first use: the pinned fingerprint is what counts (Q71)
+    tls.setTimeout(15);
+    if (!tls.connect(u.host.c_str(), u.portOrDefault())) {
+        page.error = "Cannot connect to " + u.host;
+        return;
+    }
+    track();
+    uint8_t sha[32];
+    if (!tls.getFingerprintSHA256(sha)) {
+        page.error = "No certificate from " + u.host;
+        return;
+    }
+    std::string presented = hex(sha, sizeof sha), pinned;
+    std::string key = pinKey(u.host, u.portOrDefault());
+    if (!store_.getString(key.c_str(), pinned)) store_.putString(key.c_str(), presented);  // first use
+    else if (pinned != presented) {
+        page.outcome = GeminiPage::Outcome::CertificateChanged;
+        page.host = u.host;
+        page.port = u.portOrDefault();
+        page.pinned = pinned;
+        page.presented = presented;
+        page.error = "The certificate of " + u.host + " changed";
+        return;
+    }
+
+    tls.print((gemini::requestUrl(url) + "\r\n").c_str());
+    std::string header;
+    bool inBody = false;
+    uint32_t last = millis();
+    char buf[512];
+    while (!cancel_) {
+        int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
+        if (n <= 0) {
+            if (!tls.connected()) break;
+            if (millis() - last > kIdleTimeoutMs) {
+                page.error = "Timed out";
+                return;
+            }
+            delay(5);
+            continue;
+        }
+        last = millis();
+        int i = 0;
+        while (!inBody && i < n) {
+            char c = buf[i++];
+            if (c == '\n') inBody = true;
+            else if (c != '\r') header += c;
+            if (header.size() > 1100) {
+                page.error = "Malformed response header";
+                return;
+            }
+        }
+        if (!inBody) continue;
+        if (!gemini::parseHeader(header, page.header)) {
+            page.error = "Malformed response header";
+            return;
+        }
+        if (page.header.category() != gemini::Category::Success) break;  // no body to read
+        // With a card, the body goes there while the connection is open (only 4 KB in RAM),
+        // and comes back into RAM once the connection's memory is free again.
+        if (storage_.state().present) {
+            if (!receiveToCard(tls, buf + i, n - i, page)) return;
+            tls.stop();
+            loadFromCard(page, freeBefore);
+            page.outcome = GeminiPage::Outcome::Ok;
+            return;
+        }
+        // Q86, two floors: once the connection closes, its ~50 KB comes back, so the page may use
+        // what was free before minus the steady floor; meanwhile the heap must stay above the
+        // transient floor. Both counted with the next 4 KB chunk and the line index.
+        size_t take = std::min<size_t>(n - i, kMaxBody - page.text.bytes());
+        size_t pageCost = page.text.bytes() + page.text.lineCount() * 8 + TextBuffer::kChunk + take;
+        if (take < static_cast<size_t>(n - i)) page.truncatedWhy = "longer than 64 KB";
+        else if (freeBefore < kSteadyFloor + pageCost) page.truncatedWhy = "not enough memory to keep it";
+        else if (esp_get_free_heap_size() < kTransientFloor + TextBuffer::kChunk + take)
+            page.truncatedWhy = "not enough memory while receiving";
+        if (!page.truncatedWhy.empty()) {
+            page.truncated = true;
+            break;
+        }
+        page.text.append(buf + i, take);
+        track();
+    }
+    if (cancel_) {
+        page.error = "Cancelled";
+        return;
+    }
+    if (!inBody && !gemini::parseHeader(header, page.header)) {  // a header with no line end
+        page.error = header.empty() ? "Empty response" : "Malformed response header";
+        return;
+    }
+    page.text.finish();
+    page.outcome = GeminiPage::Outcome::Ok;
+}
+
+// Streams the rest of the body to kCachePath in 1 KB pieces, each written by the storage task:
+// with IRC's TLS connection and this one both open, every KB of RAM counts (Q86).
+bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page) {
+    struct Cache {
+        File file;
+    };
+    auto cache = std::make_shared<Cache>();
+    bool opened = storage_.runAndWait([cache]() {
+        SD.mkdir("/gemini");
+        SD.mkdir("/gemini/cache");
+        cache->file = SD.open(kCachePath, FILE_WRITE);
+    });
+    if (!opened || !cache->file) {
+        page.error = "Can't write to the SD card";
+        return false;
+    }
+    std::string pending(first, len);
+    constexpr size_t kPiece = 1024;
+    pending.reserve(kPiece + 512);
+    bool ok = true;
+    auto flush = [&]() {
+        if (pending.empty() || !ok) return;
+        ok = storage_.runAndWait([cache, &pending, &ok]() {
+            ok = cache->file.write(reinterpret_cast<const uint8_t*>(pending.data()), pending.size()) == pending.size();
+        }) && ok;
+        page.receivedBytes += pending.size();
+        pending.clear();
+    };
+    uint32_t last = millis();
+    char buf[512];
+    while (!cancel_ && ok) {
+        if (pending.size() >= kPiece) flush();
+        if (page.receivedBytes + pending.size() >= kMaxOnCard) {
+            page.truncated = true;
+            page.truncatedWhy = "longer than 1 MB";
+            break;
+        }
+        int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
+        if (n <= 0) {
+            if (!tls.connected()) break;
+            if (millis() - last > kIdleTimeoutMs) break;  // keep what came
+            delay(5);
+            continue;
+        }
+        last = millis();
+        pending.append(buf, n);
+        lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size());
+    }
+    flush();
+    storage_.runAndWait([cache]() { cache->file.close(); });
+    if (cancel_) page.error = "Cancelled";
+    else if (!ok) page.error = "Writing to the SD card failed";
+    else page.cachePath = kCachePath;
+    return !cancel_ && ok;
+}
+
+// Back into RAM, now that the connection's memory is free again: as much as the steady floor
+// allows (Q86); the rest stays on the card.
+void GeminiService::loadFromCard(GeminiPage& page, size_t freeBefore) {
+    size_t budget = freeBefore > kSteadyFloor + TextBuffer::kChunk ? freeBefore - kSteadyFloor - TextBuffer::kChunk : 0;
+    budget = std::min(budget, kMaxBody);
+    storage_.runAndWait([&]() {
+        File f = SD.open(kCachePath);
+        if (!f) return;
+        char buf[512];
+        int n;
+        while ((n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
+            size_t cost = page.text.bytes() + page.text.lineCount() * 8 + n;
+            if (cost > budget) {
+                page.truncated = true;
+                if (page.truncatedWhy.empty())
+                    page.truncatedWhy = "only part of it fits in memory; the whole page is on the card";
+                break;
+            }
+            page.text.append(buf, n);
+        }
+        f.close();
+    });
+    page.text.finish();
+}
+
+void GeminiService::report(const GeminiPage& page, size_t heapBefore, size_t heapLowest) {
+    console.printf("gemini: %s -> %s, %lu ms\n", page.requested.c_str(), page.url.c_str(), (unsigned long)page.ms);
+    if (page.outcome != GeminiPage::Outcome::Ok) console.printf("gemini: %s\n", page.error.c_str());
+    else console.printf("gemini: %d %s, %u bytes, %u lines%s\n", page.header.status, page.header.meta.c_str(),
+                        (unsigned)page.text.bytes(), (unsigned)page.text.lineCount(),
+                        page.truncated ? (" (truncated: " + page.truncatedWhy + ")").c_str() : "");
+    if (page.outcome == GeminiPage::Outcome::CertificateChanged)
+        console.printf("gemini: pinned %s\ngemini: now    %s\n", page.pinned.c_str(), page.presented.c_str());
+    for (size_t i = 0; i < page.text.lineCount() && i < 6; i++) console.printf("  %s\n", page.text.line(i).substr(0, 100).c_str());
+    if (!page.cachePath.empty())
+        console.printf("gemini: %u bytes on the card in %s\n", (unsigned)page.receivedBytes, page.cachePath.c_str());
+    console.printf("gemini: heap %u before, %u lowest seen, %u after\n", (unsigned)heapBefore, (unsigned)heapLowest,
+                   (unsigned)esp_get_free_heap_size());
+}
+
 }  // namespace roro

src/services/gemini_service.h

@@ -1,21 +1,82 @@
 #pragma once
 
+#include <freertos/FreeRTOS.h>
+#include <freertos/semphr.h>
+
 #include <string>
 
+#include "gemini_response.h"
+#include "key_value_store.h"
+#include "services/storage_service.h"
+#include "text_buffer.h"
+
+class NetworkClientSecure;
+
 namespace roro {
 
+// What a fetch came back with.
+struct GeminiPage {
+    enum class Outcome { Ok, Failed, CertificateChanged };
+    Outcome outcome = Outcome::Failed;
+    std::string requested;    // what was asked for
+    std::string url;          // where it ended up, after redirects
+    gemini::Header header;    // the final response's header
+    gemini::TextBuffer text;  // 2x only: the body as lines, up to kMaxBody
+    bool truncated = false;   // the body was cut: see truncatedWhy
+    std::string truncatedWhy;
+    std::string error;        // Failed: why, for a human
+    std::string host;         // CertificateChanged: whose, and both fingerprints
+    int port = 1965;
+    std::string pinned, presented;
+    std::string cachePath;    // with a card: the whole body as received (Saved Pages copy it)
+    size_t receivedBytes = 0; // the whole body, which may be more than `text` holds
+    uint32_t ms = 0;
+};
+
 // Gemini (docs/milestones/G1.md). Each fetch runs on a short-lived task of its own: a TLS
 // handshake can block for seconds, which the main loop's watchdog wouldn't allow, and an idle
-// client should cost no stack.
+// client should cost no stack. Certificates are trusted on first use, per host and port (Q71).
 class GeminiService {
    public:
-    // `gemini get <url>`: fetches and prints the header, size, certificate fingerprint, the first
-    // lines and the heap around the fetch to the console. False if a fetch is already running.
+    static constexpr size_t kMaxBody = 64 * 1024;        // Q74
+    static constexpr size_t kStartFloor = 55 * 1024;     // Q86: no fetch below this
+    static constexpr size_t kSteadyFloor = 40 * 1024;    // Q86: what must be left once the page is in
+    static constexpr size_t kTransientFloor = 20 * 1024; // Q86: while the TLS connection is open
+    static constexpr int kMaxRedirects = 5;              // Q72
+
+    static constexpr size_t kMaxOnCard = 1024 * 1024;    // a text page larger than this is cut
+    static constexpr const char* kCachePath = "/gemini/cache/page.gmi";
+
+    GeminiService(KeyValueStore& store, StorageService& storage);
+
+    // Starts a fetch; false if one is running. The result comes from takeResult().
+    bool fetch(const std::string& url);
+    bool busy() const { return busy_; }
+    bool takeResult(GeminiPage& out);
+    void cancel() { cancel_ = true; }
+    // After the user accepts a changed certificate.
+    void trust(const std::string& host, int port, const std::string& fingerprint);
+
+    // `gemini get <url>`: the same fetch, reported on the console instead of to the App.
     bool fetchToConsole(const std::string& url);
 
    private:
-    static void consoleTask(void* arg);
-    volatile bool busy_ = false;
+    static void taskEntry(void* self);
+    void run();
+    void fetchOne(const std::string& url, GeminiPage& page);
+    void report(const GeminiPage& page, size_t heapBefore, size_t heapLowest);
+    std::string pinKey(const std::string& host, int port) const;
+
+    bool receiveToCard(::NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page);
+    void loadFromCard(GeminiPage& page, size_t freeBefore);
+
+    KeyValueStore& store_;
+    StorageService& storage_;
+    SemaphoreHandle_t lock_;
+    std::string pendingUrl_;
+    GeminiPage result_;
+    volatile bool busy_ = false, ready_ = false, cancel_ = false, toConsole_ = false;
+    size_t lowest_ = 0;
 };
 
 }  // namespace roro

src/services/storage_service.cpp

@@ -5,6 +5,7 @@
 #include <sd_diskio.h>
 
 #include <algorithm>
+#include <atomic>
 #include <memory>
 
 #include "platform/pins.h"
@@ -82,6 +83,30 @@ void StorageService::runJob(std::function<void()> job) {
     if (task_) xTaskNotifyGive(task_);
 }
 
+bool StorageService::runAndWait(std::function<void()> job) {
+    // Shared with the job, which outlives this wait if the card is missing and it never starts.
+    // Exactly one side wins the state change: the job (Queued -> Running) or the wait giving up
+    // (Queued -> Abandoned), so an abandoned job can never touch this stack frame.
+    enum : int { Queued, Running, Abandoned };
+    struct Run {
+        std::atomic<int> state{Queued};
+        SemaphoreHandle_t done = xSemaphoreCreateBinary();
+        ~Run() { vSemaphoreDelete(done); }
+    };
+    auto run = std::make_shared<Run>();
+    runJob([run, job]() {
+        int expected = Queued;
+        if (!run->state.compare_exchange_strong(expected, Running)) return;
+        job();
+        xSemaphoreGive(run->done);
+    });
+    for (int waited = 0; xSemaphoreTake(run->done, pdMS_TO_TICKS(500)) != pdTRUE; waited += 500) {
+        int expected = Queued;
+        if (waited >= 5000 && run->state.compare_exchange_strong(expected, Abandoned)) return false;
+    }
+    return true;
+}
+
 bool StorageService::requestFormat() {
     if (formatRequested_ || !task_) return false;
     formatRequested_ = true;

src/services/storage_service.h

@@ -42,6 +42,9 @@ class StorageService : public Service {
 
     // Runs `job` on the storage task, where card access is safe (e.g. reading an Update File).
     void runJob(std::function<void()> job);
+    // The same, and waits for it (from another task, never the storage task itself). False if it
+    // never started within 5 s: no card mounted, and then it never will run.
+    bool runAndWait(std::function<void()> job);
 
     // Erases the whole card: one partition spanning the card, formatted FAT32.
     bool requestFormat();