diff --git a/docs/milestones/G1.md b/docs/milestones/G1.md index 097f6f2..c6c9a3c 100644 --- a/docs/milestones/G1.md +++ b/docs/milestones/G1.md @@ -24,12 +24,14 @@ Gemini (geminiprotocol.net): one request per TLS connection on port 1965, the re | Q83 | `S` saves the page and the pages it links to, one level deep: gemtext only, same host only, at most 30 pages, in the background with a progress Toast. | | Q84 | The start page lists Saved Pages, newest first, grouped by capsule; they open with no network. In a Saved Page, a link to another Saved Page opens the saved copy; other links fetch online if Wi-Fi is up, or say "not saved, offline". A Saved Page shows when it was saved; `r` refreshes it. | | Q86 | *Decided after step 1.* **Two floors:** free heap stays above 40 KB in steady state, and above 20 KB for the second or two of a TLS handshake (measured: 24 KB with IRC connected). A fetch refuses to start below **55 KB** free ("not enough memory: stop IRC or retry"), so nothing pushes lower. | +| Q87 | *Decided in step 3.* **With a card, every page streams to `/gemini/cache/page.gmi`** in 1 KB pieces while its TLS connection is open; once the connection closes and its ~45 KB is back, the page is loaded into RAM as far as the 40 KB floor allows. The whole page stays on the card (Saved Pages copy it). Without a card, the page goes straight to RAM under the same two floors. Pages are held as lines in 4 KB chunks, never one large block (the largest free block with IRC connected is about 31 KB). | | Q85 | Saved Pages are deleted from the App only (`d`, with confirmation), never by Storage Clean-up's age rules, like Notes. | ## Measured (step 1) - `gemini://geminiprotocol.net/`: `20 text/gemini`, 1,184 bytes, TLS handshake 0.7–1.1 s, whole fetch 0.7–1.1 s; kennedy.gemi.dev 1.9 s. The fetch task's stack peaks at about 3.6 KB of 6. - **Heap, Debug Build, IRC connected over TLS:** about 68 KB free before a fetch. After the handshake the fetch holds about 32 KB (36–40 KB left); the handshake itself (certificate chain parsed with the 16 KB receive buffer allocated) dips to about **24 KB** for a second or two. Nothing leaks: the heap after matches the heap before. +- **Step 3, Cosmos (31.6 KB) with IRC connected:** first stopped at 4.6 KB (RAM only, the transfer's 20 KB floor). Streamed to the card: the whole page on the card, 20 KB of it loaded, lowest free heap 19.5 KB during the transfer and 43 KB once loaded. Without IRC: the whole page in RAM. Redirects (Cosmos `31`), input (`10`), not found (`51`) and a changed certificate (refused, both fingerprints shown) all checked on the device. - Antenna (`warmedal.se`) doesn't answer, from the PC either; the default aggregator becomes Cosmos (`gemini://skyjake.fi/~Cosmos/`, which redirects to `cosmos.skyjake.fi`). ## Done when diff --git a/lib/gemini/src/gemtext.cpp b/lib/gemini/src/gemtext.cpp index a9c24a2..6348be6 100644 --- a/lib/gemini/src/gemtext.cpp +++ b/lib/gemini/src/gemtext.cpp @@ -11,6 +11,43 @@ std::string trim(const std::string& s) { } } // namespace +GemLine parseGemLine(const std::string& line, bool& pre) { + GemLine g; + if (line.compare(0, 3, "```") == 0) { + g.type = LineType::PreToggle; + g.text = trim(line.substr(3)); + pre = !pre; + } else if (pre) { + g.type = LineType::Preformatted; + g.text = line; + } else if (line.compare(0, 2, "=>") == 0) { + g.type = LineType::Link; + std::string rest = trim(line.substr(2)); + size_t gap = rest.find_first_of(" \t"); + g.url = rest.substr(0, gap); + g.text = gap == std::string::npos ? "" : trim(rest.substr(gap)); + if (g.text.empty()) g.text = g.url; + } else if (line.compare(0, 3, "###") == 0) { + g.type = LineType::Heading3; + g.text = trim(line.substr(3)); + } else if (line.compare(0, 2, "##") == 0) { + g.type = LineType::Heading2; + g.text = trim(line.substr(2)); + } else if (line.compare(0, 1, "#") == 0) { + g.type = LineType::Heading1; + g.text = trim(line.substr(1)); + } else if (line.compare(0, 2, "* ") == 0) { + g.type = LineType::ListItem; + g.text = trim(line.substr(2)); + } else if (line.compare(0, 1, ">") == 0) { + g.type = LineType::Quote; + g.text = trim(line.substr(1)); + } else { + g.text = line; + } + return g; +} + std::vector parseGemtext(const std::string& document) { std::vector lines; bool pre = false; @@ -21,41 +58,7 @@ std::vector parseGemtext(const std::string& document) { std::string line = document.substr(start, end - start); if (!line.empty() && line.back() == '\r') line.pop_back(); start = end + 1; - - GemLine g; - if (line.compare(0, 3, "```") == 0) { - g.type = LineType::PreToggle; - g.text = trim(line.substr(3)); - pre = !pre; - } else if (pre) { - g.type = LineType::Preformatted; - g.text = line; - } else if (line.compare(0, 2, "=>") == 0) { - g.type = LineType::Link; - std::string rest = trim(line.substr(2)); - size_t gap = rest.find_first_of(" \t"); - g.url = rest.substr(0, gap); - g.text = gap == std::string::npos ? "" : trim(rest.substr(gap)); - if (g.text.empty()) g.text = g.url; - } else if (line.compare(0, 3, "###") == 0) { - g.type = LineType::Heading3; - g.text = trim(line.substr(3)); - } else if (line.compare(0, 2, "##") == 0) { - g.type = LineType::Heading2; - g.text = trim(line.substr(2)); - } else if (line.compare(0, 1, "#") == 0) { - g.type = LineType::Heading1; - g.text = trim(line.substr(1)); - } else if (line.compare(0, 2, "* ") == 0) { - g.type = LineType::ListItem; - g.text = trim(line.substr(2)); - } else if (line.compare(0, 1, ">") == 0) { - g.type = LineType::Quote; - g.text = trim(line.substr(1)); - } else { - g.text = line; - } - lines.push_back(std::move(g)); + lines.push_back(parseGemLine(line, pre)); } return lines; } diff --git a/lib/gemini/src/gemtext.h b/lib/gemini/src/gemtext.h index edb0e6f..2b4e881 100644 --- a/lib/gemini/src/gemtext.h +++ b/lib/gemini/src/gemtext.h @@ -14,6 +14,8 @@ struct GemLine { std::string text, url; }; +// One line; `preformatted` carries the ``` state from line to line (start with false). +GemLine parseGemLine(const std::string& line, bool& preformatted); std::vector parseGemtext(const std::string& document); // For the Latin-1 fonts (Q79): valid UTF-8 up to U+00FF kept, anything else '?', and tabs diff --git a/lib/gemini/src/text_buffer.cpp b/lib/gemini/src/text_buffer.cpp new file mode 100644 index 0000000..448432e --- /dev/null +++ b/lib/gemini/src/text_buffer.cpp @@ -0,0 +1,47 @@ +#include "text_buffer.h" + +#include + +namespace roro::gemini { + +void TextBuffer::append(const char* data, size_t len) { + bytes_ += len; + for (size_t i = 0; i < len; i++) { + if (data[i] == '\n') endLine(); + else partial_ += data[i]; + } +} + +void TextBuffer::finish() { + if (!partial_.empty()) endLine(); +} + +void TextBuffer::endLine() { + if (!partial_.empty() && partial_.back() == '\r') partial_.pop_back(); + size_t len = std::min(partial_.size(), 0xFFFF); + // A line never spans two chunks: a new chunk when it doesn't fit (bigger if the line is). + if (chunks_.empty() || chunks_.back().size() + len > chunks_.back().capacity()) { + chunks_.emplace_back(); + chunks_.back().reserve(std::max(kChunk, len)); + } + std::string& chunk = chunks_.back(); + lines_.push_back({static_cast(chunks_.size() - 1), static_cast(chunk.size()), + static_cast(len)}); + chunk.append(partial_, 0, len); + partial_.clear(); +} + +std::string TextBuffer::line(size_t i) const { + if (i >= lines_.size()) return ""; + const Ref& r = lines_[i]; + return chunks_[r.chunk].substr(r.offset, r.length); +} + +void TextBuffer::clear() { + chunks_.clear(); + lines_.clear(); + partial_.clear(); + bytes_ = 0; +} + +} // namespace roro::gemini diff --git a/lib/gemini/src/text_buffer.h b/lib/gemini/src/text_buffer.h new file mode 100644 index 0000000..4e04a74 --- /dev/null +++ b/lib/gemini/src/text_buffer.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include +#include + +namespace roro::gemini { + +// A page's text, kept as lines in 4 KB chunks: no large contiguous block (with IRC connected the +// largest free block is about 31 KB) and no copy when it grows. About 4 bytes per line on top of +// the text itself. Bytes are fed as they arrive; lines end at LF, a CR before it is dropped. +class TextBuffer { + public: + static constexpr size_t kChunk = 4096; + + void append(const char* data, size_t len); + void finish(); // the last line, if it has no line end + size_t lineCount() const { return lines_.size(); } + std::string line(size_t i) const; + size_t bytes() const { return bytes_; } + void clear(); + + private: + struct Ref { + uint16_t chunk; + uint16_t offset; + uint16_t length; + }; + void endLine(); + + std::vector chunks_; + std::vector lines_; + std::string partial_; // the line being received + size_t bytes_ = 0; +}; + +} // namespace roro::gemini diff --git a/src/main.cpp b/src/main.cpp index 6e8b493..ca7be09 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -55,7 +55,7 @@ static StorageService* storageService; static PowerService* power; static ClockService* clockService; static GnssService* gnssService; -static GeminiService gemini; +static GeminiService* geminiService; static SavedNetworks* savedNetworks; static WifiService* wifi; static IrcService* irc; @@ -147,6 +147,7 @@ void setup() { storageService = new StorageService(bus); power = new PowerService(settings); clockService = new ClockService(settings, bus); + geminiService = new GeminiService(nvs, *storageService); gnssService = new GnssService(settings, *clockService, *storageService, bus); savedNetworks = new SavedNetworks(nvs); savedNetworks->load(); @@ -465,8 +466,13 @@ static void runCommand(String line) { storageService->requestListing(); listingWanted = true; } + if (line.startsWith("gemini trust ")) { // gemini trust : accept a changed certificate + char host[96] = "", fp[80] = ""; + int port = 1965; + if (geminiService && sscanf(line.substring(13).c_str(), "%95s %d %79s", host, &port, fp) == 3) geminiService->trust(host, port, fp); + } if (line.startsWith("gemini get ")) - if (!gemini.fetchToConsole(line.substring(11).c_str())) console.println("gemini: a fetch is already running"); + if (!geminiService || !geminiService->fetchToConsole(line.substring(11).c_str())) console.println("gemini: busy, or unavailable in Safe Mode"); if (line == "irc start") irc->connect(); if (line == "irc stop") irc->disconnect(); if (line.startsWith("irc say ")) { // irc say diff --git a/src/services/debug_console.cpp b/src/services/debug_console.cpp index d2e3f64..d57afbb 100644 --- a/src/services/debug_console.cpp +++ b/src/services/debug_console.cpp @@ -187,29 +187,7 @@ bool DebugConsole::binaryCommand(NetworkClient& client, const std::string& line) return true; } -bool DebugConsole::onStorage(std::function job) { - // Shared with the job, which outlives this wait if the card is missing and it never starts. - // Exactly one side wins the state change: the job (Queued -> Running) or the wait giving up - // (Queued -> Abandoned), so an abandoned job can never touch this stack frame. - enum : int { Queued, Running, Abandoned }; - struct Run { - std::atomic state{Queued}; - SemaphoreHandle_t done = xSemaphoreCreateBinary(); - ~Run() { vSemaphoreDelete(done); } - }; - auto run = std::make_shared(); - storage_.runJob([run, job]() { - int expected = Queued; - if (!run->state.compare_exchange_strong(expected, Running)) return; - job(); - xSemaphoreGive(run->done); - }); - for (int waited = 0; xSemaphoreTake(run->done, pdMS_TO_TICKS(500)) != pdTRUE; waited += 500) { - int expected = Queued; - if (waited >= 5000 && run->state.compare_exchange_strong(expected, Abandoned)) return false; - } - return true; -} +bool DebugConsole::onStorage(std::function job) { return storage_.runAndWait(std::move(job)); } void DebugConsole::get(NetworkClient& client, const std::string& path) { bool ran = onStorage([&]() { diff --git a/src/services/gemini_service.cpp b/src/services/gemini_service.cpp index d6576c7..25f71a0 100644 --- a/src/services/gemini_service.cpp +++ b/src/services/gemini_service.cpp @@ -1,22 +1,22 @@ #include "services/gemini_service.h" #include +#include #include #include -#include +#include "gemini_url.h" #include "platform/console.h" +#include "sha256.h" + +using roro::gemini::TextBuffer; namespace roro { namespace { -struct ConsoleFetch { - GeminiService* service; - std::string url; - volatile bool* busy; -}; +constexpr uint32_t kIdleTimeoutMs = 15000; std::string hex(const uint8_t* data, size_t len) { static const char* digits = "0123456789abcdef"; @@ -30,75 +30,301 @@ std::string hex(const uint8_t* data, size_t len) { } // namespace -bool GeminiService::fetchToConsole(const std::string& url) { +GeminiService::GeminiService(KeyValueStore& store, StorageService& storage) + : store_(store), storage_(storage), lock_(xSemaphoreCreateMutex()) {} + +// NVS keys are 15 characters at most: "gm" and 12 hex digits of SHA-256("host:port"). +std::string GeminiService::pinKey(const std::string& host, int port) const { + std::string id = host + ":" + std::to_string(port); + uint8_t digest[32]; + Sha256::hash(reinterpret_cast(id.data()), id.size(), digest); + return "gm" + hex(digest, 6); +} + +void GeminiService::trust(const std::string& host, int port, const std::string& fingerprint) { + store_.putString(pinKey(host, port).c_str(), fingerprint); +} + +bool GeminiService::fetch(const std::string& url) { if (busy_) return false; busy_ = true; - auto* job = new ConsoleFetch{this, url, &busy_}; - if (xTaskCreate(consoleTask, "gemini", 6144, job, 1, nullptr) != pdPASS) { - delete job; + ready_ = cancel_ = false; + pendingUrl_ = url; + if (xTaskCreate(taskEntry, "gemini", 6144, this, 1, nullptr) != pdPASS) { // peak 3.6 KB (step 1) busy_ = false; return false; } return true; } -void GeminiService::consoleTask(void* arg) { - std::unique_ptr job(static_cast(arg)); - // Until step 2's URL parser: gemini://host[:port]/path - std::string rest = job->url.substr(job->url.find("://") == std::string::npos ? 0 : job->url.find("://") + 3); - std::string authority = rest.substr(0, rest.find('/')); - std::string host = authority.substr(0, authority.find(':')); - uint16_t port = authority.find(':') == std::string::npos ? 1965 : atoi(authority.substr(authority.find(':') + 1).c_str()); - - size_t before = esp_get_free_heap_size(), lowest = before; - auto track = [&lowest]() { lowest = std::min(lowest, esp_get_free_heap_size()); }; - uint32_t start = millis(); - { - NetworkClientSecure tls; - tls.setInsecure(); // trust on first use: the fingerprint is what counts (Q71) - tls.setTimeout(15); - if (!tls.connect(host.c_str(), port)) { - console.printf("gemini: cannot connect to %s:%u\n", host.c_str(), port); - } else { - uint32_t connected = millis(); - track(); // the handshake's buffers are allocated now - uint8_t sha[32]; - std::string fingerprint = tls.getFingerprintSHA256(sha) ? hex(sha, 32) : "?"; - tls.print((job->url + "\r\n").c_str()); - std::string header, sample; - size_t body = 0; - bool inBody = false; - uint32_t last = millis(); - while (tls.connected() || tls.available()) { - int c = tls.read(); - if (c < 0) { - if (millis() - last > 15000) break; - delay(5); - continue; - } - last = millis(); - if ((body & 255) == 0) track(); - if (!inBody) { - if (c == '\n') inBody = true; - else if (c != '\r' && header.size() < 1100) header += static_cast(c); - } else { - body++; - if (sample.size() < 400) sample += static_cast(c); - } - } - console.printf("gemini: %s\ngemini: header '%s', %u bytes, TLS %lu ms, total %lu ms\n", job->url.c_str(), - header.c_str(), (unsigned)body, (unsigned long)(connected - start), - (unsigned long)(millis() - start)); - console.printf("gemini: certificate sha256 %s\n", fingerprint.c_str()); - console.printf("--- first bytes ---\n%s\n--- end ---\n", sample.c_str()); - } - tls.stop(); +bool GeminiService::fetchToConsole(const std::string& url) { + if (busy_) return false; + toConsole_ = true; + if (!fetch(url)) { + toConsole_ = false; + return false; } - console.printf("gemini: heap %u before, %u lowest during, %u after; stack left %u\n", (unsigned)before, - (unsigned)lowest, (unsigned)esp_get_free_heap_size(), (unsigned)uxTaskGetStackHighWaterMark(nullptr)); - *job->busy = false; - job.reset(); + return true; +} + +bool GeminiService::takeResult(GeminiPage& out) { + if (!ready_) return false; + xSemaphoreTake(lock_, portMAX_DELAY); + out = std::move(result_); + result_ = GeminiPage(); + ready_ = false; + xSemaphoreGive(lock_); + return true; +} + +void GeminiService::taskEntry(void* self) { + static_cast(self)->run(); vTaskDelete(nullptr); } +void GeminiService::run() { + GeminiPage page; + page.requested = pendingUrl_; + size_t before = esp_get_free_heap_size(); + lowest_ = before; + uint32_t start = millis(); + std::string url = pendingUrl_; + if (before < kStartFloor) { + page.error = "Not enough memory (" + std::to_string(before / 1024) + " KB free): stop IRC or retry"; + } else { + for (int hop = 0;; hop++) { + fetchOne(url, page); + if (page.outcome != GeminiPage::Outcome::Ok || page.header.category() != gemini::Category::Redirect) break; + std::string next = gemini::resolve(url, page.header.meta); + if (!gemini::isGemini(next)) { + page.outcome = GeminiPage::Outcome::Failed; + page.error = "Redirect to another protocol: " + next; + break; + } + if (hop + 1 > kMaxRedirects) { + page.outcome = GeminiPage::Outcome::Failed; + page.error = "Too many redirects"; + break; + } + url = next; + } + } + page.url = url; + page.ms = millis() - start; + if (toConsole_) { + report(page, before, lowest_); + toConsole_ = false; + } else { + xSemaphoreTake(lock_, portMAX_DELAY); + result_ = std::move(page); + ready_ = true; + xSemaphoreGive(lock_); + } + busy_ = false; +} + +void GeminiService::fetchOne(const std::string& url, GeminiPage& page) { + page.outcome = GeminiPage::Outcome::Failed; + page.text.clear(); + page.truncated = false; + gemini::Url u; + if (!gemini::parseUrl(url, u) || u.scheme != "gemini") { + page.error = "Not a Gemini URL: " + url; + return; + } + auto track = [this]() { lowest_ = std::min(lowest_, esp_get_free_heap_size()); }; + + size_t freeBefore = esp_get_free_heap_size(); // what's left once the connection closes again + NetworkClientSecure tls; + tls.setInsecure(); // trust on first use: the pinned fingerprint is what counts (Q71) + tls.setTimeout(15); + if (!tls.connect(u.host.c_str(), u.portOrDefault())) { + page.error = "Cannot connect to " + u.host; + return; + } + track(); + uint8_t sha[32]; + if (!tls.getFingerprintSHA256(sha)) { + page.error = "No certificate from " + u.host; + return; + } + std::string presented = hex(sha, sizeof sha), pinned; + std::string key = pinKey(u.host, u.portOrDefault()); + if (!store_.getString(key.c_str(), pinned)) store_.putString(key.c_str(), presented); // first use + else if (pinned != presented) { + page.outcome = GeminiPage::Outcome::CertificateChanged; + page.host = u.host; + page.port = u.portOrDefault(); + page.pinned = pinned; + page.presented = presented; + page.error = "The certificate of " + u.host + " changed"; + return; + } + + tls.print((gemini::requestUrl(url) + "\r\n").c_str()); + std::string header; + bool inBody = false; + uint32_t last = millis(); + char buf[512]; + while (!cancel_) { + int n = tls.available() ? tls.read(reinterpret_cast(buf), sizeof buf) : 0; + if (n <= 0) { + if (!tls.connected()) break; + if (millis() - last > kIdleTimeoutMs) { + page.error = "Timed out"; + return; + } + delay(5); + continue; + } + last = millis(); + int i = 0; + while (!inBody && i < n) { + char c = buf[i++]; + if (c == '\n') inBody = true; + else if (c != '\r') header += c; + if (header.size() > 1100) { + page.error = "Malformed response header"; + return; + } + } + if (!inBody) continue; + if (!gemini::parseHeader(header, page.header)) { + page.error = "Malformed response header"; + return; + } + if (page.header.category() != gemini::Category::Success) break; // no body to read + // With a card, the body goes there while the connection is open (only 4 KB in RAM), + // and comes back into RAM once the connection's memory is free again. + if (storage_.state().present) { + if (!receiveToCard(tls, buf + i, n - i, page)) return; + tls.stop(); + loadFromCard(page, freeBefore); + page.outcome = GeminiPage::Outcome::Ok; + return; + } + // Q86, two floors: once the connection closes, its ~50 KB comes back, so the page may use + // what was free before minus the steady floor; meanwhile the heap must stay above the + // transient floor. Both counted with the next 4 KB chunk and the line index. + size_t take = std::min(n - i, kMaxBody - page.text.bytes()); + size_t pageCost = page.text.bytes() + page.text.lineCount() * 8 + TextBuffer::kChunk + take; + if (take < static_cast(n - i)) page.truncatedWhy = "longer than 64 KB"; + else if (freeBefore < kSteadyFloor + pageCost) page.truncatedWhy = "not enough memory to keep it"; + else if (esp_get_free_heap_size() < kTransientFloor + TextBuffer::kChunk + take) + page.truncatedWhy = "not enough memory while receiving"; + if (!page.truncatedWhy.empty()) { + page.truncated = true; + break; + } + page.text.append(buf + i, take); + track(); + } + if (cancel_) { + page.error = "Cancelled"; + return; + } + if (!inBody && !gemini::parseHeader(header, page.header)) { // a header with no line end + page.error = header.empty() ? "Empty response" : "Malformed response header"; + return; + } + page.text.finish(); + page.outcome = GeminiPage::Outcome::Ok; +} + +// Streams the rest of the body to kCachePath in 1 KB pieces, each written by the storage task: +// with IRC's TLS connection and this one both open, every KB of RAM counts (Q86). +bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page) { + struct Cache { + File file; + }; + auto cache = std::make_shared(); + bool opened = storage_.runAndWait([cache]() { + SD.mkdir("/gemini"); + SD.mkdir("/gemini/cache"); + cache->file = SD.open(kCachePath, FILE_WRITE); + }); + if (!opened || !cache->file) { + page.error = "Can't write to the SD card"; + return false; + } + std::string pending(first, len); + constexpr size_t kPiece = 1024; + pending.reserve(kPiece + 512); + bool ok = true; + auto flush = [&]() { + if (pending.empty() || !ok) return; + ok = storage_.runAndWait([cache, &pending, &ok]() { + ok = cache->file.write(reinterpret_cast(pending.data()), pending.size()) == pending.size(); + }) && ok; + page.receivedBytes += pending.size(); + pending.clear(); + }; + uint32_t last = millis(); + char buf[512]; + while (!cancel_ && ok) { + if (pending.size() >= kPiece) flush(); + if (page.receivedBytes + pending.size() >= kMaxOnCard) { + page.truncated = true; + page.truncatedWhy = "longer than 1 MB"; + break; + } + int n = tls.available() ? tls.read(reinterpret_cast(buf), sizeof buf) : 0; + if (n <= 0) { + if (!tls.connected()) break; + if (millis() - last > kIdleTimeoutMs) break; // keep what came + delay(5); + continue; + } + last = millis(); + pending.append(buf, n); + lowest_ = std::min(lowest_, esp_get_free_heap_size()); + } + flush(); + storage_.runAndWait([cache]() { cache->file.close(); }); + if (cancel_) page.error = "Cancelled"; + else if (!ok) page.error = "Writing to the SD card failed"; + else page.cachePath = kCachePath; + return !cancel_ && ok; +} + +// Back into RAM, now that the connection's memory is free again: as much as the steady floor +// allows (Q86); the rest stays on the card. +void GeminiService::loadFromCard(GeminiPage& page, size_t freeBefore) { + size_t budget = freeBefore > kSteadyFloor + TextBuffer::kChunk ? freeBefore - kSteadyFloor - TextBuffer::kChunk : 0; + budget = std::min(budget, kMaxBody); + storage_.runAndWait([&]() { + File f = SD.open(kCachePath); + if (!f) return; + char buf[512]; + int n; + while ((n = f.read(reinterpret_cast(buf), sizeof buf)) > 0) { + size_t cost = page.text.bytes() + page.text.lineCount() * 8 + n; + if (cost > budget) { + page.truncated = true; + if (page.truncatedWhy.empty()) + page.truncatedWhy = "only part of it fits in memory; the whole page is on the card"; + break; + } + page.text.append(buf, n); + } + f.close(); + }); + page.text.finish(); +} + +void GeminiService::report(const GeminiPage& page, size_t heapBefore, size_t heapLowest) { + console.printf("gemini: %s -> %s, %lu ms\n", page.requested.c_str(), page.url.c_str(), (unsigned long)page.ms); + if (page.outcome != GeminiPage::Outcome::Ok) console.printf("gemini: %s\n", page.error.c_str()); + else console.printf("gemini: %d %s, %u bytes, %u lines%s\n", page.header.status, page.header.meta.c_str(), + (unsigned)page.text.bytes(), (unsigned)page.text.lineCount(), + page.truncated ? (" (truncated: " + page.truncatedWhy + ")").c_str() : ""); + if (page.outcome == GeminiPage::Outcome::CertificateChanged) + console.printf("gemini: pinned %s\ngemini: now %s\n", page.pinned.c_str(), page.presented.c_str()); + for (size_t i = 0; i < page.text.lineCount() && i < 6; i++) console.printf(" %s\n", page.text.line(i).substr(0, 100).c_str()); + if (!page.cachePath.empty()) + console.printf("gemini: %u bytes on the card in %s\n", (unsigned)page.receivedBytes, page.cachePath.c_str()); + console.printf("gemini: heap %u before, %u lowest seen, %u after\n", (unsigned)heapBefore, (unsigned)heapLowest, + (unsigned)esp_get_free_heap_size()); +} + } // namespace roro diff --git a/src/services/gemini_service.h b/src/services/gemini_service.h index 6a47244..8c99964 100644 --- a/src/services/gemini_service.h +++ b/src/services/gemini_service.h @@ -1,21 +1,82 @@ #pragma once +#include +#include + #include +#include "gemini_response.h" +#include "key_value_store.h" +#include "services/storage_service.h" +#include "text_buffer.h" + +class NetworkClientSecure; + namespace roro { +// What a fetch came back with. +struct GeminiPage { + enum class Outcome { Ok, Failed, CertificateChanged }; + Outcome outcome = Outcome::Failed; + std::string requested; // what was asked for + std::string url; // where it ended up, after redirects + gemini::Header header; // the final response's header + gemini::TextBuffer text; // 2x only: the body as lines, up to kMaxBody + bool truncated = false; // the body was cut: see truncatedWhy + std::string truncatedWhy; + std::string error; // Failed: why, for a human + std::string host; // CertificateChanged: whose, and both fingerprints + int port = 1965; + std::string pinned, presented; + std::string cachePath; // with a card: the whole body as received (Saved Pages copy it) + size_t receivedBytes = 0; // the whole body, which may be more than `text` holds + uint32_t ms = 0; +}; + // Gemini (docs/milestones/G1.md). Each fetch runs on a short-lived task of its own: a TLS // handshake can block for seconds, which the main loop's watchdog wouldn't allow, and an idle -// client should cost no stack. +// client should cost no stack. Certificates are trusted on first use, per host and port (Q71). class GeminiService { public: - // `gemini get `: fetches and prints the header, size, certificate fingerprint, the first - // lines and the heap around the fetch to the console. False if a fetch is already running. + static constexpr size_t kMaxBody = 64 * 1024; // Q74 + static constexpr size_t kStartFloor = 55 * 1024; // Q86: no fetch below this + static constexpr size_t kSteadyFloor = 40 * 1024; // Q86: what must be left once the page is in + static constexpr size_t kTransientFloor = 20 * 1024; // Q86: while the TLS connection is open + static constexpr int kMaxRedirects = 5; // Q72 + + static constexpr size_t kMaxOnCard = 1024 * 1024; // a text page larger than this is cut + static constexpr const char* kCachePath = "/gemini/cache/page.gmi"; + + GeminiService(KeyValueStore& store, StorageService& storage); + + // Starts a fetch; false if one is running. The result comes from takeResult(). + bool fetch(const std::string& url); + bool busy() const { return busy_; } + bool takeResult(GeminiPage& out); + void cancel() { cancel_ = true; } + // After the user accepts a changed certificate. + void trust(const std::string& host, int port, const std::string& fingerprint); + + // `gemini get `: the same fetch, reported on the console instead of to the App. bool fetchToConsole(const std::string& url); private: - static void consoleTask(void* arg); - volatile bool busy_ = false; + static void taskEntry(void* self); + void run(); + void fetchOne(const std::string& url, GeminiPage& page); + void report(const GeminiPage& page, size_t heapBefore, size_t heapLowest); + std::string pinKey(const std::string& host, int port) const; + + bool receiveToCard(::NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page); + void loadFromCard(GeminiPage& page, size_t freeBefore); + + KeyValueStore& store_; + StorageService& storage_; + SemaphoreHandle_t lock_; + std::string pendingUrl_; + GeminiPage result_; + volatile bool busy_ = false, ready_ = false, cancel_ = false, toConsole_ = false; + size_t lowest_ = 0; }; } // namespace roro diff --git a/src/services/storage_service.cpp b/src/services/storage_service.cpp index 269cb80..9512223 100644 --- a/src/services/storage_service.cpp +++ b/src/services/storage_service.cpp @@ -5,6 +5,7 @@ #include #include +#include #include #include "platform/pins.h" @@ -82,6 +83,30 @@ void StorageService::runJob(std::function job) { if (task_) xTaskNotifyGive(task_); } +bool StorageService::runAndWait(std::function job) { + // Shared with the job, which outlives this wait if the card is missing and it never starts. + // Exactly one side wins the state change: the job (Queued -> Running) or the wait giving up + // (Queued -> Abandoned), so an abandoned job can never touch this stack frame. + enum : int { Queued, Running, Abandoned }; + struct Run { + std::atomic state{Queued}; + SemaphoreHandle_t done = xSemaphoreCreateBinary(); + ~Run() { vSemaphoreDelete(done); } + }; + auto run = std::make_shared(); + runJob([run, job]() { + int expected = Queued; + if (!run->state.compare_exchange_strong(expected, Running)) return; + job(); + xSemaphoreGive(run->done); + }); + for (int waited = 0; xSemaphoreTake(run->done, pdMS_TO_TICKS(500)) != pdTRUE; waited += 500) { + int expected = Queued; + if (waited >= 5000 && run->state.compare_exchange_strong(expected, Abandoned)) return false; + } + return true; +} + bool StorageService::requestFormat() { if (formatRequested_ || !task_) return false; formatRequested_ = true; diff --git a/src/services/storage_service.h b/src/services/storage_service.h index 6f9327d..1f75b4e 100644 --- a/src/services/storage_service.h +++ b/src/services/storage_service.h @@ -42,6 +42,9 @@ class StorageService : public Service { // Runs `job` on the storage task, where card access is safe (e.g. reading an Update File). void runJob(std::function job); + // The same, and waits for it (from another task, never the storage task itself). False if it + // never started within 5 s: no card mounted, and then it never will run. + bool runAndWait(std::function job); // Erases the whole card: one partition spanning the card, formatted FAT32. bool requestFormat(); diff --git a/test/test_gemini/test_gemini.cpp b/test/test_gemini/test_gemini.cpp index 250d3f0..261ae6f 100644 --- a/test/test_gemini/test_gemini.cpp +++ b/test/test_gemini/test_gemini.cpp @@ -5,6 +5,7 @@ #include "gemini_response.h" #include "gemini_url.h" #include "gemtext.h" +#include "text_buffer.h" using namespace roro::gemini; @@ -174,6 +175,34 @@ void test_saved_page_paths() { savedPath("gemini://example.org:1966/a?q:x#frag").c_str()); } +void test_text_buffer_keeps_lines_across_any_feed() { + TextBuffer b; + std::string doc = "# Title\r\nline two\n\nlast without end"; + for (size_t i = 0; i < doc.size(); i += 3) b.append(doc.data() + i, std::min(3, doc.size() - i)); + b.finish(); + TEST_ASSERT_EQUAL(4, static_cast(b.lineCount())); + TEST_ASSERT_EQUAL_STRING("# Title", b.line(0).c_str()); + TEST_ASSERT_EQUAL_STRING("line two", b.line(1).c_str()); + TEST_ASSERT_EQUAL_STRING("", b.line(2).c_str()); + TEST_ASSERT_EQUAL_STRING("last without end", b.line(3).c_str()); + TEST_ASSERT_EQUAL(doc.size(), b.bytes()); +} + +void test_text_buffer_spreads_over_chunks_without_splitting_lines() { + TextBuffer b; + std::string line(1000, 'x'); + for (int i = 0; i < 20; i++) { + std::string l = std::to_string(i) + line + "\n"; + b.append(l.data(), l.size()); + } + std::string big(6000, 'y'); // longer than a chunk + big += "\n"; + b.append(big.data(), big.size()); + TEST_ASSERT_EQUAL(21, static_cast(b.lineCount())); + for (int i = 0; i < 20; i++) TEST_ASSERT_EQUAL_STRING((std::to_string(i) + line).c_str(), b.line(i).c_str()); + TEST_ASSERT_EQUAL(6000, static_cast(b.line(20).size())); +} + int main() { UNITY_BEGIN(); RUN_TEST(test_rfc3986_normal_examples); @@ -188,5 +217,7 @@ int main() { RUN_TEST(test_gemtext_handles_crlf_and_a_missing_last_newline); RUN_TEST(test_display_text_keeps_latin1_and_replaces_the_rest); RUN_TEST(test_saved_page_paths); + RUN_TEST(test_text_buffer_keeps_lines_across_any_feed); + RUN_TEST(test_text_buffer_spreads_over_chunks_without_splitting_lines); return UNITY_END(); }