Public Access
Everything touching the network or the card is a job on the Gemini task (a missing card can block 5 s, past the main loop's watchdog): about:start is composed from /gemini/bookmarks.gmi and the Saved Pages (by capsule, newest first); file:// opens a Saved Page; s, S, r, d, b and downloads report one line to the URL bar, S with progress Toasts. A Saved Page is the cached body with a first line "> Saved from <url> on <date>": it shows as a quote and gives relative links their base; inside one, links to other Saved Pages open the saved copy, others go online or say "Not saved, and offline". Input prompts (11 masked) request the same URL with the answer as its query. Fixed on the way: re-wrapping indented lines rebuilt the text from its rows, inserting a space where a long word had been cut; refreshing loaded the whole Saved Page next to a TLS connection (heap down to 436 bytes), now it reads one line and every fetch checks the 55 KB floor. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT
693 lines
26 KiB
C++
693 lines
26 KiB
C++
#include "services/gemini_service.h"
|
|
|
|
#include <NetworkClientSecure.h>
|
|
#include <SD.h>
|
|
#include <WiFi.h>
|
|
#include <esp_heap_caps.h>
|
|
|
|
#include <algorithm>
|
|
#include <map>
|
|
|
|
#include "gemini_url.h"
|
|
#include "gemtext.h"
|
|
#include "platform/console.h"
|
|
#include "sha256.h"
|
|
#include "storage_paths.h"
|
|
|
|
namespace roro {
|
|
|
|
using gemini::TextBuffer;
|
|
|
|
namespace {
|
|
|
|
constexpr uint32_t kIdleTimeoutMs = 15000;
|
|
constexpr const char* kSavedRoot = "/gemini/saved";
|
|
constexpr const char* kSavedMark = "> Saved from "; // first line of a Saved Page (Q82)
|
|
constexpr const char* kDefaults =
|
|
"## Start here\n"
|
|
"=> gemini://geminiprotocol.net/ Project Gemini\n"
|
|
"=> gemini://kennedy.gemi.dev/ Kennedy: search Geminispace\n"
|
|
"=> gemini://skyjake.fi/~Cosmos/ Cosmos: what's new on Geminispace\n"
|
|
"=> gemini://bbs.geminispace.org/ Bubble: a bulletin board\n";
|
|
|
|
std::string hex(const uint8_t* data, size_t len) {
|
|
static const char* digits = "0123456789abcdef";
|
|
std::string out;
|
|
for (size_t i = 0; i < len; i++) {
|
|
out += digits[data[i] >> 4];
|
|
out += digits[data[i] & 15];
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// Folders for a file path, one at a time (SD.mkdir makes one level).
|
|
void makeParents(const std::string& path) {
|
|
for (size_t slash = path.find('/', 1); slash != std::string::npos; slash = path.find('/', slash + 1)) {
|
|
std::string dir = path.substr(0, slash);
|
|
if (!SD.exists(dir.c_str())) SD.mkdir(dir.c_str());
|
|
}
|
|
}
|
|
|
|
// Copies `from` to `to`, optionally after a first line. On the storage task.
|
|
bool copyFile(const char* from, const std::string& to, const std::string& firstLine) {
|
|
File in = SD.open(from);
|
|
if (!in) return false;
|
|
makeParents(to);
|
|
File out = SD.open(to.c_str(), FILE_WRITE);
|
|
if (!out) return false;
|
|
bool ok = firstLine.empty() || out.print((firstLine + "\n").c_str()) > 0;
|
|
uint8_t buf[512];
|
|
int n;
|
|
while (ok && (n = in.read(buf, sizeof buf)) > 0) ok = out.write(buf, n) == static_cast<size_t>(n);
|
|
out.close();
|
|
in.close();
|
|
if (!ok) SD.remove(to.c_str());
|
|
return ok;
|
|
}
|
|
|
|
struct SavedEntry {
|
|
std::string path; // /gemini/saved/host/…/name.gmi
|
|
time_t when;
|
|
};
|
|
|
|
void listSaved(const std::string& dir, std::vector<SavedEntry>& out) {
|
|
File d = SD.open(dir.c_str());
|
|
if (!d || !d.isDirectory()) return;
|
|
for (File f = d.openNextFile(); f && out.size() < 200; f = d.openNextFile()) {
|
|
std::string path = dir + "/" + f.name();
|
|
if (f.isDirectory()) listSaved(path, out);
|
|
else out.push_back({path, f.getLastWrite()});
|
|
}
|
|
}
|
|
|
|
} // namespace
|
|
|
|
GeminiService::GeminiService(KeyValueStore& store, StorageService& storage, ClockService& clock, EventBus& bus)
|
|
: store_(store), storage_(storage), clock_(clock), bus_(bus), lock_(xSemaphoreCreateMutex()) {}
|
|
|
|
// NVS keys are 15 characters at most: "gm" and 12 hex digits of SHA-256("host:port").
|
|
std::string GeminiService::pinKey(const std::string& host, int port) const {
|
|
std::string id = host + ":" + std::to_string(port);
|
|
uint8_t digest[32];
|
|
Sha256::hash(reinterpret_cast<const uint8_t*>(id.data()), id.size(), digest);
|
|
return "gm" + hex(digest, 6);
|
|
}
|
|
|
|
void GeminiService::trust(const std::string& host, int port, const std::string& fingerprint) {
|
|
store_.putString(pinKey(host, port).c_str(), fingerprint);
|
|
}
|
|
|
|
std::string GeminiService::now() const {
|
|
int64_t t = clock_.utcNow();
|
|
return t < 0 ? "an unknown date" : ClockModel::formatLocalDate(t) + " " + ClockModel::formatLocalTime(t);
|
|
}
|
|
|
|
void GeminiService::notify(const std::string& text) {
|
|
bus_.publish(Event::withText(EventType::Notification, text.c_str(), static_cast<int32_t>(NotificationLevel::Info)));
|
|
}
|
|
|
|
bool GeminiService::start(Job job) {
|
|
if (busy_) return false;
|
|
busy_ = true;
|
|
ready_ = messageReady_ = cancel_ = false;
|
|
job_ = job;
|
|
if (xTaskCreate(taskEntry, "gemini", 6144, this, 1, nullptr) != pdPASS) { // peak 3.6 KB (step 1)
|
|
busy_ = false;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool GeminiService::fetch(const std::string& url, bool preferSaved) {
|
|
if (busy_) return false;
|
|
arg_ = url;
|
|
preferSaved_ = preferSaved;
|
|
return start(Job::Fetch);
|
|
}
|
|
|
|
bool GeminiService::fetchToConsole(const std::string& url) {
|
|
if (busy_) return false;
|
|
toConsole_ = true;
|
|
if (!fetch(url)) {
|
|
toConsole_ = false;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
bool GeminiService::save(const std::string& url) {
|
|
if (busy_) return false;
|
|
arg_ = url;
|
|
return start(Job::Save);
|
|
}
|
|
|
|
bool GeminiService::saveWithLinks(const std::string& url, std::vector<std::string> links) {
|
|
if (busy_) return false;
|
|
arg_ = url;
|
|
links_ = std::move(links);
|
|
return start(Job::SaveWithLinks);
|
|
}
|
|
|
|
bool GeminiService::refresh(const std::string& savedPath) {
|
|
if (busy_) return false;
|
|
arg_ = savedPath;
|
|
return start(Job::Refresh);
|
|
}
|
|
|
|
bool GeminiService::deleteSaved(const std::string& savedPath) {
|
|
if (busy_) return false;
|
|
arg_ = savedPath;
|
|
return start(Job::Delete);
|
|
}
|
|
|
|
bool GeminiService::addBookmark(const std::string& url, const std::string& title) {
|
|
if (busy_) return false;
|
|
arg_ = url;
|
|
arg2_ = title;
|
|
return start(Job::Bookmark);
|
|
}
|
|
|
|
bool GeminiService::download(const std::string& url) {
|
|
if (busy_) return false;
|
|
arg_ = url;
|
|
return start(Job::Download);
|
|
}
|
|
|
|
bool GeminiService::takeResult(GeminiPage& out) {
|
|
if (!ready_) return false;
|
|
xSemaphoreTake(lock_, portMAX_DELAY);
|
|
out = std::move(result_);
|
|
result_ = GeminiPage();
|
|
ready_ = false;
|
|
xSemaphoreGive(lock_);
|
|
return true;
|
|
}
|
|
|
|
bool GeminiService::takeMessage(std::string& out) {
|
|
if (!messageReady_) return false;
|
|
xSemaphoreTake(lock_, portMAX_DELAY);
|
|
out = std::move(message_);
|
|
messageReady_ = false;
|
|
xSemaphoreGive(lock_);
|
|
return true;
|
|
}
|
|
|
|
void GeminiService::finishWithMessage(const std::string& text) {
|
|
xSemaphoreTake(lock_, portMAX_DELAY);
|
|
message_ = text;
|
|
messageReady_ = true;
|
|
xSemaphoreGive(lock_);
|
|
}
|
|
|
|
void GeminiService::taskEntry(void* self) {
|
|
static_cast<GeminiService*>(self)->run();
|
|
vTaskDelete(nullptr);
|
|
}
|
|
|
|
void GeminiService::run() {
|
|
switch (job_) {
|
|
case Job::Fetch: {
|
|
GeminiPage page;
|
|
runFetch(page);
|
|
if (!toConsole_) {
|
|
xSemaphoreTake(lock_, portMAX_DELAY);
|
|
result_ = std::move(page);
|
|
ready_ = true;
|
|
xSemaphoreGive(lock_);
|
|
}
|
|
toConsole_ = false;
|
|
break;
|
|
}
|
|
case Job::Save: {
|
|
std::string why = saveCached(arg_);
|
|
finishWithMessage(!why.empty() ? why : replaced_ ? "Saved again (replaced the older copy)" : "Saved for offline reading");
|
|
break;
|
|
}
|
|
case Job::SaveWithLinks: {
|
|
std::string why = saveCached(arg_);
|
|
if (!why.empty()) {
|
|
finishWithMessage(why);
|
|
break;
|
|
}
|
|
int saved = 1, failed = 0, total = static_cast<int>(links_.size()) + 1;
|
|
notify("Saving " + std::to_string(total) + " pages for offline reading");
|
|
for (const std::string& link : links_) {
|
|
if (cancel_) break;
|
|
GeminiPage page;
|
|
fetchFollowing(link, page, false);
|
|
bool text = page.outcome == GeminiPage::Outcome::Ok &&
|
|
page.header.category() == gemini::Category::Success && page.header.mimeType() == "text/gemini";
|
|
if (text && saveCached(link).empty()) saved++;
|
|
else failed++;
|
|
if ((saved + failed) % 5 == 0) notify("Saved " + std::to_string(saved) + " of " + std::to_string(total));
|
|
}
|
|
std::string done = "Saved " + std::to_string(saved) + " of " + std::to_string(total) + " pages" +
|
|
(failed ? " (" + std::to_string(failed) + " not gemtext or failed)" : "");
|
|
notify(done);
|
|
finishWithMessage(done);
|
|
break;
|
|
}
|
|
case Job::Refresh: {
|
|
// Only the first line: where it came from. (Loading the whole page here, next to the
|
|
// App's copy and a TLS connection, once took the heap down to 436 bytes.)
|
|
std::string from = savedFrom(arg_);
|
|
if (from.empty()) {
|
|
finishWithMessage("Not a Saved Page");
|
|
break;
|
|
}
|
|
GeminiPage page;
|
|
fetchFollowing(from, page, false);
|
|
std::string why = page.outcome != GeminiPage::Outcome::Ok ? page.error
|
|
: page.header.category() != gemini::Category::Success ? page.header.meta
|
|
: saveCached(from);
|
|
finishWithMessage(why.empty() ? "Refreshed" : "Not refreshed: " + why);
|
|
break;
|
|
}
|
|
case Job::Delete: {
|
|
bool removed = false;
|
|
bool ran = storage_.runAndWait([&]() { removed = SD.remove(arg_.c_str()); });
|
|
finishWithMessage(!ran ? "No SD card" : removed ? "Deleted" : "Not deleted");
|
|
break;
|
|
}
|
|
case Job::Bookmark: {
|
|
std::string line = "=> " + arg_ + (arg2_.empty() ? "" : " " + arg2_), already;
|
|
bool ran = storage_.runAndWait([&]() {
|
|
File f = SD.open(kBookmarks);
|
|
while (f && f.available()) {
|
|
String l = f.readStringUntil('\n');
|
|
if (l.startsWith(("=> " + arg_ + " ").c_str()) || l == ("=> " + arg_).c_str()) already = "yes";
|
|
}
|
|
if (f) f.close();
|
|
if (!already.empty()) return;
|
|
makeParents(kBookmarks);
|
|
File out = SD.open(kBookmarks, FILE_APPEND);
|
|
if (out) {
|
|
out.print((line + "\n").c_str());
|
|
out.close();
|
|
}
|
|
});
|
|
finishWithMessage(!ran ? "No SD card" : !already.empty() ? "Already bookmarked" : "Bookmarked");
|
|
break;
|
|
}
|
|
case Job::Download: {
|
|
gemini::Url u;
|
|
std::string name = gemini::parseUrl(arg_, u) ? u.path.substr(u.path.rfind('/') + 1) : "";
|
|
if (name.empty()) name = "download";
|
|
std::string to = "/gemini/downloads/" + storage::sanitize(name);
|
|
bool copied = false;
|
|
if (cacheHolds(arg_)) storage_.runAndWait([&]() { copied = copyFile(kCachePath, to, ""); });
|
|
finishWithMessage(copied ? "Saved to " + to : "Not saved (no card, or the page is gone)");
|
|
break;
|
|
}
|
|
}
|
|
busy_ = false;
|
|
}
|
|
|
|
void GeminiService::runFetch(GeminiPage& page) {
|
|
page.requested = arg_;
|
|
size_t before = esp_get_free_heap_size();
|
|
lowest_ = before;
|
|
uint32_t started = millis();
|
|
const std::string& url = arg_;
|
|
if (url == kStartUrl) composeStart(page);
|
|
else if (url.rfind("file://", 0) == 0) openSaved(url.substr(7), page);
|
|
else {
|
|
// Inside a Saved Page, a link to another Saved Page opens the saved copy (Q84).
|
|
bool savedCopy = false;
|
|
if (preferSaved_) {
|
|
std::string path = gemini::savedPath(url);
|
|
storage_.runAndWait([&]() { savedCopy = SD.exists(path.c_str()); });
|
|
if (savedCopy) openSaved(path, page);
|
|
}
|
|
if (!savedCopy) {
|
|
if (preferSaved_ && !WiFi.isConnected()) {
|
|
page.url = url;
|
|
page.error = "Not saved, and offline";
|
|
} else
|
|
fetchFollowing(url, page, true);
|
|
}
|
|
}
|
|
page.ms = millis() - started;
|
|
if (toConsole_) report(page, before, lowest_);
|
|
}
|
|
|
|
// One request, following up to kMaxRedirects redirects.
|
|
void GeminiService::fetchFollowing(const std::string& first, GeminiPage& page, bool loadIntoRam) {
|
|
std::string url = first;
|
|
size_t free = esp_get_free_heap_size();
|
|
if (free < kStartFloor) { // Q86, for every fetch: pages, saves, refreshes
|
|
page.url = url;
|
|
page.outcome = GeminiPage::Outcome::Failed;
|
|
page.error = "Not enough memory (" + std::to_string(free / 1024) + " KB free): stop IRC or retry";
|
|
return;
|
|
}
|
|
for (int hop = 0;; hop++) {
|
|
fetchOne(url, page, loadIntoRam);
|
|
if (page.outcome != GeminiPage::Outcome::Ok || page.header.category() != gemini::Category::Redirect) break;
|
|
std::string next = gemini::resolve(url, page.header.meta);
|
|
if (!gemini::isGemini(next)) {
|
|
page.outcome = GeminiPage::Outcome::Failed;
|
|
page.error = "Redirect to another protocol: " + next;
|
|
break;
|
|
}
|
|
if (hop + 1 > kMaxRedirects) {
|
|
page.outcome = GeminiPage::Outcome::Failed;
|
|
page.error = "Too many redirects";
|
|
break;
|
|
}
|
|
url = next;
|
|
}
|
|
page.url = url;
|
|
}
|
|
|
|
// A Saved Page's first line: "> Saved from <url> on <date>" (Q82). Empty if it isn't one.
|
|
std::string GeminiService::savedFirstLine(const std::string& path) {
|
|
std::string first;
|
|
storage_.runAndWait([&]() {
|
|
File f = SD.open(path.c_str());
|
|
if (!f) return;
|
|
char buf[600];
|
|
size_t n = f.readBytesUntil('\n', buf, sizeof buf - 1);
|
|
buf[n] = 0;
|
|
first = buf;
|
|
f.close();
|
|
});
|
|
if (!first.empty() && first.back() == '\r') first.pop_back();
|
|
return first.rfind(kSavedMark, 0) == 0 ? first : "";
|
|
}
|
|
|
|
std::string GeminiService::savedFrom(const std::string& path) {
|
|
std::string first = savedFirstLine(path);
|
|
if (first.empty()) return "";
|
|
std::string rest = first.substr(strlen(kSavedMark));
|
|
return rest.substr(0, rest.find(" on "));
|
|
}
|
|
|
|
// A Saved Page from the card, as far as memory allows; its first line says where it came from.
|
|
void GeminiService::openSaved(const std::string& path, GeminiPage& page) {
|
|
page.url = "file://" + path;
|
|
page.header.status = 20;
|
|
page.header.meta = "text/gemini";
|
|
bool found = false;
|
|
storage_.runAndWait([&]() { found = SD.exists(path.c_str()); });
|
|
std::string first = found ? savedFirstLine(path) : "";
|
|
if (!found) {
|
|
page.error = "This Saved Page is gone";
|
|
return;
|
|
}
|
|
if (first.rfind(kSavedMark, 0) == 0) {
|
|
std::string rest = first.substr(strlen(kSavedMark));
|
|
size_t on = rest.find(" on ");
|
|
page.savedFrom = rest.substr(0, on);
|
|
page.savedOn = on == std::string::npos ? "" : rest.substr(on + 4);
|
|
}
|
|
loadFromCard(page, esp_get_free_heap_size(), path.c_str());
|
|
page.outcome = GeminiPage::Outcome::Ok;
|
|
}
|
|
|
|
// about:start (Q78, Q84): bookmarks, then Saved Pages by capsule, newest first, then defaults.
|
|
void GeminiService::composeStart(GeminiPage& page) {
|
|
std::string doc = "# Gemini\nTab picks a link, Enter follows it, Back returns, g types an address. "
|
|
"On a page: b bookmarks it, s saves it to read offline, S with the pages it links to.\n";
|
|
std::string bookmarks;
|
|
std::vector<SavedEntry> saved;
|
|
bool card = storage_.runAndWait([&]() {
|
|
File f = SD.open(kBookmarks);
|
|
while (f && f.available() && bookmarks.size() < 8192) {
|
|
String l = f.readStringUntil('\n');
|
|
if (l.startsWith("=>")) bookmarks += std::string(l.c_str()) + "\n";
|
|
}
|
|
if (f) f.close();
|
|
listSaved(kSavedRoot, saved);
|
|
});
|
|
doc += "## Bookmarks\n";
|
|
doc += !card ? "No SD card: bookmarks and Saved Pages live on it.\n"
|
|
: bookmarks.empty() ? "None yet: b on a page adds it here.\n" : bookmarks;
|
|
if (!saved.empty()) {
|
|
// Capsules ordered by their newest Saved Page, pages newest first within each.
|
|
std::sort(saved.begin(), saved.end(), [](const SavedEntry& a, const SavedEntry& b) { return a.when > b.when; });
|
|
std::vector<std::string> order;
|
|
std::map<std::string, std::string> byHost;
|
|
size_t root = strlen(kSavedRoot) + 1;
|
|
for (auto& e : saved) {
|
|
std::string rel = e.path.substr(root);
|
|
std::string host = rel.substr(0, rel.find('/'));
|
|
if (!byHost.count(host)) order.push_back(host);
|
|
std::string label = rel.substr(host.size() + 1);
|
|
if (label.size() > 4 && label.compare(label.size() - 4, 4, ".gmi") == 0) label.resize(label.size() - 4);
|
|
char day[24] = "";
|
|
if (e.when > 1000000000) {
|
|
struct tm t;
|
|
localtime_r(&e.when, &t);
|
|
strftime(day, sizeof day, " (%d %b)", &t);
|
|
}
|
|
byHost[host] += "=> file://" + e.path + " " + label + day + "\n";
|
|
}
|
|
doc += "## Saved pages\n";
|
|
for (auto& host : order) doc += "### " + host + "\n" + byHost[host];
|
|
}
|
|
doc += kDefaults;
|
|
page.url = kStartUrl;
|
|
page.header.status = 20;
|
|
page.header.meta = "text/gemini";
|
|
page.text.append(doc.data(), doc.size());
|
|
page.text.finish();
|
|
page.outcome = GeminiPage::Outcome::Ok;
|
|
}
|
|
|
|
// The cached body of `url` (fetched again if the cache holds something else) to its Saved Page.
|
|
std::string GeminiService::saveCached(const std::string& url) {
|
|
if (!gemini::isGemini(url)) return "Only Gemini pages can be saved";
|
|
if (!storage_.state().present) return "No SD card";
|
|
if (!cacheHolds(url)) {
|
|
GeminiPage page;
|
|
fetchFollowing(url, page, false);
|
|
if (page.outcome != GeminiPage::Outcome::Ok) return page.error;
|
|
if (page.header.category() != gemini::Category::Success) return page.header.meta;
|
|
if (!cacheHolds(page.url)) return "Not saved";
|
|
}
|
|
std::string to = gemini::savedPath(url);
|
|
bool existed = false, ok = false;
|
|
replaced_ = false;
|
|
std::string header = std::string(kSavedMark) + url + " on " + now();
|
|
storage_.runAndWait([&]() {
|
|
existed = SD.exists(to.c_str());
|
|
ok = copyFile(kCachePath, to, header);
|
|
});
|
|
if (!ok) return "Writing to the SD card failed";
|
|
replaced_ = existed; // Q82: saving again replaces it, and says so
|
|
return "";
|
|
}
|
|
|
|
void GeminiService::fetchOne(const std::string& url, GeminiPage& page, bool loadIntoRam) {
|
|
page.outcome = GeminiPage::Outcome::Failed;
|
|
page.text.clear();
|
|
page.truncated = false;
|
|
gemini::Url u;
|
|
if (!gemini::parseUrl(url, u) || u.scheme != "gemini") {
|
|
page.error = "Not a Gemini URL: " + url;
|
|
return;
|
|
}
|
|
auto track = [this]() { lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size()); };
|
|
|
|
size_t freeBefore = esp_get_free_heap_size(); // what's left once the connection closes again
|
|
NetworkClientSecure tls;
|
|
tls.setInsecure(); // trust on first use: the pinned fingerprint is what counts (Q71)
|
|
tls.setTimeout(15);
|
|
if (!tls.connect(u.host.c_str(), u.portOrDefault())) {
|
|
page.error = "Cannot connect to " + u.host;
|
|
return;
|
|
}
|
|
track();
|
|
uint8_t sha[32];
|
|
if (!tls.getFingerprintSHA256(sha)) {
|
|
page.error = "No certificate from " + u.host;
|
|
return;
|
|
}
|
|
std::string presented = hex(sha, sizeof sha), pinned;
|
|
std::string key = pinKey(u.host, u.portOrDefault());
|
|
if (!store_.getString(key.c_str(), pinned)) store_.putString(key.c_str(), presented); // first use
|
|
else if (pinned != presented) {
|
|
page.outcome = GeminiPage::Outcome::CertificateChanged;
|
|
page.host = u.host;
|
|
page.port = u.portOrDefault();
|
|
page.pinned = pinned;
|
|
page.presented = presented;
|
|
page.error = "The certificate of " + u.host + " changed";
|
|
return;
|
|
}
|
|
|
|
tls.print((gemini::requestUrl(url) + "\r\n").c_str());
|
|
std::string header;
|
|
bool inBody = false;
|
|
uint32_t last = millis();
|
|
char buf[512];
|
|
while (!cancel_) {
|
|
int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
|
|
if (n <= 0) {
|
|
if (!tls.connected()) break;
|
|
if (millis() - last > kIdleTimeoutMs) {
|
|
page.error = "Timed out";
|
|
return;
|
|
}
|
|
delay(5);
|
|
continue;
|
|
}
|
|
last = millis();
|
|
int i = 0;
|
|
while (!inBody && i < n) {
|
|
char c = buf[i++];
|
|
if (c == '\n') inBody = true;
|
|
else if (c != '\r') header += c;
|
|
if (header.size() > 1100) {
|
|
page.error = "Malformed response header";
|
|
return;
|
|
}
|
|
}
|
|
if (!inBody) continue;
|
|
if (!gemini::parseHeader(header, page.header)) {
|
|
page.error = "Malformed response header";
|
|
return;
|
|
}
|
|
if (page.header.category() != gemini::Category::Success) break; // no body to read
|
|
// With a card, the body goes there while the connection is open (only 4 KB in RAM),
|
|
// and comes back into RAM once the connection's memory is free again.
|
|
if (storage_.state().present) {
|
|
cachedUrl_.clear();
|
|
if (!receiveToCard(tls, buf + i, n - i, page)) return;
|
|
tls.stop();
|
|
cachedUrl_ = url;
|
|
if (loadIntoRam) loadFromCard(page, freeBefore, kCachePath);
|
|
page.outcome = GeminiPage::Outcome::Ok;
|
|
return;
|
|
}
|
|
// Q86, two floors: once the connection closes, its ~50 KB comes back, so the page may use
|
|
// what was free before minus the steady floor; meanwhile the heap must stay above the
|
|
// transient floor. Both counted with the next 4 KB chunk and the line index.
|
|
size_t take = std::min<size_t>(n - i, kMaxBody - page.text.bytes());
|
|
size_t pageCost = page.text.bytes() + page.text.lineCount() * 8 + TextBuffer::kChunk + take;
|
|
if (take < static_cast<size_t>(n - i)) page.truncatedWhy = "longer than 64 KB";
|
|
else if (freeBefore < kSteadyFloor + pageCost) page.truncatedWhy = "not enough memory to keep it";
|
|
else if (esp_get_free_heap_size() < kTransientFloor + TextBuffer::kChunk + take)
|
|
page.truncatedWhy = "not enough memory while receiving";
|
|
if (!page.truncatedWhy.empty()) {
|
|
page.truncated = true;
|
|
break;
|
|
}
|
|
page.text.append(buf + i, take);
|
|
track();
|
|
}
|
|
if (cancel_) {
|
|
page.error = "Cancelled";
|
|
return;
|
|
}
|
|
if (!inBody && !gemini::parseHeader(header, page.header)) { // a header with no line end
|
|
page.error = header.empty() ? "Empty response" : "Malformed response header";
|
|
return;
|
|
}
|
|
page.text.finish();
|
|
page.outcome = GeminiPage::Outcome::Ok;
|
|
}
|
|
|
|
// Streams the rest of the body to kCachePath in 1 KB pieces, each written by the storage task:
|
|
// with IRC's TLS connection and this one both open, every KB of RAM counts (Q86).
|
|
|
|
bool GeminiService::receiveToCard(NetworkClientSecure& tls, const char* first, size_t len, GeminiPage& page) {
|
|
struct Cache {
|
|
File file;
|
|
};
|
|
auto cache = std::make_shared<Cache>();
|
|
bool opened = storage_.runAndWait([cache]() {
|
|
SD.mkdir("/gemini");
|
|
SD.mkdir("/gemini/cache");
|
|
cache->file = SD.open(kCachePath, FILE_WRITE);
|
|
});
|
|
if (!opened || !cache->file) {
|
|
page.error = "Can't write to the SD card";
|
|
return false;
|
|
}
|
|
std::string pending(first, len);
|
|
constexpr size_t kPiece = 1024;
|
|
pending.reserve(kPiece + 512);
|
|
bool ok = true;
|
|
auto flush = [&]() {
|
|
if (pending.empty() || !ok) return;
|
|
ok = storage_.runAndWait([cache, &pending, &ok]() {
|
|
ok = cache->file.write(reinterpret_cast<const uint8_t*>(pending.data()), pending.size()) == pending.size();
|
|
}) && ok;
|
|
page.receivedBytes += pending.size();
|
|
pending.clear();
|
|
};
|
|
uint32_t last = millis();
|
|
char buf[512];
|
|
while (!cancel_ && ok) {
|
|
if (pending.size() >= kPiece) flush();
|
|
if (page.receivedBytes + pending.size() >= kMaxOnCard) {
|
|
page.truncated = true;
|
|
page.truncatedWhy = "longer than 1 MB";
|
|
break;
|
|
}
|
|
int n = tls.available() ? tls.read(reinterpret_cast<uint8_t*>(buf), sizeof buf) : 0;
|
|
if (n <= 0) {
|
|
if (!tls.connected()) break;
|
|
if (millis() - last > kIdleTimeoutMs) break; // keep what came
|
|
delay(5);
|
|
continue;
|
|
}
|
|
last = millis();
|
|
pending.append(buf, n);
|
|
lowest_ = std::min<size_t>(lowest_, esp_get_free_heap_size());
|
|
}
|
|
flush();
|
|
storage_.runAndWait([cache]() { cache->file.close(); });
|
|
if (cancel_) page.error = "Cancelled";
|
|
else if (!ok) page.error = "Writing to the SD card failed";
|
|
else page.cachePath = kCachePath;
|
|
return !cancel_ && ok;
|
|
}
|
|
|
|
// Back into RAM, now that the connection's memory is free again: as much as the steady floor
|
|
// allows (Q86); the rest stays on the card.
|
|
|
|
void GeminiService::loadFromCard(GeminiPage& page, size_t freeBefore, const char* path) {
|
|
size_t budget = freeBefore > kSteadyFloor + TextBuffer::kChunk ? freeBefore - kSteadyFloor - TextBuffer::kChunk : 0;
|
|
budget = std::min(budget, kMaxBody);
|
|
storage_.runAndWait([&]() {
|
|
File f = SD.open(path);
|
|
if (!f) return;
|
|
char buf[512];
|
|
int n;
|
|
while ((n = f.read(reinterpret_cast<uint8_t*>(buf), sizeof buf)) > 0) {
|
|
size_t cost = page.text.bytes() + page.text.lineCount() * 8 + n;
|
|
if (cost > budget) {
|
|
page.truncated = true;
|
|
if (page.truncatedWhy.empty())
|
|
page.truncatedWhy = "only part of it fits in memory; the whole page is on the card";
|
|
break;
|
|
}
|
|
page.text.append(buf, n);
|
|
}
|
|
f.close();
|
|
});
|
|
page.text.finish();
|
|
}
|
|
|
|
|
|
void GeminiService::report(const GeminiPage& page, size_t heapBefore, size_t heapLowest) {
|
|
console.printf("gemini: %s -> %s, %lu ms\n", page.requested.c_str(), page.url.c_str(), (unsigned long)page.ms);
|
|
if (page.outcome != GeminiPage::Outcome::Ok) console.printf("gemini: %s\n", page.error.c_str());
|
|
else console.printf("gemini: %d %s, %u bytes, %u lines%s\n", page.header.status, page.header.meta.c_str(),
|
|
(unsigned)page.text.bytes(), (unsigned)page.text.lineCount(),
|
|
page.truncated ? (" (truncated: " + page.truncatedWhy + ")").c_str() : "");
|
|
if (page.outcome == GeminiPage::Outcome::CertificateChanged)
|
|
console.printf("gemini: pinned %s\ngemini: now %s\n", page.pinned.c_str(), page.presented.c_str());
|
|
for (size_t i = 0; i < page.text.lineCount() && i < 6; i++) console.printf(" %s\n", page.text.line(i).substr(0, 100).c_str());
|
|
if (!page.cachePath.empty())
|
|
console.printf("gemini: %u bytes on the card in %s\n", (unsigned)page.receivedBytes, page.cachePath.c_str());
|
|
console.printf("gemini: heap %u before, %u lowest seen, %u after\n", (unsigned)heapBefore, (unsigned)heapLowest,
|
|
(unsigned)esp_get_free_heap_size());
|
|
}
|
|
|
|
|
|
} // namespace roro
|