Public Access
G1 step 2: Gemini parsers (host-tested), and the memory decision (Q86)
lib/gemini: URLs split per RFC 3986 appendix B and resolved per section 5.2 (all 32 reference examples of 5.4 pass, with gemini:// for http://), the request form (no fragment, lower-case host, never an empty path), query encoding for input prompts, Saved Page paths; the response header (status, category, MIME type and parameters, 1024-byte meta limit); gemtext's line types; and display text for the Latin-1 fonts. 12 tests. Q86, decided after step 1: free heap stays above 40 KB in steady state and 20 KB during a handshake; a fetch won't start below 55 KB. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EhqxQ49eCju4CzKYNjZzwT
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,59 @@
|
||||
#include "gemini_response.h"
|
||||
|
||||
#include <cctype>
|
||||
|
||||
namespace roro::gemini {
|
||||
|
||||
namespace {
|
||||
std::string trim(const std::string& s) {
|
||||
size_t a = s.find_first_not_of(" \t"), b = s.find_last_not_of(" \t");
|
||||
return a == std::string::npos ? "" : s.substr(a, b - a + 1);
|
||||
}
|
||||
std::string lower(std::string s) {
|
||||
for (auto& c : s) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
|
||||
return s;
|
||||
}
|
||||
} // namespace
|
||||
|
||||
bool parseHeader(const std::string& line, Header& out) {
|
||||
if (line.size() < 2 || !std::isdigit(static_cast<unsigned char>(line[0])) ||
|
||||
!std::isdigit(static_cast<unsigned char>(line[1])))
|
||||
return false;
|
||||
if (line.size() > 2 && line[2] != ' ') return false;
|
||||
std::string meta = line.size() > 3 ? line.substr(3) : "";
|
||||
if (meta.size() > 1024) return false;
|
||||
out.status = (line[0] - '0') * 10 + (line[1] - '0');
|
||||
out.meta = meta;
|
||||
return true;
|
||||
}
|
||||
|
||||
Category Header::category() const {
|
||||
switch (status / 10) {
|
||||
case 1: return Category::Input;
|
||||
case 2: return Category::Success;
|
||||
case 3: return Category::Redirect;
|
||||
case 4: return Category::TemporaryFailure;
|
||||
case 5: return Category::PermanentFailure;
|
||||
case 6: return Category::ClientCertificate;
|
||||
default: return Category::Unknown;
|
||||
}
|
||||
}
|
||||
|
||||
std::string Header::mimeType() const {
|
||||
std::string type = lower(trim(meta.substr(0, meta.find(';'))));
|
||||
return type.empty() ? "text/gemini" : type;
|
||||
}
|
||||
|
||||
std::string Header::parameter(const std::string& name) const {
|
||||
size_t pos = meta.find(';');
|
||||
while (pos != std::string::npos) {
|
||||
size_t next = meta.find(';', pos + 1);
|
||||
std::string item = trim(meta.substr(pos + 1, next == std::string::npos ? std::string::npos : next - pos - 1));
|
||||
size_t eq = item.find('=');
|
||||
if (eq != std::string::npos && lower(trim(item.substr(0, eq))) == lower(name)) return trim(item.substr(eq + 1));
|
||||
pos = next;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
} // namespace roro::gemini
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace roro::gemini {
|
||||
|
||||
enum class Category { Input, Success, Redirect, TemporaryFailure, PermanentFailure, ClientCertificate, Unknown };
|
||||
|
||||
// The response header: "<two digits> <meta>", at most 1024 bytes of meta.
|
||||
struct Header {
|
||||
int status = 0;
|
||||
std::string meta;
|
||||
|
||||
Category category() const;
|
||||
bool sensitiveInput() const { return status == 11; }
|
||||
// 2x: the MIME type, lower-cased, without parameters ("text/gemini" when meta is empty).
|
||||
std::string mimeType() const;
|
||||
std::string parameter(const std::string& name) const; // e.g. "charset"
|
||||
};
|
||||
|
||||
bool parseHeader(const std::string& line, Header& out);
|
||||
|
||||
} // namespace roro::gemini
|
||||
@@ -0,0 +1,215 @@
|
||||
#include "gemini_url.h"
|
||||
|
||||
#include <cctype>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <vector>
|
||||
|
||||
#include "storage_paths.h"
|
||||
|
||||
namespace roro::gemini {
|
||||
|
||||
namespace {
|
||||
|
||||
// RFC 3986, appendix B: ^(([^:/?#]+):)?(//([^/?#]*))?([^?#]*)(\?([^#]*))?(#(.*))?
|
||||
struct Parts {
|
||||
std::string scheme, authority, path, query, fragment;
|
||||
bool hasScheme = false, hasAuthority = false, hasQuery = false, hasFragment = false;
|
||||
};
|
||||
|
||||
Parts split(const std::string& s) {
|
||||
Parts p;
|
||||
size_t i = 0;
|
||||
size_t colon = s.find(':');
|
||||
size_t stop = s.find_first_of("/?#");
|
||||
if (colon != std::string::npos && colon > 0 && (stop == std::string::npos || colon < stop)) {
|
||||
p.scheme = s.substr(0, colon);
|
||||
p.hasScheme = true;
|
||||
i = colon + 1;
|
||||
}
|
||||
if (s.compare(i, 2, "//") == 0) {
|
||||
size_t end = s.find_first_of("/?#", i + 2);
|
||||
if (end == std::string::npos) end = s.size();
|
||||
p.authority = s.substr(i + 2, end - i - 2);
|
||||
p.hasAuthority = true;
|
||||
i = end;
|
||||
}
|
||||
size_t end = s.find_first_of("?#", i);
|
||||
if (end == std::string::npos) end = s.size();
|
||||
p.path = s.substr(i, end - i);
|
||||
i = end;
|
||||
if (i < s.size() && s[i] == '?') {
|
||||
end = s.find('#', i);
|
||||
if (end == std::string::npos) end = s.size();
|
||||
p.query = s.substr(i + 1, end - i - 1);
|
||||
p.hasQuery = true;
|
||||
i = end;
|
||||
}
|
||||
if (i < s.size() && s[i] == '#') {
|
||||
p.fragment = s.substr(i + 1);
|
||||
p.hasFragment = true;
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
std::string lower(std::string s) {
|
||||
for (auto& c : s) c = static_cast<char>(std::tolower(static_cast<unsigned char>(c)));
|
||||
return s;
|
||||
}
|
||||
|
||||
// RFC 3986, section 5.2.4.
|
||||
std::string removeDotSegments(std::string in) {
|
||||
std::string out;
|
||||
while (!in.empty()) {
|
||||
if (in.compare(0, 3, "../") == 0) in.erase(0, 3);
|
||||
else if (in.compare(0, 2, "./") == 0) in.erase(0, 2);
|
||||
else if (in.compare(0, 3, "/./") == 0) in.replace(0, 3, "/");
|
||||
else if (in == "/.") in = "/";
|
||||
else if (in.compare(0, 4, "/../") == 0 || in == "/..") {
|
||||
in = in == "/.." ? "/" : in.substr(3);
|
||||
size_t slash = out.rfind('/');
|
||||
out.erase(slash == std::string::npos ? 0 : slash);
|
||||
} else if (in == "." || in == "..") in.clear();
|
||||
else {
|
||||
size_t next = in.find('/', in[0] == '/' ? 1 : 0);
|
||||
if (next == std::string::npos) next = in.size();
|
||||
out += in.substr(0, next);
|
||||
in.erase(0, next);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// RFC 3986, section 5.2.3.
|
||||
std::string merge(const Parts& base, const std::string& refPath) {
|
||||
if (base.hasAuthority && base.path.empty()) return "/" + refPath;
|
||||
size_t slash = base.path.rfind('/');
|
||||
return slash == std::string::npos ? refPath : base.path.substr(0, slash + 1) + refPath;
|
||||
}
|
||||
|
||||
// RFC 3986, section 5.3.
|
||||
std::string recompose(const Parts& p) {
|
||||
std::string out;
|
||||
if (p.hasScheme) out += p.scheme + ":";
|
||||
if (p.hasAuthority) out += "//" + p.authority;
|
||||
out += p.path;
|
||||
if (p.hasQuery) out += "?" + p.query;
|
||||
if (p.hasFragment) out += "#" + p.fragment;
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
bool parseUrl(const std::string& text, Url& out) {
|
||||
Parts p = split(text);
|
||||
if (!p.hasScheme || !p.hasAuthority) return false;
|
||||
out = Url();
|
||||
out.scheme = lower(p.scheme);
|
||||
out.authority = p.authority;
|
||||
std::string hostPort = p.authority.substr(p.authority.find('@') == std::string::npos ? 0 : p.authority.find('@') + 1);
|
||||
size_t colon = hostPort.rfind(':');
|
||||
if (colon != std::string::npos && hostPort.find(']', colon) == std::string::npos) {
|
||||
out.port = std::atoi(hostPort.c_str() + colon + 1);
|
||||
hostPort = hostPort.substr(0, colon);
|
||||
}
|
||||
out.host = lower(hostPort);
|
||||
if (out.host.empty()) return false;
|
||||
out.path = p.path;
|
||||
out.query = p.query;
|
||||
out.fragment = p.fragment;
|
||||
out.hasAuthority = true;
|
||||
out.hasQuery = p.hasQuery;
|
||||
out.hasFragment = p.hasFragment;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool isGemini(const std::string& url) {
|
||||
Parts p = split(url);
|
||||
return p.hasScheme && lower(p.scheme) == "gemini";
|
||||
}
|
||||
|
||||
std::string resolve(const std::string& baseText, const std::string& refText) {
|
||||
Parts base = split(baseText), r = split(refText), t;
|
||||
if (r.hasScheme) {
|
||||
t = r;
|
||||
t.path = removeDotSegments(r.path);
|
||||
} else {
|
||||
if (r.hasAuthority) {
|
||||
t.authority = r.authority;
|
||||
t.hasAuthority = true;
|
||||
t.path = removeDotSegments(r.path);
|
||||
t.query = r.query;
|
||||
t.hasQuery = r.hasQuery;
|
||||
} else {
|
||||
if (r.path.empty()) {
|
||||
t.path = base.path;
|
||||
t.query = r.hasQuery ? r.query : base.query;
|
||||
t.hasQuery = r.hasQuery || base.hasQuery;
|
||||
} else {
|
||||
t.path = removeDotSegments(r.path[0] == '/' ? r.path : merge(base, r.path));
|
||||
t.query = r.query;
|
||||
t.hasQuery = r.hasQuery;
|
||||
}
|
||||
t.authority = base.authority;
|
||||
t.hasAuthority = base.hasAuthority;
|
||||
}
|
||||
t.scheme = base.scheme;
|
||||
t.hasScheme = base.hasScheme;
|
||||
}
|
||||
t.fragment = r.fragment;
|
||||
t.hasFragment = r.hasFragment;
|
||||
return recompose(t);
|
||||
}
|
||||
|
||||
std::string requestUrl(const std::string& url) {
|
||||
Url u;
|
||||
if (!parseUrl(url, u)) return url;
|
||||
std::string out = u.scheme + "://" + u.host;
|
||||
if (u.port > 0 && u.port != 1965) out += ":" + std::to_string(u.port);
|
||||
out += u.path.empty() ? "/" : u.path;
|
||||
if (u.hasQuery) out += "?" + u.query;
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string encodeQuery(const std::string& text) {
|
||||
std::string out;
|
||||
for (unsigned char c : text) {
|
||||
if (std::isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') out += static_cast<char>(c);
|
||||
else {
|
||||
char buf[4];
|
||||
std::snprintf(buf, sizeof buf, "%%%02X", c);
|
||||
out += buf;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string savedPath(const std::string& url) {
|
||||
Url u;
|
||||
if (!parseUrl(url, u)) return "/gemini/saved/unknown.gmi";
|
||||
std::string out = "/gemini/saved/" + storage::sanitize(u.host + (u.port > 0 && u.port != 1965 ? "_" + std::to_string(u.port) : ""));
|
||||
std::string path = u.path.empty() ? "/" : u.path;
|
||||
std::vector<std::string> parts;
|
||||
size_t start = 1;
|
||||
while (start <= path.size()) {
|
||||
size_t slash = path.find('/', start);
|
||||
if (slash == std::string::npos) slash = path.size();
|
||||
parts.push_back(path.substr(start, slash - start));
|
||||
start = slash + 1;
|
||||
}
|
||||
if (parts.empty() || parts.back().empty()) {
|
||||
if (!parts.empty()) parts.pop_back();
|
||||
parts.push_back("index");
|
||||
}
|
||||
for (size_t i = 0; i < parts.size(); i++) {
|
||||
std::string name = parts[i];
|
||||
if (i + 1 == parts.size()) {
|
||||
if (u.hasQuery) name += "~" + u.query;
|
||||
if (name.size() < 4 || name.compare(name.size() - 4, 4, ".gmi") != 0) name += ".gmi";
|
||||
}
|
||||
out += "/" + storage::sanitize(name);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace roro::gemini
|
||||
@@ -0,0 +1,28 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
namespace roro::gemini {
|
||||
|
||||
// A URL split per RFC 3986 (appendix B). For Gemini, the host is lower-cased and the port
|
||||
// defaults to 1965.
|
||||
struct Url {
|
||||
std::string scheme, authority, host, path, query, fragment;
|
||||
int port = -1;
|
||||
bool hasAuthority = false, hasQuery = false, hasFragment = false;
|
||||
int portOrDefault() const { return port > 0 ? port : 1965; }
|
||||
};
|
||||
|
||||
// False unless it has a scheme and a host: only absolute URLs are fetched.
|
||||
bool parseUrl(const std::string& text, Url& out);
|
||||
bool isGemini(const std::string& url);
|
||||
// `ref` against `base`, per RFC 3986 section 5.2 (dot segments included).
|
||||
std::string resolve(const std::string& base, const std::string& ref);
|
||||
// What goes on the wire: no fragment, lower-case host, never an empty path.
|
||||
std::string requestUrl(const std::string& url);
|
||||
// For input prompts: everything but unreserved characters percent-encoded (UTF-8 bytes).
|
||||
std::string encodeQuery(const std::string& text);
|
||||
// Where a Saved Page lives (Q82): /gemini/saved/<host>[_<port>]/<path>[~<query>].gmi
|
||||
std::string savedPath(const std::string& url);
|
||||
|
||||
} // namespace roro::gemini
|
||||
@@ -0,0 +1,97 @@
|
||||
#include "gemtext.h"
|
||||
|
||||
#include <cstdint>
|
||||
|
||||
namespace roro::gemini {
|
||||
|
||||
namespace {
|
||||
std::string trim(const std::string& s) {
|
||||
size_t a = s.find_first_not_of(" \t"), b = s.find_last_not_of(" \t");
|
||||
return a == std::string::npos ? "" : s.substr(a, b - a + 1);
|
||||
}
|
||||
} // namespace
|
||||
|
||||
std::vector<GemLine> parseGemtext(const std::string& document) {
|
||||
std::vector<GemLine> lines;
|
||||
bool pre = false;
|
||||
size_t start = 0;
|
||||
while (start < document.size()) {
|
||||
size_t end = document.find('\n', start);
|
||||
if (end == std::string::npos) end = document.size();
|
||||
std::string line = document.substr(start, end - start);
|
||||
if (!line.empty() && line.back() == '\r') line.pop_back();
|
||||
start = end + 1;
|
||||
|
||||
GemLine g;
|
||||
if (line.compare(0, 3, "```") == 0) {
|
||||
g.type = LineType::PreToggle;
|
||||
g.text = trim(line.substr(3));
|
||||
pre = !pre;
|
||||
} else if (pre) {
|
||||
g.type = LineType::Preformatted;
|
||||
g.text = line;
|
||||
} else if (line.compare(0, 2, "=>") == 0) {
|
||||
g.type = LineType::Link;
|
||||
std::string rest = trim(line.substr(2));
|
||||
size_t gap = rest.find_first_of(" \t");
|
||||
g.url = rest.substr(0, gap);
|
||||
g.text = gap == std::string::npos ? "" : trim(rest.substr(gap));
|
||||
if (g.text.empty()) g.text = g.url;
|
||||
} else if (line.compare(0, 3, "###") == 0) {
|
||||
g.type = LineType::Heading3;
|
||||
g.text = trim(line.substr(3));
|
||||
} else if (line.compare(0, 2, "##") == 0) {
|
||||
g.type = LineType::Heading2;
|
||||
g.text = trim(line.substr(2));
|
||||
} else if (line.compare(0, 1, "#") == 0) {
|
||||
g.type = LineType::Heading1;
|
||||
g.text = trim(line.substr(1));
|
||||
} else if (line.compare(0, 2, "* ") == 0) {
|
||||
g.type = LineType::ListItem;
|
||||
g.text = trim(line.substr(2));
|
||||
} else if (line.compare(0, 1, ">") == 0) {
|
||||
g.type = LineType::Quote;
|
||||
g.text = trim(line.substr(1));
|
||||
} else {
|
||||
g.text = line;
|
||||
}
|
||||
lines.push_back(std::move(g));
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
std::string displayText(const std::string& s) {
|
||||
std::string out;
|
||||
size_t column = 0;
|
||||
for (size_t i = 0; i < s.size();) {
|
||||
unsigned char c = static_cast<unsigned char>(s[i]);
|
||||
if (c == '\t') {
|
||||
do out += ' ';
|
||||
while (++column % 4);
|
||||
i++;
|
||||
continue;
|
||||
}
|
||||
if (c < 0x80) {
|
||||
out += static_cast<char>(c);
|
||||
i++;
|
||||
} else {
|
||||
int len = (c & 0xE0) == 0xC0 ? 2 : (c & 0xF0) == 0xE0 ? 3 : (c & 0xF8) == 0xF0 ? 4 : 0;
|
||||
bool valid = len > 0 && i + len <= s.size();
|
||||
for (int k = 1; valid && k < len; k++) valid = (static_cast<unsigned char>(s[i + k]) & 0xC0) == 0x80;
|
||||
if (!valid) {
|
||||
out += '?';
|
||||
i++;
|
||||
} else {
|
||||
uint32_t cp = len == 2 ? (c & 0x1F) : len == 3 ? (c & 0x0F) : (c & 0x07);
|
||||
for (int k = 1; k < len; k++) cp = (cp << 6) | (static_cast<unsigned char>(s[i + k]) & 0x3F);
|
||||
if (cp <= 0xFF) out.append(s, i, len); // Latin-1: the fonts have it
|
||||
else out += '?';
|
||||
i += len;
|
||||
}
|
||||
}
|
||||
column++;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
} // namespace roro::gemini
|
||||
@@ -0,0 +1,23 @@
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace roro::gemini {
|
||||
|
||||
enum class LineType { Text, Link, Heading1, Heading2, Heading3, ListItem, Quote, PreToggle, Preformatted };
|
||||
|
||||
// One gemtext line. Link: `url` and its label in `text` (the URL itself without a label).
|
||||
// PreToggle: the ``` line, with its alt text. Preformatted: kept exactly.
|
||||
struct GemLine {
|
||||
LineType type = LineType::Text;
|
||||
std::string text, url;
|
||||
};
|
||||
|
||||
std::vector<GemLine> parseGemtext(const std::string& document);
|
||||
|
||||
// For the Latin-1 fonts (Q79): valid UTF-8 up to U+00FF kept, anything else '?', and tabs
|
||||
// expanded to 4-column stops.
|
||||
std::string displayText(const std::string& utf8);
|
||||
|
||||
} // namespace roro::gemini
|
||||
Reference in New Issue
Block a user