roro9stack/site/tools/gen_dev_docs.py

#!/usr/bin/env python3
"""Writes the developer pages of the site from the repository's own documents (docs/milestones/W1.md, phase 4).

    python3 site/tools/gen_dev_docs.py            writes site/content/dev/... (committed, so the server only runs `zola build`)
    python3 site/tools/gen_dev_docs.py --check    changes nothing; exits 1 if a generated page is out of date (CI)

Zola cannot read a file outside its own folder, so the pages are generated and committed. What is generated:
  decisions/  one page for each docs/adr/*.md
  milestones/ one page for each docs/milestones/*.md
  build/build-and-test, build/flash   sections of README.md
  debug/commands   the commands the firmware's `help` prints (parsed from src/main.cpp), then README's table of them
  site/data/keys.toml   every screen's keys, from lib/core/src/app_keys.h: what the help panel (Fn+h) shows on the
                   device, for the `keys` shortcode of the user guide (issue #72)
Every generated page says where it comes from; edit that file, not the page. Hand-written pages sit next to them.
"""
import json
import os
import re
import sys
from pathlib import Path

HERE = Path(__file__).resolve().parent
SITE = HERE.parent
REPO = SITE.parent
OUT = SITE / "content" / "dev"
REPO_URL = re.search(r'repo\s*=\s*"([^"]+)"', (SITE / "config.toml").read_text()).group(1)

MILESTONES = ["OTA", "M2", "G1", "M3", "S1", "F1", "R1", "W1", "U1"]  # in the order they were done
# Left out on purpose: docs/milestones/M0.md, M1.md and CONTEXT.md (the glossary) describe Wi-Fi monitoring, which this site does not publish.
# They stay in the repository.


def plain(md, limit=230):
    """One line of plain text from the start of some Markdown, for a description."""
    text = re.sub(r"`([^`]*)`", r"\1", md)
    text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
    text = re.sub(r"\*{1,3}", "", text)
    text = re.sub(r"\s+", " ", text).strip()
    text = text[:1].upper() + text[1:]
    if len(text) <= limit:
        return text
    cut = text[:limit].rsplit(" ", 1)[0].rstrip(",;:")
    return cut + "…"


def front(title, description, weight, source, tag=None, template=None):
    lines = ["+++", f"title = {json.dumps(title, ensure_ascii=False)}", f"description = {json.dumps(description, ensure_ascii=False)}", f"weight = {weight}"]
    if template:
        lines.append(f'template = "{template}"')
    lines += ["", "[extra]", "docs = true", f"source = {json.dumps(source)}"]
    if tag:
        lines.append(f"tag = {json.dumps(tag, ensure_ascii=False)}")
    lines += ["+++", ""]
    return "\n".join(lines)


def relink(md, source):
    """Links between the repository's documents become links between the site's pages, or to Gitea."""
    here = Path(source).parent

    def fix(m):
        label, target = m.group(1), m.group(2)
        if re.match(r"[a-z]+:|#|/", target):
            return m.group(0)
        path, _, frag = target.partition("#")
        full = os.path.normpath(here / path)
        frag = "#" + frag if frag else ""
        m_adr = re.fullmatch(r"docs/adr/(.+)\.md", full)
        m_ms = re.fullmatch(r"docs/milestones/(.+)\.md", full)
        if m_adr:
            return f"[{label}](/dev/decisions/{m_adr.group(1)}/{frag})"
        if m_ms:
            return f"[{label}](/dev/milestones/{m_ms.group(1).lower()}/{frag})"
        if full == "CONTEXT.md":
            return f"[{label}](/dev/glossary/{frag})"
        return f"[{label}]({REPO_URL}/src/branch/main/{full}{frag})"

    return re.sub(r"\[([^\]]*)\]\(([^)\s]+)\)", fix, md)


def title_and_body(text):
    m = re.match(r"#\s+(.+)\n", text)
    return (m.group(1).strip(), text[m.end():].lstrip("\n")) if m else ("", text)


def first_paragraph(body):
    for block in re.split(r"\n\s*\n", body):
        b = block.strip()
        if b and not b.startswith(("#", "|", "```", "- ", "1.", "![")):
            return b
    return ""


def goal_or_first(body):
    m = re.search(r"\*\*Goal:\*\*\s*(.+?)(?:\n\s*\n|\Z)", body, re.S)
    return m.group(1) if m else first_paragraph(re.sub(r"^\*\*Status:\*\*.*\n", "", body, flags=re.M))


def sections(readme):
    """README's `## ` sections by heading, each with its heading line and everything up to the next `## `."""
    out, name, buf = {}, None, []
    for line in readme.splitlines(keepends=True):
        if line.startswith("## "):
            if name:
                out[name] = "".join(buf).rstrip() + "\n"
            name, buf = line[3:].strip(), [line]
        elif name:
            buf.append(line)
    if name:
        out[name] = "".join(buf).rstrip() + "\n"
    return out


def help_text():
    """The two lists `help` prints: for every build, and for Debug Builds only, from kHelp in src/main.cpp."""
    src = (REPO / "src" / "main.cpp").read_text()
    m = re.search(r"static const char\* const kHelp =(.*?)\n\s*;", src, re.S)
    if not m:
        sys.exit("gen_dev_docs: kHelp not found in src/main.cpp")
    common, debug, in_debug = [], [], False
    for line in m.group(1).splitlines():
        stripped = line.strip()
        if stripped.startswith("#ifdef RORO_DEBUG"):
            in_debug = True
        elif stripped.startswith("#endif"):
            in_debug = False
        else:
            for lit in re.findall(r'"((?:[^"\\]|\\.)*)"', line):
                (debug if in_debug else common).append(lit.replace("\\n", "\n").replace('\\"', '"'))
    return "".join(common).rstrip("\n"), "".join(debug).rstrip("\n")


def safe_mode_commands():
    src = (REPO / "src" / "main.cpp").read_text()
    m = re.search(r"static bool safeModeCommand\(const String& line\) \{(.*?)\n\}", src, re.S)
    if not m:
        sys.exit("gen_dev_docs: safeModeCommand not found in src/main.cpp")
    body = m.group(1)
    exact = re.findall(r'line == "([^"]+)"', body)
    prefix = re.findall(r'line\.startsWith\("([^"]+)"\)', body)
    return exact, prefix


def key_tables():
    """Every table of lib/core/src/app_keys.h: [(id, title, [(keys, action), ...])], in the file's order."""
    src = (REPO / "lib" / "core" / "src" / "app_keys.h").read_text()

    def text(literal):  # a C string literal's contents: \xHH bytes are UTF-8
        raw = re.sub(r"\\x([0-9A-Fa-f]{2})", lambda m: chr(int(m.group(1), 16)), literal).replace('\\"', '"')
        return raw.encode("latin-1").decode("utf-8")

    tables = []
    for m in re.finditer(r"// ([a-z0-9-]+): ([^\n]+)\ninline constexpr KeyHelp k\w+\[\] = \{\n(.*?)\n\};", src, re.S):
        rows = [(text(k), text(a)) for k, a in re.findall(r'\{"((?:[^"\\]|\\.)*)", "((?:[^"\\]|\\.)*)"\},', m.group(3))]
        if len(rows) != len([l for l in m.group(3).splitlines() if l.strip()]):
            sys.exit(f"gen_dev_docs: a row of `{m.group(1)}` in app_keys.h isn't in the form {{\"keys\", \"action\"}},")
        tables.append((m.group(1), m.group(2).strip(), rows))
    declared = len(re.findall(r"^inline constexpr KeyHelp k\w+\[\]", src, re.M))
    if len(tables) != declared or len({t[0] for t in tables}) != len(tables):
        sys.exit(f"gen_dev_docs: app_keys.h has {declared} tables, {len(tables)} with an `// id: Title` comment and a unique id")
    return tables


def keys_toml():
    out = ["# Generated by site/tools/gen_dev_docs.py from lib/core/src/app_keys.h: the keys of every screen, as the",
           "# help panel (Fn+h) lists them on the device. Edit that header, not this file.", ""]
    for ident, title, rows in key_tables():
        out += ["[[scope]]", f"id = {json.dumps(ident)}", f"title = {json.dumps(title, ensure_ascii=False)}", "rows = ["]
        out += [f"    [{json.dumps(k, ensure_ascii=False)}, {json.dumps(a, ensure_ascii=False)}]," for k, a in rows]
        out += ["]", ""]
    return "\n".join(out)


def build():
    pages = {"../../data/keys.toml": keys_toml()}

    for path in sorted((REPO / "docs" / "adr").glob("*.md")):
        title, body = title_and_body(path.read_text())
        number = path.stem[:4]
        source = f"docs/adr/{path.name}"
        pages[f"decisions/{path.stem}.md"] = front(title, plain(first_paragraph(body)), int(number), source, tag=f"ADR {number}") + relink(body, source)

    for i, code in enumerate(MILESTONES):
        path = REPO / "docs" / "milestones" / f"{code}.md"
        title, body = title_and_body(path.read_text())
        title = re.sub(rf"^{code}\s*[—:-]\s*", "", title)
        source = f"docs/milestones/{path.name}"
        pages[f"milestones/{code.lower()}.md"] = front(title, plain(goal_or_first(body)), (i + 1) * 10, source, tag=code) + relink(body, source)

    readme = sections((REPO / "README.md").read_text())
    pages["build/build-and-test.md"] = (
        front("Build, test and release", "Docker is the only tool you need. How the firmware is built, how the host tests run, and what CI does on a pull request and on a tag.", 1, "README.md", tag="Build")
        + relink("\n".join(readme[k] for k in ("Requirements", "Build and test (local CI)", "CI and releases")), "README.md"))
    pages["build/flash.md"] = (
        front("Flash and update", "Put the firmware on a Cardputer over USB, then update it over Wi-Fi or from the SD card, and from the project's releases.", 2, "README.md", tag="Flash")
        + relink("\n".join(readme[k] for k in ("Flash", "Firmware Updates over Wi-Fi (OTA)")), "README.md"))

    common, debug = help_text()
    if debug:
        sys.exit("gen_dev_docs: kHelp has a RORO_DEBUG part again: there is one firmware (ADR 0010)")
    exact, prefix = safe_mode_commands()
    safe = ", ".join(f"`{c}`" for c in exact) + ", and anything starting with " + ", ".join(f"`{p.strip()}`" for p in prefix)
    dev = readme["Development aids"].split("### The Debug Console")[0]
    dev = re.sub(r"^## Development aids\n", "", dev).strip() + "\n"
    pages["debug/commands.md"] = (
        front("Command reference", "Every command the firmware understands, over USB serial or the Debug Console: what `help` prints, then what each one does.", 30, "src/main.cpp and README.md", tag="Reference")
        + "## What `help` prints\n\n"
        + "The firmware's own list, read from `src/main.cpp`. Every firmware has all of them, over USB serial and over the Debug Console. The ones marked *Debug Console only* exist only over Wi-Fi, where [`rdbg.py`](/dev/debug/files-and-screens/) speaks them, and the ones marked *USB serial only* only over the cable:\n\n```\n" + common + "\n```\n\n"
        + f"In **Safe Mode** (see [Crashes and Safe Mode](/dev/debug/crashes/)) only a few run: {safe}. Anything else answers `not available in Safe Mode`.\n\n"
        + "## What they do\n\n" + dev)
    return pages


def unknown_scopes():
    """Scopes a page asks the `keys` shortcode for that app_keys.h doesn't have."""
    known = {t[0] for t in key_tables()}
    bad = []
    for path in sorted((SITE / "content").rglob("*.md")):
        for call in re.findall(r"keys\(scopes=\[([^\]]*)\]", path.read_text()):
            for ident in re.findall(r'"([^"]+)"', call):
                if ident not in known:
                    bad.append(f"{path.relative_to(SITE)}: no key table `{ident}` in lib/core/src/app_keys.h")
    return bad


def main():
    check = "--check" in sys.argv
    pages = build()
    for problem in unknown_scopes():
        print("gen_dev_docs:", problem)
    if unknown_scopes():
        sys.exit(1)
    stale = []
    for rel, text in sorted(pages.items()):
        path = OUT / rel
        have = path.read_text() if path.exists() else None
        if have != text:
            stale.append(rel)
            if not check:
                path.parent.mkdir(parents=True, exist_ok=True)
                path.write_text(text)
    if check:
        for rel in stale:
            print(f"gen_dev_docs: {os.path.normpath(os.path.join('site/content/dev', rel))} is out of date: run site/tools/gen_dev_docs.py and commit the result")
        print(f"gen_dev_docs: {len(pages)} files, {len(stale)} out of date")
        sys.exit(1 if stale else 0)
    print(f"gen_dev_docs: {len(pages)} files, {len(stale)} written")


if __name__ == "__main__":
    main()