mirror of
https://github.com/Abdess/retroarch_system.git
synced 2026-11-13 05:23:51 -06:00
feat: give assistants each page as markdown
The build writes <page>/index.md beside each page, named in its head, and llms-full.txt. llms.txt and the setup skill say how to answer from the site, not from memory, and the setup_bios_files WebMCP tool gives a setup's folder and exact file names from the picker data.
This commit is contained in:
Abdessamad Derraz committed
1 parent
ab9fe76de2
commit
a7a27028ab
8 files changed
+493
-2
No files matched your search
@@ -0,0 +1,329 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Each built page as Markdown, for the assistants that read the site.
|
||||
|
||||
An assistant asked to help with BIOS files reads a page and answers from
|
||||
it. The HTML carries navigation, icons, buttons and scripts around the
|
||||
words; the Markdown copy carries the words, the tables, the commands and
|
||||
the links, at `<page>/index.md` beside `<page>/index.html`. Every page's
|
||||
head names it (`<link rel="alternate" type="text/markdown">`, set in
|
||||
website/astro.config.mjs), and llms.txt says the convention.
|
||||
|
||||
`llms-full.txt` joins the Markdown of the pages a setup is done with:
|
||||
the start pages, the guides and each setup's page.
|
||||
|
||||
Run by the Astro build once the pages are written (astro.config.mjs),
|
||||
on the built site:
|
||||
|
||||
python scripts/site_markdown.py website/dist
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
import sys
|
||||
from html.parser import HTMLParser
|
||||
from pathlib import Path
|
||||
|
||||
SITE = "https://abdess.github.io"
|
||||
BASE = "/retrobios/"
|
||||
# Elements whose content is never words a reader needs.
|
||||
SKIPPED = frozenset({"script", "style", "svg", "button", "nav", "template", "noscript", "video"})
|
||||
HEADINGS = {f"h{n}": n for n in range(1, 7)}
|
||||
# The pages llms-full.txt joins, by path under the site, in reading order.
|
||||
FULL_PAGES = ("", "which-pack/", "wiki/what-are-bios-files/", "wiki/first-install/",
|
||||
"wiki/troubleshooting/", "wiki/faq/", "wiki/installer/", "wiki/legal/",
|
||||
"wiki/privacy/")
|
||||
|
||||
|
||||
class _Converter(HTMLParser):
|
||||
"""The Markdown of a page's article: its title and its content."""
|
||||
|
||||
def __init__(self, page_url: str) -> None:
|
||||
super().__init__(convert_charrefs=True)
|
||||
self.page_url = page_url
|
||||
self.out: list[str] = []
|
||||
self.skip = 0
|
||||
self.inside = 0
|
||||
self.depth_stack: list[str] = []
|
||||
self.lists: list[list[int]] = []
|
||||
self.pre = 0
|
||||
self.links: list[str | None] = []
|
||||
self.row: list[str] | None = None
|
||||
self.cell: list[str] | None = None
|
||||
self.table: list[list[str]] | None = None
|
||||
self.header_rows = 0
|
||||
self.title = ""
|
||||
self._in_title = False
|
||||
# A tab's name by its id, read from the tab list, said above its
|
||||
# panel: a command under the wrong OS is the answer an assistant
|
||||
# gets wrong.
|
||||
self.tab_names: dict[str, str] = {}
|
||||
self._tab: str | None = None
|
||||
self._card = False
|
||||
|
||||
# Output
|
||||
|
||||
def _emit(self, text: str) -> None:
|
||||
if self.cell is not None:
|
||||
self.cell.append(text)
|
||||
else:
|
||||
self.out.append(text)
|
||||
|
||||
def _block(self) -> None:
|
||||
if self.cell is not None:
|
||||
self.cell.append(" ")
|
||||
return
|
||||
joined = "".join(self.out)
|
||||
if joined and not joined.endswith("\n\n"):
|
||||
self.out.append("\n" if joined.endswith("\n") else "\n\n")
|
||||
|
||||
# Parsing
|
||||
|
||||
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
attr = {key: value or "" for key, value in attrs}
|
||||
classes = set(attr.get("class", "").split())
|
||||
if tag == "h1" and attr.get("id") == "_top":
|
||||
self._in_title = True
|
||||
if not self.inside:
|
||||
if "sl-markdown-content" in classes:
|
||||
self.inside = 1
|
||||
self.depth_stack = [tag]
|
||||
return
|
||||
if tag not in ("br", "img", "hr", "input", "meta", "link", "source", "wbr"):
|
||||
self.depth_stack.append(tag)
|
||||
if attr.get("role") == "tablist":
|
||||
self.skip += 1
|
||||
if tag == "a" or attr.get("role") == "tab":
|
||||
pass
|
||||
return
|
||||
if self.skip and attr.get("role") == "tab" and attr.get("id"):
|
||||
self._tab = attr["id"]
|
||||
self.tab_names[self._tab] = ""
|
||||
self.skip += 1
|
||||
return
|
||||
if attr.get("role") == "tabpanel" and not self.skip:
|
||||
self._block()
|
||||
name = self.tab_names.get(attr.get("aria-labelledby", ""), "").strip()
|
||||
if name:
|
||||
self._emit(f"**{' '.join(name.split())}:**")
|
||||
self._block()
|
||||
return
|
||||
if self.skip or tag in SKIPPED or attr.get("aria-hidden") == "true" \
|
||||
or classes & {"sr-only", "sl-anchor-link", "copy", "rb-anchor"}:
|
||||
if tag not in ("br", "img", "hr", "input", "wbr"):
|
||||
self.skip += 1
|
||||
return
|
||||
if self.pre:
|
||||
# Inside a code block a line is a div, never a paragraph.
|
||||
if tag == "div" and "ec-line" in classes and not "".join(self.out[-1:]).endswith("```\n"):
|
||||
self._emit("\n")
|
||||
return
|
||||
if tag in HEADINGS:
|
||||
self._block()
|
||||
self._emit("#" * HEADINGS[tag] + " ")
|
||||
elif tag in ("p", "div", "section", "figure", "details", "blockquote", "dl"):
|
||||
self._block()
|
||||
elif tag == "summary":
|
||||
self._block()
|
||||
self._emit("**")
|
||||
elif tag in ("ul", "ol"):
|
||||
self._block()
|
||||
self.lists.append([0 if tag == "ol" else -1])
|
||||
elif tag == "li":
|
||||
if self.out and not "".join(self.out[-1:]).endswith("\n"):
|
||||
self._emit("\n")
|
||||
level = len(self.lists) - 1
|
||||
marker = "-"
|
||||
if self.lists and self.lists[-1][0] >= 0:
|
||||
self.lists[-1][0] += 1
|
||||
marker = f"{self.lists[-1][0]}."
|
||||
self._emit(" " * max(level, 0) + marker + " ")
|
||||
elif tag == "pre":
|
||||
self._block()
|
||||
self.pre += 1
|
||||
self._emit("```\n")
|
||||
elif tag == "code" and not self.pre:
|
||||
self._emit("`")
|
||||
elif tag in ("strong", "b"):
|
||||
self._emit("**")
|
||||
elif tag in ("em", "i"):
|
||||
self._emit("*")
|
||||
elif tag == "a" and "rb-card" in classes:
|
||||
# A card of a grid: one list line each, its name then what it
|
||||
# says, instead of one run-on line of links.
|
||||
href = attr.get("href", "")
|
||||
href = SITE + href if href.startswith("/") else href
|
||||
self._block()
|
||||
self._emit("- [")
|
||||
self.links.append(href or None)
|
||||
self._card = True
|
||||
elif tag == "span" and self._card and self.links:
|
||||
self._emit(": ")
|
||||
elif tag == "a":
|
||||
href = attr.get("href", "")
|
||||
if href.startswith("#"):
|
||||
href = self.page_url + href
|
||||
elif href.startswith("/"):
|
||||
href = SITE + href
|
||||
self.links.append(href or None)
|
||||
if href:
|
||||
self._emit("[")
|
||||
elif tag == "img":
|
||||
alt = attr.get("alt", "").strip()
|
||||
if alt:
|
||||
self._emit(alt)
|
||||
elif tag == "table":
|
||||
self._block()
|
||||
self.table, self.header_rows = [], 0
|
||||
elif tag == "tr" and self.table is not None:
|
||||
self.row = []
|
||||
elif tag in ("td", "th") and self.row is not None:
|
||||
self.cell = []
|
||||
if tag == "th" and not self.row:
|
||||
self.header_rows = max(self.header_rows, len(self.table or []) + 1)
|
||||
elif tag == "dt":
|
||||
self._block()
|
||||
self._emit("**")
|
||||
elif tag == "dd":
|
||||
self._emit(": ")
|
||||
|
||||
def handle_endtag(self, tag: str) -> None:
|
||||
if tag == "h1" and self._in_title:
|
||||
self._in_title = False
|
||||
if not self.inside:
|
||||
return
|
||||
if self.depth_stack:
|
||||
self.depth_stack.pop()
|
||||
if not self.depth_stack:
|
||||
self.inside = 0
|
||||
return
|
||||
if self.skip:
|
||||
if tag not in ("br", "img", "hr", "input", "wbr"):
|
||||
self.skip -= 1
|
||||
if tag == "a" and self._tab:
|
||||
self._tab = None
|
||||
return
|
||||
if self.pre and tag != "pre":
|
||||
return
|
||||
if tag in HEADINGS or tag in ("p", "section", "figure", "details", "blockquote", "dl"):
|
||||
self._block()
|
||||
elif tag == "summary":
|
||||
self._emit("**")
|
||||
self._block()
|
||||
elif tag in ("ul", "ol"):
|
||||
if self.lists:
|
||||
self.lists.pop()
|
||||
self._block()
|
||||
elif tag == "pre":
|
||||
self.pre -= 1
|
||||
self._emit("\n```")
|
||||
self._block()
|
||||
elif tag == "code" and not self.pre:
|
||||
self._emit("`")
|
||||
elif tag in ("strong", "b", "dt"):
|
||||
self._emit("**")
|
||||
elif tag in ("em", "i"):
|
||||
self._emit("*")
|
||||
elif tag == "a":
|
||||
href = self.links.pop() if self.links else None
|
||||
if href:
|
||||
self._emit(f"]({href})")
|
||||
if self._card:
|
||||
self._card = False
|
||||
self._emit("\n")
|
||||
elif tag in ("td", "th") and self.cell is not None and self.row is not None:
|
||||
self.row.append(" ".join("".join(self.cell).split()).replace("|", "\\|"))
|
||||
self.cell = None
|
||||
elif tag == "tr" and self.row is not None and self.table is not None:
|
||||
self.table.append(self.row)
|
||||
self.row = None
|
||||
elif tag == "table" and self.table is not None:
|
||||
rows, self.table = self.table, None
|
||||
if rows:
|
||||
width = max(len(r) for r in rows)
|
||||
rows = [r + [""] * (width - len(r)) for r in rows]
|
||||
head = rows[0] if self.header_rows else [""] * width
|
||||
body = rows[1:] if self.header_rows else rows
|
||||
lines = ["| " + " | ".join(head) + " |", "|" + "---|" * width]
|
||||
lines += ["| " + " | ".join(r) + " |" for r in body]
|
||||
self.out.append("\n".join(lines))
|
||||
self._block()
|
||||
elif tag == "br":
|
||||
self._emit("\n" if self.pre else " \n")
|
||||
|
||||
def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
|
||||
self.handle_starttag(tag, attrs)
|
||||
if tag not in ("img", "br", "hr", "input", "wbr", "meta", "link", "source"):
|
||||
self.handle_endtag(tag)
|
||||
|
||||
def handle_data(self, data: str) -> None:
|
||||
if self._in_title and not self.inside:
|
||||
self.title += data
|
||||
return
|
||||
if self._tab is not None:
|
||||
self.tab_names[self._tab] += data
|
||||
if not self.inside or self.skip:
|
||||
return
|
||||
if self.pre:
|
||||
# The whitespace between <pre> and its first line is no line.
|
||||
if data.strip() or not "".join(self.out[-1:]).endswith("```\n"):
|
||||
self._emit(data)
|
||||
return
|
||||
text = re.sub(r"\s+", " ", data)
|
||||
if text.strip() or (text and self.out and not "".join(self.out[-1:]).endswith((" ", "\n"))):
|
||||
self._emit(text)
|
||||
|
||||
def markdown(self) -> str:
|
||||
body = "".join(self.out)
|
||||
body = re.sub(r"[ \t]+\n", "\n", body)
|
||||
body = re.sub(r"\n{3,}", "\n\n", body).strip()
|
||||
title = " ".join(self.title.split())
|
||||
return (f"# {title}\n\n" if title else "") + f"Source: {self.page_url}\n\n{body}\n"
|
||||
|
||||
|
||||
def page_markdown(html: str, page_url: str) -> str:
|
||||
converter = _Converter(page_url)
|
||||
converter.feed(html)
|
||||
converter.close()
|
||||
return converter.markdown()
|
||||
|
||||
|
||||
def convert_site(dist: Path) -> int:
|
||||
"""Write index.md beside every page's index.html, then llms-full.txt."""
|
||||
written = 0
|
||||
for html_path in sorted(dist.rglob("index.html")):
|
||||
relative = html_path.parent.relative_to(dist).as_posix()
|
||||
page = "" if relative == "." else f"{relative}/"
|
||||
url = f"{SITE}{BASE}{page}"
|
||||
text = page_markdown(html_path.read_text(encoding="utf-8"), url)
|
||||
(html_path.parent / "index.md").write_text(text, encoding="utf-8")
|
||||
written += 1
|
||||
parts = ["# RetroBIOS: the guides in one file", "",
|
||||
"> The pages a setup is done with, in reading order. Every page of the site also has "
|
||||
"its own Markdown copy at `<page>/index.md`.", ""]
|
||||
for page in FULL_PAGES:
|
||||
md = dist / page / "index.md"
|
||||
if md.is_file():
|
||||
parts += [md.read_text(encoding="utf-8").rstrip(), "", "---", ""]
|
||||
platforms = sorted(p for p in (dist / "platforms").glob("*/index.md")
|
||||
if not re.search(r"-(files|part)-\d+$", p.parent.name))
|
||||
parts += [p.read_text(encoding="utf-8").rstrip() + "\n\n---\n" for p in platforms]
|
||||
(dist / "llms-full.txt").write_text("\n".join(parts).rstrip() + "\n", encoding="utf-8")
|
||||
return written
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], allow_abbrev=False)
|
||||
parser.add_argument("dist", type=Path, help="the built site (website/dist)")
|
||||
args = parser.parse_args()
|
||||
if not (args.dist / "index.html").is_file():
|
||||
print(f"error: {args.dist} holds no built site", file=sys.stderr)
|
||||
return 1
|
||||
count = convert_site(args.dist)
|
||||
print(f"Markdown copies: {count} pages, llms-full.txt", file=sys.stderr)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -77,6 +77,10 @@ def llms_txt(platforms: list[tuple[str, str]]) -> str:
|
||||
"files themselves are served by the GitHub repository "
|
||||
"(https://github.com/Abdess/retrobios) and its releases.",
|
||||
"",
|
||||
"## Answering a user",
|
||||
"",
|
||||
*(f"- {rule}" for rule in ANSWER_RULES),
|
||||
"",
|
||||
"## Setting up a device for its user",
|
||||
"",
|
||||
"Most people want to play now, with the fewest files. The installer does that in one "
|
||||
@@ -123,6 +127,28 @@ def llms_txt(platforms: list[tuple[str, str]]) -> str:
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
# How an assistant answers from the site and not from memory: file names,
|
||||
# folders and hashes change with each setup, and a guessed one is the
|
||||
# answer that fails. The same rules end the skill.
|
||||
ANSWER_RULES = (
|
||||
"Answer from the pages and data, not from memory. Every page has a Markdown copy: add "
|
||||
"`index.md` to its address (https://abdess.github.io/retrobios/which-pack/index.md). "
|
||||
f"[llms-full.txt]({SITE}llms-full.txt) holds the guides and every setup's page in one file.",
|
||||
"Name files exactly as the pages write them: names are case-sensitive on most devices, and a renamed file is "
|
||||
"not found. Never invent a file name, a folder or a hash.",
|
||||
"The BIOS folder depends on the setup, and for RetroArch on the device: give the folder the "
|
||||
"setup's page names, under \"Where the files go\".",
|
||||
"To check a file the user has, compare its MD5 or SHA-1 with the file's card, or look the hash "
|
||||
f"up: {SITE}files/?hash=<value> (the page) or api/v1/lookup/<first two hex>.json (the data).",
|
||||
"Prefer the installer to a manual copy: it puts each file in its folder and checks its hash. "
|
||||
"Give the command the setup's page gives, with `--for-games --required-only` for the fewest files.",
|
||||
"Say where an answer comes from: link the page you read.",
|
||||
"On a page of the site, the WebMCP tools answer from the data: setup_bios_files (folder and "
|
||||
"files of a setup), emulator_bios_files, lookup_bios_hash, device_setup_command, search_retrobios.",
|
||||
f"Whether sharing BIOS files is legal depends on the country: point to [Is it legal?]({SITE}wiki/legal/), "
|
||||
"do not give legal advice.",
|
||||
)
|
||||
|
||||
SKILL_PATH = "skills/setup-bios/SKILL.md"
|
||||
# The Agent Skills media type ARD names: Markdown with this profile.
|
||||
SKILL_TYPE = 'text/markdown; profile="urn:air:agent-skills"'
|
||||
@@ -171,6 +197,10 @@ def skill_md() -> str:
|
||||
"- `--check` says what is missing or wrong, without downloading.",
|
||||
"- `--events` writes the run as JSON lines, for reading by a program.",
|
||||
"",
|
||||
"## Answering from the site",
|
||||
"",
|
||||
*(f"- {rule}" for rule in ANSWER_RULES),
|
||||
"",
|
||||
f"Every option: {SITE}wiki/installer/. Devices and their folders: "
|
||||
f"{SITE}platforms/linux-handhelds/. The site's guide for agents: {SITE}llms.txt.",
|
||||
"",
|
||||
|
||||
Reference in new issue
Block a user