chore: merge origin/main

scripts/generate_games_index.py

@@ -294,6 +294,36 @@ def resolve_folder(
     return Resolution(sorted({resolver.canonical(s) for s in own}), f"systems of {profile}")
 
 
+# The emulator column of a row names the frontend or a launch mode, never
+# a program whose files are read: these never answer for a core.
+_NOT_PROGRAMS = frozenset({"retroarch", "retroarch32", "libretro", "external", "standalone",
+                           "port", "ports", "native", "sh", "os-shell", "none"})
+
+
+def emulator_profile(row: dict, platform: str, cores: ProfileResolver,
+                     resolver: SystemResolver, profiles: Profiles) -> str | None:
+    """The profile of the program a row's emulator column names, when its
+    core column names a mode, a machine or a plugin of that program: RetroBat
+    runs demul for core hikaru, AmberELEC MAME for core gamate, Batocera
+    mupen64plus for core glide64mk2.
+
+    Kept only when that profile declares a system of the folder: MAME's
+    profile has no GameKing, and answering MAME for its folder would leave
+    out the ROM the core needs. Unresolved, the folder keeps the files of
+    every core of its systems, as it did.
+    """
+    emulator = str(row.get("emulator") or "").strip()
+    if not emulator or emulator.lower() in _NOT_PROGRAMS or emulator.lower().startswith("standalone"):
+        return None
+    candidate = cores.resolve(emulator)
+    if not candidate:
+        return None
+    systems = resolver.resolve(folder_names(row), platform).systems
+    own = {str(s) for s in profiles.profiles.get(candidate, {}).get("systems") or []}
+    declared = {spelling for sid in own for spelling in resolver.equivalents(sid)}
+    return candidate if any(resolver.equivalents(sid) & declared for sid in systems) else None
+
+
 def build_setups(
     setups: dict[str, dict],
     resolver: SystemResolver,
@@ -321,7 +351,8 @@ def build_setups(
         counts = {"folders": 0, "resolved": 0, "profile": 0, "fallback": 0}
         for row in doc.get("folders", []):
             counts["folders"] += 1
-            profile = cores.resolve(row.get("core"))
+            profile = (cores.resolve(row.get("core"))
+                       or emulator_profile(row, platform, cores, resolver, profiles))
             resolution = resolve_folder(row, platform, profile, resolver, profiles, arcade)
             published: dict = {
                 "folder": row["folder"],

scripts/generate_site.py

@@ -11584,7 +11584,7 @@ def generate_not_found() -> str:
         "- Browse [platforms](platforms/index.md), [systems](systems/index.md) "
         "or [emulators](emulators/index.md)",
         "",
-        '<p id="rb-notfound-hint" hidden></p>',
+        '<div id="rb-notfound-hint" hidden></div>',
         "",
     ])
 

scripts/romfolders.py

@@ -172,7 +172,10 @@ class ProfileResolver:
             return None
         # RetroPie names a libretro package lr-<core> (scriptmodules/libretrocores).
         bare = core.removeprefix("lr-")
-        spellings = (core, core.lower(), bare, bare.replace("-", "_"), bare.lower().replace("-", "_"))
+        # Either separator in either direction: plumOS names flycast_xtreme
+        # the core flycast2021 lists as flycast-xtreme.
+        spellings = (core, core.lower(), bare, bare.replace("-", "_"), bare.lower().replace("-", "_"),
+                     bare.replace("_", "-"), bare.lower().replace("_", "-"))
         for name in dict.fromkeys(spellings):
             found = preferred_profile(self.index, name)
             if found:

scripts/site_markdown.py

@@ -0,0 +1,345 @@
+#!/usr/bin/env python3
+"""Each built page as Markdown, for the assistants that read the site.
+
+An assistant asked to help with BIOS files reads a page and answers from
+it. The HTML carries navigation, icons, buttons and scripts around the
+words. The Markdown copy carries the words, the tables, the commands and
+the links, at `<page>/index.md` beside `<page>/index.html`. Every page's
+head names it (`<link rel="alternate" type="text/markdown">`, set in
+website/astro.config.mjs), and llms.txt says the convention.
+
+`llms-full.txt` joins the Markdown of the pages a setup is done with:
+the start pages, the guides and each setup's page.
+
+Run by the Astro build once the pages are written (astro.config.mjs),
+on the built site:
+
+    python scripts/site_markdown.py website/dist
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from html.parser import HTMLParser
+from pathlib import Path
+
+SITE = "https://abdess.github.io"
+BASE = "/retrobios/"
+# Elements whose content is never words a reader needs.
+SKIPPED = frozenset({"script", "style", "svg", "button", "nav", "template", "noscript", "video"})
+HEADINGS = {f"h{n}": n for n in range(1, 7)}
+# The pages llms-full.txt joins, by path under the site, in reading order.
+FULL_PAGES = ("", "which-pack/", "wiki/what-are-bios-files/", "wiki/first-install/",
+              "wiki/troubleshooting/", "wiki/faq/", "wiki/installer/", "wiki/legal/",
+              "wiki/privacy/")
+
+
+class _Converter(HTMLParser):
+    """The Markdown of a page's article: its title and its content."""
+
+    def __init__(self, page_url: str) -> None:
+        super().__init__(convert_charrefs=True)
+        self.page_url = page_url
+        self.out: list[str] = []
+        self.skip = 0
+        self.inside = 0
+        self.depth_stack: list[str] = []
+        self.lists: list[list[int]] = []
+        self.pre = 0
+        self.links: list[str | None] = []
+        self.row: list[str] | None = None
+        self.cell: list[str] | None = None
+        self.table: list[list[str]] | None = None
+        self.header_rows = 0
+        self.title = ""
+        self._in_title = False
+        # A tab's name by its id, read from the tab list, said above its
+        # panel: a command under the wrong OS is the answer an assistant
+        # gets wrong.
+        self.tab_names: dict[str, str] = {}
+        self._tab: str | None = None
+        self._card = False
+
+    # Output
+
+    def _emit(self, text: str) -> None:
+        if self.cell is not None:
+            self.cell.append(text)
+        else:
+            self.out.append(text)
+
+    def _block(self) -> None:
+        if self.cell is not None:
+            self.cell.append(" ")
+            return
+        joined = "".join(self.out)
+        if joined and not joined.endswith("\n\n"):
+            self.out.append("\n" if joined.endswith("\n") else "\n\n")
+
+    # Parsing
+
+    def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        attr = {key: value or "" for key, value in attrs}
+        classes = set(attr.get("class", "").split())
+        if tag == "h1" and attr.get("id") == "_top":
+            self._in_title = True
+        if not self.inside:
+            if "sl-markdown-content" in classes:
+                self.inside = 1
+                self.depth_stack = [tag]
+            return
+        if tag not in ("br", "img", "hr", "input", "meta", "link", "source", "wbr"):
+            self.depth_stack.append(tag)
+        if attr.get("role") == "tablist":
+            self.skip += 1
+            if tag == "a" or attr.get("role") == "tab":
+                pass
+            return
+        if self.skip and attr.get("role") == "tab" and attr.get("id"):
+            self._tab = attr["id"]
+            self.tab_names[self._tab] = ""
+            self.skip += 1
+            return
+        if attr.get("role") == "tabpanel" and not self.skip:
+            self._block()
+            name = self.tab_names.get(attr.get("aria-labelledby", ""), "").strip()
+            if name:
+                self._emit(f"**{' '.join(name.split())}:**")
+                self._block()
+            return
+        if self.skip or tag in SKIPPED or attr.get("aria-hidden") == "true" \
+                or classes & {"sr-only", "sl-anchor-link", "copy", "rb-anchor"}:
+            if tag not in ("br", "img", "hr", "input", "wbr"):
+                self.skip += 1
+            return
+        if self.pre:
+            # Inside a code block a line is a div, never a paragraph.
+            if tag == "div" and "ec-line" in classes and not "".join(self.out[-1:]).endswith("```\n"):
+                self._emit("\n")
+            return
+        if tag in HEADINGS:
+            self._block()
+            self._emit("#" * HEADINGS[tag] + " ")
+        elif tag in ("p", "div", "section", "figure", "details", "blockquote", "dl"):
+            self._block()
+        elif tag == "summary":
+            self._block()
+            self._emit("**")
+        elif tag in ("ul", "ol"):
+            self._block()
+            self.lists.append([0 if tag == "ol" else -1])
+        elif tag == "li":
+            if self.out and not "".join(self.out[-1:]).endswith("\n"):
+                self._emit("\n")
+            level = len(self.lists) - 1
+            marker = "-"
+            if self.lists and self.lists[-1][0] >= 0:
+                self.lists[-1][0] += 1
+                marker = f"{self.lists[-1][0]}."
+            self._emit("  " * max(level, 0) + marker + " ")
+        elif tag == "pre":
+            self._block()
+            self.pre += 1
+            self._emit("```\n")
+        elif tag == "code" and not self.pre:
+            self._emit("`")
+        elif tag in ("strong", "b"):
+            self._emit("**")
+        elif tag in ("em", "i"):
+            self._emit("*")
+        elif tag == "a" and "rb-card" in classes:
+            # A card of a grid: one list line each, its name then what it
+            # says, instead of one run-on line of links.
+            href = attr.get("href", "")
+            href = SITE + href if href.startswith("/") else href
+            self._block()
+            self._emit("- [")
+            self.links.append(href or None)
+            self._card = True
+        elif tag == "span" and self._card and self.links:
+            self._emit(": ")
+        elif tag == "a":
+            href = attr.get("href", "")
+            if href.startswith("#"):
+                href = self.page_url + href
+            elif href.startswith("/"):
+                href = SITE + href
+            self.links.append(href or None)
+            if href:
+                self._emit("[")
+        elif tag == "img":
+            alt = attr.get("alt", "").strip()
+            if alt:
+                self._emit(alt)
+        elif tag == "table":
+            self._block()
+            self.table, self.header_rows = [], 0
+        elif tag == "tr" and self.table is not None:
+            self.row = []
+        elif tag in ("td", "th") and self.row is not None:
+            self.cell = []
+            if tag == "th" and not self.row:
+                self.header_rows = max(self.header_rows, len(self.table or []) + 1)
+        elif tag == "dt":
+            self._block()
+            self._emit("**")
+        elif tag == "dd":
+            self._emit(": ")
+
+    def handle_endtag(self, tag: str) -> None:
+        if tag == "h1" and self._in_title:
+            self._in_title = False
+        if not self.inside:
+            return
+        if self.depth_stack:
+            self.depth_stack.pop()
+        if not self.depth_stack:
+            self.inside = 0
+            return
+        if self.skip:
+            if tag not in ("br", "img", "hr", "input", "wbr"):
+                self.skip -= 1
+            if tag == "a" and self._tab:
+                self._tab = None
+            return
+        if self.pre and tag != "pre":
+            return
+        if tag in HEADINGS or tag in ("p", "section", "figure", "details", "blockquote", "dl"):
+            self._block()
+        elif tag == "summary":
+            self._emit("**")
+            self._block()
+        elif tag in ("ul", "ol"):
+            if self.lists:
+                self.lists.pop()
+            self._block()
+        elif tag == "pre":
+            self.pre -= 1
+            self._emit("\n```")
+            self._block()
+        elif tag == "code" and not self.pre:
+            self._emit("`")
+        elif tag in ("strong", "b", "dt"):
+            self._emit("**")
+        elif tag in ("em", "i"):
+            self._emit("*")
+        elif tag == "a":
+            href = self.links.pop() if self.links else None
+            if href:
+                self._emit(f"]({href})")
+            if self._card:
+                self._card = False
+                self._emit("\n")
+        elif tag in ("td", "th") and self.cell is not None and self.row is not None:
+            self.row.append(" ".join("".join(self.cell).split()).replace("|", "\\|"))
+            self.cell = None
+        elif tag == "tr" and self.row is not None and self.table is not None:
+            self.table.append(self.row)
+            self.row = None
+        elif tag == "table" and self.table is not None:
+            rows, self.table = self.table, None
+            if rows:
+                width = max(len(r) for r in rows)
+                rows = [r + [""] * (width - len(r)) for r in rows]
+                head = rows[0] if self.header_rows else [""] * width
+                body = rows[1:] if self.header_rows else rows
+                lines = ["| " + " | ".join(head) + " |", "|" + "---|" * width]
+                lines += ["| " + " | ".join(r) + " |" for r in body]
+                self.out.append("\n".join(lines))
+                self._block()
+        elif tag == "br":
+            self._emit("\n" if self.pre else "  \n")
+
+    def handle_startendtag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
+        self.handle_starttag(tag, attrs)
+        if tag not in ("img", "br", "hr", "input", "wbr", "meta", "link", "source"):
+            self.handle_endtag(tag)
+
+    def handle_data(self, data: str) -> None:
+        if self._in_title and not self.inside:
+            self.title += data
+            return
+        if self._tab is not None:
+            self.tab_names[self._tab] += data
+        if not self.inside or self.skip:
+            return
+        if self.pre:
+            # The whitespace between <pre> and its first line is no line.
+            if data.strip() or not "".join(self.out[-1:]).endswith("```\n"):
+                self._emit(data)
+            return
+        text = re.sub(r"\s+", " ", data)
+        if text.strip() or (text and self.out and not "".join(self.out[-1:]).endswith((" ", "\n"))):
+            self._emit(text)
+
+    def markdown(self) -> str:
+        body = "".join(self.out)
+        body = re.sub(r"[ \t]+\n", "\n", body)
+        body = re.sub(r"\n{3,}", "\n\n", body).strip()
+        title = " ".join(self.title.split())
+        return (f"# {title}\n\n" if title else "") + f"Source: {self.page_url}\n\n{body}\n"
+
+
+def page_markdown(html: str, page_url: str) -> str:
+    converter = _Converter(page_url)
+    converter.feed(html)
+    converter.close()
+    return converter.markdown()
+
+
+def convert_site(dist: Path) -> int:
+    """Write index.md beside every page's index.html, then llms-full.txt."""
+    written = 0
+    for html_path in sorted(dist.rglob("index.html")):
+        relative = html_path.parent.relative_to(dist).as_posix()
+        page = "" if relative == "." else f"{relative}/"
+        url = f"{SITE}{BASE}{page}"
+        text = page_markdown(html_path.read_text(encoding="utf-8"), url)
+        (html_path.parent / "index.md").write_text(text, encoding="utf-8")
+        written += 1
+    write_full(dist)
+    return written
+
+
+def write_full(dist: Path) -> None:
+    """llms-full.txt from the pages' Markdown copies."""
+    parts = ["# RetroBIOS: the guides in one file", "",
+             "> The pages a setup is done with, in reading order. Every page of the site also has "
+             "its own Markdown copy at `<page>/index.md`.", ""]
+    for page in FULL_PAGES:
+        md = dist / page / "index.md"
+        if md.is_file():
+            parts += [md.read_text(encoding="utf-8").rstrip(), "", "---", ""]
+    # Each setup's own page (the setups the picker lists, not the parts a
+    # long page is cut into), up to its reference of every file: what to
+    # do and where, not the full lists, which its own copy keeps. All of
+    # them was 6.9 MB, past what an assistant reads at once.
+    import json  # noqa: PLC0415
+
+    index = dist / "data" / "picker" / "index.json"
+    setups = [str(item["platform"]) for item in json.loads(index.read_text(encoding="utf-8"))] \
+        if index.is_file() else []
+    for setup in [*setups, "linux-handhelds"]:
+        md = dist / "platforms" / setup / "index.md"
+        if md.is_file():
+            text = md.read_text(encoding="utf-8")
+            parts += [text.split("\n## Reference", 1)[0].rstrip(), "", "---", ""]
+    (dist / "llms-full.txt").write_text("\n".join(parts).rstrip() + "\n", encoding="utf-8")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0], allow_abbrev=False)
+    parser.add_argument("dist", type=Path, help="the built site (website/dist)")
+    args = parser.parse_args()
+    if not (args.dist / "index.html").is_file():
+        print(f"error: {args.dist} holds no built site", file=sys.stderr)
+        return 1
+    count = convert_site(args.dist)
+    print(f"Markdown copies: {count} pages, llms-full.txt", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

scripts/siteagents.py

@@ -77,6 +77,10 @@ def llms_txt(platforms: list[tuple[str, str]]) -> str:
         "files themselves are served by the GitHub repository "
         "(https://github.com/Abdess/retrobios) and its releases.",
         "",
+        "## Answering a user",
+        "",
+        *(f"- {rule}" for rule in ANSWER_RULES),
+        "",
         "## Setting up a device for its user",
         "",
         "Most people want to play now, with the fewest files. The installer does that in one "
@@ -123,6 +127,28 @@ def llms_txt(platforms: list[tuple[str, str]]) -> str:
     return "\n".join(lines) + "\n"
 
 
+# How an assistant answers from the site and not from memory: file names,
+# folders and hashes change with each setup, and a guessed one is the
+# answer that fails. The same rules end the skill.
+ANSWER_RULES = (
+    "Answer from the pages and data, not from memory. Every page has a Markdown copy: add "
+    "`index.md` to its address (https://abdess.github.io/retrobios/which-pack/index.md). "
+    f"[llms-full.txt]({SITE}llms-full.txt) holds the guides and every setup's page in one file.",
+    "Name files exactly as the pages write them: names are case-sensitive on most devices, and a renamed file is "
+    "not found. Never invent a file name, a folder or a hash.",
+    "The BIOS folder depends on the setup, and for RetroArch on the device: give the folder the "
+    "setup's page names, under \"Where the files go\".",
+    "To check a file the user has, compare its MD5 or SHA-1 with the file's card, or look the hash "
+    f"up: {SITE}files/?hash=<value> (the page) or api/v1/lookup/<first two hex>.json (the data).",
+    "Prefer the installer to a manual copy: it puts each file in its folder and checks its hash. "
+    "Give the command the setup's page gives, with `--for-games --required-only` for the fewest files.",
+    "Say where an answer comes from: link the page you read.",
+    "On a page of the site, the WebMCP tools answer from the data: setup_bios_files (folder and "
+    "files of a setup), emulator_bios_files, lookup_bios_hash, device_setup_command, search_retrobios.",
+    f"Whether sharing BIOS files is legal depends on the country: point to [Is it legal?]({SITE}wiki/legal/), "
+    "do not give legal advice.",
+)
+
 SKILL_PATH = "skills/setup-bios/SKILL.md"
 # The Agent Skills media type ARD names: Markdown with this profile.
 SKILL_TYPE = 'text/markdown; profile="urn:air:agent-skills"'
@@ -171,6 +197,10 @@ def skill_md() -> str:
         "- `--check` says what is missing or wrong, without downloading.",
         "- `--events` writes the run as JSON lines, for reading by a program.",
         "",
+        "## Answering from the site",
+        "",
+        *(f"- {rule}" for rule in ANSWER_RULES),
+        "",
         f"Every option: {SITE}wiki/installer/. Devices and their folders: "
         f"{SITE}platforms/linux-handhelds/. The site's guide for agents: {SITE}llms.txt.",
         "",