From 24d7db0ab3647edb586f4851d8f35d1935cce4c9 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 19:58:42 +0100 Subject: [PATCH] feat: guide AI agents to the fewest files llms.txt and an ARD manifest name the pages and the JSON API, and lead an agent setting up a device to --for-games --required-only. Every page registers four read-only WebMCP tools: the setup command, a hash lookup, an emulator's files and the site search. --- .github/workflows/deploy-site.yml | 1 + scripts/generate_site.py | 8 +- scripts/siteagents.py | 157 +++++++++++++++++++++++++ tests/test_site_agents.py | 93 +++++++++++++++ website/astro.config.mjs | 3 + website/src/components/Footer.astro | 1 + website/src/scripts/agent-tools.ts | 170 ++++++++++++++++++++++++++++ wiki/testing-guide.md | 2 +- 8 files changed, 432 insertions(+), 3 deletions(-) create mode 100644 scripts/siteagents.py create mode 100644 tests/test_site_agents.py create mode 100644 website/src/scripts/agent-tools.ts diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index 67130fe8f..292752355 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -45,6 +45,7 @@ on: - "scripts/restore_large_files.py" - "scripts/safeparse.py" - "scripts/sect233r1.py" + - "scripts/siteagents.py" - "scripts/sitebootstrap.py" - "scripts/sitediagrams.py" - "scripts/siteentries.py" diff --git a/scripts/generate_site.py b/scripts/generate_site.py index b12626a7e..d31250d8b 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import zipfile from collections import Counter -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterable, Iterator from datetime import datetime, timezone from pathlib import Path from typing import TypeVar @@ -92,6 +92,7 @@ import sitemarkup as markup import siteentries import siteinputs import retired_files +import siteagents import sitestarlight from siteexports import generate_exports_page, load_export_reports from siteguides import ( @@ -8065,7 +8066,7 @@ def generate_emulators_index(profiles: dict) -> str: return "\n".join(lines) + "\n" -def _snippet_noun(names, firmware: int, fetched: int) -> str: +def _snippet_noun(names: Iterable[str], firmware: int, fetched: int) -> str: """What an emulator's files are called in its snippet. A Switch emulator reads keys, and "eden keys" is what readers search for; Ship of Harkinian reads game ROMs, which are no BIOS.""" @@ -12528,6 +12529,9 @@ def _generate(args: argparse.Namespace) -> None: stats = compute_stats(db, coverages, profiles) stats["composition"] = compute_composition(db) write_if_changed(str(public / "stats.json"), generate_stats(stats)) + # llms.txt and the ARD manifest an agent reads first, from the registry. + siteagents.write(public, sorted(((name, cov["platform"]) for name, cov in coverages.items()), + key=lambda row: row[1].casefold()), write_if_changed) # Computed once: the gap analysis page and the published gaps export must # report the same files. diff --git a/scripts/siteagents.py b/scripts/siteagents.py new file mode 100644 index 000000000..dbafbea99 --- /dev/null +++ b/scripts/siteagents.py @@ -0,0 +1,157 @@ +"""What the site tells an AI agent about itself. + +- ``llms.txt`` (https://llmstxt.org): the pages and data an agent should + read first, in Markdown, at the site's root. +- ``ard.json``: an Agentic Resource Discovery manifest + (https://github.com/ards-project/ard-spec) listing that file and the JSON + API, linked from every page's head with ``rel="ard"``. + +Both are written from the registry at each build, so a new platform or a +moved page cannot leave them stale. The WebMCP tools a page registers live +in website/src/scripts/agent-tools.ts. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path + +SITE = "https://abdess.github.io/retrobios/" +PUBLISHER = "abdess.github.io" +SH = "curl -fsSL https://abdess.github.io/retrobios/i | sh -s --" +PS = "& ([scriptblock]::Create((irm https://abdess.github.io/retrobios/i.ps1)))" + +START_PAGES = ( + ("Which pack do I need", "which-pack/", "pick a device or setup, get the folder its BIOS files go in, " + "one command that installs and checks them, or a ZIP pack"), + ("First install", "wiki/first-install/", "the steps from nothing to a working BIOS folder"), + ("What are BIOS files", "wiki/what-are-bios-files/", "why emulators need them and where they come from"), + ("Troubleshooting", "wiki/troubleshooting/", "a file the emulator does not find or refuses"), + ("Common questions", "wiki/faq/", "short answers"), +) + +DATA = ( + ("API index", "api/v1/index.json", "every dataset of the JSON API, versioned"), + ("Catalog", "api/v1/catalog.json", "each dataset, manifest, export and download with its size, SHA-256 and schema"), + ("Emulators", "api/v1/emulators.json", "every emulator profile: the files it reads, required or not, " + "their hashes and the source line that reads each. One document " + "per emulator at api/v1/emulators/.json"), + ("Core names", "api/v1/cores.json", "each core name and alias, with the profile it designates"), + ("Systems", "api/v1/systems.json", "each system id, its name and the section holding its files"), + ("Platforms", "api/v1/platforms.json", "each setup (RetroArch, Batocera...), its BIOS folder and files"), + ("Hash lookup", "api/v1/lookup/00.json", "what a hash (MD5, SHA-1, SHA-256, CRC32) designates, sharded " + "by its first two hex characters: api/v1/lookup/.json"), +) + +MORE = ( + ("Emulators", "emulators/", "one page per emulator"), + ("Systems", "systems/", "one page per console maker, every file with its hashes"), + ("Installer", "wiki/installer/", "every option of install.py"), + ("What we count", "wiki/privacy/", "the site's cookie-free counting"), + ("Is it legal?", "wiki/legal/", "the project's position"), + ("Contributing", "contributing/", "add a missing file or an emulator profile"), +) + + +def llms_txt(platforms: list[tuple[str, str]]) -> str: + """The site's llms.txt. ``platforms`` lists (id, display name) of every + registered setup, each with its page.""" + lines = [ + "# RetroBIOS", + "", + "> BIOS and firmware files for emulators, each checked against the emulator's source " + "code: which files an emulator reads, whether it starts without them, where each one " + "goes on every setup, their hashes, and an installer that puts them in place.", + "", + "Every claim about a file cites the line of emulator source code that reads it. The " + "files themselves are served by the GitHub repository " + "(https://github.com/Abdess/retrobios) and its releases.", + "", + "## Setting up a device for its user", + "", + "Most people want to play now, with the fewest files. The installer does that in one " + "command: it finds the setup and its BIOS folder, reads the games on the card, and with " + "`--for-games --required-only` installs only the files those games cannot start " + "without. Without these two options it installs every file of the setup's list " + "(gigabytes), even with `--yes`.", + "", + f"1. Preview, nothing written: `{SH} --for-games --required-only --dry-run`", + f"2. Install, no questions: `{SH} --for-games --required-only --yes`", + f"3. On Windows (PowerShell): `{PS} --for-games --required-only --yes`", + "", + "- Run it on the device itself (its terminal or SSH), or on a computer with the device's " + "card plugged in. The installer recognises the handheld systems on the card, among them " + "ArkOS, dArkOS, ArchR and UnofficialOS (R36S and its clones), muOS, KNULLI, ROCKNIX, " + "AmberELEC, EmuELEC, Onion, spruceOS and MinUI. " + "The [Linux handhelds page](" + SITE + "platforms/linux-handhelds/) maps each device " + "to its system and folder.", + "- From a computer, if the card is not found, name its BIOS folder: `--dest /bios` " + "(ArkOS and dArkOS: the `bios` folder of the EASYROMS part).", + "- A games folder somewhere else: `--for-games `, one subfolder per system.", + "- Later, to see what is missing or wrong: `--check`. Machine-readable progress: " + "`--events` (JSON lines, one per event, documented on the installer page).", + "- A game the installer cannot identify gets every BIOS of its system, and the run names it.", + "", + "## Start", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in START_PAGES), + "", + "## Setups", + "", + *(f"- [{name}]({SITE}platforms/{pid}/): its BIOS folder and files" for pid, name in platforms), + "", + "## Data", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in DATA), + "", + "## Optional", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in MORE), + ] + return "\n".join(lines) + "\n" + + +def ard_manifest() -> dict: + """The ARD manifest: the llms.txt and the JSON API, each with the + questions it answers.""" + return { + "entries": [ + { + "identifier": f"urn:air:{PUBLISHER}:retrobios:guide", + "displayName": "RetroBIOS guide for agents", + "type": "text/markdown", + "url": f"{SITE}llms.txt", + "description": "The pages and datasets of RetroBIOS: BIOS and firmware files for " + "emulators, checked against each emulator's source code.", + "representativeQueries": [ + "install the minimum BIOS files for the games on my R36S running dArkOS", + "which BIOS files does mGBA need", + "where do PlayStation BIOS files go on Batocera", + "how do I install BIOS files for RetroArch", + ], + "tags": ["bios", "firmware", "emulation", "retrogaming"], + }, + { + "identifier": f"urn:air:{PUBLISHER}:retrobios:api", + "displayName": "RetroBIOS JSON API", + "type": "application/json", + "url": f"{SITE}api/v1/index.json", + "description": "Versioned JSON datasets: emulator profiles, platforms, systems, core " + "names and a hash lookup.", + "representativeQueries": [ + "what file has the MD5 490f666e1afb15b7362b406ed1cea246", + "which files does the beetle_psx core read", + "list the systems Batocera supports", + ], + "capabilities": ["HashLookup", "EmulatorFiles", "PlatformFiles"], + "tags": ["api", "json", "bios"], + }, + ] + } + + +def write(public: Path, platforms: list[tuple[str, str]], + write_if_changed: Callable[[str, str], object]) -> None: + write_if_changed(str(public / "llms.txt"), llms_txt(platforms)) + write_if_changed(str(public / "ard.json"), json.dumps(ard_manifest(), indent=1) + "\n") diff --git a/tests/test_site_agents.py b/tests/test_site_agents.py new file mode 100644 index 000000000..38558e7a4 --- /dev/null +++ b/tests/test_site_agents.py @@ -0,0 +1,93 @@ +"""What the site tells an AI agent: llms.txt, the ARD manifest, the WebMCP tools. + +An agent asked to set up a handheld reads these first. They must lead it to +the fewest files the games need, and every address they give must exist. +""" +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import re +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import siteagents # noqa: E402 + +DIST = REPO_ROOT / "website" / "dist" +TOOLS = REPO_ROOT / "website" / "src" / "scripts" / "agent-tools.ts" +FOOTER = REPO_ROOT / "website" / "src" / "components" / "Footer.astro" +CONFIG = REPO_ROOT / "website" / "astro.config.mjs" + + +class TheGuideLeadsToTheFewestFiles(unittest.TestCase): + def setUp(self) -> None: + self.text = siteagents.llms_txt([("retroarch", "RetroArch"), ("batocera", "Batocera")]) + + def test_it_is_llms_txt(self) -> None: + self.assertTrue(self.text.startswith("# RetroBIOS\n\n> ")) + self.assertIn("\n## Setting up a device for its user\n", self.text) + + def test_every_command_installs_only_what_the_games_need(self) -> None: + commands = re.findall(r"`((?:curl|&) [^`]+)`", self.text) + self.assertGreaterEqual(len(commands), 3) + for command in commands: + with self.subTest(command=command): + self.assertIn("--for-games --required-only", command) + + def test_each_setup_has_its_page(self) -> None: + self.assertIn("(https://abdess.github.io/retrobios/platforms/batocera/)", self.text) + + @unittest.skipUnless((DIST / "llms.txt").is_file(), "website/dist is not built") + def test_every_address_of_the_built_guide_exists(self) -> None: + built = (DIST / "llms.txt").read_text(encoding="utf-8") + for url in re.findall(r"\((https://abdess\.github\.io/retrobios/[^)]*)\)", built): + with self.subTest(url=url): + target = DIST / url.removeprefix(siteagents.SITE) + self.assertTrue(target.is_file() or (target / "index.html").is_file(), url) + + +class TheManifestIsAnArdManifest(unittest.TestCase): + """The terms ard-entry.schema.json requires (ards-project/ard-spec).""" + + def test_entries(self) -> None: + entries = siteagents.ard_manifest()["entries"] + self.assertTrue(entries) + for entry in entries: + with self.subTest(entry=entry.get("identifier")): + self.assertRegex(entry["identifier"], r"^urn:air:[a-zA-Z0-9.-]+(:[a-zA-Z0-9._-]+)+$") + self.assertTrue(entry["displayName"]) + self.assertRegex(entry["type"], r"^[a-z]+/[\w.+-]+$") + self.assertTrue(("url" in entry) != ("data" in entry)) + self.assertTrue(2 <= len(entry["representativeQueries"]) <= 5) + self.assertTrue(entry["url"].startswith(siteagents.SITE)) + + def test_every_page_links_it(self) -> None: + self.assertIn("rel: 'ard'", CONFIG.read_text(encoding="utf-8")) + + +class TheToolsReadOnly(unittest.TestCase): + def setUp(self) -> None: + self.source = TOOLS.read_text(encoding="utf-8") + + def test_every_page_registers_them(self) -> None: + self.assertIn("import '../scripts/agent-tools.ts';", FOOTER.read_text(encoding="utf-8")) + names = re.findall(r"^\s*name: '([a-z_]+)'", self.source, re.M) + self.assertEqual(names, ["lookup_bios_hash", "emulator_bios_files", "search_retrobios", + "device_setup_command"]) + self.assertEqual(self.source.count("readOnlyHint: true"), len(names)) + + def test_the_setup_command_defaults_to_the_games(self) -> None: + self.assertIn("const games = everything ? [] :", self.source) + self.assertIn("'--required-only'", self.source) + + def test_they_reach_no_other_site(self) -> None: + code = re.sub(r"^\s*//.*$", "", self.source, flags=re.M) + self.assertEqual(set(re.findall(r"https://[\w.-]+", code)), {"https://abdess.github.io"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/website/astro.config.mjs b/website/astro.config.mjs index d5e45fa32..b2a832f29 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -77,6 +77,9 @@ export default defineConfig({ // Ownership of the URL-prefix property https://abdess.github.io/retrobios/ // in Google Search Console. Removing it unverifies the property. { tag: 'meta', attrs: { name: 'google-site-verification', content: '8SueST3hNHl8_vM-OJJP0rz-ZMlI2DALLpChrwZoGMs' } }, + // What an AI agent reads first (scripts/siteagents.py): the ARD manifest lists + // llms.txt and the JSON API. + { tag: 'link', attrs: { rel: 'ard', type: 'application/json', href: `${BASE}ard.json` } }, ], social: [{ icon: 'github', label: 'RetroBIOS on GitHub', href: 'https://github.com/Abdess/retrobios' }], customCss: ['./src/styles/retrobios.css'], diff --git a/website/src/components/Footer.astro b/website/src/components/Footer.astro index ba5c39970..05853598f 100644 --- a/website/src/components/Footer.astro +++ b/website/src/components/Footer.astro @@ -48,6 +48,7 @@ const date = releaseDate(tag);