diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index cdc72f0eb..c7997bcd0 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -46,6 +46,7 @@ on: - "scripts/restore_large_files.py" - "scripts/safeparse.py" - "scripts/sect233r1.py" + - "scripts/siteagents.py" - "scripts/sitebootstrap.py" - "scripts/sitecollection.py" - "scripts/sitediagrams.py" diff --git a/README.md b/README.md index aa2a0781c..28d15b69a 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,8 @@ curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --platform retroarch &([scriptblock]::Create((irm https://abdess.github.io/retrobios/i.ps1))) --check ``` +![Recording of the one-line command in a terminal. The installer finds an ArkOS SD card with 6 games, plans 43 BIOS files (15.1 MB) for them, installs them and shows the summary.](docs_assets/demo/install.gif) + The script opens a full-screen installer, driven with the arrow keys and Enter, or with a controller on a handheld. It finds your setup and its BIOS folder, downloads only the missing files and checks each one. It can also copy the BIOS files from your old SD card. Every option, and what the installer detects and writes, is on the [installer page](https://abdess.github.io/retrobios/wiki/installer/#passing-options). @@ -206,4 +208,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-11-06T10:21:06Z* +*Auto-generated on 2026-11-06T19:12:52Z* diff --git a/docs_assets/demo/demo.json b/docs_assets/demo/demo.json new file mode 100644 index 000000000..4c62cb74c --- /dev/null +++ b/docs_assets/demo/demo.json @@ -0,0 +1,9 @@ +{ + "width": 800, + "height": 460, + "setup": "ArkOS", + "games": 6, + "systems": 6, + "files": 43, + "size": "15.1 MB" +} diff --git a/docs_assets/demo/install.gif b/docs_assets/demo/install.gif new file mode 100644 index 000000000..72894041f Binary files /dev/null and b/docs_assets/demo/install.gif differ diff --git a/docs_assets/demo/install.mp4 b/docs_assets/demo/install.mp4 new file mode 100644 index 000000000..8b17f70e6 Binary files /dev/null and b/docs_assets/demo/install.mp4 differ diff --git a/docs_assets/demo/install.png b/docs_assets/demo/install.png new file mode 100644 index 000000000..b7922c72d Binary files /dev/null and b/docs_assets/demo/install.png differ diff --git a/docs_assets/demo/install.tape b/docs_assets/demo/install.tape new file mode 100644 index 000000000..edaf41a71 --- /dev/null +++ b/docs_assets/demo/install.tape @@ -0,0 +1,46 @@ +# The installer demo of the README and the site. Run scripts/make_demo.py, +# which builds the card, serves the repository on loopback, starts VHS on +# this file from the repository root and encodes what it writes. + +Output tmp/demo/recorded.mp4 + +Set Shell "bash" +Set FontFamily "DejaVu Sans Mono" +Set FontSize 15 +Set LineHeight 1.15 +Set Width 800 +Set Height 460 +Set Padding 16 +Set Framerate 25 +Set TypingSpeed 35ms +Set CursorBlink false +Set Theme { "name": "RetroBIOS", "background": "#12131c", "foreground": "#e6e6f0", "cursor": "#aebcff", "selection": "#3a3d52", "black": "#12131c", "red": "#ef9a9a", "green": "#a5d6a7", "yellow": "#ffcc80", "blue": "#90caf9", "magenta": "#aebcff", "cyan": "#aebcff", "white": "#cfd8dc", "brightBlack": "#7c82a3", "brightRed": "#ef9a9a", "brightGreen": "#a5d6a7", "brightYellow": "#ffcc80", "brightBlue": "#90caf9", "brightMagenta": "#c9d2ff", "brightCyan": "#c9d2ff", "brightWhite": "#e6e6f0" } + +# The card mounted at /media/EASYROMS, an empty HOME, the prompt "$ ". +Hide +Type 'exec "$RETROBIOS_DEMO_SESSION"' +Enter +Wait+Screen@20s /\$/ +Sleep 500ms +Show + +Sleep 700ms +Type "curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --for-games" +Sleep 600ms +Screenshot tmp/demo/poster.png +Sleep 600ms +Enter + +# The first screen: the card found. +Wait+Screen@30s /ArkOS SD card \(EASYROMS\)/ +Sleep 2500ms +Enter + +# The plan: only the files the games on the card need. +Wait+Screen@60s /For the games on this card: yes/ +Sleep 4500ms +Enter + +# The summary. +Wait+Screen@60s /RetroBIOS: Done/ +Sleep 4500ms diff --git a/docs_assets/demo/install.webm b/docs_assets/demo/install.webm new file mode 100644 index 000000000..95c1bf534 Binary files /dev/null and b/docs_assets/demo/install.webm differ diff --git a/emulators/fbneo.yml b/emulators/fbneo.yml index 2a2a825e9..8534afb8c 100644 --- a/emulators/fbneo.yml +++ b/emulators/fbneo.yml @@ -51,10 +51,12 @@ data_directories: - ref: fbneo-cheats unread_on: [romm] destination: fbneo/cheats + required: false source_ref: "source src/burner/conc.cpp:167" - ref: fbneo-samples unread_on: [romm] destination: fbneo + required: false source_ref: "source src/burn/snd/samples.cpp:21-35" notes: | diff --git a/install.ps1 b/install.ps1 index 28de54233..75133bb13 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4" + $defaultInstallSha256 = "b17220355845692a087bbc29cdb4a1008f68b1423d1126f34f00fa0259b53184" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 04a697af5..25a140e36 100644 --- a/install.py +++ b/install.py @@ -3724,6 +3724,8 @@ def _check_data_directory(entry: object, where: str) -> None: values = entry.get(field, []) if not isinstance(values, list) or not all(isinstance(v, str) for v in values): raise ValueError(f"{where}.{field} is invalid") + if entry.get("optional", True) is not True: + raise ValueError(f"{where}.optional is invalid") files = entry.get("files") if not isinstance(files, dict) or not files or len(files) > MAX_MANIFEST_FILES: raise ValueError(f"{where}.files is invalid") @@ -7393,11 +7395,16 @@ def _install_data_member( scratch.unlink(missing_ok=True) -def select_data_directories(manifest: dict, files: list[dict], narrowed: bool) -> list[dict]: - """The data trees this install keeps: all, or those of a kept system.""" - entries = manifest.get("data_directories") or [] +def select_data_directories(manifest: dict, files: list[dict], narrowed: bool, + required_only: bool = False) -> list[dict]: + """The data trees this install keeps: all, or those of a kept system. + --required-only leaves out a tree every core reading it runs without + (`optional`): a GBA and a SNES game took FBNeo's cheats and samples, + 3,472 files no game opened.""" + entries = [e for e in manifest.get("data_directories") or [] + if not (required_only and e.get("optional"))] if not narrowed: - return list(entries) + return entries kept = {system for f in files for system in entry_systems(f)} return [e for e in entries if kept & set(e.get("systems") or [])] @@ -8824,7 +8831,8 @@ def _prompt_custom_selection( if bios_path is not None and not dry_run and current != INSTALLED_ONLY: kept = menu_choice(files, omitted, installed, current, False)[0] kept_trees = select_data_directories({"data_directories": trees}, kept, - narrowed or current is not None) + narrowed or current is not None, + isinstance(current, GamesChoice) and current.required) if kept and all_in_place(kept, kept_trees, bios_path): # Read before the menu, as the screen reads before it asks: a # rerun was offered an install that installs nothing, then told @@ -8953,7 +8961,8 @@ def _print_menu_head( # The trees of the systems kept: 2636 data files of blueMSX, Dolphin # and PPSSPP were counted for every --system tried. kept_trees = select_data_directories({"data_directories": trees}, shown, - narrowed or current is not None) + narrowed or current is not None, + isinstance(current, GamesChoice) and current.required) data = sum(len(entry.get("files") or {}) for entry in kept_trees) data_size = sum(meta.get("size", 0) for entry in kept_trees for meta in (entry.get("files") or {}).values()) @@ -9764,6 +9773,12 @@ def games_setups(index: dict, plat: str, bios_path: Path, os_type: str = "") -> if not named: above = card_root_above(bios_path) named = above[0] if above else "" + if not named: + # --dest on a handheld's games part from a computer (ArkOS's + # EASYROMS/bios): detection never ran there, and the card's own rom + # folders sit beside the BIOS folder. + part = games_part(bios_path.parent) + named = part.name if part else "" key = _setup_key(named) if named else "" found = [name for name in sorted(setups) if key and (name == key or name.startswith(key + "-"))] if not found and plat in setups: @@ -9816,11 +9831,15 @@ def games_root(setup: dict, bios_path: Path) -> Path | None: def _may_hold_games(bios_path: Path) -> bool: """Whether a rom folder could sit near this BIOS folder, said without the - network: a folder named roms in it or up to two levels above it, or a - path that runs through one (roms, games).""" + network: a folder named roms in it or up to two levels above it, a path + that runs through one (roms, games), or a handheld's games part, whose + console folders sit beside bios (ArkOS's EASYROMS plugged into a + computer was never offered its games).""" holders = [bios_path, *list(bios_path.absolute().parents)[:2]] if any(part.casefold() in ("roms", "games") for part in bios_path.absolute().parts[-3:]): return True + if games_part(bios_path.parent) is not None: + return True return any(entry.name.casefold() == "roms" and _entry_is_dir(entry) for holder in holders for entry in _list_dir(holder)) @@ -15671,7 +15690,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: != len(manifest.get("files", [])) + len(variants if case_kept else []) or games_applied ) - data_dirs = select_data_directories(manifest, files, narrowed) + data_dirs = select_data_directories(manifest, files, narrowed, + bool(args.required_only)) # The kinds as the plan and the menu head say them (parts_words), # the data files of the folders kept among them: this head said # BIOS files and game data beside a plan of three kinds, neither @@ -21812,7 +21832,8 @@ def _plan_data(state: TuiState, check: dict) -> tuple[int, int | None]: if not _tui_estimating(state): data = check.get("data_missing", 0) return data, check.get("data_bytes") if data else None - trees = select_data_directories(state.manifest or {}, _choice_entries(state), True) + required = any(flag == "--required-only" for flag, _value in tui_flags(state)) + trees = select_data_directories(state.manifest or {}, _choice_entries(state), True, required) sizes = [meta.get("size", 0) for entry in trees for meta in (entry.get("files") or {}).values()] return len(sizes), sum(sizes) diff --git a/install.sh b/install.sh index a03428f05..768eb8535 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4" +DEFAULT_INSTALL_SHA256="b17220355845692a087bbc29cdb4a1008f68b1423d1126f34f00fa0259b53184" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/amberelec.json b/install/amberelec.json index 104db8e5e..c1390a71e 100644 --- a/install/amberelec.json +++ b/install/amberelec.json @@ -6,7 +6,7 @@ "platform": "amberelec", "display_name": "AmberELEC", "version": "prerelease-20261019_1835", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -77381,7 +77381,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -77615,7 +77616,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -77988,7 +77990,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true } ] } \ No newline at end of file diff --git a/install/batocera.json b/install/batocera.json index 583de52ee..c1f5c14e2 100644 --- a/install/batocera.json +++ b/install/batocera.json @@ -6,7 +6,7 @@ "platform": "batocera", "display_name": "Batocera", "version": "43.1", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -225291,7 +225291,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -225525,7 +225526,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true } ], "save_directory": { diff --git a/install/emudeck.json b/install/emudeck.json index 32f4ed6a2..3f01646ff 100644 --- a/install/emudeck.json +++ b/install/emudeck.json @@ -6,7 +6,7 @@ "platform": "emudeck", "display_name": "EmuDeck", "version": "2.7EA", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -123400,7 +123400,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -123634,7 +123635,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -124007,7 +124009,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "lrps2", diff --git a/install/emuelec.json b/install/emuelec.json index 32c8e5488..35ec58389 100644 --- a/install/emuelec.json +++ b/install/emuelec.json @@ -6,7 +6,7 @@ "platform": "emuelec", "display_name": "EmuELEC", "version": "4.8", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -83639,7 +83639,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -83873,7 +83874,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -84246,7 +84248,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true } ] } \ No newline at end of file diff --git a/install/knulli.json b/install/knulli.json index ccdbd73b9..fd6a9e45b 100644 --- a/install/knulli.json +++ b/install/knulli.json @@ -6,7 +6,7 @@ "platform": "knulli", "display_name": "KNULLI", "version": "20260511", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -157933,7 +157933,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -158167,7 +158168,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "lrps2", diff --git a/install/lakka.json b/install/lakka.json index 5866fe5db..09f9de198 100644 --- a/install/lakka.json +++ b/install/lakka.json @@ -6,7 +6,7 @@ "platform": "lakka", "display_name": "Lakka", "version": "v6.1", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "system", "detect": [ { @@ -130677,7 +130677,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -130911,7 +130912,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -131284,7 +131286,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "yaps2-resources", diff --git a/install/myminui.json b/install/myminui.json index 85de7b9bc..7cb542e4e 100644 --- a/install/myminui.json +++ b/install/myminui.json @@ -6,7 +6,7 @@ "platform": "myminui", "display_name": "MyMinUI", "version": "2026-08-09", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -18958,7 +18958,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-cheats", @@ -32688,7 +32689,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -32922,7 +32924,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "fbneo-samples", @@ -33156,7 +33159,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true } ] } \ No newline at end of file diff --git a/install/nextui.json b/install/nextui.json index 4518d3d5c..933ec3af5 100644 --- a/install/nextui.json +++ b/install/nextui.json @@ -6,7 +6,7 @@ "platform": "nextui", "display_name": "NextUI", "version": "2026-10-20", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -19366,7 +19366,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -19600,7 +19601,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true } ] } \ No newline at end of file diff --git a/install/recalbox.json b/install/recalbox.json index 9b97c9d50..6db9f7daa 100644 --- a/install/recalbox.json +++ b/install/recalbox.json @@ -6,7 +6,7 @@ "platform": "recalbox", "display_name": "Recalbox", "version": "10.1.1", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -62834,7 +62834,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -63068,7 +63069,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true } ], "save_directory": { diff --git a/install/retroarch.json b/install/retroarch.json index 11b60b965..5e89fef85 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -6,7 +6,7 @@ "platform": "retroarch", "display_name": "RetroArch", "version": "v1.22.2", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "system", "detect": [ { @@ -140789,7 +140789,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -141023,7 +141024,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -141396,7 +141398,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "ppsspp-assets", diff --git a/install/retrobat.json b/install/retrobat.json index ca7459771..aec7d58a0 100644 --- a/install/retrobat.json +++ b/install/retrobat.json @@ -6,7 +6,7 @@ "platform": "retrobat", "display_name": "RetroBat", "version": "8.3.1", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -89076,7 +89076,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -89310,7 +89311,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -89683,7 +89685,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true } ], "save_directory": { diff --git a/install/retrodeck.json b/install/retrodeck.json index b9b4b4a10..2938c951d 100644 --- a/install/retrodeck.json +++ b/install/retrodeck.json @@ -6,7 +6,7 @@ "platform": "retrodeck", "display_name": "RetroDECK", "version": "0.10.10b", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -153141,7 +153141,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -153375,7 +153376,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -153748,7 +153750,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "yaps2-resources", diff --git a/install/retropie.json b/install/retropie.json index 3db3da56e..b6ab50768 100644 --- a/install/retropie.json +++ b/install/retropie.json @@ -6,7 +6,7 @@ "platform": "retropie", "display_name": "RetroPie", "version": "4.8.12", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "BIOS", "detect": [ { @@ -146474,7 +146474,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -146708,7 +146709,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -147081,7 +147083,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "yaps2-resources", diff --git a/install/rocknix.json b/install/rocknix.json index f06bd3144..e15cda7e5 100644 --- a/install/rocknix.json +++ b/install/rocknix.json @@ -6,7 +6,7 @@ "platform": "rocknix", "display_name": "ROCKNIX", "version": "20261001", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -109453,7 +109453,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -109687,7 +109688,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -110060,7 +110062,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "lrps2", diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index 264c08558..a72245722 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -452,6 +452,11 @@ }, "description": "Platform systems whose cores read the tree." }, + "optional": { + "type": "boolean", + "const": true, + "description": "Every core reading the tree runs without it (required: false in its profile): --required-only leaves it out." + }, "files": { "type": "object", "additionalProperties": { diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 3934830a9..c0a20376b 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -518,7 +518,8 @@ def _data_directory_members( gets it from the tree's archive. ``refs_out``, when given, maps each yielded destination to - (registry key, system ids, directory destination), for the manifest. + (registry key, system ids, directory destination, whether the core + needs the tree), for the manifest. These are not BIOS and carry no hash of their own: they are whole trees an emulator reads, refreshed from upstream, so what ships is @@ -532,7 +533,8 @@ def _data_directory_members( ref_key = dd.get("ref", "") if not ref_key or not data_registry or ref_key not in data_registry: continue - if required_only and dd.get("required") is False: + needed = dd.get("required") is not False + if required_only and not needed: continue if ref_key in installed_data_refs(platform_name): continue @@ -567,7 +569,7 @@ def _data_directory_members( ): del reclaim[full] if refs_out is not None: - refs_out[full] = (ref_key, systems, dd_prefix) + refs_out[full] = (ref_key, systems, dd_prefix, needed) yield src, full continue if full in seen_destinations or ( @@ -581,7 +583,7 @@ def _data_directory_members( if case_insensitive: seen_lower.add(full.lower()) if refs_out is not None: - refs_out[full] = (ref_key, systems, dd_prefix) + refs_out[full] = (ref_key, systems, dd_prefix, needed) yield src, full @@ -4910,7 +4912,7 @@ def generate_manifest( # Phase 3: data directories. They are not files of the collection: the # installer fetches each from its source archive and keeps the members # whose hash the list below records, the same tree the pack carries. - data_refs: dict[str, tuple[str, tuple[str, ...], str]] = {} + data_refs: dict[str, tuple[str, tuple[str, ...], str, bool]] = {} reclaim = {full: path for full, (path, _size) in cache_held.items()} data_members = list(_data_directory_members( pack_systems, data_registry, platform_name, base_dest, @@ -5021,7 +5023,7 @@ def generate_manifest( def _manifest_data_directories( members: list[tuple[str, str]], - refs: dict[str, tuple[str, tuple[str, ...], str]], + refs: dict[str, tuple[str, tuple[str, ...], str, bool]], registry: dict, base_dest: str, ) -> list[dict]: @@ -5034,11 +5036,14 @@ def _manifest_data_directories( changed since is refused rather than installed. archive_size is the size of the archive refresh_data_dirs last downloaded, what fetching the tree costs: a tarball whose tree is 10 KB is a download of 21 MB. + ``optional`` marks a tree every core reading it runs without + (``required: false``): install.py --required-only leaves it out, as the + pack built with --required-only does. """ grouped: dict[tuple[str, str], dict] = {} versions: dict[str, dict[str, dict]] = {} for src, full in members: - ref, systems, prefix = refs[full] + ref, systems, prefix, needed = refs[full] entry = registry.get(ref, {}) key = (ref, prefix) group = grouped.get(key) @@ -5072,11 +5077,19 @@ def _manifest_data_directories( size = versions[str(recorded)].get(ref, {}).get("archive_size") if isinstance(size, int) and size > 0: group["archive_size"] = size + if needed: + group["needed"] = True group["systems"].extend(s for s in systems if s not in group["systems"]) rel = full[len(prefix):].lstrip("/") if prefix else full hashes = compute_hashes(src, frozenset({"sha1"})) group["files"][rel] = {"sha1": hashes["sha1"], "size": os.path.getsize(src)} - return [grouped[key] for key in sorted(grouped)] + trees = [] + for key in sorted(grouped): + group = grouped[key] + if not group.pop("needed", False): + group["optional"] = True + trees.append(group) + return trees def _manifest_core_index( diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 2d33474d9..f7b8aa383 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -443,6 +443,9 @@ def format_size(size: int) -> str: SITE_URL = "https://abdess.github.io/retrobios/" +# The installer demo (scripts/make_demo.py): the recording and what it shows. +DEMO_DIR = Path(__file__).resolve().parent.parent / "docs_assets" / "demo" +DEMO_GIF = "docs_assets/demo/install.gif" REPO_PATH = "../.." REPO = "Abdess/retrobios" # GitHub's "Add file > Upload files" page for bios/: a browser upload by @@ -473,6 +476,19 @@ def clone_line(db: dict) -> str: ) +def demo_facts() -> dict: + """What the installer demo shows, as make_demo.py measured it.""" + return json.loads((DEMO_DIR / "demo.json").read_text(encoding="utf-8")) + + +def demo_description(facts: dict | None = None) -> str: + """The demo in words, for a reader who cannot see it.""" + facts = facts or demo_facts() + return (f"Recording of the one-line command in a terminal. The installer finds an " + f"{facts['setup']} SD card with {facts['games']} games, plans {facts['files']} " + f"BIOS files ({facts['size']}) for them, installs them and shows the summary.") + + def _existing_contributor_block(readme_path: str = "README.md") -> list[str]: """The contributors already published, read back from the file. @@ -619,6 +635,8 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" install_command("powershell", "--check"), "```", "", + f"![{demo_description()}]({DEMO_GIF})", + "", "The script opens a full-screen installer, driven with the arrow keys and Enter, or with a" " controller on a handheld. It finds your setup and its BIOS folder, downloads only the missing" " files and checks each one. It can also copy the BIOS files from your old SD card.", diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 83b370da7..399d583b3 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -82,6 +82,8 @@ from generate_readme import ( NO_PACK, asset_links, contributing_text, + demo_description, + demo_facts, download_links, extract_notes, extract_targets, @@ -92,6 +94,7 @@ import sitemarkup as markup import siteentries import siteinputs import retired_files +import siteagents import sitestarlight from siteexports import generate_exports_page, load_export_reports from siteguides import ( @@ -515,6 +518,7 @@ def generate_home( "**Run one command.** Some setups have a ZIP.", "**Start a game.**", ]), + *install_demo(), "## Pick your device", "", *setup_picker(registry, display), @@ -4154,6 +4158,20 @@ def first_screen_words() -> list[str]: f"and **{installer.TUI_FOLDER_TITLE}**), then offers **{installer.TUI_INSTALL_ROW}**."] +# The demo files the site serves (docs_assets/demo/, scripts/make_demo.py): +# the GIF is the README's, the tape and demo.json are inputs. +DEMO_ASSETS = ("install.webm", "install.mp4", "install.png") + + +def install_demo() -> list[str]: + """The installer demo: a video that loads once it is on screen, its + description for screen readers.""" + facts = demo_facts() + text = html.escape(demo_description(facts), quote=True) + return [f'', ""] + + def check_pass_line() -> str: """The pass line of "How to check": the words `--check` prints only when no file is missing or wrong (install.py EVERYTHING_IN_PLACE), read from @@ -11956,6 +11974,7 @@ def generate_which_pack() -> str: "", *first_screen_words(), "", + *install_demo(), *trust_box(""), "**Check your folder later.** Add `--check`. It changes nothing.", "", @@ -12315,6 +12334,10 @@ def _generate(args: argparse.Namespace) -> None: source = Path("docs_assets") / folder if source.is_dir(): shutil.copytree(source, public / "assets" / dest, dirs_exist_ok=True) + demo_dest = public / "assets" / "demo" + demo_dest.mkdir(parents=True, exist_ok=True) + for name in DEMO_ASSETS: + shutil.copy2(Path("docs_assets") / "demo" / name, demo_dest / name) assets_src = Path(".github") / "assets" for name, dest_name in [("logo.png", "logo.png"), ("favicon.png", "favicon.png")]: src = assets_src / name @@ -12368,6 +12391,9 @@ def _generate(args: argparse.Namespace) -> None: stats = compute_stats(db, coverages, profiles) stats["composition"] = compute_composition(db) write_if_changed(str(public / "stats.json"), generate_stats(stats)) + # llms.txt and the ARD manifest an agent reads first, from the registry. + siteagents.write(public, sorted(((name, cov["platform"]) for name, cov in coverages.items()), + key=lambda row: row[1].casefold()), write_if_changed) # Computed once: the gap analysis page and the published gaps export must # report the same files. diff --git a/scripts/make_demo.py b/scripts/make_demo.py new file mode 100644 index 000000000..570b1b1cf --- /dev/null +++ b/scripts/make_demo.py @@ -0,0 +1,295 @@ +#!/usr/bin/env python3 +"""Record the installer demo shown in the README and on the site, once. + +The recording is the full-screen installer run from the one-line command on +an ArkOS SD card that holds six games: it finds the card, plans only the BIOS +files those games need, installs them and shows the summary. Everything it +reads is local and the same on every machine: + +- The card is built under tmp/demo/ (bios/, the folders ArkOS writes on its + games partition, one game per console) and mounted at /media/EASYROMS in a + mount namespace of this user (unshare), on a tmpfs of a card's size. /media + and /mnt are emptied there, so no drive of this machine is shown. +- The repository is served on 127.0.0.1 and named with RETROBIOS_BASE_URL: + the file lists, install/games/ and the BIOS files are this checkout's. +- The one-line command runs this checkout's install.sh and install.py. The + shell of the recording answers `curl -fsSL /i` with a line that + starts the local install.sh, which skips the download of the installer and + its SHA-256 check and runs the rest of the bootstrap as the site's copy does. +- HOME is an empty folder and the prompt is "$ ". + +docs_assets/demo/install.tape drives VHS (github.com/charmbracelet/vhs, with +ttyd, ffmpeg and a Chromium on PATH). This script encodes what it records: + + docs_assets/demo/install.gif the README + docs_assets/demo/install.webm the site, with install.mp4 as fallback + docs_assets/demo/install.png the first frame shown, the video's poster + docs_assets/demo/demo.json what the recording shows, for the alt text + +The site build copies the video files and never runs this script. Run it +again when a screen of the installer or the file list of its games changes: + + python scripts/make_demo.py +""" + +from __future__ import annotations + +import argparse +import functools +import hashlib +import http.server +import importlib.util +import json +import os +import re +import shutil +import subprocess +import sys +import threading +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +OUT = ROOT / "docs_assets" / "demo" +TAPE = OUT / "install.tape" +WORK = ROOT / "tmp" / "demo" +# What the tape writes, read back here (paths relative to ROOT, the tape's +# working directory). +RECORDED = WORK / "recorded.mp4" +POSTER_SHOT = WORK / "poster.png" +MOUNT = "/media/EASYROMS" +CARD_SIZE = "29g" +SETUP = "ArkOS" + +# The card: the folders ArkOS's default tree writes on EASYROMS (install.py +# _GAMES_PART_RULES reads launchimages/), and one game per console folder, +# named as the folder's extensions in install/games/setups/arkos.json allow. +CARD_FOLDERS = ("bios", "bgmusic", "launchimages", "themes/es-theme-nes-box") +GAMES = { + "psx": "Sky Courier (USA).iso", + "segacd": "Night Shift (USA).iso", + "megadrive": "Old Harbour (World).md", + "mastersystem": "Tide Runner (Europe).sms", + "gamegear": "Pocket Quest (World).gg", + "pcengine": "Star Lancer (Japan).pce", +} +GAME_BYTES = 256 * 1024 +# A PlayStation disc carries Sony's licence text in sector 4, the text the +# cores read the region from (install/games/systems/sony-playstation.json). +PSX_LICENCE = b" Licensed by Sony Computer Entertainment Amer ica " + +# Size targets of the outputs, in bytes: the README and two site pages. +LIMITS = {"install.gif": 1_500_000, "install.webm": 1_000_000, + "install.mp4": 1_000_000, "install.png": 150_000} +GIF_FPS = 10 +VIDEO_FPS = 25 + + +def game_bytes(name: str) -> bytes: + """Bytes of a game no catalog names, the same on every run.""" + seed = hashlib.sha256(name.encode("utf-8")).digest() + blocks, counter = [], 0 + while len(blocks) * 32 < GAME_BYTES: + blocks.append(hashlib.sha256(seed + counter.to_bytes(4, "big")).digest()) + counter += 1 + return b"".join(blocks)[:GAME_BYTES] + + +def build_card(card: Path) -> None: + if card.exists(): + shutil.rmtree(card) + for folder in CARD_FOLDERS: + (card / folder).mkdir(parents=True) + (card / "launchimages" / "loading.jpg").write_bytes(b"\xff\xd8\xff\xe0" + bytes(60) + b"\xff\xd9") + (card / "themes" / "es-theme-nes-box" / "theme.xml").write_text( + "\n\n", encoding="utf-8") + for folder, name in GAMES.items(): + data = bytearray(game_bytes(name)) + if folder == "psx": + data[4 * 2048:4 * 2048 + len(PSX_LICENCE)] = PSX_LICENCE + (card / folder).mkdir() + (card / folder / name).write_bytes(bytes(data)) + + +class QuietHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + pass + + +@contextmanager +def served_repository() -> Iterator[str]: + """The repository on a loopback port, the base URL install.py accepts + without HTTPS.""" + handler = functools.partial(QuietHandler, directory=str(ROOT)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_address[1]}" + finally: + server.shutdown() + server.server_close() + + +def write_session(work: Path, names: list[str]) -> Path: + """The program the recording's shell becomes: the card mounted in a + namespace of its own, then a clean interactive bash, or the command + given as arguments, with only the variables names lists.""" + rc = work / "bashrc" + rc.write_text( + "PS1='$ '\n" + "unset HISTFILE PROMPT_COMMAND\n" + "curl() {\n" + ' if [ "$*" = "-fsSL https://abdess.github.io/retrobios/i" ]; then\n' + ' printf \'exec %s "$@"\\n\' "$RETROBIOS_DEMO_BOOTSTRAP"\n' + " else\n" + ' command curl "$@"\n' + " fi\n" + "}\n" + "clear\n", + encoding="utf-8", + ) + session = work / "session.sh" + session.write_text( + "#!/bin/sh\n" + "# Written by scripts/make_demo.py.\n" + "set -eu\n" + "inner='set -eu\n" + "mount -t tmpfs -o mode=0755 demo /media\n" + "mount -t tmpfs -o mode=0755 demo /mnt\n" + f"mkdir {MOUNT}\n" + f"mount -t tmpfs -o size={CARD_SIZE},mode=0755 EASYROMS {MOUNT}\n" + f'cp -a "$RETROBIOS_DEMO_CARD/." {MOUNT}/\n' + 'cd "$HOME"\n' + 'exec unshare --user --map-user="$RETROBIOS_DEMO_UID" ' + '--map-group="$RETROBIOS_DEMO_GID" env -i ' + + " ".join(f'{name}="${name}"' for name in names) + ' "$@"\'\n' + 'if [ "$#" -eq 0 ]; then\n' + f' set -- bash --noprofile --rcfile "{rc}" -i\n' + "fi\n" + 'exec unshare --user --mount --map-root-user /bin/sh -c "$inner" sh "$@"\n', + encoding="utf-8", + ) + session.chmod(0o755) + return session + + +def session_env(base_url: str, work: Path, card: Path) -> dict[str, str]: + home = work / "home" + home.mkdir(parents=True, exist_ok=True) + return { + "HOME": str(home), + "PATH": "/usr/local/bin:/usr/bin:/bin", + "LANG": "C.UTF-8", + "TERM": "xterm-256color", + "TMPDIR": str(work / "scratch"), + "RETROBIOS_OS": "linux", + "RETROBIOS_BASE_URL": base_url, + # What the downloaded bootstrap sets: commands given back are the + # one-line command, not a path to a temporary copy. + "RETROBIOS_BOOTSTRAP": "sh", + "RETROBIOS_DEMO_BOOTSTRAP": str(ROOT / "install.sh"), + "RETROBIOS_DEMO_CARD": str(card), + "RETROBIOS_DEMO_UID": str(os.getuid()), + "RETROBIOS_DEMO_GID": str(os.getgid()), + } + + +def format_size(size: int) -> str: + """The size as the installer writes it.""" + spec = importlib.util.spec_from_file_location("retrobios_install", ROOT / "install.py") + if spec is None or spec.loader is None: + raise RuntimeError("install.py cannot be loaded") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return str(module.format_size(size)) + + +def measure(session: Path, env: dict[str, str]) -> dict: + """What the recording shows, read from the same run without the screen: + the games read on the card and the files planned for them.""" + command = [str(session), sys.executable, str(ROOT / "install.py"), "--platform", "retroarch", + "--dest", f"{MOUNT}/bios", "--for-games", "--dry-run", "--events"] + result = subprocess.run(command, env=env, capture_output=True, text=True, check=False) + if result.returncode != 0: + raise RuntimeError(f"the dry run failed ({result.returncode}):\n{result.stderr}") + events = [json.loads(line) for line in result.stdout.splitlines() if line.startswith("{")] + games = next(e for e in events if e["event"] == "games") + check = next(e for e in events if e["event"] == "check") + size = {key.lower(): int(value) for key, value in + re.findall(r"^Set (Width|Height) (\d+)$", TAPE.read_text(encoding="utf-8"), re.M)} + return { + "width": size["width"], + "height": size["height"], + "setup": SETUP, + "games": games["games"], + "systems": len(games["found"]), + "files": check["missing"], + "size": format_size(check["bytes_needed"]), + } + + +def ffmpeg(*args: str) -> None: + subprocess.run(["ffmpeg", "-hide_banner", "-loglevel", "error", "-y", *args], check=True) + + +def encode(recorded: Path, poster: Path, out: Path, scratch: Path) -> None: + palette = scratch / "palette.png" + ffmpeg("-i", str(recorded), "-vf", f"fps={GIF_FPS},palettegen=max_colors=48:stats_mode=full", + str(palette)) + ffmpeg("-i", str(recorded), "-i", str(palette), "-lavfi", + f"fps={GIF_FPS}[v];[v][1:v]paletteuse=dither=none:diff_mode=rectangle", + "-loop", "0", str(out / "install.gif")) + ffmpeg("-i", str(recorded), "-an", "-r", str(VIDEO_FPS), "-c:v", "libvpx-vp9", + "-b:v", "0", "-crf", "42", "-row-mt", "1", "-deadline", "good", + "-pix_fmt", "yuv420p", str(out / "install.webm")) + ffmpeg("-i", str(recorded), "-an", "-r", str(VIDEO_FPS), "-c:v", "libx264", + "-preset", "veryslow", "-crf", "30", "-tune", "animation", "-pix_fmt", "yuv420p", + "-movflags", "+faststart", str(out / "install.mp4")) + ffmpeg("-i", str(poster), "-pred", "mixed", "-compression_level", "100", + str(out / "install.png")) + + +def check_sizes(out: Path) -> list[str]: + return [f"{name}: {(out / name).stat().st_size:,} bytes, over {limit:,}" + for name, limit in LIMITS.items() if (out / name).stat().st_size > limit] + + +def main() -> None: + parser = argparse.ArgumentParser(allow_abbrev=False, + description="Record the installer demo (VHS).") + parser.parse_args() + missing = [tool for tool in ("vhs", "ttyd", "ffmpeg", "unshare") if shutil.which(tool) is None] + if missing: + sys.exit(f"Missing on PATH: {', '.join(missing)} (VHS: github.com/charmbracelet/vhs)") + if WORK.exists(): + shutil.rmtree(WORK) + (WORK / "scratch").mkdir(parents=True) + try: + card = WORK / "card" + build_card(card) + with served_repository() as base_url: + env = session_env(base_url, WORK, card) + session = write_session(WORK, sorted(env)) + facts = measure(session, env) + vhs_env = {**os.environ, **env, "PATH": os.environ.get("PATH", ""), + "HOME": os.environ.get("HOME", ""), + "RETROBIOS_DEMO_SESSION": str(session)} + subprocess.run(["vhs", str(TAPE.relative_to(ROOT))], cwd=ROOT, env=vhs_env, + check=True) + encode(RECORDED, POSTER_SHOT, OUT, WORK / "scratch") + (OUT / "demo.json").write_text(json.dumps(facts, indent=2) + "\n", encoding="utf-8") + for name in (*LIMITS, "demo.json"): + print(f"{(OUT / name).relative_to(ROOT)} {(OUT / name).stat().st_size:,} bytes") + over = check_sizes(OUT) + if over: + sys.exit("Over the size targets:\n " + "\n ".join(over)) + finally: + shutil.rmtree(WORK, ignore_errors=True) + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/scripts/siteagents.py b/scripts/siteagents.py new file mode 100644 index 000000000..ffed73260 --- /dev/null +++ b/scripts/siteagents.py @@ -0,0 +1,225 @@ +"""What the site tells an AI agent about itself. + +- ``llms.txt`` (https://llmstxt.org): the pages and data an agent should + read first, in Markdown, at the site's root. +- ``skills/setup-bios/SKILL.md``: an Agent Skill that sets up a device's + BIOS files with the fewest downloads. +- ``ard.json``: an Agentic Resource Discovery manifest + (https://github.com/ards-project/ard-spec) listing that skill and the + JSON API, linked from every page's head with ``rel="ard"``. + +Both are written from the registry at each build, so a new platform or a +moved page cannot leave them stale. The WebMCP tools a page registers live +in website/src/scripts/agent-tools.ts. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path + +SITE = "https://abdess.github.io/retrobios/" +PUBLISHER = "abdess.github.io" +SH = "curl -fsSL https://abdess.github.io/retrobios/i | sh -s --" +PS = "& ([scriptblock]::Create((irm https://abdess.github.io/retrobios/i.ps1)))" +# What a folder put into a command may not hold, as agent-tools.ts refuses +# it: PowerShell also ends a single-quoted string on the typographic quotes. +REFUSED = "any quote, straight or typographic (' \" ‘ ’ ‚ ‛ “ ” „), a backtick or a control character" +QUOTING = (f"A folder goes in single quotes, attached to its option. One that holds {REFUSED}, " + "or starts with `-`, is a question for the user, never a value to escape into the " + "command. On a page of the site, the `device_setup_command` WebMCP tool writes the " + "command and refuses such a folder.") + +START_PAGES = ( + ("Which pack do I need", "which-pack/", "pick a device or setup, get the folder its BIOS files go in, " + "one command that installs and checks them, or a ZIP pack"), + ("First install", "wiki/first-install/", "the steps from nothing to a working BIOS folder"), + ("What are BIOS files", "wiki/what-are-bios-files/", "why emulators need them and where they come from"), + ("Troubleshooting", "wiki/troubleshooting/", "a file the emulator does not find or refuses"), + ("Common questions", "wiki/faq/", "short answers"), +) + +DATA = ( + ("API index", "api/v1/index.json", "every dataset of the JSON API, versioned"), + ("Catalog", "api/v1/catalog.json", "each dataset, manifest, export and download with its size, SHA-256 and schema"), + ("Emulators", "api/v1/emulators.json", "every emulator profile: the files it reads, required or not, " + "their hashes and the source line that reads each. One document " + "per emulator at api/v1/emulators/.json"), + ("Core names", "api/v1/cores.json", "each core name and alias, with the profile it designates"), + ("Systems", "api/v1/systems.json", "each system id, its name and the section holding its files"), + ("Platforms", "api/v1/platforms.json", "each setup (RetroArch, Batocera...), its BIOS folder and files"), + ("Hash lookup", "api/v1/lookup/00.json", "what a hash (MD5, SHA-1, SHA-256, CRC32) designates, sharded " + "by its first two hex characters: api/v1/lookup/.json"), +) + +MORE = ( + ("Emulators", "emulators/", "one page per emulator"), + ("Systems", "systems/", "one page per console maker, every file with its hashes"), + ("Installer", "wiki/installer/", "every option of install.py"), + ("What we count", "wiki/privacy/", "the site's cookie-free counting"), + ("Is it legal?", "wiki/legal/", "the project's position"), + ("Contributing", "contributing/", "add a missing file or an emulator profile"), +) + + +def llms_txt(platforms: list[tuple[str, str]]) -> str: + """The site's llms.txt. ``platforms`` lists (id, display name) of every + registered setup, each with its page.""" + lines = [ + "# RetroBIOS", + "", + "> BIOS and firmware files for emulators, each checked against the emulator's source " + "code: which files an emulator reads, whether it starts without them, where each one " + "goes on every setup, their hashes, and an installer that puts them in place.", + "", + "Every claim about a file cites the line of emulator source code that reads it. The " + "files themselves are served by the GitHub repository " + "(https://github.com/Abdess/retrobios) and its releases.", + "", + "## Setting up a device for its user", + "", + "Most people want to play now, with the fewest files. The installer does that in one " + "command: it finds the setup and its BIOS folder, reads the games on the card, and with " + "`--for-games --required-only` installs only the files those games cannot start " + "without. Without these two options it installs every file of the setup's list " + "(gigabytes), even with `--yes`.", + "", + f"1. Preview, nothing written: `{SH} --for-games --required-only --dry-run`", + f"2. Install, no questions: `{SH} --for-games --required-only --yes`", + f"3. On Windows (PowerShell): `{PS} --for-games --required-only --yes`", + "", + "- Run it on the device itself (its terminal or SSH), or on a computer with the device's " + "card plugged in. The installer recognises the handheld systems on the card, among them " + "ArkOS, dArkOS, ArchR and UnofficialOS (R36S and its clones), muOS, KNULLI, ROCKNIX, " + "AmberELEC, EmuELEC, Onion, spruceOS and MinUI. " + "The [Linux handhelds page](" + SITE + "platforms/linux-handhelds/) maps each device " + "to its system and folder.", + "- From a computer, if the card is not found, name its BIOS folder: `--dest='/bios'` " + "(ArkOS and dArkOS: the `bios` folder of the EASYROMS part).", + "- A games folder somewhere else: `--for-games=''`, one subfolder per system.", + f"- {QUOTING}", + "- Later, to see what is missing or wrong: `--check`. Machine-readable progress: " + "`--events` (JSON lines, one per event, documented on the installer page).", + "- A game the installer cannot identify gets every BIOS of its system, and the run names it.", + f"- The same steps as an Agent Skill: [SKILL.md]({SITE}{SKILL_PATH}).", + "", + "## Start", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in START_PAGES), + "", + "## Setups", + "", + *(f"- [{name}]({SITE}platforms/{pid}/): its BIOS folder and files" for pid, name in platforms), + "", + "## Data", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in DATA), + "", + "## Optional", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in MORE), + ] + return "\n".join(lines) + "\n" + + +SKILL_PATH = "skills/setup-bios/SKILL.md" +# The Agent Skills media type ARD names: Markdown with this profile. +SKILL_TYPE = 'text/markdown; profile="urn:air:agent-skills"' + + +def skill_md() -> str: + """An Agent Skill (SKILL.md: YAML front matter, then instructions) that + sets up a device's BIOS files with the fewest downloads.""" + return "\n".join([ + "---", + "name: setup-bios", + "description: Install the BIOS and firmware files a retrogaming device or computer needs " + "(RetroArch, Batocera, EmuDeck, ArkOS, dArkOS, muOS, KNULLI, ROCKNIX and others), only " + "what the games on its card need to start. Use when a user asks to set up emulators, " + "a handheld (R36S, Miyoo, Anbernic), BIOS files or firmware.", + "---", + "", + "# Set up BIOS files with RetroBIOS", + "", + "Most users want to play now, with the fewest files. Install only what the games on the " + "card need, unless the user asks for everything.", + "", + "## Steps", + "", + "1. Ask where the games are: on the device itself (you have its terminal or SSH), or on " + "its card plugged into this computer.", + f"2. Preview, nothing is written: `{SH} --for-games --required-only --dry-run` " + f"(Windows PowerShell: `{PS} --for-games --required-only --dry-run`).", + "3. Show the user the plan it prints: the setup found, the BIOS folder, the files and " + "their size. Ask before going on if the setup or the folder is not the one expected.", + "4. Install with the same command, `--yes` in place of `--dry-run`.", + "5. Report the summary line. A game the installer could not identify got every BIOS of " + "its system: say which.", + "", + "## When the setup or the card is not found", + "", + "- From a computer, name the card's BIOS folder: `--dest='/bios'`. ArkOS and dArkOS " + "keep it on the EASYROMS part.", + "- Games outside the card: `--for-games=''`, one subfolder per system.", + "- A setup not detected: `--platform=`, a name exactly as `--list-platforms` prints " + "it (letters, digits, `-` and `_`).", + f"- {QUOTING}", + "", + "## Later", + "", + "- `--check` says what is missing or wrong, without downloading.", + "- `--events` writes the run as JSON lines, for reading by a program.", + "", + f"Every option: {SITE}wiki/installer/. Devices and their folders: " + f"{SITE}platforms/linux-handhelds/. The site's guide for agents: {SITE}llms.txt.", + "", + ]) + + +def ard_manifest() -> dict: + """The ARD manifest: the setup skill and the JSON API, each with the + questions it answers. llms.txt is found at the site root, as its + convention says.""" + return { + "entries": [ + { + "identifier": f"urn:air:{PUBLISHER}:retrobios:setup-bios", + "displayName": "Set up BIOS files with RetroBIOS", + "type": SKILL_TYPE, + "url": f"{SITE}{SKILL_PATH}", + "description": "An agent skill that installs the BIOS files a device needs, only " + "what the games on its card need to start.", + "representativeQueries": [ + "install the minimum BIOS files for the games on my R36S running dArkOS", + "set up BIOS files for RetroArch on my Steam Deck", + "my emulator says BIOS missing, fix it", + ], + "capabilities": ["SetupBios"], + "tags": ["bios", "firmware", "emulation", "retrogaming"], + }, + { + "identifier": f"urn:air:{PUBLISHER}:retrobios:api", + "displayName": "RetroBIOS JSON API", + "type": "application/json", + "url": f"{SITE}api/v1/index.json", + "description": "Versioned JSON datasets: emulator profiles, platforms, systems, core " + "names and a hash lookup.", + "representativeQueries": [ + "what file has the MD5 490f666e1afb15b7362b406ed1cea246", + "which files does the beetle_psx core read", + "where do PlayStation BIOS files go on Batocera", + ], + "capabilities": ["HashLookup", "EmulatorFiles", "PlatformFiles"], + "tags": ["api", "json", "bios"], + }, + ] + } + + +def write(public: Path, platforms: list[tuple[str, str]], + write_if_changed: Callable[[str, str], object]) -> None: + write_if_changed(str(public / "llms.txt"), llms_txt(platforms)) + (public / SKILL_PATH).parent.mkdir(parents=True, exist_ok=True) + write_if_changed(str(public / SKILL_PATH), skill_md()) + write_if_changed(str(public / "ard.json"), json.dumps(ard_manifest(), indent=1) + "\n") diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py index 7bd717e95..b42be73a8 100644 --- a/scripts/sitestarlight.py +++ b/scripts/sitestarlight.py @@ -29,6 +29,7 @@ COMPONENTS = { "TabItem": "@astrojs/starlight/components", "FilePicker": "components/FilePicker.astro", "MeasurementsExplorer": "components/MeasurementsExplorer.astro", + "InstallDemo": "components/InstallDemo.astro", } # Pages of file cards: level 3 headings by the hundred, so the table of @@ -330,7 +331,7 @@ OG_DIR = Path(__file__).resolve().parent.parent / "docs_assets" / "og" # An emulator page's card is that of the maker of the first system it runs, # by page path: "emulators/mgba.mdx" -> "nintendo". One card per emulator -# would be some 900 images in the repository. The maker's names the +# would be some 900 images in the repository, and the maker's names the # consoles. Filled by generate_site.record_emulator_page. EMULATOR_CARDS: dict[str, str] = {} diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 555bbd50a..9cddfef64 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -6923,9 +6923,11 @@ class RequiredOnlyReachesDataDirectories(unittest.TestCase): def test_every_tree_walk_reads_the_flag(self): source = (ROOT / "scripts" / "generate_pack.py").read_text(encoding="utf-8") - self.assertEqual( - source.count('if required_only and dd.get("required") is False:'), 2 - ) + # The pack's walk and the manifest's, which records the flag for + # install.py --required-only. + self.assertEqual(source.count('if required_only and dd.get("required") is False:'), 1) + self.assertIn('needed = dd.get("required") is not False\n if required_only and not needed:', + source) self.assertIn("seen_parents, required_only=required_only,", source) self.assertIn("data_refs, required_only=required_only,", source) diff --git a/tests/test_install.py b/tests/test_install.py index 9effcdd49..7e6689029 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3880,6 +3880,20 @@ class DataDirectoriesAreInstalled(unittest.TestCase): }, } + def test_required_only_leaves_out_an_optional_tree(self): + """A GBA and a SNES game with --for-games --required-only took + FBNeo's cheats and samples, 3,472 files no game opened.""" + cheats = {**self._entry(), "ref": "fbneo-cheats", "systems": ["snes"], "optional": True} + sys_tree = {**self._entry(), "systems": ["snes"]} + manifest = {"data_directories": [cheats, sys_tree]} + files = [{"dest": "a.rom", "systems": ["snes"]}] + self.assertEqual(install.select_data_directories(manifest, files, True), [cheats, sys_tree]) + self.assertEqual(install.select_data_directories(manifest, files, True, True), [sys_tree]) + self.assertEqual(install.select_data_directories(manifest, files, False, True), [sys_tree]) + install._check_data_directory(cheats, "data_directories[0]") + with self.assertRaises(ValueError): + install._check_data_directory({**cheats, "optional": False}, "data_directories[0]") + def test_the_archive_is_drawn_and_counted(self): """82 MB of archives came with no progress line, and the run ended on "310.5 KB downloaded", the BIOS files alone.""" diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index a7584e71b..610c70221 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -12912,6 +12912,21 @@ class DriveIsACardByItsFiles(unittest.TestCase): self.assertEqual([install.card_system(mount) for mount in mounts], [None, None, None]) + def test_a_dest_on_a_games_part_reads_its_rom_folders(self): + """--dest EASYROMS/bios --for-games from a computer: detection never + ran on the card, the generic RetroArch folders were tried, and the + run kept every file of the list.""" + arkos = games_part(self.root / "media" / "EASYROMS") + index = {"setups": {"arkos": {}, "retroarch": {}}} + with mock.patch.object(install, "_mount_points", return_value=frozenset()), \ + mock.patch.object(install, "_drive_not_removable", return_value=False): + self.assertEqual(install.games_setups(index, "retroarch", arkos / "bios"), ["arkos"]) + self.assertEqual(install.games_setups(index, "retroarch", self.root / "bios"), + ["retroarch"]) + # Its games are offered without --for-games: EASYROMS keeps no + # roms folder, its console folders sit beside bios. + self.assertTrue(install._may_hold_games(arkos / "bios")) + def test_a_card_beside_a_data_drive_comes_first(self): """The ArkOS card and the data drive mounted, the data drive sorted first and took Enter. The card comes first in diff --git a/tests/test_site_agents.py b/tests/test_site_agents.py new file mode 100644 index 000000000..ac081c6b0 --- /dev/null +++ b/tests/test_site_agents.py @@ -0,0 +1,140 @@ +"""What the site tells an AI agent: llms.txt, the ARD manifest, the WebMCP tools. + +An agent asked to set up a handheld reads these first. They must lead it to +the fewest files the games need, and every address they give must exist. +""" +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import re +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import siteagents # noqa: E402 + +DIST = REPO_ROOT / "website" / "dist" +TOOLS = REPO_ROOT / "website" / "src" / "scripts" / "agent-tools.ts" +FOOTER = REPO_ROOT / "website" / "src" / "components" / "Footer.astro" +CONFIG = REPO_ROOT / "website" / "astro.config.mjs" + + +class TheGuideLeadsToTheFewestFiles(unittest.TestCase): + def setUp(self) -> None: + self.text = siteagents.llms_txt([("retroarch", "RetroArch"), ("batocera", "Batocera")]) + + def test_it_is_llms_txt(self) -> None: + self.assertTrue(self.text.startswith("# RetroBIOS\n\n> ")) + self.assertIn("\n## Setting up a device for its user\n", self.text) + + def test_every_command_installs_only_what_the_games_need(self) -> None: + commands = re.findall(r"`((?:curl|&) [^`]+)`", self.text) + self.assertGreaterEqual(len(commands), 3) + for command in commands: + with self.subTest(command=command): + self.assertIn("--for-games --required-only", command) + + def test_each_setup_has_its_page(self) -> None: + self.assertIn("(https://abdess.github.io/retrobios/platforms/batocera/)", self.text) + + @unittest.skipUnless((DIST / "llms.txt").is_file(), "website/dist is not built") + def test_every_address_of_the_built_guide_exists(self) -> None: + built = (DIST / "llms.txt").read_text(encoding="utf-8") + for url in re.findall(r"\((https://abdess\.github\.io/retrobios/[^)]*)\)", built): + with self.subTest(url=url): + target = DIST / url.removeprefix(siteagents.SITE) + self.assertTrue(target.is_file() or (target / "index.html").is_file(), url) + + +class TheSkillIsAnAgentSkill(unittest.TestCase): + """ARD reserves text/markdown for agent skills (Lighthouse warns on a + plain Markdown entry): the setup steps are published as one.""" + + def test_front_matter_then_instructions(self) -> None: + import yaml # noqa: PLC0415 + + text = siteagents.skill_md() + head, _, body = text.removeprefix("---\n").partition("\n---\n") + front = yaml.safe_load(head) + self.assertEqual(set(front), {"name", "description"}) + self.assertRegex(front["name"], r"^[a-z0-9-]{1,64}$") + self.assertLessEqual(len(front["description"]), 1024) + self.assertTrue(body.lstrip().startswith("# ")) + for command in re.findall(r"`((?:curl|&) [^`]+)`", body): + with self.subTest(command=command): + self.assertIn("--for-games --required-only", command) + + def test_a_folder_is_quoted_and_a_hostile_one_asked_about(self) -> None: + for text in (siteagents.skill_md(), siteagents.llms_txt([])): + with self.subTest(text=text[:20]): + self.assertIn("--dest='/bios'", text) + self.assertIn("--for-games=''", text) + self.assertIn("starts with `-`", text) + # The characters agent-tools.ts refuses, typographic quotes included. + # The backtick is named in words: Markdown would read it. + self.assertIn("a backtick", siteagents.REFUSED) + for char in "'\"‘’‚‛“”„": + self.assertIn(char, siteagents.REFUSED) + self.assertIn(char, TOOLS.read_text(encoding="utf-8")) + self.assertIn(siteagents.REFUSED, text) + + def test_the_manifest_types_it_as_a_skill(self) -> None: + skill = siteagents.ard_manifest()["entries"][0] + self.assertEqual(skill["type"], 'text/markdown; profile="urn:air:agent-skills"') + self.assertTrue(skill["url"].endswith(siteagents.SKILL_PATH)) + + +class TheManifestIsAnArdManifest(unittest.TestCase): + """The terms ard-entry.schema.json requires (ards-project/ard-spec).""" + + def test_entries(self) -> None: + entries = siteagents.ard_manifest()["entries"] + self.assertTrue(entries) + for entry in entries: + with self.subTest(entry=entry.get("identifier")): + self.assertRegex(entry["identifier"], r"^urn:air:[a-zA-Z0-9.-]+(:[a-zA-Z0-9._-]+)+$") + self.assertTrue(entry["displayName"]) + self.assertRegex(entry["type"], r'^[a-z]+/[\w.+-]+(; [a-z]+="[^"]+")?$') + self.assertTrue(("url" in entry) != ("data" in entry)) + self.assertTrue(2 <= len(entry["representativeQueries"]) <= 5) + self.assertTrue(entry["url"].startswith(siteagents.SITE)) + + def test_every_page_links_it(self) -> None: + self.assertIn("rel: 'ard'", CONFIG.read_text(encoding="utf-8")) + + +class TheToolsReadOnly(unittest.TestCase): + def setUp(self) -> None: + self.source = TOOLS.read_text(encoding="utf-8") + + def test_every_page_registers_them(self) -> None: + self.assertIn("import '../scripts/agent-tools.ts';", FOOTER.read_text(encoding="utf-8")) + names = re.findall(r"^\s*name: '([a-z_]+)'", self.source, re.M) + self.assertEqual(names, ["lookup_bios_hash", "emulator_bios_files", "search_retrobios", + "device_setup_command"]) + self.assertEqual(self.source.count("readOnlyHint: true"), len(names)) + + def test_the_setup_command_defaults_to_the_games(self) -> None: + self.assertIn("const games = everything ? [] :", self.source) + self.assertIn("'--required-only'", self.source) + + def test_a_value_that_could_escape_the_command_is_refused(self) -> None: + """The command is pasted into a shell: PowerShell also ends a single-quoted + string on the typographic quotes, and a path starting with "-" reads as + an option.""" + self.assertIn("const unsafe = /[\\u0000-\\u001f\\u007f'\"`‘’‚‛“”„]/;", self.source) + self.assertIn("value.startsWith('-')", self.source) + self.assertIn("/^[a-z0-9_-]+$/.test(platform)", self.source) + self.assertIn("`--dest=${quote(bios_folder as string)}`", self.source) + + def test_they_reach_no_other_site(self) -> None: + code = re.sub(r"^\s*//.*$", "", self.source, flags=re.M) + self.assertEqual(set(re.findall(r"https://[\w.-]+", code)), {"https://abdess.github.io"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_site_demo.py b/tests/test_site_demo.py new file mode 100644 index 000000000..3d653e213 --- /dev/null +++ b/tests/test_site_demo.py @@ -0,0 +1,165 @@ +"""The installer demo of the README and the site (scripts/make_demo.py). + +The home page and Get your files score 98 to 100 on a phone: the video must +cost nothing before it is on screen, keep still under reduced motion, and say +in words what it shows. The files are committed, so their sizes are held +here, and the words come from what the recording measured. +""" + +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import json +import re +import struct +import sys +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(ROOT / "scripts")) + +import generate_readme # noqa: E402 +import make_demo # noqa: E402 +import sitestarlight # noqa: E402 + +DEMO = ROOT / "docs_assets" / "demo" +COMPONENT = ROOT / "website" / "src" / "components" / "InstallDemo.astro" +DIST = ROOT / "website" / "dist" + + +def _facts() -> dict: + return json.loads((DEMO / "demo.json").read_text(encoding="utf-8")) + + +class TheFilesAreThereAndSmall(unittest.TestCase): + def test_each_output_is_under_its_target(self): + for name, limit in make_demo.LIMITS.items(): + with self.subTest(name=name): + self.assertLessEqual((DEMO / name).stat().st_size, limit) + + def test_the_gif_and_the_poster_have_the_recorded_size(self): + facts = _facts() + head = (DEMO / "install.gif").read_bytes()[:10] + self.assertEqual(head[:6], b"GIF89a") + self.assertEqual(struct.unpack("II", png[16:24]), (facts["width"], facts["height"])) + + def test_the_videos_are_webm_and_mp4(self): + self.assertEqual((DEMO / "install.webm").read_bytes()[:4], b"\x1a\x45\xdf\xa3") + self.assertEqual((DEMO / "install.mp4").read_bytes()[4:8], b"ftyp") + + +class TheTapeAndTheDriverAgree(unittest.TestCase): + def setUp(self): + self.tape = make_demo.TAPE.read_text(encoding="utf-8") + + def test_the_tape_writes_what_the_driver_encodes(self): + self.assertIn(f"Output {make_demo.RECORDED.relative_to(ROOT).as_posix()}\n", self.tape) + self.assertIn(f"Screenshot {make_demo.POSTER_SHOT.relative_to(ROOT).as_posix()}\n", + self.tape) + + def test_the_size_in_demo_json_is_the_tapes(self): + facts = _facts() + self.assertIn(f"Set Width {facts['width']}\n", self.tape) + self.assertIn(f"Set Height {facts['height']}\n", self.tape) + + def test_the_command_typed_is_the_one_the_pages_give(self): + from siteguides import install_command # noqa: PLC0415 + + self.assertIn(f'Type "{install_command("sh", "--for-games")}"\n', self.tape) + + def test_the_shell_answers_only_the_site_address(self): + typed = re.search(r'Type "curl -fsSL (\S+) \|', self.tape) + self.assertIsNotNone(typed) + driver = Path(make_demo.__file__).read_text(encoding="utf-8") + self.assertIn(f'"$*" = "-fsSL {typed.group(1)}"', driver) + + +class TheReadmeShowsTheGif(unittest.TestCase): + def test_the_gif_sits_under_the_commands_with_its_description(self): + readme = (ROOT / "README.md").read_text(encoding="utf-8") + line = f"![{generate_readme.demo_description()}]({generate_readme.DEMO_GIF})" + self.assertIn(line, readme) + self.assertLess(readme.index("## Quick install"), readme.index(line)) + self.assertLess(readme.index(line), readme.index("## Download BIOS packs")) + + def test_the_description_names_what_the_recording_measured(self): + facts = _facts() + words = generate_readme.demo_description(facts) + for value in (facts["setup"], f"{facts['games']} games", f"{facts['files']} BIOS files", + facts["size"]): + self.assertIn(str(value), words) + + +class TheVideoLoadsOnlyOnScreen(unittest.TestCase): + def setUp(self): + self.source = COMPONENT.read_text(encoding="utf-8") + self.markup = self.source.split(" diff --git a/website/src/scripts/agent-tools.ts b/website/src/scripts/agent-tools.ts new file mode 100644 index 000000000..51de884a1 --- /dev/null +++ b/website/src/scripts/agent-tools.ts @@ -0,0 +1,183 @@ +// WebMCP tools (https://developer.chrome.com/docs/ai/webmcp): what an AI +// agent browsing the site can ask it directly, instead of reading pages. +// Each tool reads the JSON the site already publishes (api/v1/) or its +// search index, changes nothing and sends nothing anywhere else. A browser +// without WebMCP skips the whole file. + +type ToolResult = string; +type Tool = { + name: string; + description: string; + inputSchema: Record; + annotations?: { readOnlyHint?: boolean }; + execute: (input: Record) => Promise; +}; +type ModelContext = { registerTool: (tool: Tool) => Promise | void }; + +const BASE = import.meta.env.BASE_URL.replace(/\/?$/, '/'); +const SITE = new URL(BASE, location.origin); +const json = async (path: string): Promise => { + const response = await fetch(new URL(path, SITE)); + if (!response.ok) throw new Error(`${path}: HTTP ${response.status}`); + return response.json() as Promise; +}; +const absolute = (url: string) => new URL(url, location.origin).href; + +type LookupItem = { algorithm: string; layer: string; name?: string; value: string; url: string; owner?: string; sha1?: string }; +type ProfileFile = Record & { name?: string; required?: boolean }; +type EmulatorDoc = { items: { id: string; profile: { emulator?: string; systems?: string[]; files?: ProfileFile[] } }[] }; +type CoreIndex = { items: { name: string; profile: string; preferred?: boolean }[] }; + +const HASH = /^(?:[0-9a-f]{8}|[0-9a-f]{32}|[0-9a-f]{40}|[0-9a-f]{64})$/; + +const lookupHash: Tool = { + name: 'lookup_bios_hash', + description: + 'Find which BIOS or firmware file a hash designates in the RetroBIOS collection, and the emulators or setups that ' + + 'declare it. Accepts a CRC32 (8 hex characters), MD5 (32), SHA-1 (40) or SHA-256 (64).', + inputSchema: { + type: 'object', + properties: { hash: { type: 'string', description: 'The hash, in hexadecimal.' } }, + required: ['hash'], + }, + annotations: { readOnlyHint: true }, + async execute({ hash }) { + const value = String(hash ?? '').trim().toLowerCase().replace(/^0x/, ''); + if (!HASH.test(value)) return 'Not a CRC32, MD5, SHA-1 or SHA-256 value.'; + const shard = await json<{ items: LookupItem[] }>(`api/v1/lookup/${value.slice(0, 2)}.json`); + const found = shard.items.filter((item) => item.value === value); + if (!found.length) return `No file in the collection or its profiles has the ${value.length === 8 ? 'CRC32' : 'hash'} ${value}.`; + return JSON.stringify(found.map((item) => ({ ...item, url: absolute(item.url) })), null, 1); + }, +}; + +const FILE_FIELDS = ['name', 'path', 'required', 'system', 'size', 'md5', 'sha1', 'crc32', 'sha256', 'region', 'note']; + +const emulatorFiles: Tool = { + name: 'emulator_bios_files', + description: + 'List the BIOS and firmware files an emulator or libretro core reads, from its profile checked against its source code: ' + + 'each file name, where it goes, whether the emulator starts without it, and its hashes.', + inputSchema: { + type: 'object', + properties: { + emulator: { type: 'string', description: 'The emulator or core name, for example "mGBA", "beetle_psx" or "DuckStation".' }, + }, + required: ['emulator'], + }, + annotations: { readOnlyHint: true }, + async execute({ emulator }) { + const asked = String(emulator ?? '').trim(); + const key = asked.toLowerCase().replace(/[\s_]+/g, '_'); + const cores = await json('api/v1/cores.json'); + const named = cores.items.filter((core) => [core.name.toLowerCase(), core.profile].includes(key) + || core.name.toLowerCase().replace(/[\s-]+/g, '_') === key); + const id = (named.find((core) => core.preferred) ?? named[0])?.profile ?? key.replace(/_/g, '-'); + let doc: EmulatorDoc; + try { + doc = await json(`api/v1/emulators/${encodeURIComponent(id)}.json`); + } catch { + return `No emulator profile named "${asked}". The list is at ${absolute(`${BASE}emulators/`)}.`; + } + const { profile } = doc.items[0]; + const files = (profile.files ?? []).map((file) => + Object.fromEntries(FILE_FIELDS.filter((field) => file[field] !== undefined).map((field) => [field, file[field]]))); + return JSON.stringify({ + emulator: profile.emulator ?? id, + page: absolute(`${BASE}emulators/${id}/`), + systems: profile.systems ?? [], + files, + }, null, 1); + }, +}; + +type Pagefind = { search: (query: string) => Promise<{ results: { data: () => Promise<{ url: string; meta: { title?: string }; excerpt: string }> }[] }> }; + +const searchSite: Tool = { + name: 'search_retrobios', + description: 'Search the RetroBIOS site: emulators, consoles, setups (RetroArch, Batocera, EmuDeck...) and guides. Returns the best pages.', + inputSchema: { + type: 'object', + properties: { query: { type: 'string', description: 'Words to search for, for example "dreamcast bios batocera".' } }, + required: ['query'], + }, + annotations: { readOnlyHint: true }, + async execute({ query }) { + const words = String(query ?? '').trim(); + if (!words) return 'Nothing to search for.'; + const pagefind = (await import(/* @vite-ignore */ `${BASE}pagefind/pagefind.js`)) as Pagefind; + const { results } = await pagefind.search(words); + const top = await Promise.all(results.slice(0, 8).map((result) => result.data())); + if (!top.length) return `No page matches "${words}".`; + return JSON.stringify(top.map((page) => ({ + title: page.meta.title ?? '', + url: absolute(page.url), + excerpt: page.excerpt.replace(/<[^>]+>/g, ''), + })), null, 1); + }, +}; + +// The command an agent runs to set up its user's device, from the same +// options wiki/installer.md documents: the fewest files the games on the +// card need, unless the user asks for everything. +const SH = 'curl -fsSL https://abdess.github.io/retrobios/i | sh -s --'; +const PS = '& ([scriptblock]::Create((irm https://abdess.github.io/retrobios/i.ps1)))'; + +const setupCommand: Tool = { + name: 'device_setup_command', + description: + 'The one command that installs the BIOS files a device or computer needs, with RetroBIOS\'s installer. By default it ' + + 'installs only what the games on the card need to start, which is what most players want. It finds the setup ' + + '(RetroArch, Batocera, ArkOS, dArkOS, muOS, KNULLI, ROCKNIX...) and its BIOS folder itself.', + inputSchema: { + type: 'object', + properties: { + shell: { type: 'string', enum: ['sh', 'powershell'], description: '"sh" on Linux, macOS and handhelds (the default), "powershell" on Windows.' }, + everything: { type: 'boolean', description: 'Install every file of the setup\'s list instead of only what the games need. Gigabytes.' }, + bios_folder: { type: 'string', description: 'The BIOS folder, when run on a computer and the card is not found, for example "/media/user/EASYROMS/bios".' }, + games_folder: { type: 'string', description: 'A games folder that is not on the card, one subfolder per system.' }, + platform: { type: 'string', description: 'The setup, only when it is not detected (names from the installer\'s --list-platforms).' }, + }, + }, + annotations: { readOnlyHint: true }, + async execute({ shell, everything, bios_folder, games_folder, platform }) { + const powershell = shell === 'powershell'; + // The command is pasted into a shell: a value that could close its + // quotes (PowerShell also ends a string on the typographic ones) or + // start another option is refused, never escaped. + const unsafe = /[\u0000-\u001f\u007f'"`‘’‚‛“”„]/; + for (const [field, value] of [['bios_folder', bios_folder], ['games_folder', games_folder]] as const) { + if (value !== undefined && (typeof value !== 'string' || !value || unsafe.test(value) || value.startsWith('-'))) { + return `${field} must be a folder path without quotes or control characters.`; + } + } + if (platform !== undefined && (typeof platform !== 'string' || !/^[a-z0-9_-]+$/.test(platform))) { + return 'platform must be a setup name such as retroarch or batocera.'; + } + const run = powershell ? PS : SH; + const quote = (value: string) => `'${value}'`; + const place: string[] = []; + if (platform) place.push(`--platform=${platform}`); + if (bios_folder) place.push(`--dest=${quote(bios_folder as string)}`); + const games = everything ? [] : [games_folder ? `--for-games=${quote(games_folder as string)}` : '--for-games', '--required-only']; + const command = (...more: string[]) => [run, ...place, ...more].join(' '); + return JSON.stringify({ + preview: command(...games, '--dry-run'), + install: command(...games, '--yes'), + check_later: command('--check'), + notes: [ + 'Run the preview first: it writes nothing and lists what would be downloaded.', + 'Run on the device itself, or on a computer with the device\'s card plugged in.', + 'A game the installer cannot identify gets every BIOS of its system, and the run names it.', + `Guide: ${absolute(`${BASE}llms.txt`)}`, + ], + }, null, 1); + }, +}; + +const context = (document as Document & { modelContext?: ModelContext }).modelContext; +if (context?.registerTool) { + for (const tool of [setupCommand, lookupHash, emulatorFiles, searchSite]) { + Promise.resolve(context.registerTool(tool)).catch((error) => console.warn(`WebMCP: ${tool.name}:`, error)); + } +} diff --git a/wiki/installer.md b/wiki/installer.md index 6378b191a..1f109d861 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -405,7 +405,7 @@ lists record these files as omitted. -A RetroArch dry run into an empty folder, with the list of 2026-11-05, printed this: +A RetroArch dry run into an empty folder, with the list of 2026-11-06, printed this: ``` Fetching file index for retroarch... @@ -435,7 +435,7 @@ the installer can fetch them. Each line a first install prints, and what it means. The run below is a -Batocera install into an empty folder, from the list of 2026-11-05. +Batocera install into an empty folder, from the list of 2026-11-06. The figures change with each release. @@ -531,7 +531,7 @@ would save, once that saving is over half. -RetroArch with `--system sony-playstation`, from the list of 2026-11-05: +RetroArch with `--system sony-playstation`, from the list of 2026-11-06: ``` duckstation pulls 316.5 MB of this 329.0 MB download for sony-playstation. --core pcsx1 keeps 11.0 MB of it and saves 318.0 MB (96%). @@ -876,7 +876,9 @@ BIOS sets of its parent chain (`neogeo.zip` for `mslug.zip`). A game that is not recognised gets every BIOS of its system, and the run names it. So does an arcade set the core does not know. **Only what starts** -(`--required-only`) leaves out the files the cores mark optional. +(`--required-only`) leaves out the files the cores mark optional, and the data +folders every core reading them runs without (FinalBurn Neo's cheats and +samples). The games are read without hashing whole discs, and a cache keeps what was read, so a second run over the same card reads only new files. It is a diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index 9c8d73b1f..52fd0dce4 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -69,7 +69,7 @@ Every module in `tests/`, by the area it guards. | Pack probe | the cores a platform runs for a system, staging grown to a fixed point, runs mapped to pack and manifest entries, the line and its schema, the probe lock per run, that no page shows its lines. One class builds a real pack narrowed to PlayStation and lays it out both ways, from the archive and with `install.py --dest`, and skips without the collection | `test_pack_probe` | | Automation runner, disk cleanup, asset sync and bit-flip repair | fake jobs, git repositories with a bare origin and session worktrees, a committed file with one bit flipped, canned CI runs in place of `gh` | `test_autorun` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | -| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_inputs`, `test_site_insights`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_privacy`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` | +| Scrapers, upstream and provenance | inline sources, synthetic | `test_batocera_md5_list`, `test_check_freshness`, `test_cited_lines`, `test_contributing_text`, `test_coverage_cache`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_dedup`, `test_doc_pages`, `test_doc_surfaces`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_optional_dependencies`, `test_pin_build`, `test_profile_sync`, `test_prose_style`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_retired_files`, `test_romset_recipes`, `test_schema_semantic_gate`, `test_scraper_contract`, `test_site_agents`, `test_site_arcade_boards`, `test_site_availability`, `test_site_bootstrap`, `test_site_built_gate`, `test_site_components`, `test_site_console_folders`, `test_site_content_data`, `test_site_copy`, `test_site_cores`, `test_site_css`, `test_site_demo`, `test_site_density_cards`, `test_site_emulator_page`, `test_site_emulators`, `test_site_entry`, `test_site_entry_tables`, `test_site_exports`, `test_site_funnel`, `test_site_images`, `test_site_inputs`, `test_site_insights`, `test_site_integration`, `test_site_nav`, `test_site_picker`, `test_site_platform_density`, `test_site_platform_hashes`, `test_site_privacy`, `test_site_reference`, `test_site_render`, `test_site_search`, `test_site_system_links`, `test_site_system_names`, `test_site_system_sections`, `test_site_systems`, `test_site_validation`, `test_site_weight`, `test_site_wiki`, `test_site_write`, `test_source_ref_targets`, `test_upstream`, `test_upstream_index`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_version_basis`, `test_workflow_paths` | | Games on the card | small DATs, listxml and setup sources, a fake fetcher, fixture install lists, and the committed data against its schemas | `test_arcade_parents`, `test_game_catalogs`, `test_game_headers`, `test_game_regions`, `test_games_index`, `test_romfolders_card`, `test_romfolders_es` | ## Test architecture diff --git a/wiki/tools.md b/wiki/tools.md index a99c6ba6c..3b3c3cbb5 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1373,6 +1373,7 @@ before a release (wiki/release-process.md). Its lines stay in | `generate_site.py` | Generate the site (this documentation) into the Starlight project in `website/` | | `sitesources.py` | Record which cited source paths each pinned upstream tree holds, for the site's source links (`--fetch` reads a tree no cache holds) | | `site_insights.py` | What readers look for and do not find, from totals only: searches that show the site but are rarely clicked, file names searched and whether the collection holds them, site searches that found nothing, broken addresses, "Did this page help?" answers, GitHub traffic and release downloads (`--days`, `--skip gsc,goatcounter,github`, `--output`). Reads Search Console, GoatCounter and GitHub with credentials kept outside the repository. [What the site counts](privacy.md) | +| `make_demo.py` | Record the installer demo of the README and the site into `docs_assets/demo/`: an ArkOS card with six games built under `tmp/demo/` and mounted in a mount namespace of the user, this checkout served on loopback, `docs_assets/demo/install.tape` run by VHS (needs `vhs`, `ttyd`, `ffmpeg`, `unshare`), then the GIF, WebM, MP4 and poster encoded and held to their size targets. `demo.json` records what the recording shows, for the alt text. Never run by the build | | `validate_site.py` | Validate rendered metadata, heading order, image alternatives, JSON-LD, local resources, links and fragments, and the page size and search index budgets | | `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held | | `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch ` reads FBNeo at that tag, branch or commit |