diff --git a/.github/workflows/deploy-site.yml b/.github/workflows/deploy-site.yml index cdc72f0eb..c7997bcd0 100644 --- a/.github/workflows/deploy-site.yml +++ b/.github/workflows/deploy-site.yml @@ -46,6 +46,7 @@ on: - "scripts/restore_large_files.py" - "scripts/safeparse.py" - "scripts/sect233r1.py" + - "scripts/siteagents.py" - "scripts/sitebootstrap.py" - "scripts/sitecollection.py" - "scripts/sitediagrams.py" diff --git a/emulators/fbneo.yml b/emulators/fbneo.yml index 2a2a825e9..8534afb8c 100644 --- a/emulators/fbneo.yml +++ b/emulators/fbneo.yml @@ -51,10 +51,12 @@ data_directories: - ref: fbneo-cheats unread_on: [romm] destination: fbneo/cheats + required: false source_ref: "source src/burner/conc.cpp:167" - ref: fbneo-samples unread_on: [romm] destination: fbneo + required: false source_ref: "source src/burn/snd/samples.cpp:21-35" notes: | diff --git a/install.ps1 b/install.ps1 index 28de54233..01c164290 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4" + $defaultInstallSha256 = "3a78bf2e5a910d0f5c21bc2dc490aa600fc3273c56ae01e9ce5061d9b3578f07" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 04a697af5..a5cf46c28 100644 --- a/install.py +++ b/install.py @@ -3724,6 +3724,8 @@ def _check_data_directory(entry: object, where: str) -> None: values = entry.get(field, []) if not isinstance(values, list) or not all(isinstance(v, str) for v in values): raise ValueError(f"{where}.{field} is invalid") + if entry.get("optional", True) is not True: + raise ValueError(f"{where}.optional is invalid") files = entry.get("files") if not isinstance(files, dict) or not files or len(files) > MAX_MANIFEST_FILES: raise ValueError(f"{where}.files is invalid") @@ -7393,11 +7395,16 @@ def _install_data_member( scratch.unlink(missing_ok=True) -def select_data_directories(manifest: dict, files: list[dict], narrowed: bool) -> list[dict]: - """The data trees this install keeps: all, or those of a kept system.""" - entries = manifest.get("data_directories") or [] +def select_data_directories(manifest: dict, files: list[dict], narrowed: bool, + required_only: bool = False) -> list[dict]: + """The data trees this install keeps: all, or those of a kept system. + --required-only leaves out a tree every core reading it runs without + (`optional`): a GBA and a SNES game took FBNeo's cheats and samples, + 3,472 files no game opened.""" + entries = [e for e in manifest.get("data_directories") or [] + if not (required_only and e.get("optional"))] if not narrowed: - return list(entries) + return entries kept = {system for f in files for system in entry_systems(f)} return [e for e in entries if kept & set(e.get("systems") or [])] @@ -8824,7 +8831,8 @@ def _prompt_custom_selection( if bios_path is not None and not dry_run and current != INSTALLED_ONLY: kept = menu_choice(files, omitted, installed, current, False)[0] kept_trees = select_data_directories({"data_directories": trees}, kept, - narrowed or current is not None) + narrowed or current is not None, + isinstance(current, GamesChoice) and current.required) if kept and all_in_place(kept, kept_trees, bios_path): # Read before the menu, as the screen reads before it asks: a # rerun was offered an install that installs nothing, then told @@ -8953,7 +8961,8 @@ def _print_menu_head( # The trees of the systems kept: 2636 data files of blueMSX, Dolphin # and PPSSPP were counted for every --system tried. kept_trees = select_data_directories({"data_directories": trees}, shown, - narrowed or current is not None) + narrowed or current is not None, + isinstance(current, GamesChoice) and current.required) data = sum(len(entry.get("files") or {}) for entry in kept_trees) data_size = sum(meta.get("size", 0) for entry in kept_trees for meta in (entry.get("files") or {}).values()) @@ -9764,6 +9773,12 @@ def games_setups(index: dict, plat: str, bios_path: Path, os_type: str = "") -> if not named: above = card_root_above(bios_path) named = above[0] if above else "" + if not named: + # --dest on a handheld's games part from a computer (ArkOS's + # EASYROMS/bios): detection never ran there, and the card's own rom + # folders sit beside the BIOS folder. + part = games_part(bios_path.parent) + named = part.name if part else "" key = _setup_key(named) if named else "" found = [name for name in sorted(setups) if key and (name == key or name.startswith(key + "-"))] if not found and plat in setups: @@ -15671,7 +15686,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: != len(manifest.get("files", [])) + len(variants if case_kept else []) or games_applied ) - data_dirs = select_data_directories(manifest, files, narrowed) + data_dirs = select_data_directories(manifest, files, narrowed, + bool(args.required_only)) # The kinds as the plan and the menu head say them (parts_words), # the data files of the folders kept among them: this head said # BIOS files and game data beside a plan of three kinds, neither @@ -21812,7 +21828,8 @@ def _plan_data(state: TuiState, check: dict) -> tuple[int, int | None]: if not _tui_estimating(state): data = check.get("data_missing", 0) return data, check.get("data_bytes") if data else None - trees = select_data_directories(state.manifest or {}, _choice_entries(state), True) + required = any(flag == "--required-only" for flag, _value in tui_flags(state)) + trees = select_data_directories(state.manifest or {}, _choice_entries(state), True, required) sizes = [meta.get("size", 0) for entry in trees for meta in (entry.get("files") or {}).values()] return len(sizes), sum(sizes) diff --git a/install.sh b/install.sh index a03428f05..fce43f99f 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="57e3cb6739a701181908773e603f17d2b10e1b0e2a5cde5cc4d5c97426da5dd4" +DEFAULT_INSTALL_SHA256="3a78bf2e5a910d0f5c21bc2dc490aa600fc3273c56ae01e9ce5061d9b3578f07" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/amberelec.json b/install/amberelec.json index 104db8e5e..c1390a71e 100644 --- a/install/amberelec.json +++ b/install/amberelec.json @@ -6,7 +6,7 @@ "platform": "amberelec", "display_name": "AmberELEC", "version": "prerelease-20261019_1835", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -77381,7 +77381,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -77615,7 +77616,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -77988,7 +77990,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true } ] } \ No newline at end of file diff --git a/install/batocera.json b/install/batocera.json index 583de52ee..c1f5c14e2 100644 --- a/install/batocera.json +++ b/install/batocera.json @@ -6,7 +6,7 @@ "platform": "batocera", "display_name": "Batocera", "version": "43.1", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -225291,7 +225291,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -225525,7 +225526,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true } ], "save_directory": { diff --git a/install/emudeck.json b/install/emudeck.json index 32f4ed6a2..3f01646ff 100644 --- a/install/emudeck.json +++ b/install/emudeck.json @@ -6,7 +6,7 @@ "platform": "emudeck", "display_name": "EmuDeck", "version": "2.7EA", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -123400,7 +123400,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -123634,7 +123635,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -124007,7 +124009,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "lrps2", diff --git a/install/emuelec.json b/install/emuelec.json index 32c8e5488..35ec58389 100644 --- a/install/emuelec.json +++ b/install/emuelec.json @@ -6,7 +6,7 @@ "platform": "emuelec", "display_name": "EmuELEC", "version": "4.8", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -83639,7 +83639,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -83873,7 +83874,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -84246,7 +84248,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true } ] } \ No newline at end of file diff --git a/install/knulli.json b/install/knulli.json index ccdbd73b9..fd6a9e45b 100644 --- a/install/knulli.json +++ b/install/knulli.json @@ -6,7 +6,7 @@ "platform": "knulli", "display_name": "KNULLI", "version": "20260511", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -157933,7 +157933,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -158167,7 +158168,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "lrps2", diff --git a/install/lakka.json b/install/lakka.json index 5866fe5db..09f9de198 100644 --- a/install/lakka.json +++ b/install/lakka.json @@ -6,7 +6,7 @@ "platform": "lakka", "display_name": "Lakka", "version": "v6.1", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "system", "detect": [ { @@ -130677,7 +130677,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -130911,7 +130912,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -131284,7 +131286,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "yaps2-resources", diff --git a/install/myminui.json b/install/myminui.json index 85de7b9bc..7cb542e4e 100644 --- a/install/myminui.json +++ b/install/myminui.json @@ -6,7 +6,7 @@ "platform": "myminui", "display_name": "MyMinUI", "version": "2026-08-09", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -18958,7 +18958,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-cheats", @@ -32688,7 +32689,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -32922,7 +32924,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "fbneo-samples", @@ -33156,7 +33159,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true } ] } \ No newline at end of file diff --git a/install/nextui.json b/install/nextui.json index 4518d3d5c..933ec3af5 100644 --- a/install/nextui.json +++ b/install/nextui.json @@ -6,7 +6,7 @@ "platform": "nextui", "display_name": "NextUI", "version": "2026-10-20", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -19366,7 +19366,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -19600,7 +19601,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true } ] } \ No newline at end of file diff --git a/install/recalbox.json b/install/recalbox.json index 9b97c9d50..6db9f7daa 100644 --- a/install/recalbox.json +++ b/install/recalbox.json @@ -6,7 +6,7 @@ "platform": "recalbox", "display_name": "Recalbox", "version": "10.1.1", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -62834,7 +62834,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -63068,7 +63069,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true } ], "save_directory": { diff --git a/install/retroarch.json b/install/retroarch.json index 11b60b965..5e89fef85 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -6,7 +6,7 @@ "platform": "retroarch", "display_name": "RetroArch", "version": "v1.22.2", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "system", "detect": [ { @@ -140789,7 +140789,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -141023,7 +141024,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -141396,7 +141398,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "ppsspp-assets", diff --git a/install/retrobat.json b/install/retrobat.json index ca7459771..aec7d58a0 100644 --- a/install/retrobat.json +++ b/install/retrobat.json @@ -6,7 +6,7 @@ "platform": "retrobat", "display_name": "RetroBat", "version": "8.3.1", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -89076,7 +89076,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -89310,7 +89311,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -89683,7 +89685,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true } ], "save_directory": { diff --git a/install/retrodeck.json b/install/retrodeck.json index b9b4b4a10..2938c951d 100644 --- a/install/retrodeck.json +++ b/install/retrodeck.json @@ -6,7 +6,7 @@ "platform": "retrodeck", "display_name": "RetroDECK", "version": "0.10.10b", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "", "detect": [ { @@ -153141,7 +153141,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -153375,7 +153376,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -153748,7 +153750,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "yaps2-resources", diff --git a/install/retropie.json b/install/retropie.json index 3db3da56e..b6ab50768 100644 --- a/install/retropie.json +++ b/install/retropie.json @@ -6,7 +6,7 @@ "platform": "retropie", "display_name": "RetroPie", "version": "4.8.12", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "BIOS", "detect": [ { @@ -146474,7 +146474,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -146708,7 +146709,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -147081,7 +147083,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "yaps2-resources", diff --git a/install/rocknix.json b/install/rocknix.json index f06bd3144..e15cda7e5 100644 --- a/install/rocknix.json +++ b/install/rocknix.json @@ -6,7 +6,7 @@ "platform": "rocknix", "display_name": "ROCKNIX", "version": "20261001", - "generated": "2026-11-05T23:59:37Z", + "generated": "2026-11-06T07:21:27Z", "base_destination": "bios", "detect": [ { @@ -109453,7 +109453,8 @@ } }, "strip_components": 1, - "archive_size": 5083767 + "archive_size": 5083767, + "optional": true }, { "ref": "fbneo-samples", @@ -109687,7 +109688,8 @@ } }, "strip_components": 0, - "archive_size": 53006540 + "archive_size": 53006540, + "optional": true }, { "ref": "freeintv-overlays", @@ -110060,7 +110062,8 @@ } }, "strip_components": 1, - "archive_size": 21311170 + "archive_size": 21311170, + "optional": true }, { "ref": "lrps2", diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index 264c08558..a72245722 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -452,6 +452,11 @@ }, "description": "Platform systems whose cores read the tree." }, + "optional": { + "type": "boolean", + "const": true, + "description": "Every core reading the tree runs without it (required: false in its profile): --required-only leaves it out." + }, "files": { "type": "object", "additionalProperties": { diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 3934830a9..c0a20376b 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -518,7 +518,8 @@ def _data_directory_members( gets it from the tree's archive. ``refs_out``, when given, maps each yielded destination to - (registry key, system ids, directory destination), for the manifest. + (registry key, system ids, directory destination, whether the core + needs the tree), for the manifest. These are not BIOS and carry no hash of their own: they are whole trees an emulator reads, refreshed from upstream, so what ships is @@ -532,7 +533,8 @@ def _data_directory_members( ref_key = dd.get("ref", "") if not ref_key or not data_registry or ref_key not in data_registry: continue - if required_only and dd.get("required") is False: + needed = dd.get("required") is not False + if required_only and not needed: continue if ref_key in installed_data_refs(platform_name): continue @@ -567,7 +569,7 @@ def _data_directory_members( ): del reclaim[full] if refs_out is not None: - refs_out[full] = (ref_key, systems, dd_prefix) + refs_out[full] = (ref_key, systems, dd_prefix, needed) yield src, full continue if full in seen_destinations or ( @@ -581,7 +583,7 @@ def _data_directory_members( if case_insensitive: seen_lower.add(full.lower()) if refs_out is not None: - refs_out[full] = (ref_key, systems, dd_prefix) + refs_out[full] = (ref_key, systems, dd_prefix, needed) yield src, full @@ -4910,7 +4912,7 @@ def generate_manifest( # Phase 3: data directories. They are not files of the collection: the # installer fetches each from its source archive and keeps the members # whose hash the list below records, the same tree the pack carries. - data_refs: dict[str, tuple[str, tuple[str, ...], str]] = {} + data_refs: dict[str, tuple[str, tuple[str, ...], str, bool]] = {} reclaim = {full: path for full, (path, _size) in cache_held.items()} data_members = list(_data_directory_members( pack_systems, data_registry, platform_name, base_dest, @@ -5021,7 +5023,7 @@ def generate_manifest( def _manifest_data_directories( members: list[tuple[str, str]], - refs: dict[str, tuple[str, tuple[str, ...], str]], + refs: dict[str, tuple[str, tuple[str, ...], str, bool]], registry: dict, base_dest: str, ) -> list[dict]: @@ -5034,11 +5036,14 @@ def _manifest_data_directories( changed since is refused rather than installed. archive_size is the size of the archive refresh_data_dirs last downloaded, what fetching the tree costs: a tarball whose tree is 10 KB is a download of 21 MB. + ``optional`` marks a tree every core reading it runs without + (``required: false``): install.py --required-only leaves it out, as the + pack built with --required-only does. """ grouped: dict[tuple[str, str], dict] = {} versions: dict[str, dict[str, dict]] = {} for src, full in members: - ref, systems, prefix = refs[full] + ref, systems, prefix, needed = refs[full] entry = registry.get(ref, {}) key = (ref, prefix) group = grouped.get(key) @@ -5072,11 +5077,19 @@ def _manifest_data_directories( size = versions[str(recorded)].get(ref, {}).get("archive_size") if isinstance(size, int) and size > 0: group["archive_size"] = size + if needed: + group["needed"] = True group["systems"].extend(s for s in systems if s not in group["systems"]) rel = full[len(prefix):].lstrip("/") if prefix else full hashes = compute_hashes(src, frozenset({"sha1"})) group["files"][rel] = {"sha1": hashes["sha1"], "size": os.path.getsize(src)} - return [grouped[key] for key in sorted(grouped)] + trees = [] + for key in sorted(grouped): + group = grouped[key] + if not group.pop("needed", False): + group["optional"] = True + trees.append(group) + return trees def _manifest_core_index( diff --git a/scripts/generate_site.py b/scripts/generate_site.py index b60fec0bd..bac803fc9 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -32,7 +32,7 @@ import urllib.parse import urllib.request import zipfile from collections import Counter -from collections.abc import Callable, Iterator +from collections.abc import Callable, Iterable, Iterator from datetime import datetime, timezone from pathlib import Path from typing import TypeVar @@ -92,6 +92,7 @@ import sitemarkup as markup import siteentries import siteinputs import retired_files +import siteagents import sitestarlight from siteexports import generate_exports_page, load_export_reports from siteguides import ( @@ -7907,7 +7908,7 @@ def generate_emulators_index(profiles: dict) -> str: return "\n".join(lines) + "\n" -def _snippet_noun(names, firmware: int, fetched: int) -> str: +def _snippet_noun(names: Iterable[str], firmware: int, fetched: int) -> str: """What an emulator's files are called in its snippet. A Switch emulator reads keys, and "eden keys" is what readers search for; Ship of Harkinian reads game ROMs, which are no BIOS.""" @@ -12368,6 +12369,9 @@ def _generate(args: argparse.Namespace) -> None: stats = compute_stats(db, coverages, profiles) stats["composition"] = compute_composition(db) write_if_changed(str(public / "stats.json"), generate_stats(stats)) + # llms.txt and the ARD manifest an agent reads first, from the registry. + siteagents.write(public, sorted(((name, cov["platform"]) for name, cov in coverages.items()), + key=lambda row: row[1].casefold()), write_if_changed) # Computed once: the gap analysis page and the published gaps export must # report the same files. diff --git a/scripts/siteagents.py b/scripts/siteagents.py new file mode 100644 index 000000000..ffed73260 --- /dev/null +++ b/scripts/siteagents.py @@ -0,0 +1,225 @@ +"""What the site tells an AI agent about itself. + +- ``llms.txt`` (https://llmstxt.org): the pages and data an agent should + read first, in Markdown, at the site's root. +- ``skills/setup-bios/SKILL.md``: an Agent Skill that sets up a device's + BIOS files with the fewest downloads. +- ``ard.json``: an Agentic Resource Discovery manifest + (https://github.com/ards-project/ard-spec) listing that skill and the + JSON API, linked from every page's head with ``rel="ard"``. + +Both are written from the registry at each build, so a new platform or a +moved page cannot leave them stale. The WebMCP tools a page registers live +in website/src/scripts/agent-tools.ts. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable +from pathlib import Path + +SITE = "https://abdess.github.io/retrobios/" +PUBLISHER = "abdess.github.io" +SH = "curl -fsSL https://abdess.github.io/retrobios/i | sh -s --" +PS = "& ([scriptblock]::Create((irm https://abdess.github.io/retrobios/i.ps1)))" +# What a folder put into a command may not hold, as agent-tools.ts refuses +# it: PowerShell also ends a single-quoted string on the typographic quotes. +REFUSED = "any quote, straight or typographic (' \" ‘ ’ ‚ ‛ “ ” „), a backtick or a control character" +QUOTING = (f"A folder goes in single quotes, attached to its option. One that holds {REFUSED}, " + "or starts with `-`, is a question for the user, never a value to escape into the " + "command. On a page of the site, the `device_setup_command` WebMCP tool writes the " + "command and refuses such a folder.") + +START_PAGES = ( + ("Which pack do I need", "which-pack/", "pick a device or setup, get the folder its BIOS files go in, " + "one command that installs and checks them, or a ZIP pack"), + ("First install", "wiki/first-install/", "the steps from nothing to a working BIOS folder"), + ("What are BIOS files", "wiki/what-are-bios-files/", "why emulators need them and where they come from"), + ("Troubleshooting", "wiki/troubleshooting/", "a file the emulator does not find or refuses"), + ("Common questions", "wiki/faq/", "short answers"), +) + +DATA = ( + ("API index", "api/v1/index.json", "every dataset of the JSON API, versioned"), + ("Catalog", "api/v1/catalog.json", "each dataset, manifest, export and download with its size, SHA-256 and schema"), + ("Emulators", "api/v1/emulators.json", "every emulator profile: the files it reads, required or not, " + "their hashes and the source line that reads each. One document " + "per emulator at api/v1/emulators/.json"), + ("Core names", "api/v1/cores.json", "each core name and alias, with the profile it designates"), + ("Systems", "api/v1/systems.json", "each system id, its name and the section holding its files"), + ("Platforms", "api/v1/platforms.json", "each setup (RetroArch, Batocera...), its BIOS folder and files"), + ("Hash lookup", "api/v1/lookup/00.json", "what a hash (MD5, SHA-1, SHA-256, CRC32) designates, sharded " + "by its first two hex characters: api/v1/lookup/.json"), +) + +MORE = ( + ("Emulators", "emulators/", "one page per emulator"), + ("Systems", "systems/", "one page per console maker, every file with its hashes"), + ("Installer", "wiki/installer/", "every option of install.py"), + ("What we count", "wiki/privacy/", "the site's cookie-free counting"), + ("Is it legal?", "wiki/legal/", "the project's position"), + ("Contributing", "contributing/", "add a missing file or an emulator profile"), +) + + +def llms_txt(platforms: list[tuple[str, str]]) -> str: + """The site's llms.txt. ``platforms`` lists (id, display name) of every + registered setup, each with its page.""" + lines = [ + "# RetroBIOS", + "", + "> BIOS and firmware files for emulators, each checked against the emulator's source " + "code: which files an emulator reads, whether it starts without them, where each one " + "goes on every setup, their hashes, and an installer that puts them in place.", + "", + "Every claim about a file cites the line of emulator source code that reads it. The " + "files themselves are served by the GitHub repository " + "(https://github.com/Abdess/retrobios) and its releases.", + "", + "## Setting up a device for its user", + "", + "Most people want to play now, with the fewest files. The installer does that in one " + "command: it finds the setup and its BIOS folder, reads the games on the card, and with " + "`--for-games --required-only` installs only the files those games cannot start " + "without. Without these two options it installs every file of the setup's list " + "(gigabytes), even with `--yes`.", + "", + f"1. Preview, nothing written: `{SH} --for-games --required-only --dry-run`", + f"2. Install, no questions: `{SH} --for-games --required-only --yes`", + f"3. On Windows (PowerShell): `{PS} --for-games --required-only --yes`", + "", + "- Run it on the device itself (its terminal or SSH), or on a computer with the device's " + "card plugged in. The installer recognises the handheld systems on the card, among them " + "ArkOS, dArkOS, ArchR and UnofficialOS (R36S and its clones), muOS, KNULLI, ROCKNIX, " + "AmberELEC, EmuELEC, Onion, spruceOS and MinUI. " + "The [Linux handhelds page](" + SITE + "platforms/linux-handhelds/) maps each device " + "to its system and folder.", + "- From a computer, if the card is not found, name its BIOS folder: `--dest='/bios'` " + "(ArkOS and dArkOS: the `bios` folder of the EASYROMS part).", + "- A games folder somewhere else: `--for-games=''`, one subfolder per system.", + f"- {QUOTING}", + "- Later, to see what is missing or wrong: `--check`. Machine-readable progress: " + "`--events` (JSON lines, one per event, documented on the installer page).", + "- A game the installer cannot identify gets every BIOS of its system, and the run names it.", + f"- The same steps as an Agent Skill: [SKILL.md]({SITE}{SKILL_PATH}).", + "", + "## Start", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in START_PAGES), + "", + "## Setups", + "", + *(f"- [{name}]({SITE}platforms/{pid}/): its BIOS folder and files" for pid, name in platforms), + "", + "## Data", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in DATA), + "", + "## Optional", + "", + *(f"- [{title}]({SITE}{path}): {what}" for title, path, what in MORE), + ] + return "\n".join(lines) + "\n" + + +SKILL_PATH = "skills/setup-bios/SKILL.md" +# The Agent Skills media type ARD names: Markdown with this profile. +SKILL_TYPE = 'text/markdown; profile="urn:air:agent-skills"' + + +def skill_md() -> str: + """An Agent Skill (SKILL.md: YAML front matter, then instructions) that + sets up a device's BIOS files with the fewest downloads.""" + return "\n".join([ + "---", + "name: setup-bios", + "description: Install the BIOS and firmware files a retrogaming device or computer needs " + "(RetroArch, Batocera, EmuDeck, ArkOS, dArkOS, muOS, KNULLI, ROCKNIX and others), only " + "what the games on its card need to start. Use when a user asks to set up emulators, " + "a handheld (R36S, Miyoo, Anbernic), BIOS files or firmware.", + "---", + "", + "# Set up BIOS files with RetroBIOS", + "", + "Most users want to play now, with the fewest files. Install only what the games on the " + "card need, unless the user asks for everything.", + "", + "## Steps", + "", + "1. Ask where the games are: on the device itself (you have its terminal or SSH), or on " + "its card plugged into this computer.", + f"2. Preview, nothing is written: `{SH} --for-games --required-only --dry-run` " + f"(Windows PowerShell: `{PS} --for-games --required-only --dry-run`).", + "3. Show the user the plan it prints: the setup found, the BIOS folder, the files and " + "their size. Ask before going on if the setup or the folder is not the one expected.", + "4. Install with the same command, `--yes` in place of `--dry-run`.", + "5. Report the summary line. A game the installer could not identify got every BIOS of " + "its system: say which.", + "", + "## When the setup or the card is not found", + "", + "- From a computer, name the card's BIOS folder: `--dest='/bios'`. ArkOS and dArkOS " + "keep it on the EASYROMS part.", + "- Games outside the card: `--for-games=''`, one subfolder per system.", + "- A setup not detected: `--platform=`, a name exactly as `--list-platforms` prints " + "it (letters, digits, `-` and `_`).", + f"- {QUOTING}", + "", + "## Later", + "", + "- `--check` says what is missing or wrong, without downloading.", + "- `--events` writes the run as JSON lines, for reading by a program.", + "", + f"Every option: {SITE}wiki/installer/. Devices and their folders: " + f"{SITE}platforms/linux-handhelds/. The site's guide for agents: {SITE}llms.txt.", + "", + ]) + + +def ard_manifest() -> dict: + """The ARD manifest: the setup skill and the JSON API, each with the + questions it answers. llms.txt is found at the site root, as its + convention says.""" + return { + "entries": [ + { + "identifier": f"urn:air:{PUBLISHER}:retrobios:setup-bios", + "displayName": "Set up BIOS files with RetroBIOS", + "type": SKILL_TYPE, + "url": f"{SITE}{SKILL_PATH}", + "description": "An agent skill that installs the BIOS files a device needs, only " + "what the games on its card need to start.", + "representativeQueries": [ + "install the minimum BIOS files for the games on my R36S running dArkOS", + "set up BIOS files for RetroArch on my Steam Deck", + "my emulator says BIOS missing, fix it", + ], + "capabilities": ["SetupBios"], + "tags": ["bios", "firmware", "emulation", "retrogaming"], + }, + { + "identifier": f"urn:air:{PUBLISHER}:retrobios:api", + "displayName": "RetroBIOS JSON API", + "type": "application/json", + "url": f"{SITE}api/v1/index.json", + "description": "Versioned JSON datasets: emulator profiles, platforms, systems, core " + "names and a hash lookup.", + "representativeQueries": [ + "what file has the MD5 490f666e1afb15b7362b406ed1cea246", + "which files does the beetle_psx core read", + "where do PlayStation BIOS files go on Batocera", + ], + "capabilities": ["HashLookup", "EmulatorFiles", "PlatformFiles"], + "tags": ["api", "json", "bios"], + }, + ] + } + + +def write(public: Path, platforms: list[tuple[str, str]], + write_if_changed: Callable[[str, str], object]) -> None: + write_if_changed(str(public / "llms.txt"), llms_txt(platforms)) + (public / SKILL_PATH).parent.mkdir(parents=True, exist_ok=True) + write_if_changed(str(public / SKILL_PATH), skill_md()) + write_if_changed(str(public / "ard.json"), json.dumps(ard_manifest(), indent=1) + "\n") diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 555bbd50a..9cddfef64 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -6923,9 +6923,11 @@ class RequiredOnlyReachesDataDirectories(unittest.TestCase): def test_every_tree_walk_reads_the_flag(self): source = (ROOT / "scripts" / "generate_pack.py").read_text(encoding="utf-8") - self.assertEqual( - source.count('if required_only and dd.get("required") is False:'), 2 - ) + # The pack's walk and the manifest's, which records the flag for + # install.py --required-only. + self.assertEqual(source.count('if required_only and dd.get("required") is False:'), 1) + self.assertIn('needed = dd.get("required") is not False\n if required_only and not needed:', + source) self.assertIn("seen_parents, required_only=required_only,", source) self.assertIn("data_refs, required_only=required_only,", source) diff --git a/tests/test_install.py b/tests/test_install.py index 9effcdd49..7e6689029 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3880,6 +3880,20 @@ class DataDirectoriesAreInstalled(unittest.TestCase): }, } + def test_required_only_leaves_out_an_optional_tree(self): + """A GBA and a SNES game with --for-games --required-only took + FBNeo's cheats and samples, 3,472 files no game opened.""" + cheats = {**self._entry(), "ref": "fbneo-cheats", "systems": ["snes"], "optional": True} + sys_tree = {**self._entry(), "systems": ["snes"]} + manifest = {"data_directories": [cheats, sys_tree]} + files = [{"dest": "a.rom", "systems": ["snes"]}] + self.assertEqual(install.select_data_directories(manifest, files, True), [cheats, sys_tree]) + self.assertEqual(install.select_data_directories(manifest, files, True, True), [sys_tree]) + self.assertEqual(install.select_data_directories(manifest, files, False, True), [sys_tree]) + install._check_data_directory(cheats, "data_directories[0]") + with self.assertRaises(ValueError): + install._check_data_directory({**cheats, "optional": False}, "data_directories[0]") + def test_the_archive_is_drawn_and_counted(self): """82 MB of archives came with no progress line, and the run ended on "310.5 KB downloaded", the BIOS files alone.""" diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index a7584e71b..f8a7eea48 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -12912,6 +12912,18 @@ class DriveIsACardByItsFiles(unittest.TestCase): self.assertEqual([install.card_system(mount) for mount in mounts], [None, None, None]) + def test_a_dest_on_a_games_part_reads_its_rom_folders(self): + """--dest EASYROMS/bios --for-games from a computer: detection never + ran on the card, the generic RetroArch folders were tried, and the + run kept every file of the list.""" + arkos = games_part(self.root / "media" / "EASYROMS") + index = {"setups": {"arkos": {}, "retroarch": {}}} + with mock.patch.object(install, "_mount_points", return_value=frozenset()), \ + mock.patch.object(install, "_drive_not_removable", return_value=False): + self.assertEqual(install.games_setups(index, "retroarch", arkos / "bios"), ["arkos"]) + self.assertEqual(install.games_setups(index, "retroarch", self.root / "bios"), + ["retroarch"]) + def test_a_card_beside_a_data_drive_comes_first(self): """The ArkOS card and the data drive mounted, the data drive sorted first and took Enter. The card comes first in diff --git a/tests/test_site_agents.py b/tests/test_site_agents.py new file mode 100644 index 000000000..ac081c6b0 --- /dev/null +++ b/tests/test_site_agents.py @@ -0,0 +1,140 @@ +"""What the site tells an AI agent: llms.txt, the ARD manifest, the WebMCP tools. + +An agent asked to set up a handheld reads these first. They must lead it to +the fewest files the games need, and every address they give must exist. +""" +from __future__ import annotations + +import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache + +import re +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import siteagents # noqa: E402 + +DIST = REPO_ROOT / "website" / "dist" +TOOLS = REPO_ROOT / "website" / "src" / "scripts" / "agent-tools.ts" +FOOTER = REPO_ROOT / "website" / "src" / "components" / "Footer.astro" +CONFIG = REPO_ROOT / "website" / "astro.config.mjs" + + +class TheGuideLeadsToTheFewestFiles(unittest.TestCase): + def setUp(self) -> None: + self.text = siteagents.llms_txt([("retroarch", "RetroArch"), ("batocera", "Batocera")]) + + def test_it_is_llms_txt(self) -> None: + self.assertTrue(self.text.startswith("# RetroBIOS\n\n> ")) + self.assertIn("\n## Setting up a device for its user\n", self.text) + + def test_every_command_installs_only_what_the_games_need(self) -> None: + commands = re.findall(r"`((?:curl|&) [^`]+)`", self.text) + self.assertGreaterEqual(len(commands), 3) + for command in commands: + with self.subTest(command=command): + self.assertIn("--for-games --required-only", command) + + def test_each_setup_has_its_page(self) -> None: + self.assertIn("(https://abdess.github.io/retrobios/platforms/batocera/)", self.text) + + @unittest.skipUnless((DIST / "llms.txt").is_file(), "website/dist is not built") + def test_every_address_of_the_built_guide_exists(self) -> None: + built = (DIST / "llms.txt").read_text(encoding="utf-8") + for url in re.findall(r"\((https://abdess\.github\.io/retrobios/[^)]*)\)", built): + with self.subTest(url=url): + target = DIST / url.removeprefix(siteagents.SITE) + self.assertTrue(target.is_file() or (target / "index.html").is_file(), url) + + +class TheSkillIsAnAgentSkill(unittest.TestCase): + """ARD reserves text/markdown for agent skills (Lighthouse warns on a + plain Markdown entry): the setup steps are published as one.""" + + def test_front_matter_then_instructions(self) -> None: + import yaml # noqa: PLC0415 + + text = siteagents.skill_md() + head, _, body = text.removeprefix("---\n").partition("\n---\n") + front = yaml.safe_load(head) + self.assertEqual(set(front), {"name", "description"}) + self.assertRegex(front["name"], r"^[a-z0-9-]{1,64}$") + self.assertLessEqual(len(front["description"]), 1024) + self.assertTrue(body.lstrip().startswith("# ")) + for command in re.findall(r"`((?:curl|&) [^`]+)`", body): + with self.subTest(command=command): + self.assertIn("--for-games --required-only", command) + + def test_a_folder_is_quoted_and_a_hostile_one_asked_about(self) -> None: + for text in (siteagents.skill_md(), siteagents.llms_txt([])): + with self.subTest(text=text[:20]): + self.assertIn("--dest='/bios'", text) + self.assertIn("--for-games=''", text) + self.assertIn("starts with `-`", text) + # The characters agent-tools.ts refuses, typographic quotes included. + # The backtick is named in words: Markdown would read it. + self.assertIn("a backtick", siteagents.REFUSED) + for char in "'\"‘’‚‛“”„": + self.assertIn(char, siteagents.REFUSED) + self.assertIn(char, TOOLS.read_text(encoding="utf-8")) + self.assertIn(siteagents.REFUSED, text) + + def test_the_manifest_types_it_as_a_skill(self) -> None: + skill = siteagents.ard_manifest()["entries"][0] + self.assertEqual(skill["type"], 'text/markdown; profile="urn:air:agent-skills"') + self.assertTrue(skill["url"].endswith(siteagents.SKILL_PATH)) + + +class TheManifestIsAnArdManifest(unittest.TestCase): + """The terms ard-entry.schema.json requires (ards-project/ard-spec).""" + + def test_entries(self) -> None: + entries = siteagents.ard_manifest()["entries"] + self.assertTrue(entries) + for entry in entries: + with self.subTest(entry=entry.get("identifier")): + self.assertRegex(entry["identifier"], r"^urn:air:[a-zA-Z0-9.-]+(:[a-zA-Z0-9._-]+)+$") + self.assertTrue(entry["displayName"]) + self.assertRegex(entry["type"], r'^[a-z]+/[\w.+-]+(; [a-z]+="[^"]+")?$') + self.assertTrue(("url" in entry) != ("data" in entry)) + self.assertTrue(2 <= len(entry["representativeQueries"]) <= 5) + self.assertTrue(entry["url"].startswith(siteagents.SITE)) + + def test_every_page_links_it(self) -> None: + self.assertIn("rel: 'ard'", CONFIG.read_text(encoding="utf-8")) + + +class TheToolsReadOnly(unittest.TestCase): + def setUp(self) -> None: + self.source = TOOLS.read_text(encoding="utf-8") + + def test_every_page_registers_them(self) -> None: + self.assertIn("import '../scripts/agent-tools.ts';", FOOTER.read_text(encoding="utf-8")) + names = re.findall(r"^\s*name: '([a-z_]+)'", self.source, re.M) + self.assertEqual(names, ["lookup_bios_hash", "emulator_bios_files", "search_retrobios", + "device_setup_command"]) + self.assertEqual(self.source.count("readOnlyHint: true"), len(names)) + + def test_the_setup_command_defaults_to_the_games(self) -> None: + self.assertIn("const games = everything ? [] :", self.source) + self.assertIn("'--required-only'", self.source) + + def test_a_value_that_could_escape_the_command_is_refused(self) -> None: + """The command is pasted into a shell: PowerShell also ends a single-quoted + string on the typographic quotes, and a path starting with "-" reads as + an option.""" + self.assertIn("const unsafe = /[\\u0000-\\u001f\\u007f'\"`‘’‚‛“”„]/;", self.source) + self.assertIn("value.startsWith('-')", self.source) + self.assertIn("/^[a-z0-9_-]+$/.test(platform)", self.source) + self.assertIn("`--dest=${quote(bios_folder as string)}`", self.source) + + def test_they_reach_no_other_site(self) -> None: + code = re.sub(r"^\s*//.*$", "", self.source, flags=re.M) + self.assertEqual(set(re.findall(r"https://[\w.-]+", code)), {"https://abdess.github.io"}) + + +if __name__ == "__main__": + unittest.main() diff --git a/website/astro.config.mjs b/website/astro.config.mjs index d5e45fa32..b2a832f29 100644 --- a/website/astro.config.mjs +++ b/website/astro.config.mjs @@ -77,6 +77,9 @@ export default defineConfig({ // Ownership of the URL-prefix property https://abdess.github.io/retrobios/ // in Google Search Console. Removing it unverifies the property. { tag: 'meta', attrs: { name: 'google-site-verification', content: '8SueST3hNHl8_vM-OJJP0rz-ZMlI2DALLpChrwZoGMs' } }, + // What an AI agent reads first (scripts/siteagents.py): the ARD manifest lists + // llms.txt and the JSON API. + { tag: 'link', attrs: { rel: 'ard', type: 'application/json', href: `${BASE}ard.json` } }, ], social: [{ icon: 'github', label: 'RetroBIOS on GitHub', href: 'https://github.com/Abdess/retrobios' }], customCss: ['./src/styles/retrobios.css'], diff --git a/website/src/components/Footer.astro b/website/src/components/Footer.astro index ba5c39970..05853598f 100644 --- a/website/src/components/Footer.astro +++ b/website/src/components/Footer.astro @@ -48,6 +48,7 @@ const date = releaseDate(tag);