From e54d146954f45b704fbba69d9fca6fe9cb410583 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:41:34 +0100 Subject: [PATCH 01/36] fix: keep install lists unfiltered on --target --- scripts/pipeline.py | 5 +++-- tests/test_target_names.py | 22 ++++++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/scripts/pipeline.py b/scripts/pipeline.py index 0847da15f..38d59abc6 100644 --- a/scripts/pipeline.py +++ b/scripts/pipeline.py @@ -1008,8 +1008,9 @@ def main() -> None: # Step 2 refreshed the trees once: refreshing again here let the # pack and the manifest describe two revisions of a moving tree. manifest_cmd.append("--no-refresh-data") - if args.target: - manifest_cmd.extend(["--target", args.target]) + # No --target here: install/ holds the published lists, and every + # reader (site picker, release check, record) takes each file there + # as a whole platform. The per-target core lists are step 4c. ok, _ = run(manifest_cmd, STEP_LABELS["generate_manifests"]) results["generate_manifests"] = ok all_ok = all_ok and ok diff --git a/tests/test_target_names.py b/tests/test_target_names.py index 30e5d012e..263a555d0 100644 --- a/tests/test_target_names.py +++ b/tests/test_target_names.py @@ -194,3 +194,25 @@ class ASingleTargetFiltersNothing(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +class PublishedListsAreNotNarrowed(unittest.TestCase): + """pipeline --target switch wrote install/retroarch_nintendoswitch.json + next to the published lists; the site picker then replaced RetroArch's + file list with the Switch one and check_release asked for it as an asset.""" + + def test_pipeline_writes_install_unfiltered(self): + pipeline = (REPO_ROOT / "scripts" / "pipeline.py").read_text() + self.assertNotIn('manifest_cmd.extend(["--target"', pipeline) + + def test_every_install_list_is_a_whole_platform(self): + import json # noqa: PLC0415 + + for path in sorted((REPO_ROOT / "install").glob("*.json")): + data = json.loads(path.read_text(encoding="utf-8")) + with self.subTest(path=path.name): + self.assertEqual(path.stem, data["platform"]) + self.assertNotIn("target", data) + self.assertNotIn("no_game_data", data) + self.assertEqual(data["source"], "full") + self.assertEqual(data["regions"], []) From ba150de46913746dd99bd30d95165382564bb543 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:43:55 +0100 Subject: [PATCH 02/36] fix: name RomM's size rule in every check label --- README.md | 38 ++++++++++++++++----------------- scripts/generate_readme.py | 9 ++------ scripts/generate_site.py | 23 ++++++-------------- scripts/nativemode.py | 12 +++++++++++ tests/test_site_r2_reference.py | 16 ++++++++++++++ 5 files changed, 56 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 0df5cf0f0..c5db8e4a7 100644 --- a/README.md +++ b/README.md @@ -99,24 +99,24 @@ Full list with per-file details: **[https://abdess.github.io/retrobios/](https:/ | Platform | On its BIOS list | Files its emulators load | Checked by | |----------|-----------------:|-------------------------:|------------| -| AmberELEC | 414/434 | 4,751/4,935 | MD5 hash | -| Batocera | 353/353 | 14,171/15,291 | MD5 hash | -| BizHawk | 118/118 | - | SHA1 hash | -| EmuDeck | 168/168 | 8,422/8,525 | MD5 hash | -| EmuELEC | 544/572 | 4,786/10,215 | MD5 hash | -| KNULLI | 290/290 | 9,126/15,349 | MD5 hash | -| Lakka | 532/532 | 7,910/7,996 | file presence | -| MiSTer FPGA | 81/81 | - | MD5 hash | -| MinUI | 0/0 | 78/79 | file presence | -| MyMinUI | 0/0 | 356/367 | file presence | -| NextUI | 0/0 | 343/350 | file presence | -| ROCKNIX | 38/38 | 7,421/9,134 | MD5 hash | -| Recalbox | 351/351 | 3,125/3,344 | MD5 hash | -| RetroArch | 526/526 | 7,919/7,987 | file presence | -| RetroBat | 348/348 | 5,133/9,658 | MD5 hash | -| RetroDECK | 2,007/2,008 | 6,703/6,991 | MD5 hash | -| RetroPie * | 532/532 | 8,120/12,775 | file presence | -| RomM | 383/383 | 83/873 | MD5 hash | +| AmberELEC | 414/434 | 4,751/4,935 | MD5 | +| Batocera | 353/353 | 14,171/15,291 | MD5 | +| BizHawk | 118/118 | - | SHA-1 | +| EmuDeck | 168/168 | 8,422/8,525 | MD5 | +| EmuELEC | 544/572 | 4,786/10,215 | MD5 | +| KNULLI | 290/290 | 9,126/15,349 | MD5 | +| Lakka | 532/532 | 7,910/7,996 | name only | +| MiSTer FPGA | 81/81 | - | MD5 | +| MinUI | 0/0 | 78/79 | name only | +| MyMinUI | 0/0 | 356/367 | name only | +| NextUI | 0/0 | 343/350 | name only | +| ROCKNIX | 38/38 | 7,421/9,134 | MD5 | +| Recalbox | 351/351 | 3,125/3,344 | MD5 | +| RetroArch | 526/526 | 7,919/7,987 | name only | +| RetroBat | 348/348 | 5,133/9,658 | MD5 | +| RetroDECK | 2,007/2,008 | 6,703/6,991 | MD5 | +| RetroPie * | 532/532 | 8,120/12,775 | name only | +| RomM | 383/383 | 83/873 | size, then any hash | Each fraction is what the pack has over what is needed, counting required and optional files alike since both ship. The first column is the BIOS list the platform publishes. The second counts files its emulators load that this list never mentions, found by reading their source code, and it is routinely several times larger. A short fraction means files are still missing, and they are named in the [gap analysis](https://abdess.github.io/retrobios/gaps/). That second number is a floor, not a ceiling: an emulator that accepts any file handed to it names none in its code, so nothing there can be counted. @@ -203,4 +203,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-11-03T17:50:12Z* +*Auto-generated on 2026-11-05T10:43:18Z* diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index aba4e9c9f..1d32b975e 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -34,6 +34,7 @@ from common import ( unique_emulator_profiles, write_if_changed, ) +from nativemode import check_label import release_record from siteguides import install_command from verify import verify_platform @@ -737,15 +738,9 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" ] ) - mode_labels = { - "md5": "MD5 hash", - "sha1": "SHA1 hash", - "existence": "file presence", - } - for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): display = f"{cov['platform']} *" if name in archived else cov["platform"] - checked = mode_labels.get(cov["mode"], cov["mode"]) + checked = check_label(cov["mode"], cov["config"].get("hash_rule")) core_total = ( cov["core_present"] + cov["core_missing"] + cov["core_unsourceable"] ) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 2a763163b..461cadd0a 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -64,7 +64,7 @@ from common import ( from artifacts import ArtifactLockBusy, artifact_lock from artifacts import json_text as _json_text from cross_reference import FileConsumers, held_copy -from nativemode import digest_algorithm, reads_file_contents +from nativemode import check_label, digest_algorithm, reads_file_contents @@ -3194,8 +3194,6 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st SETUP_TABLE_PAGE = "wiki/verification-modes.md" SETUP_TABLE_ANCHOR = "every-setup-at-a-glance" SETUP_TABLE_LINK = f"{SETUP_TABLE_PAGE}#{SETUP_TABLE_ANCHOR}" -# How a setup checks a file, in the words of a reader. -_HOW_IT_CHECKS = {"existence": "name only", "md5": "MD5", "sha1": "SHA-1"} def setup_table(coverages: dict, registry: dict | None = None) -> list[str]: @@ -3220,7 +3218,7 @@ def setup_table(coverages: dict, registry: dict | None = None) -> list[str]: lines.append( f"| [{cov['platform']}](../platforms/{name}.md)" f"{' (archived)' if status == 'archived' else ''} | {listed} | {zipped} | " - f"{_HOW_IT_CHECKS.get(cov['mode'], cov['mode'])} |" + f"{check_label(cov['mode'], cov['config'].get('hash_rule'))} |" ) lines += [ "", @@ -4292,9 +4290,8 @@ def _platform_guide(name: str, display: str, cov: dict, registry: dict, no_list: return lines -# How a platform checks a file, in words. -_CHECK_WORDS = {"existence": "name only", "md5": "MD5", "sha1": "SHA-1", "sha256": "SHA-256", - "crc32": "CRC32"} +# A hash a platform's list writes, in words. +_HASH_WORDS = {"md5": "MD5", "sha1": "SHA-1", "sha256": "SHA-256", "crc32": "CRC32"} def generate_platform_page( @@ -4374,7 +4371,7 @@ def generate_platform_page( *_stat(f"{cov['present']}/{cov['total']}", f"Coverage ({pct})"), *_stat(str(cov["verified"]), "Verified"), *_stat(str(cov["missing"]), "Missing"), - *_stat(f'{_CHECK_WORDS.get(mode, mode)}', + *_stat(f'{check_label(mode, config.get("hash_rule"))}', "Checked by"), "", "", @@ -4384,7 +4381,7 @@ def generate_platform_page( # The hash the list writes, when it is not the one the platform checks: # RetroArch checks presence and its System.dat writes sha1. if hash_type and hash_type != mode: - lines.append(f"| Hashes in the list | {_CHECK_WORDS.get(hash_type, hash_type)} |") + lines.append(f"| Hashes in the list | {_HASH_WORDS.get(hash_type, hash_type)} |") if version: lines.append(f"| Version | {version} |") # base_destination is only the root the platform's own list writes its @@ -10409,12 +10406,6 @@ def generate_gap_analysis( "|----------|-----------------:|-------------------------:|------------|", ]) - mode_labels = { - "md5": "MD5 hash", - "sha1": "SHA1 hash", - "existence": "file presence", - } - for pname, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]): display = cov["platform"] core_total = ( @@ -10424,7 +10415,7 @@ def generate_gap_analysis( f"| [{display}](platforms/{pname}.md) " f"| {_ratio_cell(cov['present'], cov['total'], _missing_required_firmware(cov))} " f"| {_ratio_cell(cov['core_present'], core_total)} " - f"| {mode_labels.get(cov['mode'], cov['mode'])} |" + f"| {check_label(cov['mode'], cov['config'].get('hash_rule'))} |" ) lines.extend([ "", diff --git a/scripts/nativemode.py b/scripts/nativemode.py index eae8015df..24cd56549 100644 --- a/scripts/nativemode.py +++ b/scripts/nativemode.py @@ -40,6 +40,18 @@ SIZE_AND_ANY_HASH = "size_and_any_hash" HASH_RULES = (SIZE_AND_ANY_HASH,) +#: How a platform checks a file, in a reader's words. Every table and badge +#: that says it reads this, so a page cannot call RomM's rule an MD5 check. +_CHECK_LABELS = {"existence": "name only", "md5": "MD5", "sha1": "SHA-1"} + + +def check_label(mode: str | None, hash_rule: str | None = None) -> str: + """The words for what a platform's own check compares.""" + if hash_rule == SIZE_AND_ANY_HASH: + return "size, then any hash" + return _CHECK_LABELS[normalize(mode)] + + def normalize(mode: str | None) -> str: """Return a declared mode, falling back to the schema default.""" return mode if mode in MODES else DEFAULT_MODE diff --git a/tests/test_site_r2_reference.py b/tests/test_site_r2_reference.py index cd6f9aaee..4a12df33e 100644 --- a/tests/test_site_r2_reference.py +++ b/tests/test_site_r2_reference.py @@ -201,6 +201,22 @@ class PlatformReferenceInWords(unittest.TestCase): self.assertIn("[r repository at commit aaaaaaaaaaaa](https://github.com/o/r/tree/" + "a" * 40 + ")", page) + def test_every_check_label_reads_the_hash_rule(self): + """RomM's badge and table rows said MD5 beside a sentence saying it + accepts the size and any one hash.""" + details = [{"name": "f.bin", "status": "ok", "system": "sony-playstation"}] + cov = {"platform": "RomM", "details": details, "total": 1, "verified": 1, + "present": 1, "missing": 0, "untested": 0, "mode": "md5", + "core_present": 0, "core_missing": 0, "core_unsourceable": 0, + "config": {"hash_type": "md5", "hash_rule": "size_and_any_hash", + "systems": {"sony-playstation": {"files": [{"name": "f.bin"}]}}, + "source": "https://github.com/o/r/tree/" + "a" * 40}} + page = generate_site.generate_platform_page("romm", cov, {}) + self.assertIn(">size, then any hash", page) + self.assertNotIn(">MD5", page) + table = "\n".join(generate_site.setup_table({"romm": cov}, {})) + self.assertIn("| size, then any hash |", table) + def test_systems_by_name_with_the_id_in_small_text(self): systems = {"sony-playstation": {"files": [{"name": "f.bin"}]}} page = self._page("md5", "md5", systems) From ecb56c43e59a4c00b1f7145aa5168a16485ac631 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:45:24 +0100 Subject: [PATCH 03/36] fix: link every platform's launcher citations --- scripts/generate_site.py | 16 +++++----------- tests/test_site_r2_reference.py | 21 ++++++++++++++++++++- wiki/adding-a-platform.md | 2 +- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 461cadd0a..731dee433 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3401,20 +3401,14 @@ def _served_line(f: dict) -> str: f"(sha1 `{sha1[:8]}`)") -# Platforms whose launcher records are read from the repository and revision -# of their BIOS list: batocera_scraper reads configgen at its tag, -# retrobat_scraper reads emulatorLauncher at the list's commit -# (launcher_revision). The others read another repository. -_LAUNCHERS_IN_LIST_REPO = frozenset({"batocera", "retrobat"}) - - -def _launcher_refs(platform: str, config: dict) -> dict[str, list[str]]: +def _launcher_refs(config: dict) -> dict[str, list[str]]: """Entry name, casefolded -> each launcher record that opens it, rendered.""" from launchers import _names as launcher_names from sitesources import launcher_ref_markdown, list_source - source = (list_source(str(config.get("source") or "")) - if platform in _LAUNCHERS_IN_LIST_REPO else None) + # Every scraper reads its launchers in the repository and at the revision + # of the list it scrapes, so `source` pins each source_ref. + source = list_source(str(config.get("source") or "")) found: dict[str, list[str]] = {} for record in config.get("launchers") or []: if not isinstance(record, dict) or not record.get("source_ref"): @@ -3626,7 +3620,7 @@ def _render_platform_systems( # Per-system detail sections (collapsible for large platforms) use_collapsible = len(by_system) > 10 by_system_declared = _declared_by_system(config_files) - launched = _launcher_refs(platform[0] if platform else "", config or {}) + launched = _launcher_refs(config or {}) sections: list[dict] = [] for sys_id, files in sorted(by_system.items()): diff --git a/tests/test_site_r2_reference.py b/tests/test_site_r2_reference.py index 4a12df33e..24eb2c4b5 100644 --- a/tests/test_site_r2_reference.py +++ b/tests/test_site_r2_reference.py @@ -129,13 +129,32 @@ class EvidenceLinksItsSource(unittest.TestCase): evidence = {"read_by": [{"emulator": "launcher", "proof": "launcher", "entries": [], "runs_system": True}]} config = load_platform_config("batocera", PLATFORMS) - launched = generate_site._launcher_refs("batocera", config)["scph1001.bin"] + launched = generate_site._launcher_refs(config)["scph1001.bin"] text = generate_site._category_evidence_md(evidence, launched) self.assertNotIn("(by launcher)", text) self.assertIn("https://github.com/batocera-linux/batocera.linux/blob/batocera-43.1/" "package/batocera/core/batocera-configgen/", text) self.assertIn("duckstationGenerator.py:115-136", text) + def test_every_platform_links_its_launchers(self): + """Only Batocera and RetroBat were linked: Recalbox, ROCKNIX, EmuDeck, + RetroPie and RetroDECK read their launchers in the list's repository + too, and their citations came out as plain code.""" + checked = 0 + for path in sorted(Path(PLATFORMS).glob("*.yml")): + if not path.stem[0].isalpha(): + continue + config = load_platform_config(path.stem, PLATFORMS) + if not config.get("launchers"): + continue + with self.subTest(platform=path.stem): + rendered = [text for texts in generate_site._launcher_refs(config).values() + for text in texts] + self.assertTrue(rendered) + self.assertFalse([t for t in rendered if " at `" in t]) + checked += 1 + self.assertGreaterEqual(checked, 7) + def test_a_launcher_in_another_repository_is_not_linked(self): source = list_source("https://gitlab.com/recalbox/recalbox/-/raw/10.1.1/x/es_bios.xml") self.assertEqual(launcher_ref_markdown(None, "a/b.py:3"), "`b.py:3`") diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index 40082ff6e..39a61435c 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -423,7 +423,7 @@ launchers: | `fails_scope` | `emulator` when the failure needs every record of the emulator to come up empty | | `depends_on_option` | Options or conditions that decide whether the record applies | | `device` | Devices whose shipped config the record reads, set only where a package ships configs that differ between devices | -| `source_ref` | `path:lines` at the revision `source` pins | +| `source_ref` | `path:lines` in the repository and at the revision `source` pins, which the site links | Batocera's generators are read with `ast`, RetroBat's C# generators as text. ROCKNIX and EmuDeck launch from shell scripts and configs shipped with each From 0b8e4b476750bdcdbcda47950b6e6f8e08cc1950 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:45:49 +0100 Subject: [PATCH 04/36] fix: build the site before the CI test suite --- .github/workflows/validate.yml | 13 +++++++++---- tests/test_workflow_paths.py | 5 +++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index fc6d1fc69..eca666728 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -283,6 +283,13 @@ jobs: if: ${{ !cancelled() }} run: python scripts/generate_site.py + # website/dist is gitignored too. Built after the suite, the tests that + # read the rendered pages (page budgets, sidebar, search) found none and + # skipped on every run. + - name: Build the site + if: ${{ !cancelled() }} + run: npm run build --prefix website + - name: Run test suite if: ${{ !cancelled() }} # No test downloads a release asset: 252 MB files fetched into the @@ -302,11 +309,9 @@ jobs: # heading failed it there with nothing in front of it. Run whatever the # suite returned, so one red step does not hide the other. The release # assets are not restored: they change counts, not links or headings. - - name: Build the site + - name: Check the built site if: ${{ !cancelled() }} - run: | - npm run build --prefix website - python scripts/validate_site.py --site-dir website/dist + run: python scripts/validate_site.py --site-dir website/dist label-pr: runs-on: ubuntu-latest diff --git a/tests/test_workflow_paths.py b/tests/test_workflow_paths.py index f1a791a8d..be2994464 100644 --- a/tests/test_workflow_paths.py +++ b/tests/test_workflow_paths.py @@ -142,6 +142,11 @@ class ThePullRequestBuildsTheSite(unittest.TestCase): ) suite = next(i for i, run in enumerate(runs) if "unittest discover" in run) self.assertLess(generate, suite) + # The built pages too: about fifty tests read website/dist. + build = next(i for i, run in enumerate(runs) if "npm run build" in run) + self.assertLess(generate, build) + self.assertLess(build, suite) + self.assertEqual(steps[build].get("if"), "${{ !cancelled() }}") self.assertEqual(steps[suite].get("if"), "${{ !cancelled() }}") # A failed data fetch skipped the generator, and all 228 site tests # then failed under it: the generator runs whatever came before. From 43711a2507831babaa2a5c96976c05e5e0a2aa3d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:46:46 +0100 Subject: [PATCH 05/36] feat: name --from-pack in the pack README --- scripts/packreadme.py | 22 ++++++++++++++++++++-- tests/test_pack_readme.py | 16 ++++++++++++++++ 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/scripts/packreadme.py b/scripts/packreadme.py index ca80f9899..720f46104 100644 --- a/scripts/packreadme.py +++ b/scripts/packreadme.py @@ -16,6 +16,21 @@ def _detect_systems(detect: list[dict]) -> list[str]: return [name for name in order if name in seen] +def _from_pack(platform_name: str, posix: bool, windows: bool, + dest: tuple[str, str] = ("", "")) -> list[str]: + """The same command, reading the files from the ZIP in hand. The + one-liner alone downloads every file again.""" + args = f"--platform {platform_name} --from-pack" + lines = [" Already have this ZIP? Add --from-pack and its path: the files", + " come from it, not from the network. For a pack in parts, name one part.", ""] + if posix: + lines.append(f" curl -fsSL {INSTALL_SH} | sh -s -- {args} /path/to/pack.zip{dest[0]}") + if windows: + lines.append(f" &([scriptblock]::Create((irm {INSTALL_PS1}))) {args} C:\\path\\to\\pack.zip" + f"{dest[1]}") + return [*lines, ""] + + def _card_section(platform_name: str, display: str) -> str: """The one-line install for a frontend whose files live on its SD card. @@ -32,6 +47,8 @@ def _card_section(platform_name: str, display: str) -> str: f" &([scriptblock]::Create((irm {INSTALL_PS1}))) --platform {platform_name} --dest E:\\\n\n" f" The installer writes each file where {display} hands it to the core\n" " and checks every file.\n\n" + + "\n".join(_from_pack(platform_name, True, True, (" --dest /path/to/card", " --dest E:\\"))) + + "\n" ) @@ -52,7 +69,7 @@ def _card_manual(display: str) -> str: def _automatic_section( - display: str, detect: list[dict], standalone_copies: list[dict] | None, + platform_name: str, display: str, detect: list[dict], standalone_copies: list[dict] | None, ) -> str: """The one-line install, for every system the registry detects the platform on. Written per platform by hand, it left ROCKNIX and MiSTer @@ -82,6 +99,7 @@ def _automatic_section( f" The installer finds the {display} BIOS folder on this machine and", " installs there, checking every file. --dest names another folder.", "", + *_from_pack(platform_name, bool(posix), "windows" in systems), ] # On a desktop: an emulator folder under the home of a console's own # system is not one its frontend reads. @@ -222,7 +240,7 @@ def _build_readme( else: guide = ( "INSTALLATION GUIDE\n\n" - + _automatic_section(platform_display, detect, standalone_copies) + + _automatic_section(platform_name, platform_display, detect, standalone_copies) + (pack_manual(platform_name) or _generic_manual(bios_paths)) ) diff --git a/tests/test_pack_readme.py b/tests/test_pack_readme.py index 9218843ca..e63fbbc4c 100644 --- a/tests/test_pack_readme.py +++ b/tests/test_pack_readme.py @@ -113,3 +113,19 @@ class TheOneLinerFollowsTheRegistry(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +class GuidesInstallFromTheZipInHand(unittest.TestCase): + """Option A sent a user holding the ZIP to the one-liner, which downloads + every file again. --from-pack installs from the archive itself.""" + + def test_every_automatic_guide_names_from_pack(self): + for name, data in _registry()["platforms"].items(): + detect = (data.get("install", {}) or {}).get("detect", []) + if not detect: + continue + text = _build_readme(name, name, "", 1, 1, detect=detect) + if "Option A: Automatic" not in text: + continue + with self.subTest(platform=name): + self.assertIn(f"--platform {name} --from-pack ", text) From 6876fe7ec356a4491a486b237b5af725a4a7e370 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:47:48 +0100 Subject: [PATCH 06/36] docs: show the profile check CI runs --- CONTRIBUTING.md | 6 +++++- scripts/generate_readme.py | 6 +++++- tests/test_contributing_text.py | 5 +++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2fa3a8ece..fd89caf1f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,12 +66,16 @@ the checks CI runs on a pull request (Python 3.12 or newer): pip install pyyaml "jsonschema[format-nongpl]" markdown python scripts/validate_pr.py --changed # the BIOS files the branch adds python scripts/validate_schemas.py --source-only # profiles and platform files -python scripts/profile_sync.py --changed --strict # each profile the branch adds or edits +python scripts/profile_sync.py --added --base upstream/main --strict # each profile the branch adds python scripts/refresh_data_dirs.py # data directories the pack-building tests read python scripts/generate_site.py # the pages the site tests read python -m unittest discover -s tests -t . ``` +`upstream` is this repository as a remote of your fork +(`git remote add upstream https://github.com/Abdess/retrobios`). A profile you +only edit is not held to `--strict`: its older refs are reviewed by hand. + The full pipeline is not needed for a contribution: it rebuilds every pack in `dist/`, which is why maintainers run it before a release. diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 1d32b975e..cfaec5afe 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -968,12 +968,16 @@ the checks CI runs on a pull request (Python 3.12 or newer): pip install pyyaml "jsonschema[format-nongpl]" markdown python scripts/validate_pr.py --changed # the BIOS files the branch adds python scripts/validate_schemas.py --source-only # profiles and platform files -python scripts/profile_sync.py --changed --strict # each profile the branch adds or edits +python scripts/profile_sync.py --added --base upstream/main --strict # each profile the branch adds python scripts/refresh_data_dirs.py # data directories the pack-building tests read python scripts/generate_site.py # the pages the site tests read python -m unittest discover -s tests -t . ``` +`upstream` is this repository as a remote of your fork +(`git remote add upstream https://github.com/Abdess/retrobios`). A profile you +only edit is not held to `--strict`: its older refs are reviewed by hand. + The full pipeline is not needed for a contribution: it rebuilds every pack in `dist/`, which is why maintainers run it before a release. diff --git a/tests/test_contributing_text.py b/tests/test_contributing_text.py index eed4b7945..09392caea 100644 --- a/tests/test_contributing_text.py +++ b/tests/test_contributing_text.py @@ -72,6 +72,11 @@ class BeforeOpeningAPullRequest(unittest.TestCase): self.assertNotIn("pipeline.py", block) workflow = (REPO_ROOT / ".github" / "workflows" / "validate.yml").read_text(encoding="utf-8") self.assertIn("validate_schemas.py --source-only", workflow) + # CI holds added profiles to their upstream, not edited ones: --changed + # failed a contributor on refs they never wrote. + self.assertIn("profile_sync.py --added --base", workflow) + self.assertIn("profile_sync.py --added --base upstream/main --strict", block) + self.assertNotIn("--changed --strict", block) def test_it_installs_what_the_checks_need_and_no_site_package(self): """The site is built by Node from website/package-lock.json: a pip From 442d49e4e1f6e41b2e8fd0af4b42e5af24115348 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:50:35 +0100 Subject: [PATCH 07/36] chore: drop the scheduled campaign job --- automation/jobs.yml | 29 ----------------------------- scripts/campaign.py | 13 ++----------- tests/test_autorun.py | 1 - tests/test_campaign.py | 37 +++++++------------------------------ wiki/tools.md | 14 ++++---------- 5 files changed, 13 insertions(+), 81 deletions(-) diff --git a/automation/jobs.yml b/automation/jobs.yml index 1edfe158c..34da6f385 100644 --- a/automation/jobs.yml +++ b/automation/jobs.yml @@ -94,35 +94,6 @@ jobs: stdout: tmp/autorun/bios-integrity.json report: tmp/autorun/bios-integrity.json - campaign: - schedule: "*-*-* 06:00" - network: true - max_load_wait: 7200 - timeout: 12600 - steps: - - run: [python, scripts/campaign.py, run, --budget, 150m] - ok_codes: [0, 75] - stdout: tmp/autorun/campaign.json - env: [RETROBIOS_ROMFETCH, RETROBIOS_AUTO_LOCK, ROMFETCH_CONTENT_DIR, ROMFETCH_CACHE_DIR, - ROMFETCH_DATS_DIR, ROMFETCH_RETOOL_DIR, ROMFETCH_MIN_INTERVAL, ROMFETCH_MAX_SIZE, - ROMFETCH_DISABLED_SOURCES, XDG_CONFIG_HOME] - gate: - - [python, scripts/campaign.py, gate] - commits: - - subject: "chore: add campaign measurements" - body: >- - Core probe records, findings and the pass summary written by - campaign.py, behind the measurement tests and the schemas. - paths: [measurements/*.jsonl] - - subject: "chore: apply fields the campaign proved" - body: >- - Fields the campaign's rule table settled from real runs, each with - the record ids in evidence:, behind verify and an emulator pack of - every changed profile. - paths: [emulators/*.yml] - report: tmp/autorun/campaign.json - scratch: [tmp/autorun, tmp/core_probe] - assets: schedule: "*-*-* 04:30" network: true diff --git a/scripts/campaign.py b/scripts/campaign.py index 2c85fe621..fd014113a 100644 --- a/scripts/campaign.py +++ b/scripts/campaign.py @@ -53,11 +53,9 @@ Usage: python scripts/campaign.py findings [--all] [--json] python scripts/campaign.py review KEY --status applied|rejected --reason TEXT [--set FIELD=YAML] python scripts/campaign.py gate - python scripts/campaign.py install-timer [--enable] -install-timer is opt-in: a contributor who wants a schedule on their own -machine installs the timer. It writes the units and enables nothing unless ---enable is passed. +A maintainer starts each pass for a question that names its profiles: no +job runs it on a schedule. """ from __future__ import annotations @@ -1941,18 +1939,11 @@ def main(argv: Sequence[str] | None = None) -> int: help="with --status applied, a field to write into the entry") review_p.add_argument("--index", type=int, help="the entry's position when several share its name") sub.add_parser("gate", help="tests, schemas, verify and pack of the changed profiles") - timer_p = sub.add_parser("install-timer", - help="opt-in: write the autorun units, the campaign job's timer among them") - timer_p.add_argument("--enable", action="store_true", - help="also reload systemd and enable the timers (never by default)") args = parser.parse_args(argv) paths = Paths.default() if args.command == "gate": return gate(paths.root, changed_profiles(paths.root)) - if args.command == "install-timer": - command = [sys.executable, str(ROOT / "scripts" / "autorun.py"), "install-timers"] - return subprocess.run(command + (["--enable"] if args.enable else []), check=False).returncode if args.command == "review": updates: dict[str, object] = {} for item in args.set: diff --git a/tests/test_autorun.py b/tests/test_autorun.py index 354f64a7a..aa32b1d05 100644 --- a/tests/test_autorun.py +++ b/tests/test_autorun.py @@ -275,7 +275,6 @@ class GithubTokenPerStep(Scratch): def test_the_committed_registry(self): jobs = ar.load_jobs() self.assertTrue(all(step.github_token for step in jobs["nightly"].steps)) - self.assertFalse(any(step.github_token for step in jobs["campaign"].steps + jobs["campaign"].gate)) class ReportAlert(Scratch): diff --git a/tests/test_campaign.py b/tests/test_campaign.py index 3ac9d6670..5966dd8d7 100644 --- a/tests/test_campaign.py +++ b/tests/test_campaign.py @@ -1019,24 +1019,6 @@ class ContentFile(Workspace): self.assertIn("Other Game (Japan)", out.getvalue()) -class InstallTimer(unittest.TestCase): - def test_writes_the_units_and_enables_only_when_asked(self) -> None: - calls = [] - real_run = subprocess.run - - def fake_run(argv: list[str], **kwargs: object) -> subprocess.CompletedProcess: - if "install-timers" not in argv: - return real_run(argv, **kwargs) - calls.append(argv) - return subprocess.CompletedProcess(argv, 0) - with unittest.mock.patch.object(campaign.subprocess, "run", fake_run): - self.assertEqual(campaign.main(["install-timer"]), 0) - self.assertEqual(campaign.main(["install-timer", "--enable"]), 0) - self.assertEqual([argv[2:] for argv in calls], - [["install-timers"], ["install-timers", "--enable"]]) - self.assertTrue(calls[0][1].endswith("scripts/autorun.py")) - - class Registry(unittest.TestCase): def test_no_profile_takes_a_campaign_file_name(self) -> None: stems = {p.stem for p in (REPO_ROOT / "emulators").glob("*.yml")} @@ -1054,18 +1036,13 @@ class Registry(unittest.TestCase): self.assertGreater(len(item.reason), 40) self.assertGreater(item.frames, item.frame_hash_every) - def test_the_autorun_job(self) -> None: - jobs = campaign.autorun.load_jobs() - job = jobs["campaign"] - self.assertTrue(job.lock and job.worktree and job.schedule) - self.assertEqual(job.gate[0].argv, ("python", "scripts/campaign.py", "gate")) - self.assertEqual({p for rule in job.commits for p in rule.paths}, - {"measurements/*.jsonl", "emulators/*.yml"}) - self.assertGreater(job.schedule, jobs["nightly"].schedule) - - def test_autorun_tells_its_steps_the_lock_is_held(self) -> None: - job = campaign.autorun.load_jobs()["campaign"] - self.assertEqual(campaign.autorun.job_env(job)[campaign.LOCK_HELD_ENV], "1") + def test_no_job_runs_a_standing_campaign(self) -> None: + """A daily pass over every profile contradicted the rule that probes + run on demand, for a question that names its profiles.""" + for name, job in campaign.autorun.load_jobs().items(): + for step in [*job.steps, *job.gate]: + with self.subTest(job=name): + self.assertNotIn("scripts/campaign.py", step.argv) @unittest.skipUnless(shutil.which("cc") and shutil.which("strace"), "no C compiler or no strace") diff --git a/wiki/tools.md b/wiki/tools.md index 1aa6aef87..e0c6997b6 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -772,7 +772,6 @@ until a report without one. Exit codes: 0 for `ok`, |-----|----------|--------------| | `nightly` | 02:00 | `check_freshness.py --json` saved to a file, `refresh_stale.py --findings` on its `data` and `catalogs` rows, the MAME and FBNeo hash scrapers, then `pipeline.py --offline` as the gate. `database.json` and `install/*.json` are computed a second time and compared. Commits the hash profiles, the recipes and the regenerated records | | `assets` | 04:30 | `sync_release_assets.py`, as a dry run | -| `campaign` | 06:00 | `campaign.py run --budget 150m` once the host is idle (up to two hours of waiting), gated by `campaign.py gate`. Commits the records, findings and pass line, then the profile fields the runs proved | | `bios-integrity` | Sunday 08:30 | `bios_integrity.py --apply --json` in the main checkout, once the host is idle (up to four hours of waiting). Its report is kept in the job's status | | `cleanup` | 05:30 | `cleanup_disk.py --apply --hours 72` in the main checkout | | `disk-check` | hourly | `cleanup_disk.py --check`, without the lock | @@ -1052,7 +1051,6 @@ python scripts/campaign.py findings # what waits for review python scripts/campaign.py review beetle_psx/sony-playstation/scph5501.bin/validation \ --status applied --set on_mismatch=warn --reason "logs the SHA1 and loads (libretro.c:853-860)" python scripts/campaign.py gate # the checks behind a commit -python scripts/campaign.py install-timer # opt-in: writes the autorun units, enables nothing ``` | Step | What happens | @@ -1108,10 +1106,10 @@ checks. The campaign's own `evidence:` writes do not count as a profile change. | Guard | Rule | |-------|------| -| Lock | The host-wide autorun lock. Under the `campaign` job, autorun holds it and says so with `RETROBIOS_AUTO_LOCK_HELD` | +| Lock | The host-wide autorun lock. When a caller already holds it, `RETROBIOS_AUTO_LOCK_HELD` says so | | Probe lock | Not held by the pass: each core_probe it starts takes the host probe lock for its own run, so the pass and other measurements (pack_probe, a core_probe loop) interleave. Two campaign passes still exclude each other through the automation lock, since they share `.cache/campaign/state.json`. Do not wrap `campaign.py run` in `flock` | | Environment | `core_probe.py`, git and `make` get autorun's allowlist, romfetch the same plus its `ROMFETCH_*` settings. None sees a token or a password | -| Sandbox | Fails closed: without a working bubblewrap the pass is `refused` (exit 3) and nothing third-party runs, every probe is started with `--require-sandbox`, and a build is refused. `--allow-unsandboxed` overrides. Under autorun the refusal raises the `job:campaign` alert | +| Sandbox | Fails closed: without a working bubblewrap the pass is `refused` (exit 3) and nothing third-party runs, every probe is started with `--require-sandbox`, and a build is refused. `--allow-unsandboxed` overrides | | Load | Before each run, waits while the 1-minute load is above a quarter of the CPUs, up to `--max-load-wait` seconds, then the pass stops as `paused` | | Disk | Below `--min-free-gb` (default 30) the pass stops as `paused` | | Budget | No check starts after `--budget` | @@ -1129,12 +1127,8 @@ and campaign tests, `validate_schemas.py --source-only`, then `verify.py --emulator` and `generate_pack.py --emulator` for each changed profile. A file that had uncommitted changes before the pass is left alone. -A maintainer starts each pass. Scheduling is opt-in: a contributor who wants a -schedule on their own machine installs the timer with `campaign.py -install-timer`, which writes the autorun units and enables nothing unless -`--enable` is passed. The `campaign` autorun job then runs a 150-minute pass -in the automation worktree, gates it with `campaign.py gate` and commits with -its own rules, pushing only with `--push`. +A maintainer starts each pass, for a question that names its profiles. No +autorun job runs it on a schedule. `review` closes an open finding once someone has read the core's code at the profile's pin. `--status applied` writes each `--set FIELD=YAML` into the From 329fcc0f6297df851e56d2ae6aae5f8bd8f3caa0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:52:23 +0100 Subject: [PATCH 08/36] docs: name the installer cache and its variable --- tests/test_doc_pages.py | 22 ++++++++++++++++++++++ wiki/faq.md | 3 ++- wiki/installer.md | 6 +++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 1a36ca46b..0d4510e12 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1678,5 +1678,27 @@ class InstallerPagesFollowTheCode(unittest.TestCase): self.assertEqual(len(unserved_one_liners("installer.md", old)), 2) +class InstallerVariablesAreDocumented(unittest.TestCase): + """RETROBIOS_CACHE moved a cache that --for-games writes, even under + --dry-run, and no page named it or the folder it defaults to.""" + + # Set by the full-screen interface for the runs it starts, never by a user. + INTERNAL = {"RETROBIOS_TUI_LIST"} + + def test_every_variable_install_py_reads_is_in_the_table(self): + source = (ROOT / "install.py").read_text(encoding="utf-8") + page = (WIKI / "installer.md").read_text(encoding="utf-8") + table = page[page.index("## Environment overrides"):] + table = table[:table.index("\n## ", 1)] + for name in sorted(set(re.findall(r'"(RETROBIOS_[A-Z_]+)"', source)) - self.INTERNAL): + with self.subTest(variable=name): + self.assertIn(f"`{name}`", table) + + def test_the_faq_names_the_cache(self): + faq = (WIKI / "faq.md").read_text(encoding="utf-8") + section = faq[faq.index("## Is the command safe?"):] + self.assertIn("cache", section[:section.index("\n## ", 1)]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/faq.md b/wiki/faq.md index c199069a7..58646b11b 100644 --- a/wiki/faq.md +++ b/wiki/faq.md @@ -57,7 +57,8 @@ Game sets must match the emulator's version: - It stops if the installer does not match its expected fingerprint, and checks each file. -- It writes in your BIOS folder, and some emulators' save folder. +- It writes in your BIOS folder, some emulators' save folder, and a cache + of what it read about your games. - It never touches games or saves. - It keeps each file it replaces in `retrobios-backup`, for `--restore`. diff --git a/wiki/installer.md b/wiki/installer.md index b5c6b183e..fd2536c22 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -865,7 +865,9 @@ it. So does an arcade set the core does not know. **Only what starts** (`--required-only`) leaves out the files the cores mark optional. The games are read without hashing whole discs, and a cache keeps what was -read, so a second run over the same card reads only new files. Offline, with +read, so a second run over the same card reads only new files. It is a +`retrobios` folder in the system's cache folder (`RETROBIOS_CACHE`, see +[Environment overrides](#environment-overrides)), written by `--dry-run` too. Offline, with no games index on this machine, the run keeps every file and says so. ### Firmware without game data @@ -1826,6 +1828,8 @@ RetroArch folder that holds only `scph5501.bin`, in the order written: | `RETROBIOS_PAD` | The buttons a handheld launcher's mapper sends to the full screen (`gptokeyb`, `gptokeyb_muxterm`, `muxterm`, `evmapy`, `miyoo`), so its hints name them. See [On a handheld](#on-a-handheld-the-retrobios-port) | | `RETROBIOS_IDLE_EXIT` | Seconds the full screen waits for a first button, counted while nothing runs. With none pressed, it closes with exit status 2, as a quit, or with the code of the error or result screen it closed on | | `RETROBIOS_NO_MOUSE` | Set to any value but `0` or an empty one: the full screen never asks the terminal for clicks, and the mouse selects text everywhere as it does outside the installer | +| `RETROBIOS_CACHE` | The folder that keeps the games index and what a run read of the games, for the next run. Defaults to `retrobios` in `$XDG_CACHE_HOME` (else `~/.cache`), `~/Library/Caches` on macOS, `%LOCALAPPDATA%` on Windows. Deleting it loses nothing | +| `RETROBIOS_DEBUG` | Set to anything: says why the games index could not be fetched or a cache file not written | | `HTTPS_PROXY`, `NO_PROXY` | Honoured for every download through the standard library's default proxy handling: one names the proxy, the other the hosts to reach directly | ## When something goes wrong From 8d0ba26a0f44a20abfbd9b5f555d14ffa0419059 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:53:11 +0100 Subject: [PATCH 09/36] docs: show --required-only in the README --- README.md | 5 ++++- scripts/generate_readme.py | 3 +++ tests/test_doc_pages.py | 11 +++++++++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c5db8e4a7..039968708 100644 --- a/README.md +++ b/README.md @@ -142,6 +142,9 @@ python scripts/generate_pack.py --platform retroarch --region us --one-per-slot # Firmware only: the game data a list or a core names is left out, each file named python scripts/generate_pack.py --platform emuelec --no-game-data +# The bare minimum: only the files each core needs to start +python scripts/generate_pack.py --platform batocera --required-only + # List available emulators and systems python scripts/generate_pack.py --list-emulators python scripts/generate_pack.py --list-systems @@ -203,4 +206,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-11-05T10:43:18Z* +*Auto-generated on 2026-11-05T10:52:39Z* diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index cfaec5afe..132a2eb3a 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -788,6 +788,9 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" "# Firmware only: the game data a list or a core names is left out, each file named", "python scripts/generate_pack.py --platform emuelec --no-game-data", "", + "# The bare minimum: only the files each core needs to start", + "python scripts/generate_pack.py --platform batocera --required-only", + "", "# List available emulators and systems", "python scripts/generate_pack.py --list-emulators", "python scripts/generate_pack.py --list-systems", diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 0d4510e12..3a246ec4a 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1700,5 +1700,16 @@ class InstallerVariablesAreDocumented(unittest.TestCase): self.assertIn("cache", section[:section.index("\n## ", 1)]) +class ReadmeKeepsItsPointers(unittest.TestCase): + def test_the_bare_minimum_is_built_below(self): + """The README sent a reader to 'build your own pack below' for a bare + minimum, and the block below never named --required-only.""" + readme = (ROOT / "README.md").read_text(encoding="utf-8") + self.assertIn("For a bare minimum, build your own pack below.", readme) + block = readme[readme.index("## Build your own pack"):] + block = block[:block.index("\n## ", 1)] + self.assertIn("--required-only", block) + + if __name__ == "__main__": unittest.main() From 0d60cc19c99800d27f382f6987e2161b2792bbdb Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:54:22 +0100 Subject: [PATCH 10/36] docs: quote the current verify report --- tests/test_doc_pages.py | 24 ++++++++++++++++++++++++ wiki/reading-reports.md | 8 ++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 3a246ec4a..96cfaa836 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1711,5 +1711,29 @@ class ReadmeKeepsItsPointers(unittest.TestCase): self.assertIn("--required-only", block) +class ReportExcerptIsTheRun(unittest.TestCase): + """reading-reports.md quoted 'From python scripts/verify.py --platform + retrodeck' a run that said 0 required missing after the command had started + naming one, and never showed the line that names it.""" + + def test_every_quoted_line_is_printed(self): + import subprocess # noqa: PLC0415 + + page = (WIKI / "reading-reports.md").read_text(encoding="utf-8") + intro = "From `python scripts/verify.py --platform retrodeck`:" + start = page.index("```\n", page.index(intro)) + 4 + quoted = page[start:page.index("```", start)].splitlines() + run = subprocess.run( + [sys.executable, str(ROOT / "scripts" / "verify.py"), "--platform", "retrodeck"], + capture_output=True, text=True, cwd=ROOT, check=False, + ) + printed = run.stdout.splitlines() + if not any(line.startswith("RetroDECK:") for line in printed): + self.skipTest(f"verify did not run: {run.stderr[-200:]}") + for line in quoted: + with self.subTest(line=line): + self.assertIn(line, printed) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/reading-reports.md b/wiki/reading-reports.md index 159fef7b7..4847f7215 100644 --- a/wiki/reading-reports.md +++ b/wiki/reading-reports.md @@ -107,9 +107,11 @@ RetroDECK: 2007/2008 OK, 1 untested [md5] UNTESTED (required): nintendo-ds/dsifirmware.bin: declared hash contradicted by the local file DISCREPANCY: nintendo-snes/sgb2.boot.rom: RetroDECK says OK but supersnes9x says size mismatch: got 524288, accepted [256] supersnes9x [size] - Core files: 1795 in pack, 0 required missing, 27 optional missing (--verbose lists them) + Core files: 1795 in pack, 1 required missing, 27 optional missing (--verbose lists them) Core files held but not packed: 154 (installed with its emulator) + MISSING (required): FinalBurn Neo needs sfiii2a.zip (1 file) Game data: 5099 in pack, 38 missing (--verbose lists them) + Game data held but not packed: 30 (installed with its emulator) No external files: 78 cores (--verbose lists them) Ground truth: 96/2008 files have emulator validation (5%) 439 platform-only (no emulator profile) @@ -117,7 +119,9 @@ RetroDECK: 2007/2008 OK, 1 untested [md5] The run prints more `DISCREPANCY` lines than this excerpt shows. The core and game data lines count what the emulator cores add beyond the platform's -own list. A file held but not packed is left out for the reason in parentheses. +own list. A required core file the collection lacks is named under them, with +the emulator that needs it. A file held but not packed is left out for the +reason in parentheses. `--verbose` also lists the passing entries that an emulator validates, with the check and the source lines behind it: From 8e6d6721361e79e081e477499fc11db7791839b2 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:56:50 +0100 Subject: [PATCH 11/36] fix: hold CONTRIBUTING to the CI profile check --- tests/test_profile_sync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index a807a9bc6..ce62e6cc8 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -5809,7 +5809,7 @@ class TestBranchCheckIsWired(unittest.TestCase): import generate_readme # noqa: PLC0415 text = generate_readme.generate_contributing() - self.assertIn("profile_sync.py --changed --strict", text) + self.assertIn("profile_sync.py --added --base upstream/main --strict", text) self.assertNotIn("--emulator NAME --strict", text) workflow = (root / ".github" / "workflows" / "validate.yml").read_text() self.assertIn("profile_sync.py --added --base", workflow) From 8dcf53162500cb4efd9c45bcb003942d12d24150 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:57:01 +0100 Subject: [PATCH 12/36] fix: report pinned trees the source record lacks --- scripts/check_freshness.py | 18 ++++++++++----- scripts/sitesources.py | 13 ++++++----- tests/test_profile_sync.py | 3 ++- tests/test_site_r4_merge.py | 32 ++++++++++++++++++++++++++ website/src/data/source-paths.json | 36 ++++++++++++++++++++++++++++++ wiki/tools.md | 3 ++- 6 files changed, 91 insertions(+), 14 deletions(-) diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index a1116142d..e53c286ca 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -1242,8 +1242,10 @@ def _check_installer_pins() -> Finding: def check_source_paths(profiles: dict) -> Finding: """The site links a cited path only where source-paths.json records it - in the tree the profile pins. A recorded tree no profile pins any more - means a pin moved and `sitesources.py --fetch` was not run after it.""" + in the tree the profile pins. A recorded tree no profile pins any more, + or a pinned tree the record lacks, means a pin moved or a profile was + added and `sitesources.py --fetch` was not run after it. Every tree a run + tries is recorded, read or not, so a lacking one was never tried.""" import sitesources # noqa: PLC0415 pinned = { @@ -1256,14 +1258,18 @@ def check_source_paths(profiles: dict) -> Finding: except (OSError, ValueError) as exc: return Finding("profiles", "source-paths.json", ERROR, detail=str(exc)) orphaned = sorted(recorded - pinned) + unrecorded = sorted(pinned - recorded) + problems = [f"{len(found)} {words}" for found, words in ( + (orphaned, "recorded trees no profile pins"), + (unrecorded, "pinned trees not recorded"), + ) if found] return Finding( "profiles", "source-paths.json", - STALE if orphaned else OK, + STALE if problems else OK, local=f"{len(recorded)} trees", - detail=(f"{len(orphaned)} recorded trees no profile pins: run " - "scripts/sitesources.py --fetch") if orphaned - else "every recorded tree is a pinned one", + detail=(", ".join(problems) + ": run scripts/sitesources.py --fetch") if problems + else "every pinned tree is recorded, and only those", ) diff --git a/scripts/sitesources.py b/scripts/sitesources.py index 55dc2f5e9..e631a9774 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -311,9 +311,9 @@ def build_source_paths( if tree is None: unread.add(tree_key(repo, pin)) continue - found = paths & tree - if found: - held.setdefault(tree_key(repo, pin), set()).update(found) + # Recorded even when it holds no cited path, so a pinned tree + # missing from the record is one no run has read. + held.setdefault(tree_key(repo, pin), set()).update(paths & tree) return {key: sorted(paths) for key, paths in sorted(held.items())}, sorted(unread) @@ -323,12 +323,13 @@ def keep_unread( """Carry over what the last record held for the trees this run could not read. A tree key names a pinned sha, so what it held then it holds now. A run - that GitLab rate-limited dropped about twenty trees it never read. + that GitLab rate-limited dropped about twenty trees it never read. A tree + no run could read is recorded empty: confirming no path, it links none, + and check_freshness tells it from a pin no run has tried. """ kept = dict(held) for key in unread: - if key not in kept and key in previous: - kept[key] = previous[key] + kept.setdefault(key, previous.get(key, [])) return dict(sorted(kept.items())) diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index ce62e6cc8..54eabaed1 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -4882,7 +4882,8 @@ class AnUnreadTreeKeepsItsRecord(unittest.TestCase): kept = sitesources.keep_unread( {"a@1": ["x.c"]}, ["b@2", "c@3"], {"b@2": ["y.c"], "a@1": ["old.c"]} ) - self.assertEqual(kept, {"a@1": ["x.c"], "b@2": ["y.c"]}) + # c@3 was never read: recorded empty, it confirms no path. + self.assertEqual(kept, {"a@1": ["x.c"], "b@2": ["y.c"], "c@3": []}) class FreshRefusesOffline(unittest.TestCase): diff --git a/tests/test_site_r4_merge.py b/tests/test_site_r4_merge.py index 6b7d51031..c1d582e5a 100644 --- a/tests/test_site_r4_merge.py +++ b/tests/test_site_r4_merge.py @@ -13,6 +13,7 @@ import glob import json import re import sys +import tempfile import unittest from pathlib import Path from unittest import mock @@ -123,6 +124,37 @@ class SourcePathsFreshness(unittest.TestCase): self.assertEqual(finding.status, check_freshness.STALE) self.assertIn("sitesources.py --fetch", finding.detail) + def test_a_pinned_tree_never_read_is_stale(self): + """fbneo moved to two new pins while fbneo_cps12 kept the old ones: + no recorded tree was orphaned, so the check said OK and 174 citations + stayed unlinked.""" + import sitesources # noqa: PLC0415 + + profile = _profile("geolith") + trees = [sitesources.tree_key(r, p) for r, p in sitesources.profile_trees(profile)] + record = json.loads(sitesources.SOURCE_PATHS.read_text(encoding="utf-8")) + for key in trees: + record.pop(key, None) + # Kept pinned by another profile, as fbneo_cps12 kept the old FBNeo trees. + record.update({key: [] for key in trees[1:]}) + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "source-paths.json" + path.write_text(json.dumps(record), encoding="utf-8") + with mock.patch.object(sitesources, "SOURCE_PATHS", path): + finding = check_freshness.check_source_paths({"geolith": profile}) + self.assertEqual(finding.status, check_freshness.STALE) + self.assertIn("1 pinned trees not recorded", finding.detail) + + def test_a_tree_read_with_no_cited_path_is_recorded(self): + import sitesources # noqa: PLC0415 + + profile = {"source": "https://github.com/o/r", "source_commit": "e" * 40, + "files": [{"source_ref": "src/a.c:5"}]} + with mock.patch.object(sitesources, "_cached_tree", return_value=frozenset({"x.c"})): + held, unread = sitesources.build_source_paths({"p": profile}, [Path("tmp")]) + self.assertEqual(held, {"github.com/o/r@" + "e" * 40: []}) + self.assertEqual(unread, []) + class OptionsGlanceFromTheParser(unittest.TestCase): """--declared-only and --offline reached install.py and its Options diff --git a/website/src/data/source-paths.json b/website/src/data/source-paths.json index c7e9e7996..52daf59bf 100644 --- a/website/src/data/source-paths.json +++ b/website/src/data/source-paths.json @@ -1,7 +1,10 @@ { + "archive.softwareheritage.org/softwareheritage/revision@dc20a615275f66bee20a4fd851ef0231daca4f14": [], + "bitbucket.org/linuxwolf6/splitwolf@f6c00044cdef00a31c66307d8b7cb1fb4ddf8c67": [], "codeberg.org/iyzsong/eternal-libretro@13adf143aea822ebc1ee23722a80e93a7979e4c9": [ "libretro.c" ], + "codeberg.org/iyzsong/gam4980@32c531d3e7159081afd8dee72e8266153821095d": [], "codeberg.org/schelling/dosbox-pure-unleashed@4a11412248ca4c862751a7d9e6818023795031e9": [ "Makefile", "main.cpp" @@ -409,6 +412,7 @@ "plastic_tui/src/ui.rs", "plastic_ui/src/main.rs" ], + "github.com/AppleWin/AppleWin@1f3cadb88d1158c139d2e3a7b864d91836856948": [], "github.com/AppleWin/AppleWin@3e8054b4627624398e4589f7f27b3d40a6b9718e": [ "resource/resource.h", "source/CardManager.cpp", @@ -516,6 +520,7 @@ "source/bus-main-m68k.c", "source/bus-sub-m68k.c" ], + "github.com/Clownacy/clownmdemu-libretro@d43c2708b0a31c285ce16724b6c4a2e92af07346": [], "github.com/ColinPitrat/caprice32@8c778bf072a06fb7c23b9cccd179041db7f79a52": [ "cap32.cfg", "rom/MF2.rom", @@ -597,6 +602,7 @@ "Source/SysPosix/CMakeLists.txt", "Source/Utility/Paths.cpp" ], + "github.com/DanAlexMorton/3dsTrident@b97b0befa326105e181838bc6fa11c546de8294d": [], "github.com/DanAlexMorton/Panda3DS@69095ee6d68d21eacac8d1517f0b7b4ed27884d3": [ "src/core/crypto/aes_engine.cpp", "src/core/loader/ncch.cpp", @@ -855,6 +861,7 @@ "Hurrican/src/Texts.cpp", "Hurrican/src/Tileengine.cpp" ], + "github.com/Hydr8gon/NooDS@4cb1439c330248b26cdf5f8dbcfa4e5ac53dcda0": [], "github.com/JACoders/OpenJK@260c59c2907187af555a676fe0cc798893bf7757": [ "code/CMakeLists.txt", "code/qcommon/files.cpp", @@ -1138,6 +1145,7 @@ "pcsx2/VMManager.cpp", "pcsx2/ps2/BiosTools.cpp" ], + "github.com/PCSX2/pcsx2@c10a5c9ad951c517dc48b722b5b84eb9bf7fdb42": [], "github.com/PCSX2/pcsx2@f0523f8acb1d3a765ee73b74cffec63093569199": [ "common/FileSystem.cpp", "pcsx2/CDVD/CDVD.cpp", @@ -1198,6 +1206,7 @@ "Source/Core/DiscIO/NANDImporter.cpp", "Source/Core/DolphinQt/MenuBar.cpp" ], + "github.com/RPCS3/rpcs3@1f080485b95db2700c5cf244b1314f73b6cf564e": [], "github.com/RPCS3/rpcs3@c365ea76a552961ff078ea1a46da9d4b07729674": [ "rpcs3/Crypto/key_vault.h", "rpcs3/Emu/Cell/Modules/cellFont.cpp", @@ -1226,6 +1235,7 @@ "RSDKv5/RSDK/Core/Reader.cpp", "RSDKv5/RSDK/User/Core/UserCore.cpp" ], + "github.com/RSDKModding/Sonic-Mania-Decompilation@9dc699428420d752af9767bdb13f585ee0881bc0": [], "github.com/Rakashazi/emu-ex-plus-alpha@1c12fac5ce49badaadff2e2f210dcc30b89f4943": [ "2600.emu/CMakeLists.txt", "2600.emu/metadata/conf.mk", @@ -1730,6 +1740,7 @@ "desmume/src/NDSSystem.cpp", "desmume/src/NDSSystem.h" ], + "github.com/TASEmulators/fceux@d339f1fa0cd8b69cc637a0de3fb3f04cb28748ea": [], "github.com/TASEmulators/freej2me-plus@3933e75ef927a95eab9f5e065f0b76dabf336069": [ "build.xml", "src/javax/microedition/media/Manager.java", @@ -1864,6 +1875,7 @@ "wadsrc_lights/CMakeLists.txt", "wadsrc_widepix/CMakeLists.txt" ], + "github.com/VICE-Team/svn-mirror@46b1bc4393251f838ce3dadd241c1f567d9371d1": [], "github.com/VICE-Team/svn-mirror@81546e60c222fe4aef750b890b0dbe3a5ac280d8": [ "vice/configure.ac", "vice/data/C64/Makefile.am", @@ -2472,6 +2484,7 @@ "mia/system/neo-geo-aes.cpp", "mia/system/neo-geo-mvs.cpp" ], + "github.com/atari800/atari800@98cc2dc749867c9c7317a0271ba062a6c797f886": [], "github.com/audetto/AppleWin@0a89db3d593a74d04590b902fb7206723591f445": [ "source/frontends/common2/gnuframe.cpp" ], @@ -2770,6 +2783,7 @@ "src/sdl/main.cpp", "src/sdl/ti994a-sdl.cpp" ], + "github.com/clydeshaffer/GameTankEmulator@be32deab74b7778647273fa5616d1ea92e3e1267": [], "github.com/coelckers/prboom-plus@969515162c5aebea4ad7a125ee178dcad3d576ad": [ "prboom2/CMakeLists.txt", "prboom2/data/CMakeLists.txt", @@ -3167,6 +3181,7 @@ "core/loadrom.c", "libretro/libretro.c" ], + "github.com/emuplace/sudachi.emuplace.app@c7431bd420d66c54df277ba0a0b71de42cf0a5bf": [], "github.com/etlegacy/etlegacy@add16a92f4b10c2955136e3ab7868bfc9bff8fad": [ "README.md", "cmake/ETLBuildMod.cmake", @@ -3182,6 +3197,11 @@ "rott/rt_ted.c" ], "github.com/finalburnneo/FBNeo@697e385c8285bfc528343b96d948ee3a4f9e24c5": [ + "src/burn/burn.cpp", + "src/burn/drv/neogeo/d_neogeo.cpp", + "src/burn/hiscore.cpp" + ], + "github.com/finalburnneo/FBNeo@c110542721d8674b6ff48e691ac032c040fecbba": [ "src/burn/burn.cpp", "src/burn/drv/atari/d_atarisy1.cpp", "src/burn/drv/channelf/d_channelf.cpp", @@ -3446,6 +3466,7 @@ "src/xenia/ui/vulkan/vulkan_instance.cc", "src/xenia/ui/windowed_app_main_win.cc" ], + "github.com/has207/xenia-edge@d881a29125978b4eea6bf44f6ed8c33f8aea8c33": [], "github.com/has207/xenia-edge@f23c14c717ef274516f29c9b5e97d6a9db28643d": [ ".ci/build-windows-cross.sh", "libretro/CMakeLists.txt", @@ -3678,6 +3699,7 @@ "github.com/iyzsong/onscripter-libretro@ec4ff1842d498faaa844d570becf561d088deea0": [ "libretro.cpp" ], + "github.com/iyzsong/syobonaction-libretro@792c40cb590bcc7286cf720b5fb85807082b534e": [], "github.com/izzy2lost/Super3@c410f82f771b33094b8c1cdbf96ef7c74ec9cff1": [ "Manual.txt", "README.md", @@ -3870,6 +3892,12 @@ "libretro.c" ], "github.com/libretro/FBNeo@32958a89abd645ed20387c8d31d21ede3bc48161": [ + "src/burn/burn.cpp", + "src/burn/drv/neogeo/d_neogeo.cpp", + "src/burn/hiscore.cpp", + "src/burner/libretro/libretro.cpp" + ], + "github.com/libretro/FBNeo@da14132bab6683980b1ad7e4d0d6fd47bf06dce7": [ "src/burn/burn.cpp", "src/burn/drv/atari/d_atarisy1.cpp", "src/burn/drv/channelf/d_channelf.cpp", @@ -4657,6 +4685,7 @@ "src/drivers/cvs.c", "src/drivers/decocass.c", "src/drivers/konamigx.c", + "src/drivers/namcos11.c", "src/drivers/neogeo.c", "src/drivers/nss.c", "src/drivers/pgm.c", @@ -7910,6 +7939,7 @@ "mupen64plus-video-gliden64/upstream/src/mupenplus/Config_mupenplus.cpp", "settings.gradle" ], + "github.com/mupen64plus/mupen64plus-core@3be6336a1de6f32ccc4f57bf80c988163fe6bf7a": [], "github.com/mupen64plus/mupen64plus-core@6dca4c15370ac3e2171ce7b31426695f8f39b460": [ "projects/unix/Makefile", "src/api/config.c", @@ -8535,6 +8565,7 @@ "video/external/src/decoder/openh264.rs", "web/Cargo.toml" ], + "github.com/sarah-walker-pcem/pcem@25b54e3ff1d51cbf2469b447e55bce7a589697e9": [], "github.com/sashkinbro/EmuCoreV@b71d8cf5397b1c6394e92eeb48427cd17e542971": [ "app/build.gradle.kts", "app/src/main/AndroidManifest.xml", @@ -9444,6 +9475,7 @@ "github.com/xenia-canary/xenia-canary@72c2afa3ba2b1b232f89de71577251d6edafb1bd": [ "src/xenia/emulator.cc" ], + "github.com/xenia-canary/xenia-canary@dc561e4df578c903087f9bdf56ff61d9ef9f2a93": [], "github.com/xenios-jp/XeniOS@87b176a078c316fde3adf67a217f0c44615b0e0d": [ "src/xenia/app/CMakeLists.txt", "src/xenia/app/game_compat_db.cc", @@ -9571,6 +9603,7 @@ "src/common/filesystem.c", "src/common/header/common.h" ], + "github.com/yuzu-emu-mirror/yuzu@15e6e48bef0216480661444a8d8b348c1cca47bb": [], "github.com/zelurker/raine@8e25d5d8907ae459181610912b0030c955683015": [ "source/emudx.c", "source/emumain.c", @@ -9704,6 +9737,7 @@ "src/jcv_db.c", "src/jollycv.c" ], + "gitlab.com/jgemu/nestopia@b3adbeb42482e979ab99e16bcd9a22480f13479f": [], "gitlab.com/parallel-launcher/parallel-launcher@8dc2a83770c57bc00e1b1fae0aed71f2120cc875": [ "src/core/retroarch.cpp", "src/core/rom-finder.cpp", @@ -9748,6 +9782,7 @@ "libretro/LibretroCore.cpp", "libretro/RomData.h" ], + "gitlab.com/recalbox/packages/standalone/rb5000@7464996aa0cb1be77abf495dc901117e1af36e02": [], "gitlab.com/recalbox/packages/standalone/rb5000@8c75a4b48a5222b0332d0508600f8ddc7ac1191c": [ "source/Data.cpp", "source/Engine.cpp", @@ -9785,6 +9820,7 @@ "src/graphics/Shader.cpp", "src/graphics/SpriteAnimationSet.cpp" ], + "sourceforge.net/dgen/dgen@1c196e63d3e35704a97a2cef60def9df3c9fd1ce": [], "voidpoint.io/terminx/eduke32@ec5824db81817866f70da326d3811bb0f52b3517": [ "GNUmakefile", "source/duke3d/src/common.cpp", diff --git a/wiki/tools.md b/wiki/tools.md index e0c6997b6..577296f43 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -525,7 +525,8 @@ prose that the same pass would have moved is not reported as blocked. Drop A moved pin also moves the links the site draws into the cited sources: a path is linked only where `website/src/data/source-paths.json` records it in the tree the profile pins. Refresh that record after a pin moves or a profile -is added, and commit it with the profiles: +is added, and commit it with the profiles. Every tree a run tries is recorded, +read or not, so `check_freshness.py` reports a pinned tree the record lacks: ```bash python scripts/sitesources.py --fetch From 5e9c2c87b66bbb9aa01560255007fe2454d207d0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:57:58 +0100 Subject: [PATCH 13/36] fix: link prose citations of two-repo profiles --- scripts/sitesources.py | 25 ++++++++++++++----------- tests/test_site_exports.py | 22 ++++++++++++++++++++++ 2 files changed, 36 insertions(+), 11 deletions(-) diff --git a/scripts/sitesources.py b/scripts/sitesources.py index e631a9774..20844939b 100644 --- a/scripts/sitesources.py +++ b/scripts/sitesources.py @@ -527,16 +527,18 @@ def link_prose_citations(profile: dict, text: str) -> str: """Prose file:line citations as pinned links, where the target is certain. profile_sync keeps these citations on their pin, and the page printed - 4,328 of them as plain text. A citation links only when the profile reads - one repository and the path is rooted (it holds a directory) or is one a - structured source_ref of the profile already uses: a bare name could be - anywhere in the tree, and a link to the wrong file is a false citation. + 4,328 of them as plain text. A citation links only when the path is + rooted (it holds a directory) or is one a structured source_ref of the + profile already uses: a bare name could be anywhere in the tree, and a + link to the wrong file is a false citation. Of the repositories a profile + reads, the one its role word names is taken, else the one tree holding + the path: held by both, the citation does not say which it means. Text already in backticks or in a link is left alone. """ candidates = _forge_sources(profile) - if len(candidates) != 1 or ":" not in text: + if not candidates or ":" not in text: return text - repo, pin = candidates[0] + by_role = role_sources(profile) structured = { part.path for entry in profile.get("files") or [] @@ -554,14 +556,15 @@ def link_prose_citations(profile: dict, text: str) -> str: continue if "/" not in path and path not in structured: continue - role = parts[0].role - if role and (role_sources(profile).get(role) or (None,))[0] != repo: - # `upstream a.c:5` names a repository this link would not reach. - continue if any(lo <= path_span[0] < hi for lo, hi in guarded): continue - if not tree_holds(repo, pin, path): + role = parts[0].role + # `upstream a.c:5` names its repository: none other is linked. + named = [by_role[role]] if role in by_role else [] if role else candidates + holders = [tree for tree in named if tree_holds(*tree, path)] + if len(holders) != 1: continue + repo, pin = holders[0] for index, ((lo, hi), part) in enumerate(zip(spans, parts)): start = path_span[0] if index == 0 else lo url = _source_permalink(repo, pin, part.path, part.start, part.end) diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 09b2240f8..1a3dea8fe 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -800,6 +800,28 @@ class ProseCitationsLinkToThePin(unittest.TestCase): two = {**self.PROFILE, "upstream": "https://github.com/x/y", "upstream_commit": "a" * 40} self.assertEqual(link_prose_citations(two, "src/a.c:5"), "src/a.c:5") + def test_two_repositories_link_into_the_one_holding_the_path(self): + """1,044 citations in 86 profiles that read two repositories stayed + text, the swanstation note on FindBIOSImageInDirectory among them.""" + from sitesources import link_prose_citations + + two = {**self.PROFILE, "upstream": "https://github.com/x/y", "upstream_commit": "a" * 40} + + def holds(repo, _pin, path): + return path == "src/both.c" or (path == "src/port.c") == (repo.owner == "libretro") + + with mock.patch("sitesources.tree_holds", holds): + text = link_prose_citations(two, "src/port.c:5, src/emu.c:7 and src/both.c:9") + upstream_only = link_prose_citations(two, "upstream src/port.c:5") + named = link_prose_citations(two, "upstream src/both.c:9") + self.assertIn("[`src/both.c:9`](https://github.com/x/y/blob/", named) + self.assertIn("[`src/port.c:5`](https://github.com/libretro/FreeChaF/blob/", text) + self.assertIn("[`src/emu.c:7`](https://github.com/x/y/blob/" + "a" * 40, text) + # Held by both: the citation does not say which it means. + self.assertIn(" src/both.c:9", text) + # The role word names a repository that does not hold it. + self.assertNotIn("](", upstream_only) + class ACitationSaysWhoseCodeItIs(unittest.TestCase): """Readers could not tell whether the libretro core or the emulator asked From fbad34adbd0a17f0ea6ab7e85940a508d6a8f99e Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:59:32 +0100 Subject: [PATCH 14/36] fix: badge only the platforms that read a file --- scripts/generate_site.py | 21 ++++++++++++++++++++- tests/test_site_render.py | 20 ++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 731dee433..f6fff2aae 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -10296,6 +10296,22 @@ def _upload_folder(profile: dict) -> str: return "" +def reading_platforms(emu_name: str, profile: dict, name: str, + configs: dict[str, dict]) -> set[str]: + """Of the platforms running an emulator (name -> config), those whose + build of it reads the entry: BlastEm's cdbios.bin is read by the + standalone build only, and RetroBat hands that build other names.""" + from common import platform_layouts, standalone_set # noqa: PLC0415 + from validation import filter_files_by_mode, unread_on # noqa: PLC0415 + + return { + platform for platform, config in configs.items() + if any(f.get("name") == name and not unread_on(f, platform) + for view, standalone in platform_layouts(emu_name, profile, standalone_set(config)) + for f in filter_files_by_mode(view.get("files") or [], standalone)) + } + + def generate_gap_analysis( profiles: dict, coverages: dict, @@ -10627,6 +10643,7 @@ def generate_gap_analysis( for emu_name in matched: emu_to_platforms.setdefault(emu_name, set()).add(pname) + # One row per file to find: a name two emulators share is one file # only when both declare the same hash. Keyed by name alone, Fallout # 2's master.dat vanished behind Fallout 1's, and the three Sonic @@ -10685,7 +10702,9 @@ def generate_gap_analysis( key=lambda x: (not x["required"], x["name"]), ): plats = sorted(set().union( - *(emu_to_platforms.get(key, set()) for key in m["emu_keys"]) + *(reading_platforms(key, profiles[key], m["name"], { + p: coverages[p]["config"] for p in emu_to_platforms.get(key, set()) + }) for key in m["emu_keys"]) )) plat_badges = ( " ".join( diff --git a/tests/test_site_render.py b/tests/test_site_render.py index 2abaf0809..347d75cc4 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -8,6 +8,8 @@ import unittest from pathlib import Path from unittest import mock +import yaml + REPO_ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(REPO_ROOT / "scripts")) @@ -100,6 +102,24 @@ class BundledFilesAreNotToSource(unittest.TestCase): self.assertIn("| [ZQuest Classic](emulators/zq.md) | 4 | 1 | 0 | 0 | 2 | " '1 |', page) + def test_a_row_names_only_the_platforms_that_read_it(self): + """cdbios.bin, read by standalone BlastEm only and never on RetroBat, + was badged retroarch, lakka and retrobat.""" + import generate_site + from common import load_platform_config, resolve_platform_cores + + root = Path(__file__).resolve().parents[1] + profile = yaml.safe_load((root / "emulators" / "blastem.yml").read_text(encoding="utf-8")) + configs = {} + for name in ("retroarch", "lakka", "retrobat", "batocera"): + config = load_platform_config(name, str(root / "platforms")) + if "blastem" in resolve_platform_cores(config, {"blastem": profile}): + configs[name] = config + self.assertIn("retroarch", configs) + self.assertIn("retrobat", configs) + found = generate_site.reading_platforms("blastem", profile, "cdbios.bin", configs) + self.assertFalse(found & {"retroarch", "lakka", "retrobat"}, found) + class StructuredBlocks(unittest.TestCase): PIN = "1e0de94dc7e669c0925a22c0fccf6cdc837af0a0" From 3a306d0970799c929f5863803516b3f09e9e3772 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:00:45 +0100 Subject: [PATCH 15/36] feat: link each missing file to its platform entry --- scripts/generate_site.py | 27 +++++++++++++++++++++++++-- tests/test_audit_regressions.py | 22 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index f6fff2aae..de984fe3d 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -10022,6 +10022,16 @@ def _declared_hashes(config: dict, detail: dict) -> dict[str, list[str]]: return {} +def _declared_destination(config: dict, detail: dict) -> str: + """Where the platform's list puts a reported file, its name when the + entry names no folder.""" + system = (config.get("systems") or {}).get(detail.get("system", "")) or {} + for entry in system.get("files") or []: + if isinstance(entry, dict) and entry.get("name") == detail.get("name"): + return str(entry.get("destination") or detail.get("name", "")) + return str(detail.get("name", "")) + + def short_hash(value: str) -> str: """A hash in a table cell: its first 12 characters, the whole value in the title. Full MD5 and SHA1 values pushed the gaps tables past the @@ -10081,8 +10091,13 @@ def _render_problem_files( entry = missing_files.setdefault(fname, { "name": fname, "required": p["required"], "platforms": [], "reason": p["reason"], "categories": set(), + "where": {}, }) entry["platforms"].append(p["platform"]) + # The platform's entry for the file, and the folder it expects. + entry["where"].setdefault( + (p["platform"], p.get("platform_key", ""), p.get("system", "")), set(), + ).add(p.get("destination") or fname) entry["categories"].add(p.get("category", "bios")) if p["required"]: entry["required"] = True @@ -10130,7 +10145,9 @@ def _render_problem_files( "load, `game_data` comes from a copy of the game, `unread` is " "opened by none of them. Only a missing `bios` entry stops a " "console from starting. A name a list declares in several " - "folders is one row.", + "folders is one row. Each platform links to its entry, which " + "names the cores examined, and is followed by where its list " + "puts the file.", "", "| File | Category | Required | Platforms |", "|------|----------|----------|-----------|", @@ -10138,7 +10155,12 @@ def _render_problem_files( for fname in sorted(missing_files): f = missing_files[fname] req = "yes" if f["required"] else "no" - plats = ", ".join(sorted(set(f["platforms"]))) + plats = "; ".join( + (f"[{display}](platforms/{key}.md#{system.replace(' ', '-')})" + if key and system else display) + + " " + ", ".join(_code(dest) for dest in sorted(dests)) + for (display, key, system), dests in sorted(f["where"].items()) + ) cats = f["categories"] category = "bios" if "bios" in cats else ", ".join(sorted(cats)) lines.append(f"| `{fname}` | {category} | {req} | {plats} |") @@ -10377,6 +10399,7 @@ def generate_gap_analysis( "discrepancy": d.get("discrepancy", ""), "system": d.get("system", ""), "declared": _declared_hashes(cov.get("config") or {}, d), + "destination": _declared_destination(cov.get("config") or {}, d), "path": d.get("path") or "", }) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 4f18a8ca8..7c71c3c5a 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5015,6 +5015,28 @@ class GapsPageCountsOneWay(unittest.TestCase): self.assertIn('rb-badge-danger">1 name', text) self.assertNotIn("1 missing", text) + def test_a_missing_row_links_each_entry_and_names_its_folder(self): + """The row said `N88_KNJ1.ROM | unread | yes | AmberELEC` in plain text: + neither where the list puts it nor which cores were examined.""" + from scripts.generate_site import _render_problem_files + + problems = [{"platform": "AmberELEC", "platform_key": "amberelec", "name": "N88_KNJ1.ROM", + "status": "missing", "required": True, "reason": "", "discrepancy": "", + "category": "unread", "system": "nec-pc-88", + "destination": "bios/quasi88/N88_KNJ1.ROM"}] + text = "\n".join(_render_problem_files(problems)) + self.assertIn("| [AmberELEC](platforms/amberelec.md#nec-pc-88) " + "`bios/quasi88/N88_KNJ1.ROM` |", text) + + def test_the_destination_is_the_list_entrys(self): + from scripts.generate_site import _declared_destination + sys.path.insert(0, str(ROOT / "scripts")) + from common import load_platform_config + + config = load_platform_config("amberelec", str(ROOT / "platforms")) + detail = {"name": "N88_KNJ1.ROM", "system": "nec-pc-88"} + self.assertEqual(_declared_destination(config, detail), "bios/quasi88/N88_KNJ1.ROM") + def test_verify_marks_the_settled_entry(self): sys.path.insert(0, str(ROOT / "scripts")) import verify From 7fea45fc5410e9a50adfa7c9fc313ff6c5721a68 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:01:53 +0100 Subject: [PATCH 16/36] fix: look data directory files up by every hash --- scripts/generate_site.py | 40 ++++++++++++++++++++++++++++++++++---- tests/test_site_exports.py | 31 +++++++++++++++++++++++++++++ 2 files changed, 67 insertions(+), 4 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index de984fe3d..701f97c82 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4669,6 +4669,34 @@ def _console_system_names(profiles: dict, coverages: dict) -> dict[tuple[str, st return {key: next(iter(names)) for key, names in found.items() if len(names) == 1} +def _data_member_digests( + members: dict, data_registry: dict | None = None, root: Path = _REPO_ROOT, +) -> dict[str, dict[str, str]]: + """sha1 -> every hash the lookup takes, for each data directory member + whose copy in the tree's local cache has the sha1 the manifest records. + The manifests carry sha1 alone: the MD5 of Dolphin's ApprovedInis.json + found nothing although the pack ships it.""" + from common import load_data_dir_registry # noqa: PLC0415 + from hashing import compute_hashes # noqa: PLC0415 + + if data_registry is None: + data_registry = load_data_dir_registry(str(root / "platforms")) + found: dict[str, dict[str, str]] = {} + for ref, tree in sorted(members.items()): + cache = (data_registry.get(ref) or {}).get("local_cache") + if not cache: + continue + for member, info in sorted(tree.items()): + sha1 = str(info.get("sha1", "")) + path = root / cache / member + if not sha1 or sha1 in found or not path.is_file(): + continue + digests = compute_hashes(path, frozenset(HASH_ALGORITHMS)) + if digests.get("sha1") == sha1: + found[sha1] = {algo: digests[algo].lower() for algo in HASH_ALGORITHMS} + return found + + def file_lookup_records( db: dict, coverages: dict, @@ -4690,15 +4718,19 @@ def file_lookup_records( str(info.get("sha1", "")): member.rsplit("/", 1)[-1] for tree in (members or {}).values() for member, info in sorted(tree.items()) } + member_digests = _data_member_digests(members or {}) records: list[dict] = [] for item in locations: sha1 = item["sha1"] entry = files.get(sha1) if entry is None: - # A data directory member: the manifests record its sha1 only. - records.append({"value": sha1, "algorithm": "sha1", "layer": "data-directory", - "name": member_names.get(sha1, ""), "sha1": sha1, - "url": item["url"]}) + # A data directory member: the manifests record its sha1, the + # tree on disk the other three. + digests = member_digests.get(sha1) or {"sha1": sha1} + records.extend({"value": digests[algorithm], "algorithm": algorithm, + "layer": "data-directory", "name": member_names.get(sha1, ""), + "sha1": sha1, "url": item["url"]} + for algorithm in HASH_ALGORITHMS if algorithm in digests) continue parts = str(entry.get("path", "")).split("/") held_by = named.get((_slug(parts[1]), parts[2])) if len(parts) > 3 else None diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 1a3dea8fe..eeb910aa0 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -670,6 +670,37 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase): downloaded. files/?hash= reads one shard instead, and it must answer at least what the search answered: the card, the platforms, the profiles.""" + def test_a_data_directory_member_answers_every_hash(self): + """Dolphin's ApprovedInis.json was found by its SHA1 only: its MD5 + and CRC32 answered no match although the pack ships it.""" + import hashlib + import tempfile + import zlib + + import generate_site # noqa: PLC0415 + + body = b"approved inis\n" + sha1 = hashlib.sha1(body).hexdigest() + with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp: + member = Path(tmp, "data", "dolphin-sys", "ApprovedInis.json") + member.parent.mkdir(parents=True) + member.write_bytes(body) + stale = Path(tmp, "data", "dolphin-sys", "Stale.json") + stale.write_bytes(b"other bytes") + members = {"dolphin-sys": {"ApprovedInis.json": {"sha1": sha1}, + "Stale.json": {"sha1": "ef" * 20}}} + registry = {"dolphin-sys": {"local_cache": "data/dolphin-sys"}} + digests = generate_site._data_member_digests(members, registry, Path(tmp)) + self.assertEqual(digests, {sha1: { + "sha1": sha1, "md5": hashlib.md5(body).hexdigest(), + "sha256": hashlib.sha256(body).hexdigest(), + "crc32": f"{zlib.crc32(body):08x}"}}) + with mock.patch.object(generate_site, "_data_member_digests", return_value=digests): + records = generate_site.file_lookup_records( + {"files": {}}, {}, {}, [{"sha1": sha1, "url": "u"}], + {"dolphin-sys": {"ApprovedInis.json": {"sha1": sha1}}}) + self.assertEqual({r["algorithm"]: r["value"] for r in records}, digests[sha1]) + def test_records_cover_every_layer_and_shard_by_prefix(self): import json as _json import tempfile From 3ef7b2fbe558974da66474923a460bdc0b9e5185 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:08:26 +0100 Subject: [PATCH 17/36] fix: run every pack README test from its file --- tests/test_pack_readme.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_pack_readme.py b/tests/test_pack_readme.py index e63fbbc4c..6ee8c3b64 100644 --- a/tests/test_pack_readme.py +++ b/tests/test_pack_readme.py @@ -111,10 +111,6 @@ class TheOneLinerFollowsTheRegistry(unittest.TestCase): self.assertEqual(packreadme.INSTALL_PS1, f"{INSTALL_BASE}/{sitebootstrap.PS1_NAME}") -if __name__ == "__main__": - unittest.main() - - class GuidesInstallFromTheZipInHand(unittest.TestCase): """Option A sent a user holding the ZIP to the one-liner, which downloads every file again. --from-pack installs from the archive itself.""" @@ -129,3 +125,7 @@ class GuidesInstallFromTheZipInHand(unittest.TestCase): continue with self.subTest(platform=name): self.assertIn(f"--platform {name} --from-pack ", text) + + +if __name__ == "__main__": + unittest.main() From ea2e0b10c4b189a326ee52c62f2389a1094a5360 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:08:26 +0100 Subject: [PATCH 18/36] fix: find an archive by the ROM a list pins in it --- scripts/cross_reference.py | 10 +++++++++- scripts/generate_site.py | 10 ++++++++-- tests/test_site_exports.py | 33 +++++++++++++++++++++++++++++++++ website/src/scripts/site.ts | 8 +++++++- 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 7cf425616..9ffff4127 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -49,6 +49,7 @@ from hashing import parse_hash_list from validation import ( case_folded_slots, declaration_settles, + inner_rom_check, installed_with_emulator, outside_gap_scope, read_from_save_dir, @@ -476,7 +477,14 @@ class FileConsumers: entry, self._db, self._zip_contents, dest_hint=dest, data_dir_registry=self._data_dir_registry, ) - if not path or status in ("not_found", "hash_mismatch"): + if not path or status == "not_found": + return None + # Batocera pins naomi2.zip by the md5 of a ROM inside it: the resolver + # hands back the archive as a mismatch, and the member settles it as + # the builder does. + if status == "hash_mismatch" and not ( + entry.get("zipped_file") and inner_rom_check(entry, path) == "ok" + ): return None return self._sha1_by_path.get(path) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 701f97c82..ee0115e46 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4746,12 +4746,17 @@ def file_lookup_records( url = _site_path(f"platforms/{key}.md#{system_id.replace(' ', '-')}") system_label = _lookup_system_name(system_id) for entry in system.get("files") or []: + # A hash of a ROM inside the archive (zipped_file) designates + # no collection file by itself: the archive holding it is the one. + archive = (consumers.sha1_of(entry) + if consumers is not None and entry.get("zipped_file") else None) for algorithm in HASH_ALGORITHMS: for value in _hash_values(entry.get(algorithm)): records.append({ "value": value, "algorithm": algorithm, "layer": "platform", "owner": key, "name": str(entry.get("name", "")), "url": url, **({"system_name": system_label} if system_label else {}), + **({"sha1": archive} if archive else {}), }) for key, profile in sorted(profiles.items()): if profile.get("type") in ("alias", "test"): @@ -4843,8 +4848,9 @@ def _held_under_every_hash(records: list[dict], files: dict, indexes: dict) -> l for record in records: if record["layer"] not in ("platform", "emulator"): continue - sha1 = _collection_sha1(files, indexes, record["algorithm"], record["value"]) - if sha1 is None: + sha1 = record.get("sha1") or _collection_sha1( + files, indexes, record["algorithm"], record["value"]) + if sha1 not in files: continue record["sha1"] = sha1 for algorithm in HASH_ALGORITHMS: diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index eeb910aa0..151e078c2 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -4,6 +4,7 @@ from __future__ import annotations import hashlib import json +import os import sqlite3 import sys import tempfile @@ -670,6 +671,38 @@ class AHashFindsEveryPageThatNamesIt(unittest.TestCase): downloaded. files/?hash= reads one shard instead, and it must answer at least what the search answered: the card, the platforms, the profiles.""" + def test_an_archive_answers_for_the_rom_a_list_pins_inside_it(self): + """Batocera checks naomi2.zip by the md5 of 315-6146.bin inside it: + dropped on the lookup, the archive listed every setup but Batocera, + and 335 of its 337 zipped_file entries were unreachable that way.""" + import json as _json + + import generate_site # noqa: PLC0415 + from common import load_platform_config # noqa: PLC0415 + from cross_reference import FileConsumers # noqa: PLC0415 + from ziptools import build_zip_contents_index # noqa: PLC0415 + + root = Path(__file__).resolve().parents[1] + db = _json.loads((root / "database.json").read_text(encoding="utf-8")) + config = load_platform_config("batocera", str(root / "platforms")) + entry = next(e for system in config["systems"].values() for e in system.get("files") or [] + if e.get("name") == "naomi2.zip" and e.get("zipped_file")) + cwd = Path.cwd() + try: + os.chdir(root) + consumers = FileConsumers(db, build_zip_contents_index(db)) + archive = consumers.sha1_of(entry) + finally: + os.chdir(cwd) + if archive is None: + self.skipTest("no copy of naomi2.zip holding the pinned ROM") + coverages = {"batocera": {"config": {"systems": {"sega-naomi2": {"files": [entry]}}}}} + records = generate_site.file_lookup_records(db, coverages, {}, [], None, consumers) + found = {(r["owner"], r["value"]) for r in records if r["layer"] == "platform"} + self.assertIn(("batocera", archive), found) + pinned = next(r for r in records if r["layer"] == "platform" and r["algorithm"] == "md5") + self.assertEqual(pinned["sha1"], archive) + def test_a_data_directory_member_answers_every_hash(self): """Dolphin's ApprovedInis.json was found by its SHA1 only: its MD5 and CRC32 answered no match although the pack ships it.""" diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index 3f4470325..aa1252de9 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -502,7 +502,13 @@ const distinct = (items: Item[], key: (item: Item) => string) => { // read under every hash of that file, so the MD5, the SHA1, the CRC32 and // the SHA256 of one dump give one answer. Setups and emulators are counted // per file, by its SHA1. -const renderMatches = async (status: HTMLElement, result: HTMLElement, found: Item[]) => { +const renderMatches = async (status: HTMLElement, result: HTMLElement, matched: Item[]) => { + // A declaration that names the file holding the value: Batocera pins + // naomi2.zip by the MD5 of a ROM inside it, which no file of the + // collection has. + const searched = new Set(matched.map((item) => item.value)); + const carried = [...new Set(matched.flatMap((item) => (item.sha1 && !searched.has(item.sha1) ? [item.sha1] : [])))]; + const found = carried.length ? [...matched, ...(await lookupValues(carried).catch(() => [] as Item[]))] : matched; const held = distinct(found.filter((item) => item.layer === 'collection'), (item) => item.sha1 ?? item.url); const files = await Promise.all(held.map((item) => heldFile(item).catch(() => undefined))); const looked = new Set(found.map((item) => item.value)); From 9c17943996a8b5459a260511ceae8e09f91a649d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:09:55 +0100 Subject: [PATCH 19/36] feat: read the hash out of a checksum tool's line --- scripts/generate_site.py | 3 ++- tests/test_site_r4_components.py | 33 ++++++++++++++++++++++++++++++++ website/src/scripts/site.ts | 16 +++++++++++++++- 3 files changed, 50 insertions(+), 2 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index ee0115e46..a627dda60 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4517,8 +4517,9 @@ def generate_files_page() -> str: "", '
', ' ', + # No pattern: the line md5sum or certutil prints is taken whole, and + # the page reads the hash out of it. ' ', '', "
", diff --git a/tests/test_site_r4_components.py b/tests/test_site_r4_components.py index ea5b6b5e4..03df811e6 100644 --- a/tests/test_site_r4_components.py +++ b/tests/test_site_r4_components.py @@ -265,5 +265,38 @@ class CopyButtonInTheHeader(unittest.TestCase): self.assertIn(".expressive-code .frame.has-title .copy { inset-block-start:", source) +class AChecksumLineIsAHash(unittest.TestCase): + """The lookup and the search took a bare hash only: the line md5sum, + shasum --tag, certutil or Get-FileHash prints was refused whole.""" + + def test_the_hash_is_read_out_of_the_line(self): + source = _read(SITE_TS) + code = (source[source.index("const HASH_LENGTHS"):source.index("\n", source.index("const HASH_LENGTHS"))] + + source[source.index("const normalizeHash"):source.index("// Tables")]) + md5, sha1 = "490f666e1afb15b7362b406ed1cea246", "b05def971d8ec59f346f2d9ac21fb742e3eb6917" + spaced = " ".join(sha1[i:i + 2] for i in range(0, 40, 2)) + cases = [ + f"{md5} scph5501.bin", + f"SHA1 (scph5501.bin) = {sha1}", + f"SHA1 hash of scph5501.bin:\r\n{spaced}\r\nCertUtil: -hashfile command completed successfully.", + f"Algorithm Hash Path\n--------- ---- ----\nSHA1 {sha1.upper()} C:\\bios\\scph5501.bin", + f" 0x{md5.upper()} ", + "8d8cb7e4", + "deadbeef.bin", + f"{md5} {sha1}", + ] + out = _node(code + f"console.log(JSON.stringify({json.dumps(cases)}.map((c) => " + "[normalizeHash(c), isHash(normalizeHash(c))])));") + self.assertEqual(json.loads(out), [ + [md5, True], [sha1, True], [sha1, True], [sha1, True], [md5, True], + ["8d8cb7e4", True], ["deadbeef.bin", False], [md5 + sha1, False], + ]) + + def test_the_form_takes_the_line(self): + source = _read(REPO_ROOT / "scripts" / "generate_site.py") + start = source.index('", start)]) + + if __name__ == "__main__": unittest.main() diff --git a/website/src/scripts/site.ts b/website/src/scripts/site.ts index aa1252de9..469aebee2 100644 --- a/website/src/scripts/site.ts +++ b/website/src/scripts/site.ts @@ -24,7 +24,21 @@ type Item = { }; const siteRoot = () => new URL(import.meta.env.BASE_URL.replace(/\/?$/, '/'), location.origin); -const normalizeHash = (raw: string) => raw.trim().toLowerCase().replace(/^0x/, '').replace(/\s+/g, ''); +// A pasted hash, or the line a checksum tool prints around it: md5sum +// (` `), BSD and shasum --tag (`SHA1 () = `), +// certutil (the bytes spaced on a line of their own), Get-FileHash's table. +// The one word of an MD5, SHA1 or SHA256's length is taken: an 8-character +// word in a file name searched for is no CRC32. None or two, and the text is +// kept as typed, which isHash refuses. +const normalizeHash = (raw: string) => { + const text = raw.trim().toLowerCase(); + const bare = text.replace(/^0x/, '').replace(/\s+/g, ''); + if (/^[0-9a-f]*$/.test(bare)) return bare; + const words = text.split('\n').flatMap((line) => + /^[0-9a-f ]+$/.test(line.trim()) ? [line.replace(/\s+/g, '')] : (line.match(/\b[0-9a-f]+\b/g) ?? [])); + const found = [...new Set(words.filter((word) => word.length > 8 && word.length in HASH_LENGTHS))]; + return found.length === 1 ? found[0] : bare; +}; const isHash = (value: string) => /^[0-9a-f]+$/.test(value) && value.length in HASH_LENGTHS; // Tables From ef477036d60a38aab40e56436a5ff415635918bf Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:10:01 +0100 Subject: [PATCH 20/36] docs: say the lookup takes a checksum tool's line --- scripts/generate_site.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index a627dda60..c01fc7919 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4512,8 +4512,9 @@ def generate_files_page() -> str: "Tools such as clrmamepro and RomVault print one, and so do emulator logs " "and the BIOS lists of setups.", "", - "Paste it below. The page shows the file in the collection that has it, the " - "setups that list it and the emulators that check for it.", + "Paste it below, alone or in the line md5sum or certutil prints. The page " + "shows the file in the collection that has it, the setups that list it and " + "the emulators that check for it.", "", '
', ' ', From 398246b064f17f1a6179145a6d9e54c6a8f12c46 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:11:12 +0100 Subject: [PATCH 21/36] feat: link the change reports and the wanted CSV --- scripts/generate_site.py | 11 +++++++++++ scripts/siteexports.py | 10 ++++++++-- tests/test_export_report.py | 11 +++++++++++ tests/test_provenance.py | 6 ++++++ 4 files changed, 36 insertions(+), 2 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index c01fc7919..b8b79744a 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1847,6 +1847,15 @@ def _publish_manifests(api: Path) -> list[tuple]: _record(targets / source.name) assets.append((targets / source.name, f"Target list: {source.stem}", "application/json", "schemas/target-manifest.schema.json")) + # Every change each updated platform file carries, for a maintainer who + # reviews them by script: the Exports page linked only the files. + exports = api / "exports" + exports.mkdir(parents=True, exist_ok=True) + for source in sorted((_REPO_ROOT / "exports").glob("*.json")): + shutil.copy2(source, exports / source.name) + _record(exports / source.name) + assets.append((exports / source.name, f"Updated platform file, its changes: {source.stem}", + "application/json", None)) release = _REPO_ROOT / "release.json" if release.is_file(): shutil.copy2(release, api / "release.json") @@ -5344,6 +5353,8 @@ def generate_provenance_page( "matching one of these hashes is welcome: see " "[Contributing](contributing.md).", "", + markup.button("Download the list (CSV)", "downloads/provenance-wanted.csv"), + "", ] ) diff --git a/scripts/siteexports.py b/scripts/siteexports.py index d6be34f48..870cbe43d 100644 --- a/scripts/siteexports.py +++ b/scripts/siteexports.py @@ -17,6 +17,8 @@ EXPORTS_DIR = "exports" REPO_BLOB = "https://github.com/Abdess/retrobios/blob/main" REPO_TREE = "https://github.com/Abdess/retrobios/tree/main" REPO_RAW = "https://raw.githubusercontent.com/Abdess/retrobios/main" +# Where the site publishes each report, beside the API (generate_site). +API_EXPORTS = "api/v1/exports" # Report key, then how a line of it reads on the page. CHANGE_KINDS = ( @@ -318,16 +320,20 @@ def generate_exports_page(reports: list[dict], names: dict[str, str]) -> str: originals = report.get("originals") or {} files = report["files"] raw = [f"{REPO_RAW}/{EXPORTS_DIR}/{platform}/{name}" for name in files] + # Every change as JSON, the report the page is written from. + changes = markup.button("Every change (JSON)", f"{API_EXPORTS}/{platform}.json") if len(files) == 1: lines.extend([ markup.button("Download the updated file", raw[0], primary=True) + " " - + markup.button("See it on GitHub", f"{REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}"), + + markup.button("See it on GitHub", f"{REPO_BLOB}/{EXPORTS_DIR}/{platform}/{files[0]}") + + " " + changes, "", ]) else: lines.extend([ markup.button(f"Browse the {len(files)} updated files", - f"{REPO_TREE}/{EXPORTS_DIR}/{platform}", primary=True), + f"{REPO_TREE}/{EXPORTS_DIR}/{platform}", primary=True) + + " " + changes, "", ]) if len(originals) > 1: diff --git a/tests/test_export_report.py b/tests/test_export_report.py index 5ebf2283f..1b25ab45e 100644 --- a/tests/test_export_report.py +++ b/tests/test_export_report.py @@ -89,6 +89,17 @@ class CorrectedFilesPage(unittest.TestCase): self.page, ) + def test_every_section_links_its_changes_as_json(self): + """exports/batocera.json listed 141 changes and no page or catalog + entry named it: a maintainer had to scrape the HTML.""" + for platform in ("recalbox", "retropie"): + self.assertIn(f'href="api/v1/exports/{platform}.json"', self.page) + source = (ROOT / "scripts" / "generate_site.py").read_text(encoding="utf-8") + publish = source[source.index("def _publish_manifests("):] + publish = publish[:publish.index("\ndef ")] + self.assertIn('(_REPO_ROOT / "exports").glob("*.json")', publish) + self.assertIn('exports = api / "exports"', publish) + def test_several_files_link_to_their_directory(self): self.assertIn("tree/main/exports/retropie", self.page) self.assertIn("The 2 original files it is based on", self.page) diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 14ce31bfe..9f3eebea4 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -470,6 +470,12 @@ class TestProvenancePage(unittest.TestCase): self.assertIn("14df4f6c1e367ce097c11deae21566b4fe5647a9", page) self.assertIn("### Redump", page) + def test_the_wanted_list_links_its_csv(self): + """provenance-wanted.csv was linked from the data page alone.""" + page = self._page() + wanted = page[page.index("## Wanted: catalogued dumps"):] + self.assertIn('href="downloads/provenance-wanted.csv"', wanted[:wanted.index("###")]) + def test_page_has_anchor_targets_for_badges(self): """System-page badges link to ../provenance.md#.""" page = self._page() From 4efd64be704d6b3d25e8ec7db78b73d7a1cb9cbe Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:13:47 +0100 Subject: [PATCH 22/36] feat: publish a Logiqx DAT per setup --- scripts/generate_site.py | 87 ++++++++++++++++++++++++++++++--- tests/test_audit_regressions.py | 26 ++++++++++ 2 files changed, 105 insertions(+), 8 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index b8b79744a..8f241f97a 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1664,6 +1664,7 @@ def generate_data_exports( ), }) write_if_changed(str(downloads / "retrobios.dat"), collection_dat(db, generated_at)) + platform_dats = _write_platform_dats(downloads / "dat", db, members, generated_at) write_if_changed( str(downloads / "files.csv"), _csv_document( @@ -1741,6 +1742,7 @@ def generate_data_exports( (downloads / "files.csv", "File hashes CSV", "text/csv", None), (downloads / "retrobios.dat", "Collection DAT (Logiqx XML, for clrmamepro and RomVault)", "application/xml", None), + *((path, f"Setup DAT: {path.stem}", "application/xml", None) for path in platform_dats), (downloads / "platform-files.csv", "Platform declarations CSV", "text/csv", None), (downloads / "cross-reference.csv", "Cross-reference CSV", "text/csv", None), (downloads / "gaps.csv", "Verification and coverage gaps CSV", "text/csv", None), @@ -1988,6 +1990,14 @@ def _is_target_list(item: dict) -> bool: return item["url"].startswith("api/v1/targets/") +def _is_setup_dat(item: dict) -> bool: + return item["url"].startswith("downloads/dat/") + + +def _is_export_report(item: dict) -> bool: + return item["url"].startswith("api/v1/exports/") + + def _served_schemas() -> list[tuple[str, str]]: """(file name, title) of every schema api/v1/schemas/ serves, from the folder the export copies: the page linked 7 of 11.""" @@ -2126,7 +2136,7 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N ] for item in exports: if (item["media_type"] != "application/json" or _is_manifest(item) - or _is_target_list(item)): + or _is_target_list(item) or _is_export_report(item)): continue schema = ( f"[JSON Schema](api/v1/{item['schema']})" @@ -2233,6 +2243,9 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.json')}]({item['url']})" for item in targets) + "." if targets else "", "", + "`api/v1/exports/.json` lists every change of a setup's " + "[updated file](exports.md), the report that page is written from.", + "", "| Field | Meaning |", "|-------|---------|", *_schema_field_rows(manifest_schema, MANIFEST_TOP_FIELDS), @@ -2296,7 +2309,7 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N "|--------|--------|-----:|--------|", ]) for item in exports: - if item["media_type"] == "application/json": + if item["media_type"] == "application/json" or _is_setup_dat(item): continue lines.append( f"| [{item['title']}]({item['url']}) | `{item['media_type']}` | " @@ -2311,8 +2324,14 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N "per `bios///` folder of the collection, named " "`Manufacturer\\Console` so the tools look in that subfolder, each file a " "rom named by its path below that folder. It audits a copy of the " - "collection's `bios/` tree, or rebuilds that layout from your own dumps. A " - "platform pack has its own layout and is checked with `install.py --check`.", + "collection's `bios/` tree, or rebuilds that layout from your own dumps.", + "", + "A setup's DAT, `downloads/dat/.dat`, describes its BIOS folder " + "once installed: one game named after the folder (`bios`, `system`), each " + "file a rom named by its place below it, as its manifest lists it. It audits " + "that folder with the same tools. Setup DATs: " + ", ".join( + f"[{item['url'].rsplit('/', 1)[-1].removesuffix('.dat')}]({item['url']})" + for item in exports if _is_setup_dat(item)) + ".", "", "The SQLite file contains indexed tables for content hashes, provenance " "catalog matches, platform declarations, emulator profiles and current " @@ -2620,8 +2639,6 @@ def collection_dat(db: dict, generated_at: str) -> str: folder, subfolders joined with a backslash as Logiqx DATs write them. An auditor checking a BIOS tree against the collection had JSON, CSV and SQLite, and a converter to write. """ - import xml.etree.ElementTree as ET # noqa: PLC0415 - games: dict[str, list[dict]] = {} files = db.get("files", {}) # Every copy, not the one path the database keeps: an audit of bios/ @@ -2637,12 +2654,64 @@ def collection_dat(db: dict, generated_at: str) -> str: games.setdefault("\\".join(folder), []).append( {**files[sha1], "_rom": "\\".join(parts[1 + len(folder):])} ) + return _logiqx(SITE_NAME, f"{SITE_NAME} collection ({sum(map(len, games.values()))} files)", + games, generated_at) + + +def _write_platform_dats(folder: Path, db: dict, members: dict, generated_at: str) -> list[Path]: + """downloads/dat/.dat for each install manifest.""" + folder.mkdir(parents=True, exist_ok=True) + digests = _data_member_digests(members) + written = [] + for source in sorted((_REPO_ROOT / "install").glob("*.json")): + manifest = json.loads(source.read_text(encoding="utf-8")) + path = folder / f"{source.stem}.dat" + write_if_changed(str(path), platform_dat(manifest, db, digests, generated_at)) + written.append(path) + return written + + +def platform_dat(manifest: dict, db: dict, member_digests: dict[str, dict[str, str]], + generated_at: str) -> str: + """What one setup's BIOS folder holds once installed, as a Logiqx DAT. + + One game named after the folder (`bios`, `system`), each file a rom + named by its place below it, as the install manifest writes it: the + files of the setup's list and those its cores load, and the members of + its data directories. A Batocera user auditing share/bios against the + collection DAT met thousands of sets laid out by manufacturer. + """ + files = db.get("files", {}) + roms: list[dict] = [] + for entry in manifest.get("files") or []: + if entry.get("root") == "save": + continue # installed in the save directory, not this folder + sha1 = str(entry.get("sha1", "")) + roms.append({**files.get(sha1, {}), "sha1": sha1, "size": entry.get("size", 0), + "_rom": str(entry["dest"]).replace("/", "\\")}) + for tree in manifest.get("data_directories") or []: + dest = str(tree.get("dest") or "").strip("/") + for member, info in (tree.get("files") or {}).items(): + sha1 = str(info.get("sha1", "")) + path = f"{dest}/{member}" if dest else member + roms.append({**member_digests.get(sha1, {}), "sha1": sha1, + "size": info.get("size", 0), "_rom": path.replace("/", "\\")}) + folder = str(manifest.get("base_destination") or "bios").strip("/").rsplit("/", 1)[-1] + name = f"{SITE_NAME} - {manifest.get('display_name') or manifest['platform']}" + unique = list({rom["_rom"]: rom for rom in reversed(roms)}.values()) + return _logiqx(name, f"{name} ({len(unique)} files)", {folder: unique}, generated_at) + + +def _logiqx(name: str, description: str, games: dict[str, list[dict]], generated_at: str) -> str: + """Games of roms as a Logiqx datafile, each rom's `_rom` its name.""" + import xml.etree.ElementTree as ET # noqa: PLC0415 + root = ET.Element("datafile") header = ET.SubElement(root, "header") version = generated_at[:10].replace("-", "") for tag, text in ( - ("name", SITE_NAME), - ("description", f"{SITE_NAME} collection ({sum(map(len, games.values()))} files)"), + ("name", name), + ("description", description), ("version", version), ("date", generated_at[:10]), ("author", SITE_NAME), @@ -3759,6 +3828,8 @@ def _platform_data_links(name: str) -> str: ] if (_REPO_ROOT / "install" / f"{name}.json").is_file(): links.append(f"[install manifest]({REPO_URL}/blob/main/install/{name}.json)") + # The installed BIOS folder for clrmamepro and RomVault. + links.append(f"[DAT](../downloads/dat/{name}.dat)") return ", ".join(links) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 7c71c3c5a..a155f1fb6 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -5296,6 +5296,32 @@ class TheCollectionShipsADat(unittest.TestCase): self.assertIn(("Nintendo\\GameCube", "GC\\USA\\IPL.bin"), roms) self.assertEqual(roms[("Sony\\PlayStation", "scph5501.bin")].crc32, "8d8cb7e4") + def test_each_setup_has_a_dat_of_its_installed_folder(self): + """A Batocera user auditing share/bios against the collection DAT met + thousands of sets laid out by manufacturer, and nothing of the folder + Batocera reads.""" + sys.path.insert(0, str(ROOT / "scripts" / "scraper")) + from logiqx_parser import parse_logiqx + from scripts.generate_site import _platform_data_links, platform_dat + + db = {"files": {"a" * 40: {"size": 4, "crc32": "8D8CB7E4", "md5": "1" * 32, + "sha1": "a" * 40}}} + manifest = { + "platform": "batocera", "display_name": "Batocera", "base_destination": "bios", + "files": [{"dest": "dc/naomi2.zip", "sha1": "a" * 40, "size": 4}, + {"dest": "saves/x.bin", "sha1": "a" * 40, "size": 4, "root": "save"}], + "data_directories": [{"dest": "dolphin-emu/Sys", + "files": {"GC/font.bin": {"sha1": "c" * 40, "size": 2}}}], + } + dat = parse_logiqx(platform_dat(manifest, db, {"c" * 40: {"md5": "3" * 32}}, + "2026-11-05T00:00:00Z")) + roms = {(r.game, r.name): r for r in dat.roms} + self.assertEqual(set(roms), {("bios", "dc\\naomi2.zip"), + ("bios", "dolphin-emu\\Sys\\GC\\font.bin")}) + self.assertEqual(roms[("bios", "dc\\naomi2.zip")].md5, "1" * 32) + self.assertEqual(roms[("bios", "dolphin-emu\\Sys\\GC\\font.bin")].md5, "3" * 32) + self.assertIn("[DAT](../downloads/dat/batocera.dat)", _platform_data_links("batocera")) + class ZippedFileMd5IsParsedOnce(unittest.TestCase): From 6fb79cab6d1ffbc895dc32cc19ecd72d5ce73dff Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:15:26 +0100 Subject: [PATCH 23/36] fix: redirect every list part that goes away --- scripts/generate_site.py | 55 +++++++++++++++---- tests/test_site_r2_nav.py | 17 ++++++ ...atform-parts.json => published-parts.json} | 0 3 files changed, 60 insertions(+), 12 deletions(-) rename website/src/data/{platform-parts.json => published-parts.json} (100%) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 8f241f97a..47af25709 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3339,18 +3339,49 @@ PLATFORM_PART_BUDGET = 700 PLATFORM_REFERENCE_BUDGET = 600 -# Every part page a platform list has had, with the system it held: -# platforms/batocera-atari-st.md -> platforms/batocera.md#atari-st. When a -# budget or a list changes, a part can fold back into its platform page, -# and its address then redirects there instead of ending on the 404 page. -PLATFORM_PARTS_FILE = (Path(__file__).resolve().parent.parent - / "website" / "src" / "data" / "platform-parts.json") +# Every part page the site has published, with the page it belongs to: +# platforms/batocera-atari-st.md -> platforms/batocera.md#atari-st, +# gaps-files-to-source-7.md -> gaps.md. When a budget or a list changes, a +# part can fold back or the list shrink, and its address then redirects to +# that page instead of ending on the 404 page. +PUBLISHED_PARTS_FILE = (Path(__file__).resolve().parent.parent + / "website" / "src" / "data" / "published-parts.json") -def _published_platform_parts() -> dict[str, str]: - if not PLATFORM_PARTS_FILE.is_file(): +def _published_parts() -> dict[str, str]: + if not PUBLISHED_PARTS_FILE.is_file(): return {} - return json.loads(PLATFORM_PARTS_FILE.read_text(encoding="utf-8")) + return json.loads(PUBLISHED_PARTS_FILE.read_text(encoding="utf-8")) + + +def _parent_page(link: str) -> str: + """The page a `parent:` link names (/platforms/batocera/), under docs/.""" + path, _, fragment = link.partition("#") + stem = path.strip("/") or "index" + return f"{stem}.md" + (f"#{fragment}" if fragment else "") + + +def record_parts(docs: Path, present: set[str], published: dict[str, str]) -> dict[str, str]: + """``published`` with every page of this run that names its parent. + + Only platform parts were recorded: gaps-files-to-source-7 or + mame-files-10, gone once their list shrank, ended on the 404 page. A + part already recorded keeps its target, which may name a section. + """ + found = dict(published) + for rel in sorted(present): + page = docs / rel + if not page.is_file(): + page = page.with_suffix(".mdx") + try: + with page.open(encoding="utf-8") as handle: + head = handle.read(2048) + except OSError: + continue + link = re.search(r"^parent:\n(?: .*\n)*? link: (\"[^\"\n]*\")", head, re.M) + if link and rel not in found: + found[rel] = _parent_page(json.loads(link[1])) + return dict(sorted(found.items())) # A list moved off a reference page (gaps, provenance) is cut in parts of @@ -12156,7 +12187,7 @@ def _generate(args: argparse.Namespace) -> None: ) if HANDHELD_GUIDE: write_if_changed(str(docs / HANDHELD_GUIDE.path), generate_handheld_guide(HANDHELD_GUIDE)) - published_parts = _published_platform_parts() + published_parts = _published_parts() for name, cov in coverages.items(): # The emulators this platform runs, not every one of the system. runs = resolve_platform_cores(cov["config"], profiles) @@ -12345,8 +12376,6 @@ def _generate(args: argparse.Namespace) -> None: stale += _sweep_top_pages(docs) print("Adding page metadata and structured data...") - write_if_changed(str(PLATFORM_PARTS_FILE), - json.dumps(dict(sorted(published_parts.items())), indent=1) + "\n") lastmod = decorate_markdown_pages(docs) nav = generate_nav(coverages, manufacturers, profiles, registry, @@ -12357,6 +12386,8 @@ def _generate(args: argparse.Namespace) -> None: path.relative_to(docs).with_suffix(".md").as_posix() for path in docs.rglob("*.md*") if path.suffix in (".md", ".mdx") } - {"404.md"} + published_parts = record_parts(docs, present, published_parts) + write_if_changed(str(PUBLISHED_PARTS_FILE), json.dumps(published_parts, indent=1) + "\n") generated = site / "src" / "generated" generated.mkdir(parents=True, exist_ok=True) write_if_changed(str(generated / "sidebar.json"), diff --git a/tests/test_site_r2_nav.py b/tests/test_site_r2_nav.py index 6e3f134a7..a0175dcc1 100644 --- a/tests/test_site_r2_nav.py +++ b/tests/test_site_r2_nav.py @@ -279,6 +279,23 @@ class Redirects(unittest.TestCase): source = Path(generate_site.__file__).read_text(encoding="utf-8") self.assertIn('generated / "redirects.json"', source) + def test_a_list_part_gone_with_its_rows_redirects_to_its_page(self): + """Only platform parts were recorded: gaps-files-to-source-7, gone + once 29 files were collected, ended on the 404 page.""" + with tempfile.TemporaryDirectory() as tmp: + docs = Path(tmp) + part = '---\nlevel: reference\nparent:\n label: "Gaps"\n link: "/gaps/"\n---\n\n# P\n' + (docs / "gaps-files-to-source-7.md").write_text(part, encoding="utf-8") + (docs / "gaps.md").write_text("# Gaps\n", encoding="utf-8") + kept = {"platforms/b-atari-st.md": "platforms/b.md#atari-st"} + published = generate_site.record_parts( + docs, {"gaps-files-to-source-7.md", "gaps.md"}, kept) + self.assertEqual(published, {"gaps-files-to-source-7.md": "gaps.md", **kept}) + # The next run made six parts. + self.assertEqual( + sitestarlight.redirects({}, {"gaps.md", "platforms/b.md"}, published), + {"/gaps-files-to-source-7/": "/gaps/", "/platforms/b-atari-st/": "/platforms/b/#atari-st"}) + class PlatformShareCards(unittest.TestCase): def test_every_platform_page_shares_its_own_card(self): diff --git a/website/src/data/platform-parts.json b/website/src/data/published-parts.json similarity index 100% rename from website/src/data/platform-parts.json rename to website/src/data/published-parts.json From b62ebf384bfc76fb36e28c22162af3d8c9cb67c5 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:16:29 +0100 Subject: [PATCH 24/36] feat: reach the network share from a Mac or Linux --- scripts/generate_site.py | 30 +++++++++++++++++++++++++++++- tests/test_site_r4_entry.py | 14 ++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 47af25709..d355bc57d 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4078,6 +4078,33 @@ def _network_share(name: str, guide: Guide) -> str: return "" +def _share_tabs(name: str, share: str) -> tuple[dict[str, str], dict[str, str]]: + """The command per computer for a network folder (\\\\RECALBOX\\share\\bios), + and how a Mac or a Linux desktop opens that folder first. Only Windows + had a line: a Mac user fell back to SSH or to taking the card out. + + macOS mounts smb://recalbox/share at /Volumes/share. A Linux desktop + mounts it where its file manager chooses, so that tab names the folder. + """ + host, _, rest = share.lstrip("\\").partition("\\") + volume, _, below = rest.partition("\\") + url = f"smb://{host.lower()}/{volume}" + tail = "/" + below.replace("\\", "/") if below else "" + args = { + "windows": f"--platform {name} --dest {share}", + "macos": f"--platform {name} --dest /Volumes/{volume}{tail}", + "linux": f"--platform {name} --dest /path/to/{volume}{tail}", + } + how = { + "macos": (f"In Finder, choose Go, then Connect to Server, and enter `{url}`. " + f"It opens as `/Volumes/{volume}`. Then open **Terminal**, in " + "Applications, then Utilities. Paste the line. Press Return."), + "linux": (f"Open `{url}` in your file manager. Open a terminal, paste the line " + f"and change `/path/to/{volume}` to the folder it opened. Press Enter."), + } + return args, how + + def _offline_line(name: str) -> list[str]: """The offline line, on a setup whose ZIP a release publishes: the installer reads that ZIP with no network (`--from-pack`).""" @@ -4138,10 +4165,11 @@ def _setup_routes(name: str, display: str, page_link: bool = False) -> list[str] if share or (card and guide.run == ("device",)): computer = ["## From your computer { #from-your-computer }", ""] if share: + args, how = _share_tabs(name, share) computer += [ "Needs the device on your network. Run the command on the computer.", "", - *terminal_tabs(("windows",), {"windows": f"--platform {name} --dest {share}"}), + *terminal_tabs(("windows", "macos", "linux"), args, how=how), SAFE_LINK, "", ] diff --git a/tests/test_site_r4_entry.py b/tests/test_site_r4_entry.py index ec4d423dc..404b01946 100644 --- a/tests/test_site_r4_entry.py +++ b/tests/test_site_r4_entry.py @@ -85,6 +85,20 @@ class NoZipNoUnzipText(unittest.TestCase): self.assertIn("Needs the device on your network.", text) self.assertIn(f"--platform {name} --dest {share}\n", text) + def test_a_mac_and_a_linux_desktop_reach_the_share_too(self): + """The share route was a PowerShell tab alone: a Mac user with Recalbox + on Wi-Fi fell back to SSH or to taking the card out.""" + args, how = generate_site._share_tabs("recalbox", "\\\\RECALBOX\\share\\bios") + self.assertEqual(args["macos"], "--platform recalbox --dest /Volumes/share/bios") + self.assertIn("`smb://recalbox/share`", how["macos"]) + self.assertIn("`/Volumes/share`", how["macos"]) + self.assertEqual(args["linux"], "--platform recalbox --dest /path/to/share/bios") + self.assertIn("`smb://recalbox/share`", how["linux"]) + lines = generate_site._setup_routes("recalbox", "Recalbox") + routes = "\n".join(lines[:lines.index("## On the device { #install }")]) + for label in ("Windows", "macOS", "Linux"): + self.assertIn(f'label="{label}"', routes) + def test_the_network_route_needs_a_share_and_no_zip(self): for name in SETUPS: share = generate_site._network_share(name, siteguides.GUIDES[name]) From e3a9d6479b2d64049e19cbb3f2642526e40143cc Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:16:59 +0100 Subject: [PATCH 25/36] fix: badge a differing hash as Other hash --- scripts/generate_site.py | 8 +++++--- tests/test_site_platform_density.py | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index d355bc57d..e76d55ada 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3534,21 +3534,23 @@ def _launcher_refs(config: dict) -> dict[str, list[str]]: # The hash a row shows first, when the list does not say which it checks. _ROW_HASHES = ("sha1", "md5", "sha256", "crc32") # An untested entry whose reason is a hash the bytes do not match. -_WRONG_HASH = re.compile(r"hash|mismatch|expected|must match", re.IGNORECASE) +_OTHER_HASH = re.compile(r"hash|mismatch|expected|must match", re.IGNORECASE) # A list this long gets a filter box above it, as a table of RECORD_ROWS # rows does (website/src/plugins/rehype-stack-tables.mjs). FILTER_ROWS = 20 def _status_chip(f: dict) -> str: - """The row's status as a chip, its word and its colour saying the same.""" + """The row's status as a chip, its word and its colour saying the same. A + declared hash the held copy does not have is "Other hash": the two differ, + which says nothing of whose is right (a platform may pin its own bundle).""" status = f.get("status") if status == "ok": word, tone = "OK", "success" elif status == "missing": word, tone = "Missing", "danger" elif status == "untested": - word = "Wrong hash" if _WRONG_HASH.search(str(f.get("reason") or "")) else "Untested" + word = "Other hash" if _OTHER_HASH.search(str(f.get("reason") or "")) else "Untested" tone = "warning" else: word, tone = str(status), "muted" diff --git a/tests/test_site_platform_density.py b/tests/test_site_platform_density.py index dae52b5ec..df7d981fc 100644 --- a/tests/test_site_platform_density.py +++ b/tests/test_site_platform_density.py @@ -126,8 +126,8 @@ class ARowPerFile(unittest.TestCase): def test_status_words(self): cases = { ("missing", ""): ("Missing", "danger"), - ("untested", "declared hash contradicted by the local file"): ("Wrong hash", "warning"), - ("untested", "expected 1234... got 5678..."): ("Wrong hash", "warning"), + ("untested", "declared hash contradicted by the local file"): ("Other hash", "warning"), + ("untested", "expected 1234... got 5678..."): ("Other hash", "warning"), ("untested", "bios/x read error"): ("Untested", "warning"), } for (status, reason), (word, tone) in cases.items(): From 832b2890d4c22b99c91eaf9c0ca98a974f584084 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:17:51 +0100 Subject: [PATCH 26/36] docs: say what the unsourceable count is --- README.md | 4 ++-- scripts/generate_readme.py | 16 ++++++++-------- tests/test_doc_pages.py | 9 +++++++++ 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 039968708..bebb13a22 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ These are the files an emulator loads from disk instead of carrying inside itsel Each file is checked the way your platform checks it. Most compare a checksum, the fingerprint of a file's contents, which catches a corrupt or unexpected copy. RetroArch, Lakka and RetroPie check that the file is present under its name without reading its contents (`path_is_valid` in RetroArch's `core_info.c`), and MinUI, NextUI and MyMinUI hand each core its folder without opening a file in it: the Coverage table says which applies to you. Independently of that, the collection records five fingerprints per file, and wherever an emulator's code states an expected size or hash, that value is read from its source and rechecked here. -- **1,087 files** the platforms' emulators load are still to be found, and 138 more cannot be sourced at all (per-user keys, user-filled slots, dumps nobody has made). Both are named in the [gap analysis](https://abdess.github.io/retrobios/gaps/) +- **1,087 files** the platforms' emulators load are still to be found, and 138 more the collection does not hold and does not serve by name (retail data, per-user keys, user-filled slots), each with its reason. Both are named in the [gap analysis](https://abdess.github.io/retrobios/gaps/) - **18 platforms** supported with platform-specific verification - **623 emulators** profiled from source (RetroArch cores + standalone) - **533 systems** handled by those emulators (NES, SNES, PlayStation, Saturn, Dreamcast, ...) @@ -206,4 +206,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-11-05T10:52:39Z* +*Auto-generated on 2026-11-05T11:17:19Z* diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 132a2eb3a..3387abbb9 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -535,11 +535,10 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" emulator_count = len(profiles) comp = compute_composition(db) missing_total = len(set().union(*(c["missing_names"] for c in coverages.values()))) - # "Not in the collection yet" implies somebody could still put it there. - # Most of this figure cannot be: per-user registration keys, slots the - # emulator expects the user to fill, dumps nobody has made. Counting those - # beside genuinely acquirable files overstates what is actionable, so the - # two are reported apart. + # The entries marked `unsourceable:` are never served by name, so a + # homonym cannot take their place: retail game data, per-user keys, slots + # the user fills. Each stays an acquisition target with its reason, and + # is counted apart from the files still to be found. unsourceable_total = len( set().union(*(c["unsourceable_names"] for c in coverages.values())) ) @@ -668,9 +667,10 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" "", ( f"- **{acquirable_total:,} files** the platforms' emulators load" - f" are still to be found, and {unsourceable_total:,} more cannot" - f" be sourced at all (per-user keys, user-filled slots, dumps" - f" nobody has made). Both are named in the" + f" are still to be found, and {unsourceable_total:,} more the" + f" collection does not hold and does not serve by name (retail" + f" data, per-user keys, user-filled slots), each with its reason." + f" Both are named in the" f" [gap analysis]({SITE_URL}gaps/)" if missing_total else "- **Nothing missing**: every file the platforms'" diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 96cfaa836..8cf8ed661 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1710,6 +1710,15 @@ class ReadmeKeepsItsPointers(unittest.TestCase): block = block[:block.index("\n## ", 1)] self.assertIn("--required-only", block) + def test_unsourceable_files_are_not_said_to_exist_nowhere(self): + """The README said 138 files 'cannot be sourced at all (... dumps + nobody has made)', while most of their reasons read 'retail data the + collection does not hold'.""" + readme = (ROOT / "README.md").read_text(encoding="utf-8") + self.assertNotIn("sourced at all", readme) + self.assertNotIn("nobody has made", readme) + self.assertRegex(readme, r"more the collection does not hold and does not serve by name") + class ReportExcerptIsTheRun(unittest.TestCase): """reading-reports.md quoted 'From python scripts/verify.py --platform From 9c500ff216a2415210be4e9727947a32540837b2 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:18:51 +0100 Subject: [PATCH 27/36] docs: describe how a contested slot is decided --- tests/test_doc_pages.py | 21 +++++++++++++++++++++ wiki/architecture.md | 30 +++++++++++++++++++++++++----- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 8cf8ed661..b442d79d4 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1720,6 +1720,27 @@ class ReadmeKeepsItsPointers(unittest.TestCase): self.assertRegex(readme, r"more the collection does not hold and does not serve by name") +class ArchitectureStatesTheArbitration(unittest.TestCase): + """architecture.md said the profile takes precedence for pack generation, + while slots.arbitrate() keeps the platform's file where its frontend + checks the content: RetroBat's ATARIOSB.ROM.""" + + def test_the_page_says_what_arbitrate_decides(self): + import slots # noqa: PLC0415 + + page = (WIKI / "architecture.md").read_text(encoding="utf-8") + self.assertNotIn("the profile takes precedence for pack generation", page) + section = page[page.index("### When both layers claim one destination"):] + section = section[:section.index("\n## ")] + self.assertIn("`slots.arbitrate()`", section) + source = inspect.getsource(slots.arbitrate) + # The three outcomes the section describes. + for outcome in ("ADDRESSEE", "SERVES_BOTH", "FRONTEND_CHECKS_CONTENT"): + self.assertIn(outcome, source) + for said in ("--emulator", "`existence`", "the platform's file stays"): + self.assertIn(said, section) + + class ReportExcerptIsTheRun(unittest.TestCase): """reading-reports.md quoted 'From python scripts/verify.py --platform retrodeck' a run that said 0 required missing after the command had started diff --git a/wiki/architecture.md b/wiki/architecture.md index 590a68207..bd69cd264 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -169,11 +169,31 @@ The emulator's source code serves as ground truth for what files are needed, what names they use, and what validation the emulator performs. Platform YAML configs are scraped from upstream and record what each platform declares. A list kept by hand can lag behind an emulator release. The emulator profiles complement the -platform data by documenting what the code actually loads. When the two disagree, -the profile takes precedence for pack generation: files the code needs are included -even if the platform does not declare them. Files the platform declares but no -profile references are kept as well (flagged during cross-reference), since the -upstream may cover cases not yet profiled. +platform data by documenting what the code actually loads. Files the code needs +are included even if the platform does not declare them. Files the platform +declares but no profile references are kept as well (flagged during +cross-reference), since the upstream may cover cases not yet profiled. + +### When both layers claim one destination + +A platform and an emulator profile can each name a different file for the same +place in the BIOS folder: RetroBat puts its own `ATARIOSB.ROM` at +`bios/ATARIOSB.ROM`, where atari800 reads another dump. A claim counts only when +it rests on content or on the path. A file found by its name alone claims +nothing. The pack then answers to whoever asked for it: + +- A **platform pack** must leave the frontend's own check green, since a user + reading red concludes the pack is broken. Where the frontend only looks for + the name (`existence`), the emulator's file satisfies both, and the pack takes + it. Where the frontend checks an MD5 or SHA1, one path cannot hold both + answers: the platform's file stays, and the loss is reported. Only the + platform's own list can change what its frontend accepts there. +- A pack built with `--emulator` answers to that emulator, and takes its file. + +`scripts/slots.py` prints each contested destination and the reason for its +winner (`python scripts/slots.py --platform retrobat`). `generate_pack.py` and +`verify.py` both decide through `slots.arbitrate()`, so the pack and the report +agree. A platform YAML is regenerated by its scraper and never edited by hand: an edit would be lost at the next scrape, and the file would stop describing the From efd5f0b43c9f1914d0219f7fa5d137cf1bb3f8e1 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:19:09 +0100 Subject: [PATCH 28/36] fix: keep test prose free of semicolons --- tests/test_target_names.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_target_names.py b/tests/test_target_names.py index 263a555d0..21bea65c3 100644 --- a/tests/test_target_names.py +++ b/tests/test_target_names.py @@ -198,7 +198,7 @@ if __name__ == "__main__": class PublishedListsAreNotNarrowed(unittest.TestCase): """pipeline --target switch wrote install/retroarch_nintendoswitch.json - next to the published lists; the site picker then replaced RetroArch's + next to the published lists. The site picker then replaced RetroArch's file list with the Switch one and check_release asked for it as an asset.""" def test_pipeline_writes_install_unfiltered(self): From e835c3862ebbaf18f43780a17d677cd2275a83b8 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:19:42 +0100 Subject: [PATCH 29/36] docs: document every version_basis kind and key --- tests/test_doc_pages.py | 17 +++++++++++++++++ wiki/profiling.md | 2 +- wiki/tools.md | 8 +++++--- 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index b442d79d4..ab05252a6 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1741,6 +1741,23 @@ class ArchitectureStatesTheArbitration(unittest.TestCase): self.assertIn(said, section) +class VersionBasisReferenceIsTheSchema(unittest.TestCase): + """profiling.md listed five kinds of version_basis: `unverified`, its + `why` and the `url`, `commit` and `match` keys 18 profiles use were in + the schema alone.""" + + def test_the_row_names_every_kind_reason_and_key(self): + schema = json.loads((ROOT / "schemas" / "emulator.schema.json").read_text(encoding="utf-8")) + basis = schema["$defs"]["version_basis"]["properties"] + page = (WIKI / "profiling.md").read_text(encoding="utf-8") + row = next(line for line in page.splitlines() if line.startswith("| `version_basis` |")) + for word in [*basis["kind"]["enum"], *basis["why"]["enum"], *basis]: + with self.subTest(word=word): + self.assertIn(f"`{word}`", row) + tools = (WIKI / "tools.md").read_text(encoding="utf-8") + self.assertIn("`unverified`", tools) + + class ReportExcerptIsTheRun(unittest.TestCase): """reading-reports.md quoted 'From python scripts/verify.py --platform retrodeck' a run that said 0 required missing after the command had started diff --git a/wiki/profiling.md b/wiki/profiling.md index 0b69e0423..63780e168 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -517,7 +517,7 @@ reports the profile as uncheckable, and the CI job that runs | `upstream` | no | original emulator repository URL | | `profiled_date` | see above | date of source analysis. Quote it, or YAML parses it as a date object | | `core_version` | every profile (`tests/test_version_basis.py`) | the version of the code at the pin, read where `version_basis` says. null when the code states none, never `Git`, `SVN` or a branch name. A dict keyed by build mode when the builds differ | -| `version_basis` | every profile (`tests/test_version_basis.py`) | where `core_version` is read: `file` (`path`, `pattern`, optional `format`), `tag` (the nearest `tag` in the pin's history and its `distance`, `core_version` in git describe's form), `alias` (`of`), `package` (`artifact`, the APK or binary read) or `none`. `repo` names `upstream` when the version is the upstream tree's. `profile_sync --verify-version` reads it again, and a pin advance rewrites `core_version` from it | +| `version_basis` | every profile (`tests/test_version_basis.py`) | where `core_version` is read, by `kind`: `file` (`path`, `pattern`, optional `format`, and `url` with its `commit` for a file outside the profile's repositories, such as the package file of a build a platform ships), `tag` (the nearest `tag` in the pin's history and its `distance`, `core_version` in git describe's form, `match` the glob a tag must fit), `alias` (`of`), `package` (`artifact`, the APK or binary read, `url` where it was obtained), `none` (the code states no version: `core_version` null) or `unverified` (no reading could be made: `core_version` stays as written and is reported unconfirmed, `why` says why: `no_pin`, `host_unsupported`, `host_gone`, `host_unreachable`, `pin_off_branch`, `no_artifact`, `no_source`). `repo` names `upstream` or `source_mirror` when the version is that tree's. `profile_sync --verify-version` reads it again, and a pin advance rewrites `core_version` from it | | `source_commit` | see above | commit SHA of `source` the code was read at. Anchors every `source_ref` line number | | `upstream_commit` | no | same, for `upstream` | | `pin_frozen` | no | why `source_commit` is held on purpose rather than followed. The profile is then judged against its own revision, like one pinned to a superseded tag | diff --git a/wiki/tools.md b/wiki/tools.md index 577296f43..3f81d1340 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -591,9 +591,11 @@ shows added, removed and renamed files in the directories the refs point at. says where it is read: a pattern in one file (`file`), the nearest tag and the commits after it (`tag`, written in git describe's form), the profile an alias follows (`alias`), an artefact read outside any source tree -(`package`), or `none` when the code states nothing, with `core_version` -null. `--verify-version` reads it again at the pin and exits 1 on a -difference or a missing basis. `--set-version` writes what the code states. +(`package`), `none` when the code states nothing, with `core_version` +null, or `unverified` when no reading can be made (a forge gone, a host the +tools cannot read, no artefact to read): `core_version` stays as written, +reported unconfirmed, and `why` names the reason. `--verify-version` reads +it again at the pin and exits 1 on a difference or a missing basis. `--set-version` writes what the code states. A pin advance reads the version at the new pin in the same write and is refused when it cannot: a package version moves by hand. From 3a36350f5cde33a02c8d7699bdd96c856dfdd532 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:19:58 +0100 Subject: [PATCH 30/36] docs: state the .info hash rule one way --- tests/test_doc_pages.py | 12 ++++++++++++ wiki/profiling.md | 12 +++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index ab05252a6..3fdfe59e5 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1758,6 +1758,18 @@ class VersionBasisReferenceIsTheSchema(unittest.TestCase): self.assertIn("`unverified`", tools) +class ProfilingStatesTheHashRuleOnce(unittest.TestCase): + """profiling.md let an .info MD5 in ('source confirmation makes it more + reliable') and a file the code never loads stay ('can be left out'), + beside the strict rule of the same paragraph.""" + + def test_no_permissive_form_remains(self): + page = (WIKI / "profiling.md").read_text(encoding="utf-8") + self.assertNotIn("source confirmation makes it more reliable", page) + self.assertNotIn("it can be left out of the profile", page) + self.assertIn("An MD5 the .info or\n the docs alone give is not written", page) + + class ReportExcerptIsTheRun(unittest.TestCase): """reading-reports.md quoted 'From python scripts/verify.py --platform retrodeck' a run that said 0 required missing after the command had started diff --git a/wiki/profiling.md b/wiki/profiling.md index 63780e168..46a8e853b 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -383,8 +383,9 @@ A few field conventions that protect the toolchain: string (like emuscv's `635a978...` in memory.cpp), that goes in. If the source embeds ROM data as byte arrays (like ep128emu's roms.hpp), the bytes can be extracted and hashed. If the source performs no hash - check at all, the hash is omitted from the profile. The .info or docs - may list an MD5, but source confirmation makes it more reliable. + check at all, the hash is omitted from the profile. An MD5 the .info or + the docs alone give is not written: pokketstation's `.info` names a + revision no copy in circulation has. A data file the build generates is rebuilt from the tree and compared byte for byte with the official release before its hash is recorded (PrBoom+ builds `prboom-plus.wad` with its own tool). A firmware URL that @@ -493,9 +494,10 @@ and DSP firmware through different code paths. Checking the actual code for each core avoids assumptions based on a related profile. **Code over docs.** Wiki pages and README files sometimes reference -files from older versions or a different fork. If the source code -does not load a particular file, it can be left out of the profile -even if documentation mentions it. +files from older versions or a different fork. A file the source code +does not load is left out of the profile, even when documentation or the +`.info` mentions it (desmume2015's `.info` lists three firmware files its +code never opens). ## YAML field reference From 1e96968744c0971619617f5be1de494dc8c79d8d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:24:18 +0100 Subject: [PATCH 31/36] fix: record list parts by their front matter --- scripts/generate_site.py | 10 +- scripts/sitestarlight.py | 5 +- tests/test_site_r2_nav.py | 5 + website/src/data/published-parts.json | 260 +++++++++++++++++++++++++- 4 files changed, 272 insertions(+), 8 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index e76d55ada..3ca81db28 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3357,6 +3357,8 @@ def _published_parts() -> dict[str, str]: def _parent_page(link: str) -> str: """The page a `parent:` link names (/platforms/batocera/), under docs/.""" path, _, fragment = link.partition("#") + # Written with the site's base (/retrobios/emulators/x/) or without it. + path = path.removeprefix(sitestarlight.BASE.rstrip("/")) stem = path.strip("/") or "index" return f"{stem}.md" + (f"#{fragment}" if fragment else "") @@ -3374,13 +3376,11 @@ def record_parts(docs: Path, present: set[str], published: dict[str, str]) -> di if not page.is_file(): page = page.with_suffix(".mdx") try: - with page.open(encoding="utf-8") as handle: - head = handle.read(2048) + parent = _front_matter(page.read_text(encoding="utf-8")).get("parent") except OSError: continue - link = re.search(r"^parent:\n(?: .*\n)*? link: (\"[^\"\n]*\")", head, re.M) - if link and rel not in found: - found[rel] = _parent_page(json.loads(link[1])) + if isinstance(parent, dict) and parent.get("link") and rel not in found: + found[rel] = _parent_page(str(parent["link"])) return dict(sorted(found.items())) diff --git a/scripts/sitestarlight.py b/scripts/sitestarlight.py index dad8fa63f..bf532b9f5 100644 --- a/scripts/sitestarlight.py +++ b/scripts/sitestarlight.py @@ -819,12 +819,13 @@ def redirects(profiles: dict, present: set[str], that is already a page's address, is left out. Addresses are written under the site root, without its base. ``moved`` adds pages that moved by generation, old page to new page, the new one with an optional - #fragment.""" + #fragment. A page MOVED_PAGES names keeps that target over the part + record: cross-reference-minui is answered on MinUI's own page.""" def address(rel: str) -> str: page, _, fragment = rel.partition("#") return "/" + page_url(page)[len(BASE):] + (f"#{fragment}" if fragment else "") - out = {address(old): address(new) for old, new in {**MOVED_PAGES, **(moved or {})}.items() + out = {address(old): address(new) for old, new in {**(moved or {}), **MOVED_PAGES}.items() if old not in present and new.partition("#")[0] in present} pages = {rel.casefold() for rel in present} claims: dict[str, set[str]] = {} diff --git a/tests/test_site_r2_nav.py b/tests/test_site_r2_nav.py index a0175dcc1..4f90adb2e 100644 --- a/tests/test_site_r2_nav.py +++ b/tests/test_site_r2_nav.py @@ -291,6 +291,11 @@ class Redirects(unittest.TestCase): published = generate_site.record_parts( docs, {"gaps-files-to-source-7.md", "gaps.md"}, kept) self.assertEqual(published, {"gaps-files-to-source-7.md": "gaps.md", **kept}) + # An emulator part names its parent with the site's base. + self.assertEqual(generate_site._parent_page("/retrobios/emulators/amiberry/"), + "emulators/amiberry.md") + self.assertEqual(generate_site._parent_page("/platforms/b/#atari-st"), + "platforms/b.md#atari-st") # The next run made six parts. self.assertEqual( sitestarlight.redirects({}, {"gaps.md", "platforms/b.md"}, published), diff --git a/website/src/data/published-parts.json b/website/src/data/published-parts.json index 7c119e52a..3521fb626 100644 --- a/website/src/data/published-parts.json +++ b/website/src/data/published-parts.json @@ -1,4 +1,62 @@ { + "cross-reference-amberelec.md": "cross-reference.md", + "cross-reference-batocera.md": "cross-reference.md", + "cross-reference-emudeck.md": "cross-reference.md", + "cross-reference-emuelec.md": "cross-reference.md", + "cross-reference-knulli.md": "cross-reference.md", + "cross-reference-lakka.md": "cross-reference.md", + "cross-reference-minui.md": "cross-reference.md", + "cross-reference-myminui.md": "cross-reference.md", + "cross-reference-nextui.md": "cross-reference.md", + "cross-reference-recalbox.md": "cross-reference.md", + "cross-reference-retroarch.md": "cross-reference.md", + "cross-reference-retrobat.md": "cross-reference.md", + "cross-reference-retrodeck.md": "cross-reference.md", + "cross-reference-retropie.md": "cross-reference.md", + "cross-reference-rocknix.md": "cross-reference.md", + "cross-reference-romm.md": "cross-reference.md", + "emulators/amiberry-files-1.md": "emulators/amiberry.md", + "emulators/amiberry-files-2.md": "emulators/amiberry.md", + "emulators/data-dolphin-sys-2.md": "emulators/dolphin.md", + "emulators/data-dolphin-sys-3.md": "emulators/dolphin.md", + "emulators/data-dolphin-sys.md": "emulators/dolphin.md", + "emulators/mame-files-1.md": "emulators/mame.md", + "emulators/mame-files-10.md": "emulators/mame.md", + "emulators/mame-files-2.md": "emulators/mame.md", + "emulators/mame-files-3.md": "emulators/mame.md", + "emulators/mame-files-4.md": "emulators/mame.md", + "emulators/mame-files-5.md": "emulators/mame.md", + "emulators/mame-files-6.md": "emulators/mame.md", + "emulators/mame-files-7.md": "emulators/mame.md", + "emulators/mame-files-8.md": "emulators/mame.md", + "emulators/mame-files-9.md": "emulators/mame.md", + "emulators/vice-standalone-files-1.md": "emulators/vice-standalone.md", + "emulators/vice-standalone-files-2.md": "emulators/vice-standalone.md", + "gaps-files-to-source-2.md": "gaps.md", + "gaps-files-to-source-3.md": "gaps.md", + "gaps-files-to-source-4.md": "gaps.md", + "gaps-files-to-source-5.md": "gaps.md", + "gaps-files-to-source-6.md": "gaps.md", + "gaps-files-to-source-7.md": "gaps.md", + "gaps-files-to-source.md": "gaps.md", + "get/amberelec.md": "platforms/amberelec.md", + "get/batocera.md": "platforms/batocera.md", + "get/bizhawk.md": "platforms/bizhawk.md", + "get/emudeck.md": "platforms/emudeck.md", + "get/emuelec.md": "platforms/emuelec.md", + "get/knulli.md": "platforms/knulli.md", + "get/lakka.md": "platforms/lakka.md", + "get/minui.md": "platforms/minui.md", + "get/misterfpga.md": "platforms/misterfpga.md", + "get/myminui.md": "platforms/myminui.md", + "get/nextui.md": "platforms/nextui.md", + "get/recalbox.md": "platforms/recalbox.md", + "get/retroarch.md": "platforms/retroarch.md", + "get/retrobat.md": "platforms/retrobat.md", + "get/retrodeck.md": "platforms/retrodeck.md", + "get/retropie.md": "platforms/retropie.md", + "get/rocknix.md": "platforms/rocknix.md", + "get/romm.md": "platforms/romm.md", "platforms/batocera-acorn-bbc-micro.md": "platforms/batocera.md#acorn-bbc-micro", "platforms/batocera-apple-ii.md": "platforms/batocera.md#apple-ii", "platforms/batocera-apple-macintosh-ii.md": "platforms/batocera.md#apple-macintosh-ii", @@ -78,5 +136,205 @@ "platforms/retropie-snk-neogeo-cd.md": "platforms/retropie.md#snk-neogeo-cd", "platforms/retropie-sony-playstation-2.md": "platforms/retropie.md#sony-playstation-2", "platforms/retropie-sony-playstation.md": "platforms/retropie.md#sony-playstation", - "platforms/romm-sony-playstation-2.md": "platforms/romm.md#sony-playstation-2" + "platforms/romm-sony-playstation-2.md": "platforms/romm.md#sony-playstation-2", + "provenance-no-intro.md": "provenance.md", + "provenance-redump.md": "provenance.md", + "provenance-tosec.md": "provenance.md", + "systems/arcade-aamber-pegasus.md": "systems/arcade.md", + "systems/arcade-acclaim-zn-1.md": "systems/arcade.md", + "systems/arcade-allied-leisure-pinball.md": "systems/arcade.md", + "systems/arcade-american-laser-games-laserdisc.md": "systems/arcade.md", + "systems/arcade-andamiro-pump-it-up.md": "systems/arcade.md", + "systems/arcade-arcade-2.md": "systems/arcade.md", + "systems/arcade-arcade-3.md": "systems/arcade.md", + "systems/arcade-arcade-4.md": "systems/arcade.md", + "systems/arcade-arcade.md": "systems/arcade.md", + "systems/arcade-arcadia-systems-multi-select.md": "systems/arcade.md", + "systems/arcade-aristocrat-mk5.md": "systems/arcade.md", + "systems/arcade-aristocrat-mk6.md": "systems/arcade.md", + "systems/arcade-aristocrat-mk7.md": "systems/arcade.md", + "systems/arcade-atari-system-1.md": "systems/arcade.md", + "systems/arcade-atari-zn-1.md": "systems/arcade.md", + "systems/arcade-atlus-zn-1.md": "systems/arcade.md", + "systems/arcade-barcrest-mpu4.md": "systems/arcade.md", + "systems/arcade-brezzasoft-crystal-system.md": "systems/arcade.md", + "systems/arcade-capcom-zn-1.md": "systems/arcade.md", + "systems/arcade-capcom-zn-2.md": "systems/arcade.md", + "systems/arcade-cd-express-cubo-cd32.md": "systems/arcade.md", + "systems/arcade-century-cvs.md": "systems/arcade.md", + "systems/arcade-ces-galaxy-games.md": "systems/arcade.md", + "systems/arcade-data-east-deco-cassette-system.md": "systems/arcade.md", + "systems/arcade-efo-cedar-magnet-system.md": "systems/arcade.md", + "systems/arcade-eighting-zn-1.md": "systems/arcade.md", + "systems/arcade-exidy-max-a-flex.md": "systems/arcade.md", + "systems/arcade-game-plan-mpu-1.md": "systems/arcade.md", + "systems/arcade-gottlieb-system-1.md": "systems/arcade.md", + "systems/arcade-igs-polygame-master.md": "systems/arcade.md", + "systems/arcade-imax-macs.md": "systems/arcade.md", + "systems/arcade-incredible-technologies-eagle.md": "systems/arcade.md", + "systems/arcade-kaneko-super-nova-system.md": "systems/arcade.md", + "systems/arcade-konami-bubble-system.md": "systems/arcade.md", + "systems/arcade-konami-endeavour.md": "systems/arcade.md", + "systems/arcade-konami-gv.md": "systems/arcade.md", + "systems/arcade-konami-gx.md": "systems/arcade.md", + "systems/arcade-konami-python-2.md": "systems/arcade.md", + "systems/arcade-konami-python.md": "systems/arcade.md", + "systems/arcade-konami-system-573.md": "systems/arcade.md", + "systems/arcade-konami-twinkle.md": "systems/arcade.md", + "systems/arcade-konami-viper.md": "systems/arcade.md", + "systems/arcade-laserdisc-arcade-games.md": "systems/arcade.md", + "systems/arcade-mame-2.md": "systems/arcade.md", + "systems/arcade-mame-3.md": "systems/arcade.md", + "systems/arcade-mame-4.md": "systems/arcade.md", + "systems/arcade-mame-5.md": "systems/arcade.md", + "systems/arcade-mame-6.md": "systems/arcade.md", + "systems/arcade-mame-7.md": "systems/arcade.md", + "systems/arcade-mame-8.md": "systems/arcade.md", + "systems/arcade-mame.md": "systems/arcade.md", + "systems/arcade-midway-mcr.md": "systems/arcade.md", + "systems/arcade-namco-system-11.md": "systems/arcade.md", + "systems/arcade-namco-system-246.md": "systems/arcade.md", + "systems/arcade-namco-system-256.md": "systems/arcade.md", + "systems/arcade-nichibutsu-high-rate-dvd.md": "systems/arcade.md", + "systems/arcade-nintendo-aleck64.md": "systems/arcade.md", + "systems/arcade-nintendo-playchoice-10.md": "systems/arcade.md", + "systems/arcade-nintendo-super-system.md": "systems/arcade.md", + "systems/arcade-nmk16.md": "systems/arcade.md", + "systems/arcade-plug-and-play-tv-games.md": "systems/arcade.md", + "systems/arcade-recel-system-iii.md": "systems/arcade.md", + "systems/arcade-sammy-atomiswave.md": "systems/arcade.md", + "systems/arcade-sammy-medal-games.md": "systems/arcade.md", + "systems/arcade-sega-isg-selection-master.md": "systems/arcade.md", + "systems/arcade-sega-mega-play.md": "systems/arcade.md", + "systems/arcade-sega-mega-tech.md": "systems/arcade.md", + "systems/arcade-sega-model-2.md": "systems/arcade.md", + "systems/arcade-sega-naomi-2.md": "systems/arcade.md", + "systems/arcade-sega-naomi-gd-rom.md": "systems/arcade.md", + "systems/arcade-sega-naomi.md": "systems/arcade.md", + "systems/arcade-sega-shooting-zone.md": "systems/arcade.md", + "systems/arcade-sega-system-sp.md": "systems/arcade.md", + "systems/arcade-sielectronics-system-y2.md": "systems/arcade.md", + "systems/arcade-snk-hyper-neo-geo-64.md": "systems/arcade.md", + "systems/arcade-tab-austria-silverball.md": "systems/arcade.md", + "systems/arcade-taito-c-chip.md": "systems/arcade.md", + "systems/arcade-taito-fx-1.md": "systems/arcade.md", + "systems/arcade-taito-g-net.md": "systems/arcade.md", + "systems/arcade-taito-type-zero.md": "systems/arcade.md", + "systems/arcade-tecmo-tps.md": "systems/arcade.md", + "systems/arcade-tourvision.md": "systems/arcade.md", + "systems/arcade-video-system-zn-1.md": "systems/arcade.md", + "systems/arcade-virtuality-su2000.md": "systems/arcade.md", + "systems/arcade-visual-pinball-tables.md": "systems/arcade.md", + "systems/bungie-marathon-2.md": "systems/bungie.md", + "systems/bungie-marathon.md": "systems/bungie.md", + "systems/commodore-amiga-2.md": "systems/commodore.md", + "systems/commodore-amiga.md": "systems/commodore.md", + "systems/commodore-c128.md": "systems/commodore.md", + "systems/epic-megagames-jazz-jackrabbit-2-2.md": "systems/epic-megagames.md", + "systems/epic-megagames-jazz-jackrabbit-2.md": "systems/epic-megagames.md", + "systems/game-engines-c-dogs-sdl-10.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-11.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-12.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-2.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-3.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-4.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-5.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-6.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-7.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-8.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl-9.md": "systems/game-engines.md", + "systems/game-engines-c-dogs-sdl.md": "systems/game-engines.md", + "systems/game-engines-hurrican-2.md": "systems/game-engines.md", + "systems/game-engines-hurrican-3.md": "systems/game-engines.md", + "systems/game-engines-hurrican-4.md": "systems/game-engines.md", + "systems/game-engines-hurrican.md": "systems/game-engines.md", + "systems/game-engines-hydra-castle-labyrinth-2.md": "systems/game-engines.md", + "systems/game-engines-hydra-castle-labyrinth-3.md": "systems/game-engines.md", + "systems/game-engines-hydra-castle-labyrinth.md": "systems/game-engines.md", + "systems/game-engines-ikemen-go.md": "systems/game-engines.md", + "systems/game-engines-theme-hospital-2.md": "systems/game-engines.md", + "systems/game-engines-theme-hospital.md": "systems/game-engines.md", + "systems/game-engines-thextech-10.md": "systems/game-engines.md", + "systems/game-engines-thextech-11.md": "systems/game-engines.md", + "systems/game-engines-thextech-2.md": "systems/game-engines.md", + "systems/game-engines-thextech-3.md": "systems/game-engines.md", + "systems/game-engines-thextech-4.md": "systems/game-engines.md", + "systems/game-engines-thextech-5.md": "systems/game-engines.md", + "systems/game-engines-thextech-6.md": "systems/game-engines.md", + "systems/game-engines-thextech-7.md": "systems/game-engines.md", + "systems/game-engines-thextech-8.md": "systems/game-engines.md", + "systems/game-engines-thextech-9.md": "systems/game-engines.md", + "systems/game-engines-thextech.md": "systems/game-engines.md", + "systems/magnavox-odyssey2-2.md": "systems/magnavox.md", + "systems/magnavox-odyssey2.md": "systems/magnavox.md", + "systems/microsoft-msx.md": "systems/microsoft.md", + "systems/nintendo-3ds.md": "systems/nintendo.md", + "systems/nintendo-game-boy.md": "systems/nintendo.md", + "systems/nintendo-gamecube.md": "systems/nintendo.md", + "systems/nintendo-nes.md": "systems/nintendo.md", + "systems/nintendo-nintendo-64dd.md": "systems/nintendo.md", + "systems/nintendo-nintendo-ds.md": "systems/nintendo.md", + "systems/nintendo-nintendo-dsi.md": "systems/nintendo.md", + "systems/nintendo-snes.md": "systems/nintendo.md", + "systems/nintendo-wii-u.md": "systems/nintendo.md", + "systems/other-altirra.md": "systems/other.md", + "systems/other-armsx2.md": "systems/other.md", + "systems/other-c64-emu.md": "systems/other.md", + "systems/other-clock-signal.md": "systems/other.md", + "systems/other-cxbx-reloaded.md": "systems/other.md", + "systems/other-eka2l1.md": "systems/other.md", + "systems/other-epsxe.md": "systems/other.md", + "systems/other-fbneo.md": "systems/other.md", + "systems/other-future-pinball-fploader.md": "systems/other.md", + "systems/other-gopher2600.md": "systems/other.md", + "systems/other-idtech4a.md": "systems/other.md", + "systems/other-ines.md": "systems/other.md", + "systems/other-mame-2.md": "systems/other.md", + "systems/other-mame-3.md": "systems/other.md", + "systems/other-mame-4.md": "systems/other.md", + "systems/other-mame-5.md": "systems/other.md", + "systems/other-mame.md": "systems/other.md", + "systems/other-mame4droid.md": "systems/other.md", + "systems/other-mariani.md": "systems/other.md", + "systems/other-mednafen.md": "systems/other.md", + "systems/other-msx-emu.md": "systems/other.md", + "systems/other-nethersx2.md": "systems/other.md", + "systems/other-nuance.md": "systems/other.md", + "systems/other-opentyrian.md": "systems/other.md", + "systems/other-pcem.md": "systems/other.md", + "systems/other-pom2.md": "systems/other.md", + "systems/other-project64-2.md": "systems/other.md", + "systems/other-project64-3.md": "systems/other.md", + "systems/other-project64.md": "systems/other.md", + "systems/other-shadps4.md": "systems/other.md", + "systems/other-skyline-2.md": "systems/other.md", + "systems/other-skyline.md": "systems/other.md", + "systems/other-syobonaction.md": "systems/other.md", + "systems/other-xm6typeg.md": "systems/other.md", + "systems/other-yaps2.md": "systems/other.md", + "systems/other-zesarux.md": "systems/other.md", + "systems/rpg-maker-easyrpg-2.md": "systems/rpg-maker.md", + "systems/rpg-maker-easyrpg-3.md": "systems/rpg-maker.md", + "systems/rpg-maker-easyrpg-4.md": "systems/rpg-maker.md", + "systems/rpg-maker-easyrpg-5.md": "systems/rpg-maker.md", + "systems/rpg-maker-easyrpg.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z-2.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z-3.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z-4.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z-5.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z-6.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z-7.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z-8.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z-9.md": "systems/rpg-maker.md", + "systems/rpg-maker-mkxp-z.md": "systems/rpg-maker.md", + "systems/sega-arcade.md": "systems/sega.md", + "systems/sega-mega-cd.md": "systems/sega.md", + "systems/sega-saturn.md": "systems/sega.md", + "systems/sony-playstation-2.md": "systems/sony.md", + "systems/sony-playstation-3.md": "systems/sony.md", + "systems/sony-playstation-4.md": "systems/sony.md", + "systems/sony-playstation-portable.md": "systems/sony.md", + "systems/sony-playstation-vita.md": "systems/sony.md", + "systems/sony-playstation.md": "systems/sony.md", + "systems/sony-pocketstation.md": "systems/sony.md" } From dd39be0f3d841379cf3b32f9ac2bf0ef79bbcb48 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:55:52 +0100 Subject: [PATCH 32/36] fix: hold the new report line and test order --- tests/test_doc_surfaces.py | 1 + tests/test_target_names.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_doc_surfaces.py b/tests/test_doc_surfaces.py index fc5140625..962e32fd2 100644 --- a/tests/test_doc_surfaces.py +++ b/tests/test_doc_surfaces.py @@ -309,6 +309,7 @@ class ShownOutputIsPrintedOutput(unittest.TestCase): (r"^ Core files held but not packed: \d+ \(.+\)$", '" Core files held but not packed: '), (r"^ Game data: \d+ in pack, (0 missing|\d+ missing \(--verbose lists them\))$", '" Game data: {gd_packed} in pack'), + (r"^ Game data held but not packed: \d+ \(.+\)$", '" Game data held but not packed: '), (r"^ No external files: \d+ cores \(--verbose lists them\)$", '" No external files: {len(exclusions)} cores (--verbose lists them)"'), (r"^ Ground truth: \d+/\d+ files have emulator validation \(\d+%\)$", diff --git a/tests/test_target_names.py b/tests/test_target_names.py index 21bea65c3..877899c08 100644 --- a/tests/test_target_names.py +++ b/tests/test_target_names.py @@ -192,10 +192,6 @@ class ASingleTargetFiltersNothing(unittest.TestCase): checked += 1 self.assertGreaterEqual(checked, 3) -if __name__ == "__main__": - unittest.main() - - class PublishedListsAreNotNarrowed(unittest.TestCase): """pipeline --target switch wrote install/retroarch_nintendoswitch.json next to the published lists. The site picker then replaced RetroArch's @@ -216,3 +212,7 @@ class PublishedListsAreNotNarrowed(unittest.TestCase): self.assertNotIn("no_game_data", data) self.assertEqual(data["source"], "full") self.assertEqual(data["regions"], []) + + +if __name__ == "__main__": + unittest.main() From 01d5d960648a0595b93a1edf303ca68f0d4280c5 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:56:05 +0100 Subject: [PATCH 33/36] fix: name the export reports as copied files --- scripts/generate_site.py | 4 ++-- tests/test_site_r3_reference.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 3ca81db28..875af17c4 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2157,8 +2157,8 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N "", *(f"- [`{name}`](api/v1/schemas/{name}): {title}" for name, title in _served_schemas()), "", - "`database.json`, `stats.json`, `release.json` and the manifests are not " - "envelopes. Their revision is the `commit` of `catalog.json` that lists " + "`database.json`, `stats.json`, `release.json`, the manifests and the export reports " + "are not envelopes. Their revision is the `commit` of `catalog.json` that lists " "their SHA256.", "", "One emulator profile without the whole list: " diff --git a/tests/test_site_r3_reference.py b/tests/test_site_r3_reference.py index 4b15c11dd..91011a3cf 100644 --- a/tests/test_site_r3_reference.py +++ b/tests/test_site_r3_reference.py @@ -267,7 +267,7 @@ class EveryEnvelopeCarriesTheCommit(unittest.TestCase): raise unittest.SkipTest("not a git checkout") lacking = [ path.relative_to(API).as_posix() for path in sorted(API.rglob("*.json")) - if path.parent.name not in ("schemas", "manifests", "targets") + if path.parent.name not in ("schemas", "manifests", "targets", "exports") and path.name not in ("database.json", "stats.json", "release.json") and "commit" not in json.loads(path.read_text()) ] From 586fbd7ef2a9e7de5b4dc4d7a595accce76f4afa Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:05:31 +0100 Subject: [PATCH 34/36] fix: keep the FAQ in its word budget --- wiki/faq.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/wiki/faq.md b/wiki/faq.md index 58646b11b..9cd775de7 100644 --- a/wiki/faq.md +++ b/wiki/faq.md @@ -55,10 +55,9 @@ Game sets must match the emulator's version: ## Is the command safe? { #is-piping-the-installer-into-a-shell-safe } -- It stops if the installer does not match its expected fingerprint, and +- It stops if the installer does not match its fingerprint, and checks each file. -- It writes in your BIOS folder, some emulators' save folder, and a cache - of what it read about your games. +- It writes in your BIOS folder, some emulators' save folder and a games cache. - It never touches games or saves. - It keeps each file it replaces in `retrobios-backup`, for `--restore`. @@ -74,12 +73,12 @@ and download their files. ## Why is there no ZIP for my setup? ZIPs are made for the setups most people download. The command installs every -setup's files and checks each one. `--dry-run` shows the list first. +setup's files and checks them. `--dry-run` shows the list first. ## Fill an SD card from a computer? -The interface finds a muOS, Onion, spruceOS, MinUI, KNULLI or ROCKNIX card by -itself. Name your setup and the card's BIOS folder. For RetroArch on a Switch, +The interface finds a muOS, Onion, spruceOS, MinUI, KNULLI or ROCKNIX +card. Name your setup and the card's BIOS folder. For RetroArch on a Switch, with the card in drive `E:`, run: From 2722f6f9cead42165e2b745f19e616240ea11ac3 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:05:31 +0100 Subject: [PATCH 35/36] fix: keep setup pages in their budgets --- scripts/generate_site.py | 19 ++++++++++++------- tests/test_site_r4_entry.py | 4 +++- tests/test_site_r5_entry.py | 11 +++++++++-- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 875af17c4..91c760ee3 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3860,7 +3860,8 @@ def _platform_data_links(name: str) -> str: f"[JSON](../api/v1/platforms/{name}.json)", ] if (_REPO_ROOT / "install" / f"{name}.json").is_file(): - links.append(f"[install manifest]({REPO_URL}/blob/main/install/{name}.json)") + # The site's own copy, byte for byte (_publish_manifests). + links.append(f"[install manifest](../api/v1/manifests/{name}.json)") # The installed BIOS folder for clrmamepro and RomVault. links.append(f"[DAT](../downloads/dat/{name}.dat)") return ", ".join(links) @@ -4098,11 +4099,10 @@ def _share_tabs(name: str, share: str) -> tuple[dict[str, str], dict[str, str]]: "linux": f"--platform {name} --dest /path/to/{volume}{tail}", } how = { - "macos": (f"In Finder, choose Go, then Connect to Server, and enter `{url}`. " - f"It opens as `/Volumes/{volume}`. Then open **Terminal**, in " - "Applications, then Utilities. Paste the line. Press Return."), - "linux": (f"Open `{url}` in your file manager. Open a terminal, paste the line " - f"and change `/path/to/{volume}` to the folder it opened. Press Enter."), + "macos": (f"In Finder, Go > Connect to Server: `{url}`. Then paste the line in " + "**Terminal**."), + "linux": (f"Open `{url}` in your file manager. Paste the line in a terminal, " + f"with the folder it opened for `/path/to/{volume}`."), } return args, how @@ -4168,10 +4168,15 @@ def _setup_routes(name: str, display: str, page_link: bool = False) -> list[str] computer = ["## From your computer { #from-your-computer }", ""] if share: args, how = _share_tabs(name, share) + # The setup page stays in its word budget: its Mac and Linux + # lines are on the get/ page, the one the command is shared from. + tabs = ("windows",) if page_link else ("windows", "macos", "linux") computer += [ "Needs the device on your network. Run the command on the computer.", "", - *terminal_tabs(("windows", "macos", "linux"), args, how=how), + *terminal_tabs(tabs, args, how=how), + *([f"[Mac or Linux](../{get_page_link(name)}#from-your-computer)", ""] + if page_link else []), SAFE_LINK, "", ] diff --git a/tests/test_site_r4_entry.py b/tests/test_site_r4_entry.py index 404b01946..4e99a2ab0 100644 --- a/tests/test_site_r4_entry.py +++ b/tests/test_site_r4_entry.py @@ -91,13 +91,15 @@ class NoZipNoUnzipText(unittest.TestCase): args, how = generate_site._share_tabs("recalbox", "\\\\RECALBOX\\share\\bios") self.assertEqual(args["macos"], "--platform recalbox --dest /Volumes/share/bios") self.assertIn("`smb://recalbox/share`", how["macos"]) - self.assertIn("`/Volumes/share`", how["macos"]) self.assertEqual(args["linux"], "--platform recalbox --dest /path/to/share/bios") self.assertIn("`smb://recalbox/share`", how["linux"]) lines = generate_site._setup_routes("recalbox", "Recalbox") routes = "\n".join(lines[:lines.index("## On the device { #install }")]) for label in ("Windows", "macOS", "Linux"): self.assertIn(f'label="{label}"', routes) + # The setup page sends a Mac or Linux reader to those tabs. + lines = generate_site._setup_routes("recalbox", "Recalbox", page_link=True) + self.assertIn("[Mac or Linux](../get/recalbox.md#from-your-computer)", lines) def test_the_network_route_needs_a_share_and_no_zip(self): for name in SETUPS: diff --git a/tests/test_site_r5_entry.py b/tests/test_site_r5_entry.py index 262cb905c..55167a4f2 100644 --- a/tests/test_site_r5_entry.py +++ b/tests/test_site_r5_entry.py @@ -59,8 +59,14 @@ class GetPagesGiveTheSetupsOwnRoutes(unittest.TestCase): def test_every_command_on_a_get_page_is_on_its_setup_page(self): for name in SETUPS: get = set(COMMAND.findall(_page(f"get/{name}.md"))) - setup = set(COMMAND.findall(_page(f"platforms/{name}.md"))) + page = _page(f"platforms/{name}.md") + setup = set(COMMAND.findall(page)) with self.subTest(setup=name): + # The Mac and Linux lines of the network folder live on the + # get/ page alone, which the setup page links for them. + if f"(/retrobios/get/{name}/#from-your-computer)" in page: + get = {c for c in get if "--dest /Volumes/" not in c + and "--dest /path/to/" not in c} self.assertTrue(get <= setup, sorted(get - setup)) if not _with_zip(name): self.assertTrue(get, "a setup with no ZIP gets its command") @@ -82,7 +88,8 @@ class GetPagesGiveTheSetupsOwnRoutes(unittest.TestCase): allowed = {labels[key] for key in guide.run} allowed |= {labels[key] for key in guide.card_run or ("windows", "macos", "linux")} if generate_site._network_share(name, guide): - allowed.add(labels["windows"]) + # The network folder is reached from any computer. + allowed |= {labels[key] for key in ("windows", "macos", "linux")} with self.subTest(setup=name): self.assertLessEqual(_tab_labels(_page(f"get/{name}.md")), allowed) self.assertNotIn("Android (Termux", _page(f"get/{name}.md")) From 771e68016f579d2766798d61a8c6dfbad0df4b3b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 5 Nov 2026 13:39:41 +0100 Subject: [PATCH 36/36] fix: keep the FAQ and data page wording tests hold --- scripts/generate_site.py | 7 ++++--- wiki/faq.md | 12 ++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 91c760ee3..78bfa241b 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2157,8 +2157,8 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N "", *(f"- [`{name}`](api/v1/schemas/{name}): {title}" for name, title in _served_schemas()), "", - "`database.json`, `stats.json`, `release.json`, the manifests and the export reports " - "are not envelopes. Their revision is the `commit` of `catalog.json` that lists " + "`database.json`, `stats.json`, `release.json` and the manifests are not " + "envelopes. Their revision is the `commit` of `catalog.json` that lists " "their SHA256.", "", "One emulator profile without the whole list: " @@ -2244,7 +2244,8 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N for item in targets) + "." if targets else "", "", "`api/v1/exports/.json` lists every change of a setup's " - "[updated file](exports.md), the report that page is written from.", + "[updated file](exports.md), the report that page is written from, copied " + "byte for byte like the manifests.", "", "| Field | Meaning |", "|-------|---------|", diff --git a/wiki/faq.md b/wiki/faq.md index 9cd775de7..e27e3619f 100644 --- a/wiki/faq.md +++ b/wiki/faq.md @@ -34,7 +34,7 @@ option. ## Zip or unzip? { #zip-or-unzip } -Unzip it into the folder your setup's page names. Keep +Unzip into the folder your setup's page names. Keep inner ZIPs zipped, like `neogeo.zip`. ## Does a BIOS work on every setup? @@ -55,9 +55,9 @@ Game sets must match the emulator's version: ## Is the command safe? { #is-piping-the-installer-into-a-shell-safe } -- It stops if the installer does not match its fingerprint, and +- It stops if the installer does not match its expected fingerprint, and checks each file. -- It writes in your BIOS folder, some emulators' save folder and a games cache. +- It fills the BIOS folder, some save folders and a games cache. - It never touches games or saves. - It keeps each file it replaces in `retrobios-backup`, for `--restore`. @@ -73,12 +73,12 @@ and download their files. ## Why is there no ZIP for my setup? ZIPs are made for the setups most people download. The command installs every -setup's files and checks them. `--dry-run` shows the list first. +setup's files and checks each one. `--dry-run` shows the list first. ## Fill an SD card from a computer? -The interface finds a muOS, Onion, spruceOS, MinUI, KNULLI or ROCKNIX -card. Name your setup and the card's BIOS folder. For RetroArch on a Switch, +The interface finds a muOS, Onion, spruceOS, MinUI, KNULLI or ROCKNIX card by +itself. Name your setup and the card's BIOS folder. For RetroArch on a Switch, with the card in drive `E:`, run: