From 55ac3162f56bb040ed61fa8ff22935c8486d2937 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:13:57 +0100 Subject: [PATCH 01/30] docs: name the slot function the pack reads architecture.md said the pack and verify decide through slots.arbitrate(), while a test forbids them to call it: they read slot_choices() through pack_overrides(). build_claim_index, the name-keyed index the module docstring described, had no caller outside its own tests. --- scripts/slots.py | 33 +++++++---------------------- tests/test_doc_pages.py | 6 ++++++ tests/test_slots.py | 47 +++++++++++++++++++---------------------- wiki/architecture.md | 7 +++--- 4 files changed, 40 insertions(+), 53 deletions(-) diff --git a/scripts/slots.py b/scripts/slots.py index 3680f7c69..e9b45708b 100644 --- a/scripts/slots.py +++ b/scripts/slots.py @@ -7,11 +7,14 @@ module existed nothing compared the two: the builder resolves the platform entry first and drops the profile entry on a bare filename match, so a profile that names the right file for a slot could never win and never even be heard. -The index is keyed by destination AND by name, the name carrying the union of -what every same-named entry claims. Dolphin declares three IPL.bin separated -only by their path, so a name-only key merges them, and a path-only key answers -nothing when a candidate is known by name alone. Ambiguity resolves to the -union and is never discarded, the same rule region.py already applies. +Claims are keyed by destination, folded to one case only where the platform's +filesystem folds it. Dolphin declares three IPL.bin separated only by their +path, and each is its own slot. A profile claim resting on a bare filename +proves nothing about content and contests no slot. + +The pack builder, the install manifest and verify read slot_choices() through +pack_overrides(). arbitrate() states the same decision for one conflict and +feeds the report this module prints. """ from __future__ import annotations @@ -125,26 +128,6 @@ def _normalize(destination: str, fold: bool = True) -> str: return cleaned.casefold() if fold else cleaned -def build_claim_index(claims: list[Claim]) -> dict[str, list[Claim]]: - """Index claims by destination and by name, the name carrying the union. - - A lookup by full destination answers for one slot. A lookup by bare name - answers with every slot that shares it, which is what a candidate known - only by its filename needs in order not to be dropped. - """ - index: dict[str, list[Claim]] = {} - for claim in claims: - keys = {_normalize(claim.destination)} - if claim.name: - keys.add(_normalize(claim.name)) - base = claim.destination.rsplit("/", 1)[-1] - if base: - keys.add(_normalize(base)) - for key in keys - {""}: - index.setdefault(key, []).append(claim) - return index - - def platform_claims( config: dict, db: dict, diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 3054134d4..ee225c127 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1738,6 +1738,12 @@ class ArchitectureStatesTheArbitration(unittest.TestCase): section = page[page.index("### When both layers claim one destination"):] section = section[:section.index("\n## ")] self.assertIn("`slots.arbitrate()`", section) + # The pack, the manifest and verify decide through slot_choices, and + # a test in test_slots forbids them to call arbitrate: the page names + # the function they read, not only the one the report prints. + self.assertIn("`slots.slot_choices()`", section) + self.assertIn("`slots.pack_overrides()`", section) + self.assertNotIn("both decide through `slots.arbitrate()`", section) source = inspect.getsource(slots.arbitrate) # The three outcomes the section describes. for outcome in ("ADDRESSEE", "SERVES_BOTH", "FRONTEND_CHECKS_CONTENT"): diff --git a/tests/test_slots.py b/tests/test_slots.py index d4dabab91..70e98ca40 100644 --- a/tests/test_slots.py +++ b/tests/test_slots.py @@ -75,31 +75,6 @@ def tearDownModule(): shutil.rmtree(_ROOT, ignore_errors=True) -class TestClaimIndex(unittest.TestCase): - """Keyed by destination and by name, the name carrying the union.""" - - def _claims(self): - return [ - slots.Claim("profile", "GC/USA/IPL.bin", "IPL.bin", emulator="dolphin"), - slots.Claim("profile", "GC/JAP/IPL.bin", "IPL.bin", emulator="dolphin"), - ] - - def test_destination_key_separates_same_named_entries(self): - index = slots.build_claim_index(self._claims()) - self.assertEqual(len(index["gc/usa/ipl.bin"]), 1) - self.assertEqual(len(index["gc/jap/ipl.bin"]), 1) - - def test_bare_name_answers_with_the_union(self): - index = slots.build_claim_index(self._claims()) - self.assertEqual(len(index["ipl.bin"]), 2) - - def test_lookup_ignores_case_and_leading_separator(self): - index = slots.build_claim_index( - [slots.Claim("platform", "/BIOS/Disk.rom", "Disk.rom")] - ) - self.assertIn("bios/disk.rom", index) - - class TestConflicts(unittest.TestCase): """A proven profile claim contradicting what the baseline would ship.""" @@ -411,6 +386,28 @@ class TestArbitration(unittest.TestCase): # The hashed declaration keeps the file the frontend checks. self.assertEqual(overrides, {"IPL.bin": "bios/Console/GC/JAP/IPL.bin"}) + def test_the_report_and_the_pack_decide_alike(self): + """The report prints arbitrate(), the pack, the manifest and verify + read slot_choices(): both must name the same file for a slot.""" + for mode in ("existence", "md5"): + config = { + "verification_mode": mode, + "cores": ["dolphin"], + "systems": {"console": {"files": [ + {"name": "IPL.bin", "destination": "GC/JAP/IPL.bin", "md5": "m" * 32}, + ]}}, + } + profiles = {"dolphin": {"type": "libretro", "systems": ["console"], "files": [ + {"name": "IPL.bin", "path": "GC/JAP/IPL.bin"}, + ]}} + with self.subTest(mode=mode): + [conflict] = slots.find_conflicts(config, profiles, REGIONS_DB) + decision = slots.arbitrate(conflict, mode) + served = slots.pack_overrides(config, profiles, REGIONS_DB).get( + "GC/JAP/IPL.bin", conflict.platform_claim.local_path + ) + self.assertEqual(served, decision.winner.local_path) + def test_the_reported_line_gives_the_ground_for_the_decision(self): kept = slots.format_decision(slots.arbitrate(self._conflict(), "md5")) self.assertIn("verifies content", kept) diff --git a/wiki/architecture.md b/wiki/architecture.md index 215b16c8c..eb768e95b 100644 --- a/wiki/architecture.md +++ b/wiki/architecture.md @@ -196,9 +196,10 @@ whoever asked for it: - A pack built with `--emulator` answers to that emulator, and takes its file. `scripts/slots.py` prints each contested destination and the reason for its -winner (`python scripts/slots.py --platform retrobat`). `generate_pack.py` and -`verify.py` both decide through `slots.arbitrate()`, so the pack and the report -agree. +winner (`python scripts/slots.py --platform retrobat`), as `slots.arbitrate()` +decides it. `generate_pack.py`, the install manifest and `verify.py` all read +`slots.slot_choices()` through `slots.pack_overrides()`, which applies the same +order, so the pack, the installer and the report agree. A platform YAML is regenerated by its scraper and never edited by hand: an edit would be lost at the next scrape, and the file would stop describing the From 8c3e65323423c715a10d754fb7b909943c1526bb Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:14:38 +0100 Subject: [PATCH 02/30] fix: key the arcade sha1 cache by the database The cache compared id(db), so a database freed and replaced at the same address answered with the first one's Arcade set. --- scripts/siterender.py | 8 +++++--- tests/test_site_render.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/scripts/siterender.py b/scripts/siterender.py index ebc47e8ab..9f19a9d02 100644 --- a/scripts/siterender.py +++ b/scripts/siterender.py @@ -372,19 +372,21 @@ def _in_arcade(sha1: str, db: dict) -> bool: return sha1 in _arcade_suffix_sha1s(db) -_suffix_cache: list[tuple[int, frozenset[str]] | None] = [None] +_suffix_cache: list[tuple[dict, frozenset[str]] | None] = [None] def _arcade_suffix_sha1s(db: dict) -> frozenset[str]: + # Holding the database itself, never id(db): a database freed and + # replaced at the same address answered with the first one's set. cached = _suffix_cache[0] - if cached is not None and cached[0] == id(db): + if cached is not None and cached[0] is db: return cached[1] found = frozenset( sha1s[0] for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items() if tail.startswith("Arcade/") and len(sha1s) == 1 ) - _suffix_cache[0] = (id(db), found) + _suffix_cache[0] = (db, found) return found diff --git a/tests/test_site_render.py b/tests/test_site_render.py index 35b27dbc8..f4b11a4ac 100644 --- a/tests/test_site_render.py +++ b/tests/test_site_render.py @@ -503,5 +503,34 @@ class NoTooltipInsideAName(unittest.TestCase): self.assertIn('>BIOS', marked) +class TheArcadeCacheHoldsItsDatabase(unittest.TestCase): + """The set of sha1s filed under Arcade/ was cached by id(db): a database + freed and replaced at the same address answered with the first one's set, + and an arcade card of the second landed in another section.""" + + @staticmethod + def _db(sha1: str) -> dict: + return {"files": {}, "indexes": {"by_path_suffix": {"Arcade/x.zip": [sha1]}}} + + def test_a_new_database_at_a_reused_address_is_read_again(self): + saved = siterender._suffix_cache[0] + self.addCleanup(siterender._suffix_cache.__setitem__, 0, saved) + first = self._db("a" * 40) + self.assertTrue(siterender._in_arcade("a" * 40, first)) + address = id(first) + del first + # CPython hands a freed dict's address to a later one of its size, + # unless the cache still holds the first database. + for _ in range(1000): + second = self._db("b" * 40) + if id(second) == address: + break + del second + else: + second = self._db("b" * 40) + self.assertTrue(siterender._in_arcade("b" * 40, second)) + self.assertFalse(siterender._in_arcade("a" * 40, second)) + + if __name__ == "__main__": unittest.main() From 33db4e9c838f9137a28b765b379787cbb496c876 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:15:39 +0100 Subject: [PATCH 03/30] fix: file a device set under each of its systems The site read an entry's system: only, so the 430 MAME entries that list systems: were filed under no section. It now reads common.entry_system_ids for an entry that names its own systems. --- scripts/generate_site.py | 8 +++++--- tests/test_site_system_sections.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 6fe7b38ca..2380d3b5a 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -47,6 +47,7 @@ from common import ( GAME_DATA_TOPS, compute_composition, count_catalog_matched, + entry_system_ids, list_registered_platforms, list_release_pack_platforms, load_platform_registry, @@ -6500,9 +6501,10 @@ def _profiles_on_disk() -> dict[str, dict]: def _entry_systems(emulator: str, entry: dict) -> list[str]: - """The system an entry is read for: its own, or its profile's only one.""" - if entry.get("system"): - return [str(entry["system"])] + """The systems an entry is read for: its own `system:` or `systems:` + (a MAME device set several systems load), else its profile's only one.""" + if entry.get("system") or entry.get("systems"): + return entry_system_ids(entry, {}) systems = (_profiles_on_disk().get(emulator) or {}).get("systems") or [] return [str(systems[0])] if len(systems) == 1 else [] diff --git a/tests/test_site_system_sections.py b/tests/test_site_system_sections.py index 8a4d59583..4a3b4268e 100644 --- a/tests/test_site_system_sections.py +++ b/tests/test_site_system_sections.py @@ -304,5 +304,22 @@ class BuildCount(unittest.TestCase): self.assertIn("system ids linked", source) +class AnEntryListingSystemsIsFiledUnderEach(unittest.TestCase): + """MAME declares a device set under `systems:`, the machines of several + systems loading it. The site read only `system:`, so ym2608.zip was filed + under neither Arcade nor NEC PC-88, and kept on pages it does not serve.""" + + def test_the_site_reads_the_common_rule(self): + import generate_site as gs # noqa: PLC0415 + from common import entry_system_ids # noqa: PLC0415 + + entry = {"name": "ym2608.zip", "systems": ["arcade", "pc80"]} + self.assertEqual(gs._entry_systems("mame", entry), ["arcade", "pc80"]) + self.assertEqual( + gs._entry_systems("mame", entry), entry_system_ids(entry, {"systems": []}) + ) + self.assertEqual(gs._entry_systems("mame", {"name": "a", "system": "pc80"}), ["pc80"]) + + if __name__ == "__main__": unittest.main() From ab1866fd661a0c378578cb6c1f1f7519bbc3c9fb Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:17:31 +0100 Subject: [PATCH 04/30] feat: compare the site npm pins with the registry The ci area of check_freshness read the PyPI and action pins only. The exact versions website/package.json pins (astro, starlight) now get a row each, STALE once npm publishes a newer release. --- scripts/check_freshness.py | 53 ++++++++++++++++++++++++++++++++++- tests/test_check_freshness.py | 38 +++++++++++++++++++++++++ wiki/tools.md | 5 ++-- 3 files changed, 93 insertions(+), 3 deletions(-) diff --git a/scripts/check_freshness.py b/scripts/check_freshness.py index fac97bc71..425bd6ad3 100644 --- a/scripts/check_freshness.py +++ b/scripts/check_freshness.py @@ -79,6 +79,9 @@ TOSEC_DOWNLOADS = "https://www.tosecdev.org/downloads" MAME_REPO = "https://github.com/mamedev/mame" FBNEO_REPO = "https://github.com/libretro/FBNeo" PYPI_URL = "https://pypi.org/pypi/{name}/json" +NPM_URL = "https://registry.npmjs.org/{name}/latest" +# The site's npm dependencies, pinned to exact versions and built by Deploy Site. +SITE_PACKAGE = REPO_ROOT / "website" / "package.json" INSTALLERS = ("install.sh", "install.ps1") SHOWN_ITEMS = 4 # items named in a diff summary before ", ..." @@ -1355,6 +1358,53 @@ def pypi_latest(name: str) -> str | None: return str(info.get("version") or "") or None +def parse_npm_pins(text: str) -> dict[str, str]: + """Package -> the version a package.json names, dependencies of both kinds.""" + data = json.loads(text) + pins: dict[str, str] = {} + for key in ("dependencies", "devDependencies"): + pins.update({str(k): str(v) for k, v in (data.get(key) or {}).items()}) + return pins + + +def npm_latest(name: str) -> str | None: + payload = upstream._http_json(NPM_URL.format(name=name)) # noqa: SLF001 + if not isinstance(payload, dict): + return None + return str(payload.get("version") or "") or None + + +def _npm_findings() -> list[Finding]: + """One row per site dependency: an exact pin is stale once npm's latest + release is newer, since nothing else asks npm.""" + findings = [] + for name, pinned in sorted(parse_npm_pins(SITE_PACKAGE.read_text(encoding="utf-8")).items()): + subject = f"npm:{name}" + if not re.fullmatch(r"\d+(\.\d+)*", pinned): + findings.append(Finding("ci", subject, UNKNOWN, local=pinned, detail="a range, not an exact pin")) + continue + try: + latest = npm_latest(name) + except upstream.UpstreamError as exc: + findings.append(Finding("ci", subject, UNKNOWN, local=pinned, detail=str(exc))) + continue + if latest is None: + findings.append(Finding("ci", subject, UNKNOWN, local=pinned, detail="not on npm")) + continue + newer = _version_tuple(latest) > _version_tuple(pinned) + findings.append( + Finding( + "ci", + subject, + STALE if newer else OK, + local=pinned, + remote=latest, + detail="a newer release is out" if newer else "", + ) + ) + return findings + + def check_ci(cache_dir: str, offline: bool) -> list[Finding]: findings = [_check_installer_pins()] workflows = sorted((REPO_ROOT / ".github" / "workflows").glob("*.yml")) @@ -1366,8 +1416,9 @@ def check_ci(cache_dir: str, offline: bool) -> list[Finding]: pins.setdefault(name, spec) actions.update(parse_action_pins(text)) if offline: - findings.append(Finding("ci", "workflow pins", SKIPPED, detail="offline")) + findings.append(Finding("ci", "workflow and site pins", SKIPPED, detail="offline")) return findings + findings.extend(_npm_findings()) for name, spec in sorted(pins.items()): try: latest = pypi_latest(name) diff --git a/tests/test_check_freshness.py b/tests/test_check_freshness.py index ca72aefc9..72846872f 100644 --- a/tests/test_check_freshness.py +++ b/tests/test_check_freshness.py @@ -8,6 +8,7 @@ report is folded, since each of those decides whether a row says STALE. from __future__ import annotations +import json import os import sys import tempfile @@ -353,6 +354,43 @@ class PinParsingTests(unittest.TestCase): self.assertTrue(cf.specifier_allows(">=10.14", "12.0.1")) +class SiteNpmPins(unittest.TestCase): + """The site's npm dependencies are pinned to exact versions in + website/package.json, and the ci area compared only the PyPI and action + pins: astro 7.4.2 stayed behind 7.4.7 with no STALE row.""" + + def test_both_dependency_kinds_are_read(self): + pins = cf.parse_npm_pins( + '{"dependencies": {"astro": "7.4.2"}, "devDependencies": {"x": "^1.0.0"}}' + ) + self.assertEqual(pins, {"astro": "7.4.2", "x": "^1.0.0"}) + + def test_a_newer_release_is_stale(self): + with tempfile.TemporaryDirectory() as tmp: + package = Path(tmp) / "package.json" + package.write_text(json.dumps({"dependencies": { + "astro": "7.4.2", "@astrojs/starlight": "0.42.9", "y": "^2.0.0", + }})) + latest = {"astro": "7.4.7", "@astrojs/starlight": "0.42.9"} + def answer(url): + name = url.split("registry.npmjs.org/", 1)[1].rsplit("/latest", 1)[0] + return {"version": latest[name]} + with mock.patch.object(cf, "SITE_PACKAGE", package), \ + mock.patch.object(cf.upstream, "_http_json", side_effect=answer): + rows = {f.subject: f for f in cf._npm_findings()} + self.assertEqual(rows["npm:astro"].status, cf.STALE) + self.assertEqual(rows["npm:astro"].remote, "7.4.7") + self.assertEqual(rows["npm:@astrojs/starlight"].status, cf.OK) + self.assertEqual(rows["npm:y"].status, cf.UNKNOWN) + + def test_the_ci_area_reads_the_site_pins(self): + with mock.patch.object(cf, "_npm_findings", return_value=[]) as npm, \ + mock.patch.object(cf, "parse_pip_pins", return_value={}), \ + mock.patch.object(cf, "parse_action_pins", return_value={}): + cf.check_ci("unused", offline=False) + npm.assert_called_once() + + class RomFolderPins(unittest.TestCase): """The rom folders of a setup were pinned by hand and nothing compared the pins with the platform file or the project's releases.""" diff --git a/wiki/tools.md b/wiki/tools.md index 31086a61e..9d9c2d08a 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -688,8 +688,9 @@ directories are checked against their upstream, dump catalogs and romset snapshots against the catalog they were read from (Redump DAT versions, the No-Intro daily rebuild, the newest TOSEC pack, the newest MAME release, the git blobs of the FBNeo DATs), the game catalogs of `games/catalogs/` against -the No-Intro pack and the Redump datfiles they were read from, and the CI -toolchain against PyPI and the actions' latest releases. +the No-Intro pack and the Redump datfiles they were read from, the CI +toolchain against PyPI and the actions' latest releases, and the site's exact +npm pins in `website/package.json` against the npm registry. The `native` area needs no network. It reads the cache `export_native.py` patches from and reports an original that is absent, that came from another From 7b4c03afc246d08940c9a1cc35c38d18bcd11c2f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:20:01 +0100 Subject: [PATCH 05/30] docs: say what the archived status changes The README, the setup pages and adding-a-platform tied archived to a scraping schedule and a CI matrix that do not exist. Archived keeps a platform out of the runs that cover every platform unless --include-archived is passed. --- README.md | 4 ++-- scripts/generate_pack.py | 6 +++--- scripts/generate_readme.py | 4 ++-- scripts/generate_site.py | 4 ++-- scripts/list_platforms.py | 8 ++++---- tests/test_doc_pages.py | 29 +++++++++++++++++++++++++++++ wiki/adding-a-platform.md | 11 +++++++---- 7 files changed, 49 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 398f4b771..b325f26ae 100644 --- a/README.md +++ b/README.md @@ -72,7 +72,7 @@ The Files column counts what a pack holds once extracted, the figure a file mana ZIPs are made for the setups most people download. The others read "Use the command": the installer above installs every file for them and checks each one, and `--dry-run` shows what it would install ([why no ZIP](https://abdess.github.io/retrobios/wiki/faq/#why-is-there-no-zip-for-my-setup)). -\* Archived: the configuration is kept and the installer serves it, but upstream is no longer scraped on a schedule. +\* Archived: the installer serves it, and checks that cover every setup include it only on request. ## What's included @@ -206,4 +206,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-11-05T22:45:23Z* +*Auto-generated on 2026-11-06T02:18:34Z* diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 4039b1655..f62bb4c35 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -3639,9 +3639,9 @@ def _main(held: contextlib.ExitStack) -> None: # Platform mode if args.all and args.manifest: # Install manifests cover every registered platform: the installer - # serves each one, with or without a release pack. "Archived" means - # upstream is no longer scraped on a schedule, not that the platform - # lost its users. + # serves each one, with or without a release pack. "Archived" keeps a + # platform out of the runs that cover every platform, not out of the + # installer: it has not lost its users. platforms = list_registered_platforms(args.platforms_dir, include_archived=True) elif args.all: # A release publishes the packs the registry marks release_pack and diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 006aab519..6a15faaae 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -638,8 +638,8 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" lines.extend( [ "", - "\\* Archived: the configuration is kept and the installer serves it," - " but upstream is no longer scraped on a schedule.", + "\\* Archived: the installer serves it, and checks that cover every" + " setup include it only on request.", ] ) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 2380d3b5a..9a22b1e52 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3346,8 +3346,8 @@ def setup_table(coverages: dict, registry: dict | None = None) -> list[str]: lines += ["No ZIP: the setup is served by the command alone. " f"[Why?](../{NO_ZIP_FAQ})", ""] if archived: - lines += ["An archived setup keeps its page and the command. Its list is no longer " - "read again on a schedule.", ""] + lines += ["An archived setup keeps its page and the command. Checks that cover " + "every setup include it only on request.", ""] return lines diff --git a/scripts/list_platforms.py b/scripts/list_platforms.py index 9eaface44..4adb20fad 100644 --- a/scripts/list_platforms.py +++ b/scripts/list_platforms.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 -"""List available platforms for CI matrix strategy. +"""List the registered platforms. Respects the `status` field in _registry.yml: -- active: included in CI releases and automated scraping -- archived: excluded from CI, user can generate manually +- active: listed by default +- archived: listed with --include-archived only Usage: - python scripts/list_platforms.py # Active platforms (for CI) + python scripts/list_platforms.py # active platforms python scripts/list_platforms.py --include-archived # archived ones too python scripts/list_platforms.py >> "$GITHUB_OUTPUT" """ diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index ee225c127..2f342fcee 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1900,5 +1900,34 @@ class SkipFlagsNameTheSteps(unittest.TestCase): self.assertEqual(set(re.findall(self.STEP, text)), steps) +class ArchivedIsWhatTheToolsDo(unittest.TestCase): + """The README and the setup pages said an archived platform is no longer + scraped on a schedule, and adding-a-platform said `active` feeds a CI + matrix and the scrapers. No workflow calls list_platforms.py and no tool + reads `schedule:`: what `archived` changes is which full runs include it.""" + + def test_no_text_ties_the_status_to_automation(self): + import generate_pack # noqa: PLC0415 + import generate_readme # noqa: PLC0415 + import generate_site # noqa: PLC0415 + import list_platforms # noqa: PLC0415 + + texts = { + "adding-a-platform.md": (WIKI / "adding-a-platform.md").read_text(encoding="utf-8"), + "generate_readme": inspect.getsource(generate_readme), + "generate_site": inspect.getsource(generate_site), + "generate_pack": inspect.getsource(generate_pack), + "list_platforms": inspect.getsource(list_platforms), + } + for name, text in texts.items(): + with self.subTest(name): + for claim in ("scraped on a schedule", "read again on a schedule", + "CI matrix", "without the automation"): + self.assertNotIn(claim, text) + self.assertIn("`--include-archived`", texts["adding-a-platform.md"]) + for workflow in (ROOT / ".github" / "workflows").glob("*.yml"): + self.assertNotIn("list_platforms", workflow.read_text(encoding="utf-8")) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/adding-a-platform.md b/wiki/adding-a-platform.md index 39a61435c..a17f818d4 100644 --- a/wiki/adding-a-platform.md +++ b/wiki/adding-a-platform.md @@ -732,10 +732,13 @@ This executes in sequence, numbered as the run prints them (`STEP_LABELS` in `--offline` skips steps 2 to 2b2. -A new platform is archived-by-default nowhere: `status: active` puts it in -`list_platforms.py`, the CI matrix, and the scrapers. Set -`status: archived` to keep the configuration without the automation. Archived -platforms are excluded unless `--all` or `--include-archived` is passed. +A new platform is `status: active`. `status: archived` keeps the platform +out of the runs that cover every platform (`verify.py --all`, +`generate_pack.py --all`, `slots.py`, `pipeline.py`, `list_platforms.py`) +unless `--include-archived` is passed. The installer manifests, the site, the +native export and `check_freshness.py` cover archived platforms as well. No +tool scrapes a platform on its own, archived or not: the `schedule:` field is +the cadence a maintainer keeps by hand. A new platform has no release ZIP: `release_pack: true` is set for the setups most people download, and the installer serves every other one from its From adf969c70b98699fe46d69986b5bf58552649d64 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:26:24 +0100 Subject: [PATCH 06/30] docs: link the start pages from the README Quick install linked only the installer reference. It now names the first install, troubleshooting and FAQ pages. --- README.md | 4 +++- scripts/generate_readme.py | 4 ++++ tests/test_doc_surfaces.py | 15 +++++++++++++++ 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index b325f26ae..66077693f 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ curl -fsSL https://abdess.github.io/retrobios/i | sh -s -- --platform retroarch The script opens a full-screen installer, driven with the arrow keys and Enter, or with a controller on a handheld. It finds your setup and its BIOS folder, downloads only the missing files and checks each one. It can also copy the BIOS files from your old SD card. Every option, and what the installer detects and writes, is on the [installer page](https://abdess.github.io/retrobios/wiki/installer/#passing-options). +First time? Follow the [first install, step by step](https://abdess.github.io/retrobios/wiki/first-install/). A game does not start: [what to check](https://abdess.github.io/retrobios/wiki/troubleshooting/). Other questions: the [FAQ](https://abdess.github.io/retrobios/wiki/faq/). + ## Download BIOS packs A pack holds everything its platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **Choose systems**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below. @@ -206,4 +208,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-11-06T02:18:34Z* +*Auto-generated on 2026-11-06T02:20:28Z* diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 6a15faaae..1a3187461 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -593,6 +593,10 @@ def generate_readme(db: dict, platforms_dir: str, db_path: str = "database.json" " files and checks each one. It can also copy the BIOS files from your old SD card.", f"Every option, and what the installer detects and writes, is on the [installer page]({SITE_URL}wiki/installer/#passing-options).", "", + f"First time? Follow the [first install, step by step]({SITE_URL}wiki/first-install/)." + f" A game does not start: [what to check]({SITE_URL}wiki/troubleshooting/)." + f" Other questions: the [FAQ]({SITE_URL}wiki/faq/).", + "", "## Download BIOS packs", "", "A pack holds everything its platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. Each Download link is a page that keeps its address from one release to the next. The installer above does the same file by file: its screen installs everything, or fewer under **" + _installer_words().TUI_SYSTEMS_ROW + "**, and `--system`, `--core`, `--region` and `--target` make the same choice without questions. For a bare minimum, build your own pack below.", diff --git a/tests/test_doc_surfaces.py b/tests/test_doc_surfaces.py index 962e32fd2..2c2c28c94 100644 --- a/tests/test_doc_surfaces.py +++ b/tests/test_doc_surfaces.py @@ -248,6 +248,21 @@ class SlicesLeaveWithTheirRelease(unittest.TestCase): self.assertNotIn(".zip.001", _which_pack()) +class TheReadmeLeadsBeginnersToTheirPages(unittest.TestCase): + """The README linked the installer reference and the FAQ's no-ZIP anchor + only: a user stopped by an install error or a BIOS-not-found message had + no link to the first install or troubleshooting page.""" + + def test_quick_install_links_the_start_pages(self): + # The code block holds `#` comment lines, so the section runs to + # the next level-two heading. + section = _read("README.md").split("## Quick install", 1)[1].split("\n## ", 1)[0] + for page in ("first-install", "troubleshooting", "faq"): + with self.subTest(page=page): + self.assertIn(f"wiki/{page}/)", section) + self.assertTrue((REPO_ROOT / "wiki" / f"{page}.md").is_file()) + + class EndUsersCheckTheirOwnFolder(unittest.TestCase): """verify.py reads database.json and bios/. Advice about a user's own files points at the installer's --check, which hashes their folder.""" From 4581a5488219624225e2347e4c9a6aa3a8fed037 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:26:59 +0100 Subject: [PATCH 07/30] docs: list mame among the trees dedup keeps tools.md named 14 of the 15 NODEDUP_DIRS. The test matched the missing mame inside _mame_clones.json and now compares the list item by item. --- tests/test_audit_regressions.py | 8 +++++++- wiki/tools.md | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index d697062c4..35cecba97 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -6284,7 +6284,13 @@ class DocsDescribeTheRegistriesTheyName(unittest.TestCase): from dedup import NODEDUP_DIRS tools = (ROOT / "wiki" / "tools.md").read_text(encoding="utf-8") line = next(l for l in tools.splitlines() if "NODEDUP_DIRS" in l) - self.assertEqual(sorted(d for d in NODEDUP_DIRS if d not in line), []) + # Compared as list items: `mame` was missing while the substring + # test found it in `_mame_clones.json` on the same line. + listed = line.split("`NODEDUP_DIRS`", 1)[1].split(":", 1)[1].rstrip(" |") + self.assertEqual( + sorted(item.strip() for item in listed.split(",")), + sorted(NODEDUP_DIRS), + ) readme_gen = (ROOT / "scripts" / "generate_readme.py").read_text(encoding="utf-8") self.assertIn("NODEDUP_DIRS", readme_gen) diff --git a/wiki/tools.md b/wiki/tools.md index 9d9c2d08a..93b044523 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -1325,7 +1325,7 @@ before a release (wiki/release-process.md). Its lines stay in | Script | Purpose | |--------|---------| | `common.py` | Shared library: hash computation, file resolution, platform config loading, emulator profiles, target filtering | -| `dedup.py` | Deduplicate `bios/` (`--dry-run`, `--bios-dir`): byte-identical copies are deleted and one kept. Identical MAME sets under other names stay one file, named in `_mame_clones.json`. Trees addressed by path are left alone (`NODEDUP_DIRS`): BBK, Bungie, C-Dogs SDL, Ikemen GO, PinMAME, RPG Maker, ScummVM, Softdisk, TheXTech, splitwolf, Theme Hospital, c64-emu, opentyrian, tzdata | +| `dedup.py` | Deduplicate `bios/` (`--dry-run`, `--bios-dir`): byte-identical copies are deleted and one kept. Identical MAME sets under other names stay one file, named in `_mame_clones.json`. Trees addressed by path are left alone (`NODEDUP_DIRS`), under a folder of that name at any depth: BBK, Bungie, C-Dogs SDL, Ikemen GO, mame, PinMAME, RPG Maker, ScummVM, Softdisk, TheXTech, splitwolf, Theme Hospital, c64-emu, opentyrian, tzdata | | `identity.py` | One content filed under two identities: groups `bios/` by content and reports a group that crosses manufacturers outside `KNOWN_SHARED` (`--strict` exits 1, pipeline step 3a) | | `slots.py` | Destinations the platform and a core both claim, and how the pack settles each (`--platform`, `--include-archived`, `--target`, `--json`, `--strict`, and pipeline step 3b passes the run's own `--include-archived` and `--target`) | | `region_audit.py` | Confront every profile `region:` with the regions No-Intro names for the same dump (`--json`, `--db`, `--emulators-dir`). An opinion, never an authority: the code decides, and a disagreement is a line to read in the emulator's source | From f1f2254be8222b7749112b2c83ab7a4adfdce08d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:28:36 +0100 Subject: [PATCH 08/30] docs: drop the upstream remote from CONTRIBUTING profile_sync.py finds and fetches the project's main by itself. The documented --base upstream/main failed in any clone without that remote, where the same command without --base passes. --- CONTRIBUTING.md | 9 ++++----- scripts/generate_readme.py | 9 ++++----- tests/test_contributing_text.py | 3 ++- tests/test_profile_sync.py | 6 +++++- wiki/tools.md | 2 +- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ae2c25459..eda7d3945 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,7 +71,7 @@ the checks CI runs on a pull request (Python 3.12 or newer): pip install pyyaml "jsonschema[format-nongpl]" markdown pyte hypothesis vermin python scripts/validate_pr.py --changed # the BIOS files the branch adds python scripts/validate_schemas.py --source-only # profiles and platform files -python scripts/profile_sync.py --added --base upstream/main --strict # each profile the branch adds +python scripts/profile_sync.py --added --strict # each profile the branch adds python scripts/refresh_data_dirs.py # data directories the pack-building tests read python scripts/generate_site.py # the pages the site tests read npm ci --prefix website && npm run build --prefix website # the built site, Node 24 @@ -80,10 +80,9 @@ python scripts/validate_site.py --site-dir website/dist # links and page budget vermin --no-tips -t=3.8- --violations install.py # install.py stays on Python 3.8 ``` -`upstream` is this repository as a remote of your fork -(`git remote add upstream https://github.com/Abdess/retrobios`, then -`git fetch upstream`). A profile you only edit is not held to `--strict`: its -older refs are reviewed by hand. +`profile_sync.py` compares the branch with this repository's `main`, and +fetches it in a fork's clone. A profile you only edit is not held to +`--strict`: its older refs are reviewed by hand. The full pipeline is not needed for a contribution: it rebuilds every pack in `dist/`, which is why maintainers run it before a release. diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 1a3187461..77c5ef33d 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -981,7 +981,7 @@ the checks CI runs on a pull request (Python 3.12 or newer): pip install pyyaml "jsonschema[format-nongpl]" markdown pyte hypothesis vermin python scripts/validate_pr.py --changed # the BIOS files the branch adds python scripts/validate_schemas.py --source-only # profiles and platform files -python scripts/profile_sync.py --added --base upstream/main --strict # each profile the branch adds +python scripts/profile_sync.py --added --strict # each profile the branch adds python scripts/refresh_data_dirs.py # data directories the pack-building tests read python scripts/generate_site.py # the pages the site tests read npm ci --prefix website && npm run build --prefix website # the built site, Node 24 @@ -990,10 +990,9 @@ python scripts/validate_site.py --site-dir website/dist # links and page budget vermin --no-tips -t=3.8- --violations install.py # install.py stays on Python 3.8 ``` -`upstream` is this repository as a remote of your fork -(`git remote add upstream https://github.com/Abdess/retrobios`, then -`git fetch upstream`). A profile you only edit is not held to `--strict`: its -older refs are reviewed by hand. +`profile_sync.py` compares the branch with this repository's `main`, and +fetches it in a fork's clone. A profile you only edit is not held to +`--strict`: its older refs are reviewed by hand. The full pipeline is not needed for a contribution: it rebuilds every pack in `dist/`, which is why maintainers run it before a release. diff --git a/tests/test_contributing_text.py b/tests/test_contributing_text.py index 66ced6b53..593ad04fa 100644 --- a/tests/test_contributing_text.py +++ b/tests/test_contributing_text.py @@ -75,7 +75,8 @@ class BeforeOpeningAPullRequest(unittest.TestCase): # CI holds added profiles to their upstream, not edited ones: --changed # failed a contributor on refs they never wrote. self.assertIn("profile_sync.py --added --base", workflow) - self.assertIn("profile_sync.py --added --base upstream/main --strict", block) + self.assertIn("profile_sync.py --added --strict", block) + self.assertNotIn("upstream/main", section) self.assertNotIn("--changed --strict", block) def test_it_names_the_site_build_and_the_packages_ci_runs(self): diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index fc511761f..292ec753e 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -6017,7 +6017,11 @@ class TestBranchCheckIsWired(unittest.TestCase): import generate_readme # noqa: PLC0415 text = generate_readme.generate_contributing() - self.assertIn("profile_sync.py --added --base upstream/main --strict", text) + self.assertIn("profile_sync.py --added --strict", text) + # The default base is fetched by the tool: a documented upstream + # remote failed every clone that had origin alone. + self.assertNotIn("--base upstream/main", text) + self.assertNotIn("remote add upstream", text) self.assertNotIn("--emulator NAME --strict", text) workflow = (root / ".github" / "workflows" / "validate.yml").read_text() self.assertIn("profile_sync.py --added --base", workflow) diff --git a/wiki/tools.md b/wiki/tools.md index 93b044523..5992abbcd 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -475,7 +475,7 @@ python scripts/profile_sync.py --all --stale-before 2026-06-01 --limit 20 --tria python scripts/profile_sync.py --emulator vice --full-diff --ref vice/src/c64/c64rom.h --context 6 python scripts/profile_sync.py --emulator vice --strict python scripts/profile_sync.py --changed --strict -python scripts/profile_sync.py --added --base upstream/main --strict +python scripts/profile_sync.py --added --strict ``` Per part of a ref: `ANCHORED` (same content, same lines), `SHIFTED` (same From 81acb005363c00956f0bdbaf3071a22fd0593eaa Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:29:24 +0100 Subject: [PATCH 09/30] feat: fill the large file issue form validate_pr linked the bare missing-bios form for a file too large for git and then said to put it in the pull request and under bios/. The link now carries filename, sha1, md5 and size, and the pull request advice is left out for such a file. --- scripts/validate_pr.py | 34 ++++++++++++++++++++++++++++------ tests/test_validate_pr_flow.py | 24 ++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/scripts/validate_pr.py b/scripts/validate_pr.py index cbc6fa1bd..5a2a0401b 100644 --- a/scripts/validate_pr.py +++ b/scripts/validate_pr.py @@ -24,6 +24,7 @@ import re import shutil import subprocess import sys +import urllib.parse from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) @@ -72,6 +73,22 @@ LARGE_FILE_ISSUE_URL = ( ) +def large_file_issue_url(result: ValidationResult) -> str: + """The issue form with the values this run computed already filled in. + + GitHub fills an issue form field from the query parameter named by its + id: filename, sha1, md5 and size in missing-bios.yml. + """ + fields = { + "title": f"[BIOS] Missing: {result.filename}", + "filename": result.filename, + "sha1": result.sha1, + "md5": result.md5, + "size": str(result.size), + } + return f"{LARGE_FILE_ISSUE_URL}&{urllib.parse.urlencode(fields)}" + + class ValidationResult: def __init__(self, filepath: str) -> None: self.filepath = filepath @@ -358,14 +375,15 @@ def validate_file( if ext in BLOCKED_EXTENSIONS: result.add_check(False, f"Blocked file extension: {ext}") - if result.size > MAX_FILE_SIZE: + too_large = result.size > MAX_FILE_SIZE + if too_large: # GitHub refuses a pushed file over 100 MiB, so a pull request cannot # carry it. Only a maintainer can add an asset to the release. result.add_check( False, f"File too large for git ({result.size:,} > {MAX_FILE_SIZE:,} bytes). " - "Leave it out of the pull request and open an issue " - f"({LARGE_FILE_ISSUE_URL}) with the hashes above and where the file " + "Leave it out of the pull request and open an issue, its hashes " + f"filled in ({large_file_issue_url(result)}), saying where the file " "can be obtained: a maintainer adds it to the `large-files` release.", ) elif result.size == 0: @@ -449,8 +467,8 @@ def validate_file( result.outcome = "unknown" result.add_warning( "Not recognised: no platform list or emulator profile declares this " - "file. Say in the pull request which emulator reads it, for which " - "system, and where the dump comes from" + f"file. Say in the {'issue' if too_large else 'pull request'} which " + "emulator reads it, for which system, and where the dump comes from" ) normalized = os.path.normpath(relative) @@ -466,7 +484,11 @@ def validate_file( set(), ) expected_folders = sorted(expected) - if normalized.startswith("bios" + os.sep): + if too_large: + # The file stays out of the pull request, so no folder of bios/ is + # the contributor's to choose: the release asset is placed by hand. + pass + elif normalized.startswith("bios" + os.sep): parts = normalized.split(os.sep) here = f"bios/{parts[1]}/{parts[2]}/" if len(parts) >= 4 else "" makers = {folder.split("/")[1] for folder in expected} diff --git a/tests/test_validate_pr_flow.py b/tests/test_validate_pr_flow.py index 0cb7a8667..cab20c4b8 100644 --- a/tests/test_validate_pr_flow.py +++ b/tests/test_validate_pr_flow.py @@ -115,6 +115,30 @@ class OversizedFileAdvice(unittest.TestCase): self.assertIn("open an issue", message) self.assertIn(validate_pr.LARGE_FILE_ISSUE_URL, message) + def test_the_issue_form_is_filled_and_nothing_points_to_the_pr(self): + """The link opened an empty form, and the same report said to put the + file in the pull request and under bios/.""" + import urllib.parse # noqa: PLC0415 + + path = self.tmp / "big.bin" + path.write_bytes(b"x" * 8) + validate_pr.MAX_FILE_SIZE = 4 + result = validate_pr.validate_file( + "big.bin", None, {"sha1": set(), "md5": set(), "names": set()}, + ) + message = next(m for s, m in result.checks if s == "FAIL" and "too large" in m) + url = message.split("(", 2)[2].split(")", 1)[0] + query = urllib.parse.parse_qs(urllib.parse.urlsplit(url).query) + template = (REPO_ROOT / ".github" / "ISSUE_TEMPLATE" / "missing-bios.yml").read_text() + for field, value in (("filename", "big.bin"), ("sha1", result.sha1), + ("md5", result.md5), ("size", "8")): + with self.subTest(field=field): + self.assertIn(f"id: {field}\n", template) + self.assertEqual(query[field], [value]) + others = " ".join(m for _s, m in result.checks) + self.assertNotIn("pull request which", others) + self.assertNotIn("not under bios/", others) + def test_the_linked_issue_template_exists(self): template = validate_pr.LARGE_FILE_ISSUE_URL.rsplit("template=", 1)[1] self.assertTrue( From 9efb34c7610551f978ddf2a8ac838572b2fb0ff0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:30:12 +0100 Subject: [PATCH 10/30] fix: resolve the data directory source link The Fetched from line printed source_url with {version} unsubstituted, a 404 on six pages, and Revision showed a zip's ETag as a revision. The URL now carries the registry version, and only a git commit is shown, linked. --- scripts/generate_site.py | 38 +++++++++++++++++++++++++++--------- tests/test_site_emulators.py | 29 +++++++++++++++++++++++++++ 2 files changed, 58 insertions(+), 9 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 9a22b1e52..d90403be3 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4759,6 +4759,33 @@ def data_directory_revisions(data_dir: Path = _REPO_ROOT / "data") -> dict[str, return json.loads(path.read_text(encoding="utf-8")) +_GIT_COMMIT = re.compile(r"[0-9a-f]{40}") +_GITHUB_REPO = re.compile(r"https://github\.com/([^/]+/[^/]+)/") + + +def data_directory_origin(entry: dict, fetched: dict) -> list[str]: + """Where a data directory is fetched from, and the commit the packs hold. + + The URL is written with the `{version}` refresh_data_dirs substitutes: + printed raw it led to a 404. A zip source records the server's ETag, + which names no revision anyone can fetch, so only a git commit is shown. + """ + lines = [] + url = str(entry.get("source_url") or "") + if url: + url = url.replace("{version}", str(entry.get("version", "master"))) + lines.append(f"Fetched from <{url}>") + sha = str(fetched.get("sha", "")).strip('"') + if _GIT_COMMIT.fullmatch(sha): + repo = _GITHUB_REPO.match(url) + commit = f"[{_code(sha[:12])}](https://github.com/{repo.group(1)}/commit/{sha})" \ + if repo else _code(sha) + branch = str(fetched.get("version", "")) + lines.append(f"Revision: {commit}" + ( + f", the head of `{branch}` when fetched" if branch and branch != sha else "")) + return lines + + def page_data_directories(name: str, profiles: dict) -> list[dict]: """The data directories an emulator page lists. @@ -10075,15 +10102,8 @@ def emulator_pages( if entry.get("description"): line += f": {entry['description']}" lines.append(line) - if entry.get("source_url"): - lines.append(f" - Fetched from <{entry['source_url']}>") - fetched = _DATA_REVISIONS.get(ref) or {} - if fetched.get("version") or fetched.get("sha"): - # The URL serves whatever is current: the revision says - # which content the packs were built from. - revision = str(fetched.get("sha", "")).strip('"') - lines.append(f" - Revision: {fetched.get('version', '')}" - + (f" ({_code(revision)})" if revision else "")) + lines.extend(f" - {line}" for line in data_directory_origin( + entry, _DATA_REVISIONS.get(ref) or {})) if dd.get("mode"): lines.append(f" - Read by the {dd['mode']} build only") if dd.get("bundled"): diff --git a/tests/test_site_emulators.py b/tests/test_site_emulators.py index 79e871b40..b2d2b0bb9 100644 --- a/tests/test_site_emulators.py +++ b/tests/test_site_emulators.py @@ -763,5 +763,34 @@ class SmallFixes(unittest.TestCase): self.assertEqual(said, []) +class ADataDirectoryNamesAFetchableOrigin(unittest.TestCase): + """pcee2's page linked github.com/PCSX2/pcsx2/archive/{version}.tar.gz, + a 404, and buildbot zips showed `Revision: master (2047720058)`: a branch + that means nothing for a buildbot URL and the server's ETag.""" + + def test_the_url_is_substituted_and_a_commit_is_linked(self): + entry = {"source_url": "https://github.com/PCSX2/pcsx2/archive/{version}.tar.gz", + "version": "6db638dc0373769b4c5d5b5dd7ed6da641b9a18e"} + lines = gs.data_directory_origin(entry, {"sha": entry["version"], + "version": entry["version"]}) + self.assertEqual(lines[0], "Fetched from ") + self.assertIn("(https://github.com/PCSX2/pcsx2/commit/6db638dc0373769b4c5d5b5dd7ed6da641b9a18e)", + lines[1]) + self.assertNotIn("{version}", " ".join(lines)) + + def test_an_etag_is_no_revision(self): + entry = {"source_url": "https://buildbot.libretro.com/assets/system/Dolphin.zip"} + lines = gs.data_directory_origin(entry, {"sha": '"2047720058"', "version": "master"}) + self.assertEqual(lines, ["Fetched from "]) + + def test_every_registered_url_resolves(self): + registry = yaml.safe_load((REPO_ROOT / "platforms" / "_data_dirs.yml").read_text()) + for ref, entry in (registry.get("data_directories") or registry).items(): + if isinstance(entry, dict): + with self.subTest(ref=ref): + self.assertNotIn("{", " ".join(gs.data_directory_origin(entry, {}))) + + if __name__ == "__main__": unittest.main() From 6d6a79019abdb1c9fe843a857e6e81d2ca30e84e Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:37:17 +0100 Subject: [PATCH 11/30] feat: name the setups on each emulator page An emulator page said nothing of the setups that run it, and a file card's Platforms line named only the setups whose own list holds the file. A Run by row lists the platforms resolve_platform_cores gives, and each card names the packs that carry the file for their emulators. --- scripts/generate_site.py | 30 +++++++++++++++++++++++++++++- tests/test_site_emulators.py | 26 ++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 1 deletion(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index d90403be3..f21d5ebe2 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -6277,7 +6277,7 @@ def search_excluded(lines: list[str]) -> list[str]: SEARCH_SKIPPED = ( "- SHA1:", "- MD5:", "- CRC32:", "- SHA256:", "- Adler32", "- Size:", "- Validation", "- Priority:", "- Contents (", - "- Platforms:", "- Also installed by:", "- Source:", "- Asked for by:", + "- Platforms:", "- Packed for its emulators by:", "- Source:", "- Asked for by:", "- Collection path:", "- Collection copy:", "- Collection copies:", "- Installed by the platform's own package on:", "- Not in the platform's package", "- Never opened on these platforms", @@ -8039,6 +8039,18 @@ def _on_mismatch_lines(value: object) -> list[str]: return [] +def emulator_setups(coverages: dict, profiles: dict) -> dict[str, list[str]]: + """Each emulator with the platforms that run it, by display name.""" + setups: dict[str, list[str]] = {} + for platform, cov in coverages.items(): + for emulator in resolve_platform_cores(cov["config"], profiles): + setups.setdefault(emulator, []).append(platform) + return { + emulator: sorted(names, key=lambda p: _platform_display(p).casefold()) + for emulator, names in setups.items() + } + + def _build_rows(profile: dict) -> list[str]: """Meta card rows for who ships the build, how its files are found, why its revision is held and where its releases are announced.""" @@ -8282,6 +8294,13 @@ def _render_emulator_file( details.append("Platforms: " + ". ".join(places)) elif plat_links: details.append(f"Platforms: {', '.join(plat_links)}") + # The packs that carry the file for their emulators, as the system + # card says: handy's lynxboot.img named every setup but EmuDeck, + # ROCKNIX and NextUI, which install it for the core. + packed = _installed_places( + sha1, consumers, set(consumers.platforms.get(sha1, ()))) if sha1 else [] + if packed: + details.append("Packed for its emulators by: " + ". ".join(_places_markdown(packed))) if dest and dest != fname and dest != fpath: details.append(f"Destination: `{dest}`") @@ -9403,6 +9422,8 @@ def _said_as_list(items: list[str]) -> str: # Data directory members and revisions, read once per build. _DATA_MEMBERS: dict[str, dict[str, dict]] = {} _DATA_REVISIONS: dict[str, dict] = {} +# Emulator -> the platforms that run it, as resolve_platform_cores answers. +_EMULATOR_SETUPS: dict[str, list[str]] = {} # The measurement index items of each emulator, for the proof lines of its page. _MEASUREMENT_ITEMS: dict[str, list[dict]] = {} # The emulator pages that list each data directory, by ref. @@ -9998,6 +10019,12 @@ def emulator_pages( if revision: lines.append(f"| Revision | {revision} |") lines.extend(_build_rows(profile)) + setups = _EMULATOR_SETUPS.get(name) + if setups: + # The file cards name the setups whose own list holds a file: the + # ones that run the emulator and pack its files were said nowhere. + lines.append("| Run by | " + ", ".join( + f"[{_platform_display(p)}](../platforms/{p}.md)" for p in setups) + " |") lines.append(f"| Profiled | {profiled} |") # The YAML is the reusable document this site renders, and the JSON is the # same profile for a program. @@ -12325,6 +12352,7 @@ def _generate(args: argparse.Namespace) -> None: PAGE_DATA[f"platforms/{name}.md"] = platform_structured_data(name, cov) print("Generating emulator pages...") + _EMULATOR_SETUPS.update(emulator_setups(coverages, profiles)) _DATA_MEMBERS.update(data_directory_members()) _DATA_REVISIONS.update(data_directory_revisions()) for key in sorted(profiles): diff --git a/tests/test_site_emulators.py b/tests/test_site_emulators.py index b2d2b0bb9..f15c91374 100644 --- a/tests/test_site_emulators.py +++ b/tests/test_site_emulators.py @@ -763,6 +763,32 @@ class SmallFixes(unittest.TestCase): self.assertEqual(said, []) +class AnEmulatorPageNamesItsSetups(unittest.TestCase): + """emulators/handy named no setup that runs Handy, and its lynxboot.img + card left out EmuDeck, ROCKNIX and NextUI, which pack it for the core.""" + + def test_setups_come_from_the_core_resolution(self): + coverages = { + "b": {"config": {"cores": ["handy"]}}, + "a": {"config": {"cores": ["handy", "other"]}}, + "c": {"config": {"cores": ["other"]}}, + } + profiles = {"handy": {"type": "libretro"}, "other": {"type": "libretro"}} + self.assertEqual(gs.emulator_setups(coverages, profiles)["handy"], ["a", "b"]) + + def test_the_page_carries_both(self): + from tests import sitepages # noqa: PLC0415 + + sitepages.require() + text = sitepages.page("emulators/handy.md").read_text(encoding="utf-8") + row = next(line for line in text.splitlines() if line.startswith("| Run by |")) + for setup in ("emudeck", "rocknix", "nextui", "retroarch"): + self.assertIn(f"/platforms/{setup}/", row) + card = text.split("lynxboot.img", 1)[1].split("\n- Packed for its emulators by:", 1) + self.assertEqual(len(card), 2) + self.assertIn("EmuDeck", card[1].splitlines()[0]) + + class ADataDirectoryNamesAFetchableOrigin(unittest.TestCase): """pcee2's page linked github.com/PCSX2/pcsx2/archive/{version}.tar.gz, a 404, and buildbot zips showed `Revision: master (2047720058)`: a branch From 225cdaff2247d142f68a6994480ace1b52d43ef3 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:44:57 +0100 Subject: [PATCH 12/30] fix: show exact sizes where the code compares them A size validated by the emulator is shown with its bytes on the emulator page, and a collection card's badge gives the exact size, in whole KB or MB only when it is one. raze's three DUKE3D.GRP read 42.3 MB alike. --- scripts/generate_site.py | 39 ++++++++++++++++++++++++++------ tests/test_site_density_cards.py | 2 +- tests/test_site_emulators.py | 16 +++++++++++++ tests/test_site_systems.py | 13 +++++++++++ 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index f21d5ebe2..e996cb6b0 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -6022,7 +6022,9 @@ def _system_file_card( name = f["name"] sha1_full = f.get("sha1", "unknown") md5_full = f.get("md5", "unknown") - size = _fmt_size(f.get("size", 0)) + # The size a card names is the file's own, exact: rounded, 44,348,015 + # and 44,356,548 bytes were one 42.3 MB. + size = _card_size(int(f.get("size", 0))) # Who reads this file, not who reads a file of this name plats = sorted(consumers.platforms.get(sha1_full, ())) emus = sorted(consumers.emulators.get(sha1_full, ())) @@ -8051,6 +8053,26 @@ def emulator_setups(coverages: dict, profiles: dict) -> dict[str, list[str]]: } +def _validation_names(validation: object) -> set[str]: + """The checks a `validation:` names, a list or one list per build.""" + if isinstance(validation, dict): + return {str(v) for checks in validation.values() for v in checks or []} + return {str(v) for v in validation or []} + + +def _card_size(size: int) -> str: + """A file's size, exact: in whole MB or KB when it is one, else in B.""" + for unit, scale in (("MB", 1024 * 1024), ("KB", 1024)): + if size >= scale and size % scale == 0: + return f"{size // scale:,} {unit}" + return f"{size:,} B" + + +def _exact_size(size: int) -> str: + """A rounded size with its bytes, where the bytes are not the same figure.""" + return _fmt_size(size) if size < 1024 else f"{_fmt_size(size)} ({size:,} bytes)" + + def _build_rows(profile: dict) -> list[str]: """Meta card rows for who ships the build, how its files are found, why its revision is held and where its releases are announced.""" @@ -8184,25 +8206,28 @@ def _render_emulator_file( details.append(f"Found by: {found}") if fsystem: details.append(f"System: {_named_system_link(fsystem, '../')}") + # Where the code compares the size, the exact bytes: raze tells three + # DUKE3D.GRP apart by size, all three shown as 42.3 MB. + sized = _exact_size if "size" in _validation_names(validation) else _fmt_size if size: if isinstance(size, list): - size_str = " / ".join(_fmt_size(s) for s in size) + size_str = " / ".join(sized(s) for s in size) else: - size_str = _fmt_size(size) + size_str = sized(size) if fmin or fmax: bounds = [] if fmin: - bounds.append(f"min {_fmt_size(fmin)}") + bounds.append(f"min {sized(fmin)}") if fmax: - bounds.append(f"max {_fmt_size(fmax)}") + bounds.append(f"max {sized(fmax)}") size_str += f" ({', '.join(bounds)})" details.append(f"Size: {size_str}") elif fmin or fmax: bounds = [] if fmin: - bounds.append(f"min {_fmt_size(fmin)}") + bounds.append(f"min {sized(fmin)}") if fmax: - bounds.append(f"max {_fmt_size(fmax)}") + bounds.append(f"max {sized(fmax)}") details.append(f"Size: {', '.join(bounds)}") # Starlight folds the hashes and gives each a copy button. copyable = markup.starlight() diff --git a/tests/test_site_density_cards.py b/tests/test_site_density_cards.py index 56ddd41c7..9c031adec 100644 --- a/tests/test_site_density_cards.py +++ b/tests/test_site_density_cards.py @@ -110,7 +110,7 @@ class ASystemCardShowsTheAnswerFirst(unittest.TestCase): self.shown = self.text.replace(self.fold, "") def test_the_open_part(self) -> None: - for fact in ("ppge_atlas.zim", ">650.9 KB<", ">Download", + for fact in ("ppge_atlas.zim", ">666,539 B<", ">Download", "
On the setup's list
", "
Packed for its emulators by
", "
Emulators
", f'
SHA1
{SHA1[:12]}
'): self.assertIn(fact, self.shown) diff --git a/tests/test_site_emulators.py b/tests/test_site_emulators.py index f15c91374..03f59f620 100644 --- a/tests/test_site_emulators.py +++ b/tests/test_site_emulators.py @@ -789,6 +789,22 @@ class AnEmulatorPageNamesItsSetups(unittest.TestCase): self.assertIn("EmuDeck", card[1].splitlines()[0]) +class SizesTheCodeComparesAreExact(unittest.TestCase): + """raze validates DUKE3D.GRP by size, and its page showed 42.3 MB for + 44,356,548 and 44,348,015 bytes alike.""" + + def test_a_validated_size_carries_its_bytes(self): + profile = {"emulator": "x", "type": "libretro", "systems": ["sony-playstation"], "files": [ + {"name": "DUKE3D.GRP", "size": 44348015, "validation": ["size", "crc32"]}, + {"name": "other.bin", "size": 44356548}, + {"name": "built.bin", "size": 2304, "validation": {"core": ["size"]}}, + ]} + page = _page("x", profile) + self.assertIn("Size: 42.3 MB (44,348,015 bytes)", page) + self.assertIn("Size: 2.2 KB (2,304 bytes)", page) + self.assertNotIn("44,356,548", page) + + class ADataDirectoryNamesAFetchableOrigin(unittest.TestCase): """pcee2's page linked github.com/PCSX2/pcsx2/archive/{version}.tar.gz, a 404, and buildbot zips showed `Revision: master (2047720058)`: a branch diff --git a/tests/test_site_systems.py b/tests/test_site_systems.py index 784c17092..9f8891799 100644 --- a/tests/test_site_systems.py +++ b/tests/test_site_systems.py @@ -184,6 +184,19 @@ class ThePermalinkIsAnIcon(unittest.TestCase): self.assertEqual(page.count('href="#rb-link-icon"'), 2) +class TheCardGivesTheSizeInBytes(unittest.TestCase): + """The badge rounded to 0.1 KB and no line of a system card gave the + bytes an emulator compares.""" + + def test_the_badge_is_exact(self): + for size, shown in ((524288, "512 KB"), (44348015, "44,348,015 B"), + (2 * 1024 * 1024, "2 MB"), (2304, "2,304 B"), (10, "10 B")): + with self.subTest(size=size): + card = {**_sony_file("scph5501.bin", "4"), "size": size} + title = gs._system_file_card(card, FileConsumers({"files": {}}))[2] + self.assertIn(f'{shown}', title) + + class TheCardLinesSayWhatTheyMean(unittest.TestCase): """"Platforms:" and "Also installed by:" listed different setups, as lower-case ids, and only a code comment told them apart.""" From 00aac8dc72d2c867c77ba04bed2593056b7e3f32 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:48:37 +0100 Subject: [PATCH 13/30] docs: say what catalog.json lists index.json called the catalog every published file, while the one-item documents of emulators/, platforms/, systems/ and lookup/ are in none of its items. The description and the data page now say so. --- scripts/generate_site.py | 7 ++++++- tests/test_site_reference.py | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index e996cb6b0..5fbadd18f 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1804,7 +1804,9 @@ def generate_data_exports( # entry point is a document of its own, naming where to start. entry_points = [ {"title": "Catalog", "url": "catalog.json", - "description": "Every published file with its size, SHA256 and schema."}, + "description": "Each dataset, manifest, export and download with its size, " + "SHA256 and schema. The one-item documents of emulators/, " + "platforms/, systems/ and lookup/ are not listed."}, {"title": "System ids", "url": "systems.json", "description": "Each system id with its name and the section holding its files."}, {"title": "Core names", "url": "cores.json", @@ -2204,6 +2206,9 @@ def generate_data_page(exports: list[dict], measurements_sqlite: Path | None = N "when one is known, the `system_name` the site gives its system. " "[Find or check a file](files/index.md) reads it.", "", + "`catalog.json` lists the datasets, manifests, exports and downloads. It " + "does not list these one-item documents, which hold no SHA256 of their own.", + "", "In a browser, the SHA1 of a file the reader picks finds its records when " "the collection holds that file. A list naming a file the collection lacks " "is filed under its MD5 or CRC32 alone, and Web Crypto has no MD5: " diff --git a/tests/test_site_reference.py b/tests/test_site_reference.py index 5375a7972..408327c99 100644 --- a/tests/test_site_reference.py +++ b/tests/test_site_reference.py @@ -649,6 +649,24 @@ class TheContractsValidate(unittest.TestCase): [e.message for e in envelope.iter_errors(json.loads((API / name).read_text()))], []) +class TheIndexSaysWhatTheCatalogLists(unittest.TestCase): + """index.json called catalog.json every published file, while the 990 + one-item documents of emulators/, platforms/, systems/ and lookup/ are + in none of its 80 items.""" + + def test_the_description_matches_the_items(self): + index = json.loads(_built(API / "index.json").read_text()) + catalog = json.loads(_built(API / "catalog.json").read_text()) + described = next(i["description"] for i in index["items"] if i["url"] == "catalog.json") + self.assertNotIn("Every published file", described) + listed = {item["url"].split("/", 1)[0] for item in catalog["items"]} + for folder in ("emulators", "platforms", "systems", "lookup"): + with self.subTest(folder=folder): + self.assertIn(f"{folder}/", described) + self.assertNotIn(folder, listed) + self.assertTrue(any((API / folder).glob("*.json"))) + + class EveryEnvelopeCarriesTheCommit(unittest.TestCase): """8.5: emulators/ and systems/ lacked `commit` (716 documents).""" From 2f81aa9c0f63b41362691dd86b8516914b917385 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:50:49 +0100 Subject: [PATCH 14/30] fix: write region names in pack READMEs as labels packreadme title-cased each slug, printing asia-ntsc as Asia Ntsc. region.label() holds the words the site already used, and both read it. --- scripts/generate_site.py | 13 +++++-------- scripts/packreadme.py | 5 ++--- scripts/region.py | 13 +++++++++++++ tests/test_region.py | 18 ++++++++++++++++++ 4 files changed, 38 insertions(+), 11 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 5fbadd18f..79d23238c 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -7750,20 +7750,17 @@ def _distinct_spellings(names: list) -> list[str]: return list(seen.values()) -# Region slugs whose words are not the slug title-cased. -_REGION_WORDS = { - "uk": "UK", - "asia-ntsc": "Asia (NTSC)", - "asia-pal": "Asia (PAL)", - "world": "every region", -} +# Region slugs whose words are not the slug title-cased, beyond region.LABELS. +_REGION_WORDS = {"world": "every region"} def _region_words(value: object, most: int = 0) -> str: """A region: value in words. ``most`` caps the regions named, the rest counted: one Saturn BIOS serves seven.""" raw = value if isinstance(value, list) else [value] if value else [] - words = [_REGION_WORDS.get(str(r), str(r).replace("-", " ").title()) for r in raw] + from region import label # noqa: PLC0415 + + words = [_REGION_WORDS.get(str(r)) or label(str(r)) for r in raw] if most and len(words) > most: words = words[:most] + [f"{len(words) - most} more"] return _said_as_list(words) diff --git a/scripts/packreadme.py b/scripts/packreadme.py index 720f46104..06ee483a3 100644 --- a/scripts/packreadme.py +++ b/scripts/packreadme.py @@ -5,6 +5,7 @@ goes, and which ones the user has to rename.""" from __future__ import annotations +from region import label as region_label from siteguides import INSTALL_PS1, INSTALL_SH, pack_manual def _detect_systems(detect: list[dict]) -> list[str]: @@ -290,9 +291,7 @@ def _build_readme( region_info = "" if regions: - pretty = ", ".join( - " ".join(w.title() for w in slug.split("-")) for slug in regions - ) + pretty = ", ".join(region_label(slug) for slug in regions) region_info = ( "PACK TYPE: Region Filtered\n\n" f" Region priority: {pretty}\n\n" diff --git a/scripts/region.py b/scripts/region.py index bb7cfaf90..52e726d81 100644 --- a/scripts/region.py +++ b/scripts/region.py @@ -107,6 +107,19 @@ ALIASES: dict[str, str] = { } +# Slugs a person does not write title-cased: "Asia Ntsc" read as machine text. +LABELS: dict[str, str] = { + "uk": "UK", + "asia-ntsc": "Asia (NTSC)", + "asia-pal": "Asia (PAL)", +} + + +def label(slug: str) -> str: + """A region slug in words: north-america as North America.""" + return LABELS.get(slug) or " ".join(word.title() for word in slug.split("-")) + + def unknown_region_message(raw: str, aliases: dict[str, str], regions: frozenset[str]) -> str: """What to type instead: the closest spellings, then every short form. diff --git a/tests/test_region.py b/tests/test_region.py index 0d7fb7e72..d0611a975 100644 --- a/tests/test_region.py +++ b/tests/test_region.py @@ -1452,5 +1452,23 @@ class UsaRegionServesTheAmericas(unittest.TestCase): self.assertEqual(region.rank(jpn, requested), 1) +class RegionWordsInAPackReadme(unittest.TestCase): + """The pack README title-cased each slug: asia-ntsc read "Asia Ntsc".""" + + def test_the_priority_line_reads_as_written(self): + from packreadme import _build_readme # noqa: PLC0415 + + readme = _build_readme( + "retroarch", "RetroArch", "system", 1, 1, + regions=region.parse_requested("us,asia-ntsc,jp"), + ) + self.assertIn("Region priority: North America, Asia (NTSC), Japan", readme) + + def test_every_region_has_a_label(self): + for slug in region.REGIONS: + with self.subTest(slug=slug): + self.assertNotRegex(region.label(slug), r"\bNtsc\b|\bPal\b|\bUk\b") + + if __name__ == "__main__": unittest.main() From 657e3bc55bd56efc2da46011d52146ef7a8fc08c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:51:05 +0100 Subject: [PATCH 15/30] docs: say what archived means on the RetroPie page The RetroPie guide note still tied the archived status to a scraping schedule no tool runs. The doc test now reads _guides.yml too. --- platforms/_guides.yml | 4 ++-- tests/test_doc_pages.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/platforms/_guides.yml b/platforms/_guides.yml index aa6bf2dd1..4353b6ccc 100644 --- a/platforms/_guides.yml +++ b/platforms/_guides.yml @@ -648,8 +648,8 @@ guides: RetroPie also installs standalone emulators (openMSX, PCSX2, XRoar, Amiberry and others) whose files only the RetroPie file list carries. - >- - RetroPie is archived in this project: its configuration is kept and the command - serves it, but its upstream list is no longer read again on a schedule. + RetroPie is archived in this project: the command serves it, and checks that cover + every setup include it only on request. pack_manual: | Option B: Via network share -------------------------- diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 2f342fcee..b62cb827e 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -1918,6 +1918,7 @@ class ArchivedIsWhatTheToolsDo(unittest.TestCase): "generate_site": inspect.getsource(generate_site), "generate_pack": inspect.getsource(generate_pack), "list_platforms": inspect.getsource(list_platforms), + "_guides.yml": (ROOT / "platforms" / "_guides.yml").read_text(encoding="utf-8"), } for name, text in texts.items(): with self.subTest(name): From 6015552fb288e2d36d6616609cc475f8f489ea7b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:58:38 +0100 Subject: [PATCH 16/30] fix: offer Lakka's card route from Linux only Lakka keeps /storage on the card's ext4 partition (Lakka-LibreELEC 4696b48f scripts/mkimage:72, 95), which Windows and macOS do not open, and its page offered them a card route. card_run is now [linux], and a page with no Windows route says only Linux opens the card. --- platforms/_guides.yml | 6 ++++-- scripts/generate_site.py | 6 +++++- tests/test_site_entry.py | 13 +++++++++++++ 3 files changed, 22 insertions(+), 3 deletions(-) diff --git a/platforms/_guides.yml b/platforms/_guides.yml index 4353b6ccc..371507733 100644 --- a/platforms/_guides.yml +++ b/platforms/_guides.yml @@ -452,6 +452,9 @@ guides: Turn on **SSH**, a terminal over the network, in Lakka's **Settings > Services**. From a computer, connect with `ssh root@lakka`, password `root`. Paste the line. Press Enter. + # /storage is the card's ext4 partition, the other one FAT32 (Lakka-LibreELEC + # 4696b48f, scripts/mkimage:71-72 and 95): from a computer, only Linux opens it. + card_run: [linux] reference_notes: - The folder is reachable over SSH and over the network share. pack_manual: | @@ -648,8 +651,7 @@ guides: RetroPie also installs standalone emulators (openMSX, PCSX2, XRoar, Amiberry and others) whose files only the RetroPie file list carries. - >- - RetroPie is archived in this project: the command serves it, and checks that cover - every setup include it only on request. + RetroPie is archived here: checks of every setup include it only on request. pack_manual: | Option B: Via network share -------------------------- diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 79d23238c..ccd8e4a05 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4188,7 +4188,11 @@ def _card_route(name: str, display: str, guide: Guide) -> list[str]: lead = ("No network here? Run the command on a computer that has one, with a USB " f"drive's folder after `--dest`. Then copy it into {_folder_words(display, extract_to)}.") return [lead, "", *terminal_tabs(tabs, args, steps=False)] - lead = ("No network? Put the device's card in your computer. After `--dest`, " + # A folder on the card's Linux partition: Windows and macOS show the + # small boot partition alone, and no folder there fits --dest. + put = ("Put the device's card in your computer." if "windows" in tabs + else "Only a Linux computer opens the card.") + lead = (f"No network? {put} After `--dest`, " f"name the folder the device calls `{extract_to}`.") return [lead, "", *terminal_tabs(tabs, args, steps=False), f"[More on SD cards](../{CARD_ROUTE_FAQ})", ""] diff --git a/tests/test_site_entry.py b/tests/test_site_entry.py index 032f5d67d..18e9b27b0 100644 --- a/tests/test_site_entry.py +++ b/tests/test_site_entry.py @@ -937,6 +937,19 @@ class ARouteWithoutANetwork(unittest.TestCase): self.assertEqual(_tab_labels(card), set()) self.assertIn("**Linux**", card) + def test_a_linux_partition_card_says_why_and_offers_linux_alone(self): + """Lakka's /storage is the card's ext4 partition (Lakka-LibreELEC + scripts/mkimage): its page offered Windows and macOS tabs for a + folder neither can open.""" + for name in ("lakka", "retropie"): + with self.subTest(setup=name): + self.assertEqual(siteguides.GUIDES[name].card_run, ("linux",)) + card = _plain_page(f"platforms/{name}.md").split("No network?", 1)[1] + card = card.split("## On the device", 1)[0] + self.assertEqual(_tab_labels(card), set()) + self.assertNotIn("E:\\", card) + self.assertIn("Only a Linux computer opens the card.", card) + def test_no_zip_is_promised_for_an_unmarked_setup(self): for name in SETUPS: if REGISTRY[name].get("release_pack") is True: From f1dfe390302df9a6eb33ffb5c6f6ee6b701b395c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 03:58:46 +0100 Subject: [PATCH 17/30] chore: record the GroovyMAME part of Other Filing device sets under each of their systems gave the Other page a GroovyMAME part of its own, recorded so its address keeps a target. --- website/src/data/published-parts.json | 1 + 1 file changed, 1 insertion(+) diff --git a/website/src/data/published-parts.json b/website/src/data/published-parts.json index b59e894dd..70f1d1218 100644 --- a/website/src/data/published-parts.json +++ b/website/src/data/published-parts.json @@ -291,6 +291,7 @@ "systems/other-fbneo.md": "systems/other.md", "systems/other-future-pinball-fploader.md": "systems/other.md", "systems/other-gopher2600.md": "systems/other.md", + "systems/other-groovymame.md": "systems/other.md", "systems/other-idtech4a.md": "systems/other.md", "systems/other-ines.md": "systems/other.md", "systems/other-mame-2.md": "systems/other.md", From 28d310d8bc77cfb37deca5fd343cf857c34c3938 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 04:02:14 +0100 Subject: [PATCH 18/30] fix: preselect RetroArch in a system's install box The box showed the first setup by name, AmberELEC on 17 system pages, under Windows tabs. It now selects RetroArch, which runs on every system the box has a tab for, else a setup the release publishes a ZIP for. --- scripts/generate_site.py | 22 +++++++++++++++++++--- tests/test_site_density_cards.py | 16 ++++++++++++++-- website/src/scripts/site.ts | 2 +- 3 files changed, 34 insertions(+), 6 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index ccd8e4a05..7a1787e76 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -9114,6 +9114,21 @@ def _install_args(platform: str, system: str, shell: str) -> str: return f"--platform {platform} --system {quoted}" +def _default_setup(declared: list[tuple[str, str]]) -> tuple[str, str]: + """The setup an install box shows before the reader picks one. + + The first in alphabetical order was AmberELEC on 17 system pages, a + handheld system under a Windows tab. RetroArch runs on every OS the box + has a tab for. Without it, a setup the release publishes a ZIP for, + the ones most people download (`release_pack`). + """ + for wanted in (lambda p: p == "retroarch", has_release_pack): + chosen = next((pair for pair in declared if wanted(pair[0])), None) + if chosen: + return chosen + return declared[0] + + def _install_picker(declared: list[tuple[str, str]]) -> list[str]: """One command, for the setup the reader picks, under the OS tabs. @@ -9128,15 +9143,16 @@ def _install_picker(declared: list[tuple[str, str]]) -> list[str]: return html.escape(text, quote=False).replace('"', """) names = Counter(platform for platform, _system in declared) + lead = _default_setup(declared) options = "".join( - f'" for platform, system in declared) - lead = declared[0] args = {key: _install_args(*lead, TERMINALS[key][2]) for key in INSTALL_PICKER_OSES} return [ "Each platform names the system its own way.", diff --git a/tests/test_site_density_cards.py b/tests/test_site_density_cards.py index 9c031adec..6bea1ba41 100644 --- a/tests/test_site_density_cards.py +++ b/tests/test_site_density_cards.py @@ -193,11 +193,23 @@ class AnInstallBoxShowsOneCommand(unittest.TestCase): self.assertEqual(re.findall(r' None: + """AmberELEC led 17 system pages, a handheld system under the + Windows tab, for being first in alphabetical order.""" + self.assertEqual(gs._default_setup([("amberelec", "psx"), ("retroarch", "Sony - PlayStation")]), + ("retroarch", "Sony - PlayStation")) + self.assertEqual(gs._default_setup([("amberelec", "x"), ("batocera", "x")]), ("batocera", "x")) + self.assertEqual(gs._default_setup([("amberelec", "x"), ("knulli", "x")]), ("amberelec", "x")) + select = re.search(r"", self.text).group(1) + self.assertEqual(re.findall(r'