From 45db77cfd0c227a539c26a866ff84c7223f7b20b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:54:56 +0200 Subject: [PATCH 01/18] fix: link a system to its maker's folder The database keeps one path per content, so a system placed by the copy deduplication kept linked to another maker: nintendo-nes and nintendo-fds to Arcade > FBNeo, sega-mastersystem and sinclair-zx-spectrum to Other > zesarux. Every folder holding a copy now votes, an id that spells one console folder links to it, and a maker's folder outranks the folders that file copies by emulator. --- scripts/generate_site.py | 147 ++++++++++++++++++++++++++------ tests/test_site_system_links.py | 83 ++++++++++++++++++ 2 files changed, 206 insertions(+), 24 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 0bad09ca..efcf28d7 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -125,19 +125,95 @@ CLS_LABELS = { } +def _copy_places(db: dict) -> dict[str, set[tuple[str, str, str]]]: + """sha1 -> every (manufacturer, console, file name) holding a copy. + + The database keeps one path per content, deduplication having left one + copy. by_path_suffix still records the other folders a copy is filed in, + under the name that folder gives it. + """ + files = db.get("files", {}) + bios_dir = Path(str(db.get("bios_dir") or "bios")) + places: dict[str, set[tuple[str, str, str]]] = {} + for sha1, entry in files.items(): + parts = str(entry.get("path", "")).split("/") + if len(parts) >= 3 and parts[0] == "bios": + places.setdefault(sha1, set()).add((parts[1], parts[2], parts[-1])) + for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items(): + parts = tail.split("/") + if len(parts) < 3 or parts[1].startswith(".") or len(sha1s) != 1: + continue + if sha1s[0] in files and (bios_dir / tail).is_file(): + places.setdefault(sha1s[0], set()).add((parts[0], parts[1], parts[-1])) + return places + + +# bios/Other/ and bios/Arcade/ file a copy by +# who loads it, not by machine. A copy also filed under a machine's maker +# places the system there. +_BUCKET_FOLDERS = frozenset({"Other", "Arcade"}) + + +def _slug(manufacturer: str) -> str: + return manufacturer.lower().replace(" ", "-") + + +def _names_manufacturer(sys_id: str, manufacturer: str) -> bool: + slug = _slug(manufacturer) + return sys_id == slug or sys_id.startswith(f"{slug}-") + + +def _copies_by_path(db: dict) -> dict[str, set[tuple[str, str, str]]]: + """The copy places of each content, keyed by the path the database keeps.""" + files = db.get("files", {}) + return { + str(files[sha1].get("path", "")): where + for sha1, where in _copy_places(db).items() + } + + def _system_folder( - files: list[dict], db: dict, name_consoles: dict[str, set[tuple[str, str]]] + files: list[dict], + db: dict, + name_consoles: dict[str, set[tuple[str, str]]], + sys_id: str = "", + copies: dict[str, set[tuple[str, str, str]]] | None = None, ) -> tuple[str, str] | None: - """The bios// folder a system's files live in.""" + """The bios// folder a system's files live in. + + A content votes for every folder holding a copy under the name the system + declares, not only the one deduplication kept: disksys.rom is kept as + Arcade/FBNeo/fdsbios.nes and filed again under Nintendo/Famicom Disk + System. A copy filed under another name, outside the manufacturer the id + names, is another machine's file and votes only when no copy carries the + declared name. An id that names its manufacturer (nintendo-fds) keeps the + votes cast inside that manufacturer when there are any, and a maker's + folder outranks the folders that file copies by emulator. + """ + if copies is None: + copies = _copies_by_path(db) votes: dict[tuple[str, str], int] = {} for fe in files: local, status = resolve_local_file(fe, db) - parts = (local or "").split("/") - if len(parts) >= 4 and parts[0] == "bios" and resolution_is_hash_exact(status): - place = (parts[1], parts[2]) + if not local or not resolution_is_hash_exact(status): + continue + held = copies.get(local, set()) + if not held: + parts = local.split("/") + if len(parts) >= 4 and parts[0] == "bios": + held = {(parts[1], parts[2], parts[-1])} + declared = str(fe.get("name", "")).casefold() + named = { + (m, c) for m, c, base in held + if base.casefold() == declared or _names_manufacturer(sys_id, m) + } + for place in named or {(m, c) for m, c, _ in held}: votes[place] = votes.get(place, 0) + 1 if votes: - return max(sorted(votes), key=lambda place: votes[place]) + own = [place for place in votes if _names_manufacturer(sys_id, place[0])] + makers = [place for place in votes if place[0] not in _BUCKET_FOLDERS] + candidates = own or makers or list(votes) + return max(sorted(candidates), key=lambda place: votes[place]) for fe in files: places = name_consoles.get(fe.get("name", ""), set()) if len(places) == 1: @@ -145,6 +221,32 @@ def _system_folder( return None +def _folders_by_name(manufacturers: dict) -> dict[str, set[tuple[str, str]]]: + """Normalized system name -> the console folders that spell it. + + nintendo-nes, nes and the NES folder normalize alike, as do + sega-mastersystem and Sega/Master System. + """ + from common import _norm_system_id + + index: dict[str, set[tuple[str, str]]] = {} + for mfr, consoles in manufacturers.items(): + for console in consoles: + spelled = console.lower().replace(" ", "-") + for key in { + _norm_system_id(f"{_slug(mfr)}-{spelled}"), _norm_system_id(spelled) + }: + index.setdefault(key, set()).add((mfr, console)) + return index + + +def _folder_named(sys_id: str, by_name: dict[str, set[tuple[str, str]]]) -> tuple[str, str] | None: + from common import _norm_system_id + + places = by_name.get(_norm_system_id(sys_id), set()) + return next(iter(places)) if len(places) == 1 else None + + def _system_pages_from_data( manufacturers: dict, coverages: dict, @@ -189,21 +291,28 @@ def _system_pages_from_data( console_norm[key] = entry # Map system_id -> (manufacturer, console) via platform file entries + copies = _copies_by_path(db) + folders_by_name = _folders_by_name(manufacturers) pages: dict[str, tuple[str, str]] = {} for cov in coverages.values(): config = cov["config"] for sys_id, system in config.get("systems", {}).items(): if sys_id in pages: continue - # Strategy 1: the folder holding the files the system declares, + # Strategy 1: the one console folder the id spells (nintendo-nes, + # NES). A copy of the disk system BIOS in the NES list does not + # move the NES to the folder that copy sits in. + # Strategy 2: the folder holding the files the system declares, # found by content, and most votes win. A bare name stands in only # where the collection files it in a single console. - place = _system_folder(system.get("files", []), db, name_consoles) + place = _folder_named(sys_id, folders_by_name) or _system_folder( + system.get("files", []), db, name_consoles, sys_id, copies + ) if place is not None: mfr, console = place pages[sys_id] = (mfr.lower().replace(" ", "-"), console) continue - # Strategy 2: fuzzy match system_id against console directory names + # Strategy 3: fuzzy match system_id against console directory names if sys_id in console_norm: pages[sys_id] = console_norm[sys_id] else: @@ -228,7 +337,9 @@ def _system_pages_from_data( if isinstance(entry, dict) and (entry.get("system") == sys_id or len(systems) == 1) ] - place = _system_folder(files, db, name_consoles) + place = _folder_named(sys_id, folders_by_name) or _system_folder( + files, db, name_consoles, sys_id, copies + ) if place is not None: mfr, console = place pages[sys_id] = (mfr.lower().replace(" ", "-"), console) @@ -2151,21 +2262,9 @@ def _group_by_manufacturer(db: dict) -> dict[str, dict[str, list]]: read from the full paths by_path_suffix records. """ files = db.get("files", {}) - bios_dir = Path(str(db.get("bios_dir") or "bios")) - places: dict[str, set[tuple[str, str]]] = {} - for sha1, entry in files.items(): - parts = str(entry.get("path", "")).split("/") - if len(parts) >= 3 and parts[0] == "bios": - places.setdefault(sha1, set()).add((parts[1], parts[2])) - for tail, sha1s in db.get("indexes", {}).get("by_path_suffix", {}).items(): - parts = tail.split("/") - if len(parts) < 3 or parts[1].startswith(".") or len(sha1s) != 1: - continue - if sha1s[0] in files and (bios_dir / tail).is_file(): - places.setdefault(sha1s[0], set()).add((parts[0], parts[1])) manufacturers: dict[str, dict[str, list]] = {} - for sha1, where in places.items(): - for manufacturer, console in sorted(where): + for sha1, where in _copy_places(db).items(): + for manufacturer, console in sorted({(m, c) for m, c, _ in where}): manufacturers.setdefault(manufacturer, {}).setdefault(console, []).append(files[sha1]) return manufacturers diff --git a/tests/test_site_system_links.py b/tests/test_site_system_links.py index 17f05e3d..9e1edf78 100644 --- a/tests/test_site_system_links.py +++ b/tests/test_site_system_links.py @@ -147,6 +147,89 @@ class AnIdNamesOneCardPerPage(unittest.TestCase): self.assertEqual(page.count("rb-sys-file"), 2) +class ACopyFiledByEmulatorDoesNotMoveAMachine(unittest.TestCase): + """The database keeps one path per content, and the system links followed + it: nintendo-nes and nintendo-fds led to Arcade > FBNeo, where + disksys.rom is kept as fdsbios.nes, sega-mastersystem and + sinclair-zx-spectrum to Other > zesarux.""" + + def _db(self, tmp: str) -> dict: + for rel in ("Arcade/FBNeo/fdsbios.nes", "Nintendo/Famicom Disk System/disksys.rom", + "Nintendo/NES/other.rom"): + (Path(tmp) / rel).parent.mkdir(parents=True, exist_ok=True) + (Path(tmp) / rel).write_bytes(b"x") + return { + "bios_dir": tmp, + "files": { + "d" * 40: {"path": "bios/Arcade/FBNeo/fdsbios.nes", "sha1": "d" * 40, + "name": "fdsbios.nes"}, + "e" * 40: {"path": "bios/Nintendo/NES/other.rom", "sha1": "e" * 40, + "name": "other.rom"}, + }, + "indexes": { + "by_sha1": {}, "by_md5": {}, "by_name": {}, + "by_path_suffix": { + "Arcade/FBNeo/fdsbios.nes": ["d" * 40], + "Nintendo/Famicom Disk System/disksys.rom": ["d" * 40], + "Nintendo/NES/other.rom": ["e" * 40], + }, + }, + } + + def test_the_copy_under_the_declared_name_places_the_system(self): + import tempfile + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + db = self._db(tmp) + files = [{"name": "disksys.rom", "sha1": "d" * 40}] + self.assertEqual( + gs._system_folder(files, db, {}, "nintendo-fds"), + ("Nintendo", "Famicom Disk System"), + ) + # A file named otherwise still finds the maker's copy before the + # emulator's folder. + files = [{"name": "FDS_disksys-nintendo.rom", "sha1": "d" * 40}] + self.assertEqual( + gs._system_folder(files, db, {}, "fds"), + ("Nintendo", "Famicom Disk System"), + ) + + def test_an_id_that_spells_a_folder_links_to_it(self): + import tempfile + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + db = self._db(tmp) + manufacturers = gs._group_by_manufacturer(db) + config = {"systems": {"nintendo-nes": {"files": [ + {"name": "FDS_disksys-nintendo.rom", "sha1": "d" * 40}]}}} + pages = gs._system_pages_from_data( + manufacturers, {"p": {"config": config}}, db) + self.assertEqual(pages["nintendo-nes"], ("nintendo", "NES")) + + def test_repository_systems(self): + if not (REPO_ROOT / "database.json").exists(): + self.skipTest("database.json is not built") + previous = os.getcwd() + os.chdir(REPO_ROOT) + self.addCleanup(os.chdir, previous) + from common import load_database, load_emulator_profiles + + db = load_database("database.json") + manufacturers = gs._group_by_manufacturer(db) + pages = gs._system_pages_from_data( + manufacturers, {}, db, load_emulator_profiles("emulators", skip_aliases=False)) + expected = { + "nintendo-nes": ("nintendo", "NES"), + "nintendo-fds": ("nintendo", "Famicom Disk System"), + "sega-mastersystem": ("sega", "Master System"), + "sinclair-zx-spectrum": ("sinclair", "ZX Spectrum"), + "commodore-vic20": ("commodore", "VIC-20"), + } + for sys_id, page in expected.items(): + with self.subTest(system=sys_id): + self.assertEqual(pages.get(sys_id), page) + + class SystemEmulatorsWhenNoIdMatches(unittest.TestCase): """RetroArch files NAOMI and Atomiswave as sega-dreamcast-arcade and the Enterprise as enterprise-64-128, ids no profile uses: the emulator cell From 2e5d5e4f7984103d23e138b462b955662ee1a679 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 09:59:45 +0200 Subject: [PATCH 02/18] chore: pin upstream revisions of mame ports Each upstream URL names a mamedev/mame tag; upstream_commit is the commit that tag points at. Every upstream-role ref lands on its declared set name or hash there. --- emulators/cdi2015.yml | 1 + emulators/mame2009.yml | 1 + emulators/mame2010.yml | 1 + emulators/mame2014.yml | 1 + emulators/mame2015.yml | 1 + emulators/mame2016.yml | 1 + emulators/ume2015.yml | 1 + 7 files changed, 7 insertions(+) diff --git a/emulators/cdi2015.yml b/emulators/cdi2015.yml index 02fde949..e703ab99 100644 --- a/emulators/cdi2015.yml +++ b/emulators/cdi2015.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/mame2015-libretro" upstream: "https://github.com/mamedev/mame/tree/mame0160" profiled_date: "2026-03-23" source_commit: "7adbf440d5f554097f58f8c6ebb673adc4320a31" +upstream_commit: "0147bb4cc82e642cf75151f039f240eb20989fbc" core_version: "0.160" display_name: "Philips CDi (CDi 2015)" cores: [cdi2015] diff --git a/emulators/mame2009.yml b/emulators/mame2009.yml index 8d62395d..f78a0c27 100644 --- a/emulators/mame2009.yml +++ b/emulators/mame2009.yml @@ -5,6 +5,7 @@ source: "https://github.com/r-type/mame2009-libretro" upstream: "https://github.com/mamedev/mame/tree/mame0135u4" profiled_date: "2026-03-24" source_commit: "45048806fbc0fea82709a75a523f2eda945e0a47" +upstream_commit: "c285eb9bcd3505c14a338e29f251ba00776cc75c" core_version: "0.135u4" display_name: "Arcade (MAME 2009)" diff --git a/emulators/mame2010.yml b/emulators/mame2010.yml index 20edf39d..26b15b78 100644 --- a/emulators/mame2010.yml +++ b/emulators/mame2010.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0139" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-03-22" source_commit: "dff8aadd1c3f38215af3955746d6e19abe0ddcea" +upstream_commit: "f4f24ba7274010fb8ccb84548b5ca95c2be2f85b" core_version: "0.139" display_name: "Arcade (MAME 2010)" mame_version: "0.139" diff --git a/emulators/mame2014.yml b/emulators/mame2014.yml index 835cee5b..de767d40 100644 --- a/emulators/mame2014.yml +++ b/emulators/mame2014.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0160" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-10-20" source_commit: "c8fe6c7399f24466808b67933b2b96feea188e0b" +upstream_commit: "0147bb4cc82e642cf75151f039f240eb20989fbc" core_version: "0.160" display_name: "Arcade (MAME 2014)" mame_version: "0.160" diff --git a/emulators/mame2015.yml b/emulators/mame2015.yml index 816735dc..cd7ca70f 100644 --- a/emulators/mame2015.yml +++ b/emulators/mame2015.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0160" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-03-24" source_commit: "7adbf440d5f554097f58f8c6ebb673adc4320a31" +upstream_commit: "0147bb4cc82e642cf75151f039f240eb20989fbc" core_version: "0.160" display_name: "Arcade (MAME 2015)" mame_version: "0.160" diff --git a/emulators/mame2016.yml b/emulators/mame2016.yml index b6d30f1b..ae67052e 100644 --- a/emulators/mame2016.yml +++ b/emulators/mame2016.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0174" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-03-22" source_commit: "ae07c2f88ff2482ba9f50ffc8c9e7e6fbfe97d0a" +upstream_commit: "68785dccfee3208163c8169abc5c6d01becf36d4" core_version: "0.174" cores: diff --git a/emulators/ume2015.yml b/emulators/ume2015.yml index 71eea26e..8a94a11f 100644 --- a/emulators/ume2015.yml +++ b/emulators/ume2015.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/mamedev/mame/tree/mame0159" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-03-25" source_commit: "7adbf440d5f554097f58f8c6ebb673adc4320a31" +upstream_commit: "6ae44e46f91aa2bd74d9895e6bfd345baebd3d22" core_version: "0.159" display_name: "Multi (UME 2015)" mame_version: "0.159" From e65ece743dd3fd2b948aa7b6d2be63d81c3cd5fa Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:02:50 +0200 Subject: [PATCH 03/18] fix: name where a held profile entry sits 164 entries wore the in repo badge with no copy named: they are held as a ZIP member or in a data directory, which the database does not index. held_copy() finds that copy and the card names it. storage: large_file no longer counts as held by itself: kenji-nx's user-picked firmware package read as held although nothing collects it. --- scripts/cross_reference.py | 71 ++++++++++++++++++++++++++++----- scripts/generate_site.py | 53 ++++++++++++++++++------ scripts/verify.py | 9 ++--- tests/test_site_availability.py | 52 ++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 27 deletions(-) diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index f82966b5..5a7c755d 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -361,6 +361,52 @@ def _content_matches(location: tuple[str, str], declared: dict[str, set[str]]) - return any(actual[field] in values for field, values in declared.items()) +_tree_cache: dict[tuple[str, str], list[str]] = {} + + +def _trees(data_root: str, bios_root: str) -> list[str]: + """Every directory under data/ and bios/, as a path with a trailing slash.""" + key = (data_root, bios_root) + if key not in _tree_cache: + _tree_cache[key] = sorted( + f"{path.as_posix()}/" + for root in (data_root, bios_root) if Path(root).is_dir() + for path in Path(root).rglob("*") + if path.is_dir() + and not any(p.startswith(".") for p in path.relative_to(root).parts) + ) + return _tree_cache[key] + + +def held_copy( + entry: dict, data_root: str = "data", bios_root: str = "bios" +) -> tuple[str, str] | None: + """The data/ file, directory or bios/ ZIP member holding an entry. + + The database indexes the files under bios/, and an entry held otherwise + (specmate.rom as a member of spectrum_opus.zip, Dolphin's fonts in its + data directory) read as held with no word of where. Returns (path, + member), the member empty for a file or a directory. A file entry that + declares a hash is held only by a copy carrying it, as _data_hit judges. + """ + names = [entry.get("name", ""), entry.get("path") or "", *(entry.get("aliases") or [])] + names = [str(n) for n in names if n] + if entry.get("type") == "directory" or any(n.endswith("/") for n in names[:1]): + for name in names: + tail = "/" + name.strip("/") + "/" + for tree in _trees(data_root, bios_root): + if tree.endswith(tail): + return tree, "" + return None + declared = _declared_content(entry) + locations = _supplemental_locations(data_root, bios_root) + for name in names: + for location in locations.get(name.rsplit("/", 1)[-1].lower(), ()): + if not declared or _content_matches(location, declared): + return location + return None + + class FileConsumers: """Who reads each collected file, joined through the resolver. @@ -459,6 +505,10 @@ class FileConsumers: self.installed.setdefault(sha1, {}).setdefault( platform, set()).add(str(entry.get("dest", ""))) + def sha1_at(self, path: str) -> str | None: + """The file the database records at a collection path.""" + return self._sha1_by_path.get(path) + def path_of(self, sha1: str) -> str: """The collection path the database records for a file.""" return str(self._db.get("files", {}).get(sha1, {}).get("path", "")) @@ -742,17 +792,18 @@ def _cross_reference_profile( continue # --- resolve source provenance --- - storage = f.get("storage", "") - if storage in ("release", "large_file"): + # A release asset is held when the database records it, as any other + # file: storage says where the bytes are served from, not that the + # collection has them (kenji-nx's user-picked firmware package). + source = next( + ( + found for version in versions[slot] + if (found := entry_source(version, index)) is not None + ), + "missing", + ) + if source != "missing" and f.get("storage", "") in ("release", "large_file"): source = "large_file" - else: - source = next( - ( - found for version in versions[slot] - if (found := entry_source(version, index)) is not None - ), - "missing", - ) in_repo = source != "missing" diff --git a/scripts/generate_site.py b/scripts/generate_site.py index efcf28d7..b6c99522 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -15,6 +15,7 @@ from siterender import WIKI_SRC_DIR import argparse import csv import hashlib +import html import io import json import os @@ -48,7 +49,7 @@ from common import ( yaml_load, ) from artifacts import ArtifactLockBusy, artifact_lock -from cross_reference import FileConsumers +from cross_reference import FileConsumers, held_copy from nativemode import reads_file_contents @@ -3121,18 +3122,21 @@ def _render_emulator_file( details.extend(_directory_copies(f, consumers)) if consumers: sha1 = consumers.sha1_of(f, f.get("path") or "", owner=owner) - if sha1 and in_repo: + held = consumers.path_of(sha1) if sha1 and in_repo else "" + parts = held.split("/") + if len(parts) >= 3 and parts[0] == "bios": # Which copy answers this entry, linked to its collection card # (provenance, every hash, download): an entry declaring no hash # said "in repo" without naming the file. - held = consumers.path_of(sha1) - parts = held.split("/") - if len(parts) >= 3 and parts[0] == "bios": - page = parts[1].lower().replace(" ", "-") - details.append( - f"Collection copy: [{_code(held)}]" - f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" - ) + page = parts[1].lower().replace(" ", "-") + details.append( + f"Collection copy: [{_code(held)}]" + f"(../systems/{urllib.parse.quote(page)}.md#sha1-{sha1[:12]})" + ) + elif in_repo and not any(d.startswith("Collection copies:") for d in details): + copy = held_copy(f) + if copy: + details.append(_held_copy_line(copy, consumers)) plat_links = _platform_links(sha1, consumers) if sha1 else [] if plat_links: details.append(f"Platforms: {', '.join(plat_links)}") @@ -3189,6 +3193,33 @@ def _render_emulator_file( return lines +def _held_copy_line(copy: tuple[str, str], consumers: FileConsumers) -> str: + """Where a copy the database does not index sits: a ZIP member under + bios/, or a file or tree of a data directory the packs fetch.""" + path, member = copy + parts = path.rstrip("/").split("/") + if member: + archive = _code(path) + zsha1 = consumers.sha1_at(path) + if zsha1 and len(parts) >= 3 and parts[0] == "bios": + page = urllib.parse.quote(parts[1].lower().replace(" ", "-")) + archive = f"[{archive}](../systems/{page}.md#sha1-{zsha1[:12]})" + return f"Collection copy: member {_code(member)} of {archive}" + if parts[0] == "data" and len(parts) >= 2: + ref, inside = parts[1], "/".join(parts[2:]) + if path.endswith("/"): + where = _code(inside + "/") if inside else "the tree" + return f"Collection copy: {where} in the {_code(ref)} data directory" + info = (_DATA_MEMBERS.get(ref) or {}).get(inside) or {} + shown = _code(inside) + if info.get("sha1"): + shown = (f'' + f"{html.escape(inside)}") + return f"Collection copy: {shown} in the {_code(ref)} data directory" + tree = f"{REPO_URL}/tree/main/{urllib.parse.quote(path.rstrip('/'))}" + return f"Collection copy: [{_code(path)}]({tree})" + + def _directory_copies(entry: dict, consumers: FileConsumers) -> list[str]: """What the collection holds under a directory entry, and where. @@ -3425,8 +3456,6 @@ def _availability_check( # under the name belongs to another game or release. if not f.get("name") or f.get("unsourceable"): return False - if f.get("storage", "") in ("release", "large_file"): - return True return entry_source(f, index) is not None return _file_available diff --git a/scripts/verify.py b/scripts/verify.py index 09947723..17913623 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -644,11 +644,10 @@ def find_undeclared_files( entry["required"] = True continue - # Resolution: storage flag, then name, then path basename - storage = f.get("storage", "") - if storage in ("release", "large_file"): - in_repo = True - elif shipped_on_mismatch and (f.get("md5") or f.get("sha1")): + # Resolution: name, then path basename + # storage: release names where the bytes are served from, not + # that the collection holds them: resolved like any file. + if shipped_on_mismatch and (f.get("md5") or f.get("sha1")): # The entry states what its content should be and the builder # drops a copy that contradicts it, so content decides here # too. A name match would not do: generic names collide across diff --git a/tests/test_site_availability.py b/tests/test_site_availability.py index a1e09fbe..12fa9052 100644 --- a/tests/test_site_availability.py +++ b/tests/test_site_availability.py @@ -33,6 +33,13 @@ class UnsourceableIsNotHeld(unittest.TestCase): self.assertTrue(available({"name": "pak0.pk3"})) self.assertFalse(available({"name": "pak0.pk3", "unsourceable": "retail data"})) + def test_a_release_asset_is_held_only_when_collected(self): + """storage: large_file said "in repo" for kenji-nx's user-picked + firmware package, which the collection does not hold.""" + available = _availability_check(DB, set()) + self.assertFalse(available({"name": ".zip", "storage": "large_file"})) + self.assertTrue(available({"name": "pak0.pk3", "storage": "release"})) + def test_the_badge_says_why(self): entry = {"name": "pak0.pk3", "unsourceable": "retail data"} badges = " ".join(_file_badges(entry, False)) @@ -57,5 +64,50 @@ class UnsourceableIsNotHeld(unittest.TestCase): self.assertEqual(shown_held, []) +class AHeldEntrySaysWhere(unittest.TestCase): + """164 entries wore the "in repo" badge with no copy named: they are held + as a ZIP member (zesarux's specmate.rom in spectrum_opus.zip) or in a + data directory (Dolphin's fonts), which the database does not index.""" + + def test_member_file_and_tree(self): + import hashlib + import tempfile + import zipfile + + import generate_site + from cross_reference import FileConsumers, held_copy + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + data, bios = Path(tmp) / "data", Path(tmp) / "bios" + (data / "dolphin-sys" / "GC").mkdir(parents=True) + (data / "dolphin-sys" / "GC" / "font_western.bin").write_bytes(b"font") + (bios / "Arcade" / "MAME").mkdir(parents=True) + archive = bios / "Arcade" / "MAME" / "spectrum_opus.zip" + with zipfile.ZipFile(archive, "w") as zf: + zf.writestr("specmate.rom", b"rom") + roots = {"data_root": str(data), "bios_root": str(bios)} + member = held_copy({"name": "specmate.rom", + "sha1": hashlib.sha1(b"rom").hexdigest()}, **roots) + self.assertEqual(member, (str(archive), "specmate.rom")) + # A declared hash the copy does not carry names no copy. + self.assertIsNone(held_copy({"name": "specmate.rom", "sha1": "0" * 40}, **roots)) + self.assertEqual(held_copy({"name": "font_western.bin"}, **roots), + (str(data / "dolphin-sys/GC/font_western.bin"), "")) + self.assertEqual(held_copy({"name": "GC/", "type": "directory"}, **roots), + (f"{(data / 'dolphin-sys/GC').as_posix()}/", "")) + + consumers = FileConsumers({"files": {"z" * 40: { + "path": "bios/Arcade/MAME/spectrum_opus.zip"}}}, data_dir_registry={}) + line = generate_site._held_copy_line( + ("bios/Arcade/MAME/spectrum_opus.zip", "specmate.rom"), consumers) + self.assertEqual(line, "Collection copy: member `specmate.rom` of " + "[`bios/Arcade/MAME/spectrum_opus.zip`]" + "(../systems/arcade.md#sha1-zzzzzzzzzzzz)") + line = generate_site._held_copy_line(("data/dolphin-sys/GC/font_western.bin", ""), + consumers) + self.assertEqual(line, "Collection copy: `GC/font_western.bin` in the " + "`dolphin-sys` data directory") + + if __name__ == "__main__": unittest.main() From 377e53f4c00510eac577ab230438cc4d8d65f724 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:05:18 +0200 Subject: [PATCH 04/18] feat: show load_from, built_by and pin fields load_from, found_by, on_mismatch, built_by, pin_frozen and version_source were in the profiles and on no page: Dolphin's gba_bios.bin showed a path under system/ although the libretro core reads it from its save directory, and mame0278 did not say only Recalbox ships it. --- scripts/generate_site.py | 85 ++++++++++++++++++++++++++++++++ tests/test_site_emulator_page.py | 73 +++++++++++++++++++++++++++ wiki/testing-guide.md | 2 +- 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 tests/test_site_emulator_page.py diff --git a/scripts/generate_site.py b/scripts/generate_site.py index b6c99522..21325f7e 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2978,6 +2978,83 @@ def _first_hash_value(entry: dict) -> str: return "" +_LOAD_FROM = { + "system_dir": "the system (BIOS) directory", + "save_dir": "the save directory, not the BIOS directory", + "content_dir": "the folder of the loaded content, not the BIOS directory", + "working_directory": "the working directory, not the BIOS directory", + "data_dir": "the emulator's data directory, not the BIOS directory", + "hdd_image": "inside the emulated hard disk image, not the BIOS directory", + "internal_storage": "the app's internal storage, not the BIOS directory", +} + + +def _by_build(value: object, words: Callable[[str], str]) -> str: + """A field keyed by build, said per build, or a plain value said once.""" + if isinstance(value, dict): + return ", ".join(f"{words(str(v))} ({mode} build)" for mode, v in value.items() if v) + return words(str(value)) if value else "" + + +def _load_from_text(value: object) -> str: + """Where the code reads a file, in words. + + The card showed the Path alone: Dolphin's libretro core reads + gba_bios.bin from its save directory, and a reader put it under system/. + """ + if value in (None, "", "system_dir"): + return "" + return _by_build(value, lambda v: _LOAD_FROM.get(v, _code(v))) + + +def _found_by_text(value: object) -> str: + """How the code finds a file: by the name it opens, or by content.""" + words = { + "name": "file name (the code opens this name)", + "content": "content (the code scans the folder and identifies the file " + "by its hash, any file name works)", + } + return _by_build(value, lambda v: words.get(v, _code(v))) + + +def _on_mismatch_lines(value: object) -> list[str]: + """What the code does when a declared check fails, where it is not reject.""" + said = {"warn": "warns and runs with the file", "reject": "rejects the file"} + if value == "warn": + return [f"On a failed check: {said['warn']}"] + if isinstance(value, dict): + return [f"On a failed {check} check: {said.get(str(how), _code(str(how)))}" + for check, how in value.items()] + return [] + + +def _build_rows(profile: dict) -> list[str]: + """Meta card rows for who ships the build, how its files are found, why + its revision is held and where its releases are announced.""" + rows = [] + built_by = profile.get("built_by") or [] + if built_by: + names = ", ".join(f"[{p}](../platforms/{p}.md)" for p in map(str, built_by)) + rows.append(f"| Built by | {names}, under its core name. The libretro " + "buildbot does not publish this build. |") + found = _found_by_text(profile.get("found_by")) + if found: + rows.append(f"| Files found by | {found} |") + frozen = profile.get("pin_frozen") + if frozen: + held = _escape_tags(' '.join(str(frozen).split())).replace('|', r'\|') + rows.append(f"| Revision held | {held} |") + announced = profile.get("version_source") + if isinstance(announced, dict) and announced.get("url"): + url = str(announced["url"]) + row = f"| Releases announced at | [{url}]({url})" + if announced.get("no_genuine_copy"): + row += (f" (version {_escape_tags(str(announced['no_genuine_copy']))} " + "announced there, found only as rebuilt copies)") + rows.append(row + " |") + return rows + + def _render_emulator_file( f: dict, profile: dict, @@ -3042,6 +3119,12 @@ def _render_emulator_file( # under Firmware/, and a page naming only the core's path sent its # users to the wrong folder. details.append(f"Standalone path: `{standalone_path}`") + read_from = _load_from_text(f.get("load_from")) + if read_from: + details.append(f"Read from: {read_from}") + found = _found_by_text(f.get("found_by")) + if found: + details.append(f"Found by: {found}") if fsystem: details.append(f"System: {_system_link(fsystem, '../')}") if size: @@ -3099,6 +3182,7 @@ def _render_emulator_file( elif isinstance(validation, dict): for scope, checks in validation.items(): details.append(f"Validation ({scope}): {', '.join(checks)}") + details.extend(_on_mismatch_lines(f.get("on_mismatch"))) if f.get("installed_on"): details.append("Installed by the platform's own package on: " + ", ".join(sorted(f["installed_on"]))) @@ -3622,6 +3706,7 @@ def generate_emulator_page( revision = revision_markdown(profile) if revision: lines.append(f"| Revision | {revision} |") + lines.extend(_build_rows(profile)) lines.append(f"| Profiled | {profiled} |") # The YAML is the reusable document this site renders, and the JSON is the # same profile for a program. diff --git a/tests/test_site_emulator_page.py b/tests/test_site_emulator_page.py new file mode 100644 index 00000000..8f0e60af --- /dev/null +++ b/tests/test_site_emulator_page.py @@ -0,0 +1,73 @@ +"""What an emulator page says about a profile and its files. + +No network and no database: pages are rendered from small profiles. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import generate_site as gs # noqa: E402 + +EMPTY_DB = {"files": {}, "indexes": {"by_name": {}}} + + +def _page(name: str, profile: dict) -> str: + return gs.generate_emulator_page(name, profile, EMPTY_DB, None, set()) + + +class WhereAndHowAFileIsRead(unittest.TestCase): + """load_from, found_by and on_mismatch were in 99, 4 and 25 entries and + on no page: Dolphin's gba_bios.bin showed a Path a RetroArch user put + under system/, while the libretro core reads it from its save directory.""" + + def test_load_from(self): + page = _page("dolphin", {"type": "standalone + libretro", "files": [ + {"name": "gba_bios.bin", "path": "GBA/gba_bios.bin", + "load_from": {"libretro": "save_dir"}}, + {"name": "hdd.rom", "load_from": "hdd_image"}, + {"name": "bios.bin", "load_from": "system_dir"}, + ]}) + self.assertIn("- Read from: the save directory, not the BIOS directory " + "(libretro build)", page) + self.assertIn("- Read from: inside the emulated hard disk image", page) + self.assertEqual(page.count("Read from:"), 2) + + def test_found_by_and_on_mismatch(self): + page = _page("x", {"type": "standalone", "found_by": "content", "files": [ + {"name": "scph5501.bin", "found_by": "content", "on_mismatch": "warn"}, + {"name": "boot.bin", "on_mismatch": {"signature": "warn"}}, + ]}) + self.assertIn("| Files found by | content (the code scans the folder", page) + self.assertIn("- Found by: content (", page) + self.assertIn("- On a failed check: warns and runs with the file", page) + self.assertIn("- On a failed signature check: warns and runs with the file", page) + + +class WhoShipsTheBuildAndWhy(unittest.TestCase): + """built_by, pin_frozen and version_source were in 9, 13 and 40 profiles + and on no page: mame0278 did not say only Recalbox ships it.""" + + def test_meta_rows(self): + page = _page("mame0278", { + "type": "libretro", "built_by": ["recalbox"], + "pin_frozen": "ROCKNIX 20261001 builds this revision.", + "version_source": {"url": "https://example.org/news", "pattern": "v(\\S+)", + "no_genuine_copy": "13.0"}, + "files": [], + }) + self.assertIn("| Built by | [recalbox](../platforms/recalbox.md), under its " + "core name.", page) + self.assertIn("| Revision held | ROCKNIX 20261001 builds this revision. |", page) + self.assertIn("| Releases announced at | [https://example.org/news]" + "(https://example.org/news) (version 13.0 announced there, " + "found only as rebuilt copies) |", page) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index da47320f..794720e7 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -44,7 +44,7 @@ Every module in `tests/`, by the area it guards. | Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | -| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_exports`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | +| Site, docs and repository contracts | rendered pages, sources | `test_contributing_text`, `test_data_dir_refs`, `test_dataclass_defaults`, `test_doc_pages`, `test_doc_surfaces`, `test_optional_dependencies`, `test_prose_style`, `test_site_availability`, `test_site_cores`, `test_site_emulator_page`, `test_site_exports`, `test_site_platform_hashes`, `test_site_render`, `test_site_system_links`, `test_site_validation`, `test_site_write`, `test_validate_pr`, `test_validate_pr_flow`, `test_validate_pr_hashes`, `test_workflow_paths` | ## Test architecture From fac86858196b87a951cee76113c1093b35afd2b1 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:06:52 +0200 Subject: [PATCH 05/18] fix: give both hashes in the untested table A row said the declared hash was contradicted by the local file and named neither. The table now prints the hashes the platform declares and the path and hashes of the copy the collection holds. --- scripts/generate_site.py | 69 +++++++++++++++++++++++++++--- tests/test_site_platform_hashes.py | 26 +++++++++++ 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 21325f7e..ab49e477 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3849,6 +3849,48 @@ def _held_accepted(discrepancy: str, db: dict | None) -> str: return "" +_PROBLEM_HASHES = ("sha1", "md5", "sha256", "crc32") + + +def _declared_hashes(config: dict, detail: dict) -> dict[str, list[str]]: + """The hashes the platform declares for a reported file, by algorithm. + + The untested table said "declared hash contradicted by the local file" + and named neither: the reader could not look for the dump RetroDECK asks + for, nor report the difference. + """ + from hashing import parse_hash_list # noqa: PLC0415 + + system = (config.get("systems") or {}).get(detail.get("system", "")) or {} + for entry in system.get("files") or []: + if isinstance(entry, dict) and entry.get("name") == detail.get("name"): + return { + algo: parse_hash_list(entry.get(algo)) + for algo in _PROBLEM_HASHES if parse_hash_list(entry.get(algo)) + } + return {} + + +def _hash_cell(hashes: dict[str, list[str]]) -> str: + return "
".join( + f"{algo} " + ", ".join(_code(v) for v in values) for algo, values in hashes.items() + ) + + +def _copy_cell(path: str, algos: list[str], db: dict | None) -> str: + """The file a path holds, with its hashes in the algorithms declared.""" + if not path: + return "-" + record = next( + (rec | {"sha1": sha1} for sha1, rec in (db or {}).get("files", {}).items() + if rec.get("path") == path), + {}, + ) + hashes = {algo: [str(record[algo]).lower()] for algo in algos if record.get(algo)} + shown = f"[{_code(path)}]({_fetch_url(path)})" + return f"{shown}
{_hash_cell(hashes)}" if hashes else shown + + def _render_problem_files( platform_problems: list, settled: list | None = None, db: dict | None = None ) -> list[str]: @@ -3877,9 +3919,14 @@ def _render_problem_files( if p["required"]: entry["required"] = True elif p["status"] == "untested": - entry = untested_files.setdefault(fname, { + # Keyed by what is declared and what is held: two platforms + # asking two dumps under one name are two rows. + declared = p.get("declared") or {} + key = f"{fname}\0{sorted(declared.items())}\0{p.get('path', '')}" + entry = untested_files.setdefault(key, { "name": fname, "required": p["required"], "platforms": [], "reason": p["reason"], + "declared": declared, "path": p.get("path", ""), }) entry["platforms"].append(p["platform"]) if p.get("discrepancy"): @@ -3920,15 +3967,21 @@ def _render_problem_files( f'### Untested ' f"{_count(len(untested_files), 'file')}", "", - "Present but hash not verified.", + "Present but hash not verified. Declared is what the platform " + "asks for, the collection copy is what the path holds.", "", - "| File | Platforms | Reason |", - "|------|----------|--------|", + "| File | Platforms | Reason | Declared | Collection copy |", + "|------|----------|--------|----------|-----------------|", ]) - for fname in sorted(untested_files): - f = untested_files[fname] + for key in sorted(untested_files): + f = untested_files[key] plats = ", ".join(sorted(set(f["platforms"]))) - lines.append(f"| `{fname}` | {plats} | {f['reason']} |") + declared = f["declared"] + lines.append( + f"| {_code(f['name'])} | {plats} | {f['reason']} " + f"| {_hash_cell(declared) or '-'} " + f"| {_copy_cell(f['path'], list(declared) or ['md5'], db)} |" + ) lines.append("") if mismatch_files: @@ -4026,6 +4079,8 @@ def generate_gap_analysis( "reason": d.get("reason", ""), "discrepancy": d.get("discrepancy", ""), "system": d.get("system", ""), + "declared": _declared_hashes(cov.get("config") or {}, d), + "path": d.get("path") or "", }) pct_verified = ( diff --git a/tests/test_site_platform_hashes.py b/tests/test_site_platform_hashes.py index 672e8fe8..27b52a8a 100644 --- a/tests/test_site_platform_hashes.py +++ b/tests/test_site_platform_hashes.py @@ -44,5 +44,31 @@ class EachRowItsDeclaration(unittest.TestCase): self.assertNotIn(A, saturn) +class AnUntestedRowNamesBothHashes(unittest.TestCase): + """The gaps page said "declared hash contradicted by the local file" for + RetroDECK's dsifirmware.bin and named neither the dump RetroDECK asks for + nor the one the collection holds.""" + + def test_declared_and_held(self): + import generate_site as gs + + path = "bios/Nintendo/Nintendo DS/dsifirmware.bin" + declared, held = "1" * 64, "2" * 64 + config = {"systems": {"nintendo-ds": {"files": [ + {"name": "dsifirmware.bin", "sha256": declared}]}}} + detail = {"name": "dsifirmware.bin", "system": "nintendo-ds"} + problem = { + "platform": "RetroDECK", "name": "dsifirmware.bin", "status": "untested", + "required": True, "reason": "declared hash contradicted by the local file", + "declared": gs._declared_hashes(config, detail), "path": path, + } + db = {"files": {"f" * 40: {"path": path, "sha256": held}}} + table = "\n".join(gs._render_problem_files([problem], db=db)) + row = next(line for line in table.splitlines() if "dsifirmware" in line) + self.assertIn(f"sha256 `{declared}`", row) + self.assertIn(f"sha256 `{held}`", row) + self.assertIn(f"`{path}`", row) + + if __name__ == "__main__": unittest.main() From 73ff57468fdd871b35cd7aeb0f942665dd69ab3f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:09:26 +0200 Subject: [PATCH 06/18] fix: page descriptions and urls match the page A core that loads no file no longer promises paths and hashes in its snippet, a description is cut at a word and keeps the underscores of its code spans, and the JSON-LD url is encoded as mkdocs encodes the canonical link and the sitemap. --- scripts/generate_site.py | 30 +++++++++++++++++++--- tests/test_site_emulator_page.py | 44 ++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 3 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index ab49e477..d4313b05 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1373,10 +1373,26 @@ def _plain_markdown(text: str) -> str: text = re.sub(r"<[^>]+>", " ", text) text = re.sub(r"!\[([^]]*)\]\([^)]*\)", r"\1", text) text = re.sub(r"\[([^]]+)\]\([^)]*\)", r"\1", text) - text = re.sub(r"[`*_~]", "", text) + # Code spans keep their text whole: validate_schemas.py read + # validateschemas.py once every underscore was taken for emphasis. + pieces = re.split(r"(`[^`]*`)", text) + text = "".join( + piece.strip("`") if piece.startswith("`") + else re.sub(r"(? str: + """A description cut at a word, never inside one: six snippets ended + in "op..." or "Fo...".""" + if len(plain) <= limit: + return plain + cut = plain[: limit + 1].rsplit(" ", 1)[0] if " " in plain[:limit] else plain[:limit] + return cut.rstrip(" ,;:-(.") + "..." + + def _page_description(markdown: str, relative: Path, title: str) -> str: key = relative.as_posix() specific = { @@ -1398,6 +1414,12 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: about = PAGE_DATA.get(key, {}).get("about", {}) pinned = any(a.get("version") for a in (about if isinstance(about, list) else [about])) kind = "source-pinned emulator firmware profile" if pinned else "emulator firmware profile" + if not PAGE_DATA.get(key, {}).get("hasPart"): + # 152 cores that load no file announced paths and hashes. + kind = kind.replace(" firmware", "") + trees = (" The data directories it reads are listed." + if "\n## Data directories\n" in markdown else "") + return f"{title}: {kind}. No BIOS or firmware file is required.{trees}" return f"{title}: {kind} with paths, hashes, requirements and validation behavior." if key.startswith("systems/") and relative.stem != "index": return f"{title}: indexed firmware files, hashes, variants, provenance and platform or emulator usage." @@ -1412,7 +1434,7 @@ def _page_description(markdown: str, relative: Path, title: str) -> str: continue plain = _plain_markdown(stripped) if len(plain) >= 35: - return plain[:157].rstrip(" ,;:-") + ("..." if len(plain) > 157 else "") + return _shorten(plain) return f"{title}: RetroBIOS source-traced retrogaming firmware reference." @@ -1733,7 +1755,9 @@ def decorate_markdown_pages(docs: Path) -> None: page_path = "" if len(relative.parts) == 1 else "/".join(relative.parts[:-1]) + "/" else: page_path = relative.with_suffix("").as_posix() + "/" - canonical = urllib.parse.urljoin(SITE_URL, page_path) + # Encoded as mkdocs encodes the canonical link and the sitemap: + # idtech4a++ named .../idtech4a++/ here and .../idtech4a%2B%2B/ there. + canonical = urllib.parse.urljoin(SITE_URL, urllib.parse.quote(page_path)) schema_type = "WebSite" if relative.as_posix() == "index.md" else "TechArticle" if relative.as_posix() == "data.md": schema_type = "Dataset" diff --git a/tests/test_site_emulator_page.py b/tests/test_site_emulator_page.py index 8f0e60af..fd364aad 100644 --- a/tests/test_site_emulator_page.py +++ b/tests/test_site_emulator_page.py @@ -69,5 +69,49 @@ class WhoShipsTheBuildAndWhy(unittest.TestCase): "found only as rebuilt copies) |", page) +class WhatASearchSnippetSays(unittest.TestCase): + def test_a_core_loading_no_file_promises_no_hash(self): + """152 pages of cores that load no file announced paths and hashes.""" + rel = Path("emulators/lutro.md") + saved = dict(gs.PAGE_DATA) + self.addCleanup(lambda: (gs.PAGE_DATA.clear(), gs.PAGE_DATA.update(saved))) + gs.PAGE_DATA[rel.as_posix()] = {} + text = gs._page_description("# Lutro\n", rel, "Lutro") + self.assertNotIn("hashes", text) + self.assertIn("No BIOS or firmware file is required.", text) + gs.PAGE_DATA[rel.as_posix()] = {"hasPart": [{"name": "a.bin"}]} + self.assertIn("paths, hashes", gs._page_description("# Lutro\n", rel, "Lutro")) + + def test_a_long_paragraph_is_cut_at_a_word(self): + plain = ("files/?sha1= is the address that does not move: it opens the card " + "wherever the file is now, " * 4) + text = gs._shorten(plain) + self.assertTrue(text.endswith("...")) + self.assertLessEqual(len(text), 160) + self.assertTrue(plain.startswith(text[:-3])) + self.assertIn(plain[len(text) - 3], " ,:") + + def test_code_keeps_its_underscores(self): + self.assertEqual(gs._plain_markdown("only `validate_schemas.py` and _this_"), + "only validate_schemas.py and this") + + def test_structured_url_is_encoded_like_the_canonical(self): + """mkdocs encodes idtech4a++ as idtech4a%2B%2B in the canonical link + and the sitemap, and the JSON-LD named the unencoded URL.""" + import json + import tempfile + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + page = Path(tmp) / "emulators" / "idtech4a++.md" + page.parent.mkdir() + page.write_text("# idTech4A++\n\nSome text long enough to be a description.\n") + gs.decorate_markdown_pages(Path(tmp)) + text = page.read_text() + data = json.loads(text.split('")[0].replace("<\\/", " Date: Wed, 21 Oct 2026 10:12:08 +0200 Subject: [PATCH 07/18] chore: pin upstream revisions of forked ports upstream_commit is the merge base the compare API gives between each port at its source_commit and the upstream default branch. holani and melonds_ds take the revision their build fetches (tag 0.9.9, and the melonDS branch whose merge base is 906e9ebb). The upstream-role refs land on their declared values there, except mesence bios.col, whose line 70 is in MesenCE's own FirmwareTypeExtensions.cs. --- emulators/armsx2.yml | 1 + emulators/fbalpha2019.yml | 1 + emulators/fbneo.yml | 1 + emulators/groovymame.yml | 1 + emulators/holani.yml | 1 + emulators/kronos.yml | 1 + emulators/melonds_ds.yml | 1 + emulators/mesen-s.yml | 1 + emulators/mesence.yml | 1 + emulators/np2kai.yml | 1 + emulators/scummvm.yml | 1 + emulators/snes9x.yml | 1 + 12 files changed, 12 insertions(+) diff --git a/emulators/armsx2.yml b/emulators/armsx2.yml index da416bae..b9f3c099 100644 --- a/emulators/armsx2.yml +++ b/emulators/armsx2.yml @@ -6,6 +6,7 @@ source: "https://github.com/ARMSX2/ARMSX2" upstream: "https://github.com/PCSX2/pcsx2" profiled_date: "2026-10-04" source_commit: "d383b3636947565d9201f81e32178d0288623d8d" +upstream_commit: "36907f4b70d149b05c2700acae687bedb720a7af" core_version: "2.8" display_name: "Sony - PlayStation 2 (ARMSX2)" cores: diff --git a/emulators/fbalpha2019.yml b/emulators/fbalpha2019.yml index b385302e..2f64305e 100644 --- a/emulators/fbalpha2019.yml +++ b/emulators/fbalpha2019.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/fbalpha" upstream: "https://github.com/barry65536/FBAlpha" profiled_date: "2026-10-17" source_commit: "0581797db6fdffd826086b053ced4b6b29bb6678" +upstream_commit: "416948158d88ba17f87813419ddc3c1209ef68c0" pin_frozen: "ROCKNIX 20261001 builds this revision as fbalpha2019. Later commits on libretro/fbalpha are not part of that build." core_version: "v0.2.97.44" display_name: "Arcade (FB Alpha 2019)" diff --git a/emulators/fbneo.yml b/emulators/fbneo.yml index 77a8c3a0..9028f2c7 100644 --- a/emulators/fbneo.yml +++ b/emulators/fbneo.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/finalburnneo/FBNeo" logo: "https://raw.githubusercontent.com/finalburnneo/FBNeo/master/projectfiles/xcode/Emulator/Assets.xcassets/AppIcon.appiconset/icon_512.png" profiled_date: "2026-03-23" source_commit: "63c4190785cadd5ff84483399375871ed6e98754" +upstream_commit: "8d47450deff3802a61559108f12bb0c846c98e36" core_version: "v1.0.0.2" display_name: "Arcade (FinalBurn Neo)" cores: diff --git a/emulators/groovymame.yml b/emulators/groovymame.yml index 187622a6..604e1215 100644 --- a/emulators/groovymame.yml +++ b/emulators/groovymame.yml @@ -4,6 +4,7 @@ core_classification: enhanced_fork source: "https://github.com/antonioginer/GroovyMAME" upstream: "https://github.com/mamedev/mame" source_commit: "953db38f4aaab1d75faa4ac79e1b45686214f987" +upstream_commit: "f34f02505e32c1993c6a782b6814232cbfc74e36" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-08-08" core_version: "0.288" diff --git a/emulators/holani.yml b/emulators/holani.yml index b85d17c1..ef05f37b 100644 --- a/emulators/holani.yml +++ b/emulators/holani.yml @@ -5,6 +5,7 @@ source: "https://github.com/LLeny/holani-retro" upstream: "https://github.com/LLeny/holani" profiled_date: "2026-03-24" source_commit: "19b974f77247a937369adf48280d2b34a5a66245" +upstream_commit: "4fa0ee920c63a9cf8d07b974ae77fb15161ff967" core_version: "0.9.9" display_name: "Atari - Lynx (Holani)" cores: [holani] diff --git a/emulators/kronos.yml b/emulators/kronos.yml index 9f0a90e9..4f55dc45 100644 --- a/emulators/kronos.yml +++ b/emulators/kronos.yml @@ -6,6 +6,7 @@ source_branch: kronos upstream: "https://github.com/FCare/Kronos" profiled_date: "2026-03-24" source_commit: "3791ffb2221fda61d70aec5cca376418caf05669" +upstream_commit: "d0384bc33aad06bfa9cf5cd308c4bd7ae6cfc814" core_version: "v2.6.0" display_name: "Sega - Saturn/ST-V (Kronos)" cores: diff --git a/emulators/melonds_ds.yml b/emulators/melonds_ds.yml index 64fcd777..066fd054 100644 --- a/emulators/melonds_ds.yml +++ b/emulators/melonds_ds.yml @@ -6,6 +6,7 @@ source: "https://github.com/JesseTG/melonds-ds" upstream: "https://github.com/melonDS-emu/melonDS" profiled_date: "2026-10-04" source_commit: "f394adbacb5722ee97c1b37c8064da9a25818310" +upstream_commit: "906e9ebb27da8c6a715cd7abab4abfe8a8d29427" core_version: "1.4.0" display_name: "Nintendo - DS (melonDS DS)" systems: diff --git a/emulators/mesen-s.yml b/emulators/mesen-s.yml index f23f5035..6bec6c52 100644 --- a/emulators/mesen-s.yml +++ b/emulators/mesen-s.yml @@ -9,6 +9,7 @@ source: "https://github.com/libretro/Mesen-S" upstream: "https://github.com/SourMesen/Mesen-S" profiled_date: "2026-03-24" source_commit: "9e4fdeb9b336470bc96beb8765b2e79c86a2da1e" +upstream_commit: "89199f298a95345a893d814525b6668567a6d490" core_version: "0.4.0" cores: [mesen-s, mesen_s] systems: [nintendo-snes, nintendo-gb, nintendo-gbc, nintendo-super-game-boy, nintendo-satellaview] diff --git a/emulators/mesence.yml b/emulators/mesence.yml index c31bbf94..16b48963 100644 --- a/emulators/mesence.yml +++ b/emulators/mesence.yml @@ -9,6 +9,7 @@ profiled_date: "2026-09-17" source_commit: standalone: "20f497c9620a7f4e55b7752e1d22734f9d7492fa" libretro: "4c4e069f16a8ad6ed79e046046c56d51558c7daf" +upstream_commit: "fabc9a62174f8734a113df6d244f5539ef6b8fcf" core_version: standalone: "2.2.1" libretro: "2.0.0" diff --git a/emulators/np2kai.yml b/emulators/np2kai.yml index e86f43d0..1f2214ab 100644 --- a/emulators/np2kai.yml +++ b/emulators/np2kai.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/NP2kai" upstream: "https://github.com/AZO234/NP2kai" profiled_date: "2026-03-24" source_commit: "1ea561cd7fd62b33f4e7b3f1c2e30fc3e6089f28" +upstream_commit: "5d47f3a04b0b8ea78f625eeec354ac132c80df06" core_version: "0.86" display_name: "NEC - PC-98 (Neko Project II Kai)" cores: [np2kai] diff --git a/emulators/scummvm.yml b/emulators/scummvm.yml index b8ce7478..3d166653 100644 --- a/emulators/scummvm.yml +++ b/emulators/scummvm.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/scummvm/scummvm" logo: "https://raw.githubusercontent.com/scummvm/scummvm/master/icons/scummvm.svg" profiled_date: "2026-03-25" source_commit: "fcbce3ae815269dacdc309092bc92ccc6d3e13bb" +upstream_commit: "fcbce3ae815269dacdc309092bc92ccc6d3e13bb" core_version: "2.8.0git" display_name: "ScummVM" cores: [scummvm] diff --git a/emulators/snes9x.yml b/emulators/snes9x.yml index b488d503..d1dcc1b3 100644 --- a/emulators/snes9x.yml +++ b/emulators/snes9x.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/snes9xgit/snes9x" logo: "https://raw.githubusercontent.com/snes9xgit/snes9x/master/gtk/data/snes9x.svg" profiled_date: "2026-03-25" source_commit: "fae2fea08f74180759ef540ee94259213f503480" +upstream_commit: "85591435763dede7ef111aab8070cdf2d5a7da49" core_version: "1.61" display_name: "Nintendo - SNES / SFC (Snes9x)" systems: [nintendo-snes, nintendo-satellaview, nintendo-sufami-turbo] From 2239df669f6239288d2ffb68b030815297ab142c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:12:26 +0200 Subject: [PATCH 08/18] fix: address a hashed file card by its hash An emulator card's anchor changed when a second entry with the same path was added, so cited links landed at the page top. A card whose entry declares a hash is now addressed by it, and the first card of a path keeps the plain file- id as an alias. --- scripts/generate_site.py | 57 ++++++++++++++++++++++++++------ tests/test_site_emulator_page.py | 25 ++++++++++++++ 2 files changed, 71 insertions(+), 11 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index d4313b05..785c16fb 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1615,11 +1615,14 @@ def dataset_structured_data(exports: list[dict], db: dict) -> dict: def emulator_file_anchors(files: list[dict]) -> list[str]: """The id of each file card on an emulator page, in profile order. - A path declared once keeps its plain anchor. One declared for several - versions (raze's nine DUKE3D.GRP) takes what tells the version apart: - its hash, else its size, else, when even the size is shared, its - description. The page and its JSON-LD both read this, so a structured - part links to its card. + An entry that declares a hash is addressed by it: its anchor depends on + its own content, never on the entries beside it, and a second version + added under the same path moves nothing. An entry without one keeps + its plain anchor when its path is declared once, and otherwise takes + its size, else, when even the size is shared, its description. The + page and its JSON-LD both read this, so a structured part links to its + card. The plain anchor of a hashed card stays reachable through + emulator_file_aliases. """ bases = [ "file-" + ( @@ -1638,12 +1641,15 @@ def emulator_file_anchors(files: list[dict]) -> list[str]: return digest[:8] return f"{f['size']}b" if f.get("size") else "" - keys = [(b, _version_key(f)) if b in repeated else (b, "") for f, b in zip(files, bases)] + keys = [ + (b, _version_key(f)) if b in repeated or _first_hash_value(f) else (b, "") + for f, b in zip(files, bases) + ] used: dict[str, int] = {} anchors = [] for f, (base, key) in zip(files, keys): anchor = f"{base}-{key}" if key else base - if base in repeated and keys.count((base, key)) > 1 and f.get("description"): + if keys.count((base, key)) > 1 and f.get("description"): described = re.sub(r"[^a-z0-9]+", "-", str(f["description"]).lower()).strip("-") anchor = f"{anchor}-{described[:40].strip('-')}" used[anchor] = used.get(anchor, 0) + 1 @@ -1653,6 +1659,30 @@ def emulator_file_anchors(files: list[dict]) -> list[str]: return anchors +def emulator_file_aliases(files: list[dict], anchors: list[str]) -> list[str]: + """The plain file- id each card also answers to, or "". + + A link cited as emulators/raze/#file-duke3d-grp must still land once the + card is addressed by its hash: the first card of that path keeps the + plain id, unless another card already is it. + """ + taken = set(anchors) + aliases = [] + for f, anchor in zip(files, anchors): + base = "file-" + ( + re.sub( + r"[^a-z0-9]+", "-", + str(f.get("path") or f.get("standalone_path") or f.get("name", "")).lower(), + ).strip("-") or "entry" + ) + if base in taken or anchor == base: + aliases.append("") + else: + taken.add(base) + aliases.append(base) + return aliases + + def emulator_structured_data(profile: dict) -> dict: """What an emulator page states, for a reader of its JSON-LD. @@ -3087,10 +3117,12 @@ def _render_emulator_file( _file_available: Callable[[dict], bool], anchor: str = "", owner: str = "", + alias: str = "", ) -> list[str]: """Render one row of an emulator profile's file table. - `anchor` is the card's id on the page, so one file can be cited by URL. + `anchor` is the card's id on the page, so one file can be cited by URL, + and `alias` a second id it answers to. Split out of generate_emulator_page, where it was a 228-line loop body carrying most of that function's branching: thirty-odd optional fields, @@ -3128,7 +3160,8 @@ def _render_emulator_file( badge_str = " ".join(badges) lines.append('
') lines.append("") - lines.append(_card_title(f"{_code(fname)} {badge_str}", anchor)) + marker = f'' if alias else "" + lines.append(_card_title(f"{marker}{_code(fname)} {badge_str}", anchor)) lines.append("") if desc: lines.extend([_escape_tags(str(desc)), ""]) @@ -3782,10 +3815,12 @@ def generate_emulator_page( lines.extend(_emulator_file_summary(files, _file_available)) - for f, anchor in zip(files, emulator_file_anchors(files)): + anchors = emulator_file_anchors(files) + for f, anchor, alias in zip(files, anchors, emulator_file_aliases(files, anchors)): lines.extend( _render_emulator_file( - f, profile, consumers, files, _file_available, anchor, name + f, profile, consumers, files, _file_available, anchor, name, + alias, ) ) diff --git a/tests/test_site_emulator_page.py b/tests/test_site_emulator_page.py index fd364aad..9c5df978 100644 --- a/tests/test_site_emulator_page.py +++ b/tests/test_site_emulator_page.py @@ -69,6 +69,31 @@ class WhoShipsTheBuildAndWhy(unittest.TestCase): "found only as rebuilt copies) |", page) +class ACardKeepsItsAddress(unittest.TestCase): + """emulators/raze/#file-duke3d-grp stopped pointing at its card once a + second version of the path was declared: the anchor of the first one + became file-duke3d-grp-.""" + + def test_a_second_version_moves_nothing(self): + one = {"name": "DUKE3D.GRP", "md5": "aaaa1111bbbb"} + two = {"name": "DUKE3D.GRP", "md5": "cccc2222dddd"} + before = gs.emulator_file_anchors([one]) + after = gs.emulator_file_anchors([two, one]) + self.assertEqual(before, ["file-duke3d-grp-aaaa1111"]) + self.assertEqual(after[1], before[0]) + + def test_the_plain_address_still_lands(self): + files = [{"name": "DUKE3D.GRP", "md5": "aaaa1111bbbb"}] + page = _page("raze", {"type": "standalone", "files": files}) + self.assertIn('', page) + self.assertIn("{ #file-duke3d-grp-aaaa1111 .rb-file-title }", page) + # A card already holding the plain id gives it to no one else. + files = [{"name": "x.bin"}, {"name": "x.bin", "sha1": "ab" * 20}] + anchors = gs.emulator_file_anchors(files) + self.assertEqual(anchors, ["file-x-bin", "file-x-bin-abababab"]) + self.assertEqual(gs.emulator_file_aliases(files, anchors), ["", ""]) + + class WhatASearchSnippetSays(unittest.TestCase): def test_a_core_loading_no_file_promises_no_hash(self): """152 pages of cores that load no file announced paths and hashes.""" From 69965ebb2512c4670671075a564dbd06b45a4c28 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:15:11 +0200 Subject: [PATCH 09/18] feat: link each platform and system to its data An emulator page linked its YAML and api/v1 JSON, platform and system pages nothing. Each platform page now links its YAML, its own api/v1/platforms/.json and its install manifest, and each system page its api/v1/systems/.json, both validated by the envelope schema. --- schemas/site-api-envelope.schema.json | 71 +++++++++++++++++++++++++++ scripts/generate_site.py | 69 ++++++++++++++++++++++++-- scripts/validate_schemas.py | 4 +- tests/test_site_exports.py | 37 ++++++++++++++ 4 files changed, 176 insertions(+), 5 deletions(-) diff --git a/schemas/site-api-envelope.schema.json b/schemas/site-api-envelope.schema.json index df64e518..18bf24ff 100644 --- a/schemas/site-api-envelope.schema.json +++ b/schemas/site-api-envelope.schema.json @@ -25,6 +25,7 @@ "catalog", "platforms", "emulators", + "systems", "verification-and-coverage-gaps", "file-locations" ] @@ -261,6 +262,76 @@ } } } + }, + { + "if": { + "properties": { + "kind": { + "const": "systems" + } + } + }, + "then": { + "properties": { + "items": { + "items": { + "type": "object", + "required": [ + "consoles", + "id", + "name" + ], + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "consoles": { + "type": "object", + "additionalProperties": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "path", + "sha1" + ], + "properties": { + "name": { + "type": "string" + }, + "path": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "sha1": { + "type": "string", + "pattern": "^[0-9a-f]{40}$" + }, + "md5": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "crc32": { + "type": "string" + } + } + } + } + } + } + } + } + } + } } ] } diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 785c16fb..13cefaea 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -1060,6 +1060,35 @@ def _write_sqlite_export( os.replace(temp_path, destination) +def write_single_item(path: Path, kind: str, item: dict, generated_at: str) -> None: + """One item in the envelope of its collection, for a page to link.""" + path.parent.mkdir(parents=True, exist_ok=True) + write_if_changed( + str(path), + json.dumps( + {"schema_version": 1, "generated_at": generated_at, "kind": kind, + "count": 1, "items": [item]}, + indent=2, sort_keys=True, ensure_ascii=False, default=str, + ) + "\n", + ) + + +def system_export_item(slug: str, manufacturer: str, consoles: dict[str, list]) -> dict: + """A system page's files by console, with the hashes its cards show.""" + fields = ("name", "path", "size", "sha1", "md5", "sha256", "crc32") + return { + "id": slug, + "name": manufacturer, + "consoles": { + console: [ + {field: entry[field] for field in fields if entry.get(field) is not None} + for entry in sorted(entries, key=lambda e: (e.get("name", ""), e.get("sha1", ""))) + ] + for console, entries in sorted(consoles.items()) + }, + } + + def generate_data_exports( docs: Path, db: dict, @@ -1100,6 +1129,11 @@ def generate_data_exports( str(api / filename), json.dumps(document, ensure_ascii=False, indent=2, sort_keys=True) + "\n", ) + # One platform without the others, as api/v1/emulators/.json is one + # profile: platforms.json is 2.8 MB for twelve platforms. + for item in platform_items: + write_single_item(api / "platforms" / f"{item['id']}.json", + "platforms", item, generated_at) write_if_changed(str(api / "stats.json"), generate_stats(stats)) write_if_changed( str(api / "database.json"), @@ -1280,7 +1314,9 @@ def generate_data_page(exports: list[dict]) -> str: "`api/v1/emulators/.json`, the envelope of `emulators.json` holding " "that one item, its `profile` valid against " "[`emulator.schema.json`](api/v1/schemas/emulator.schema.json). Each " - "emulator page links its own.", + "emulator page links its own. `api/v1/platforms/.json` holds one " + "platform of `platforms.json`, and `api/v1/systems/.json` the files " + "of one system page by console. Each platform and system page links its own.", "", "## Bulk downloads", "", @@ -2000,6 +2036,22 @@ def _render_platform_systems( _EXPORTED: set[str] = set() +def _platform_data_links(name: str) -> str: + """The platform's own data, machine-readable: the YAML transcribed from + its upstream list, this page's JSON and the manifest the installer reads. + + An emulator page linked its YAML and JSON, a platform page nothing: a + tool had to download all twelve platforms in platforms.json and filter. + """ + links = [ + f"[YAML]({REPO_URL}/blob/main/platforms/{name}.yml)", + f"[JSON](../api/v1/platforms/{name}.json)", + ] + if (_REPO_ROOT / "install" / f"{name}.json").is_file(): + links.append(f"[install manifest]({REPO_URL}/blob/main/install/{name}.json)") + return ", ".join(links) + + def generate_platform_page( name: str, cov: dict, @@ -2093,6 +2145,7 @@ def generate_platform_page( lines.append(f"| Transcribed from | <{transcribed}>{note} |") if name in _EXPORTED: lines.append(f"| Updated file | [Download it and see the changes](../exports.md#{name}) |") + lines.append(f"| Data | {_platform_data_links(name)} |") contrib_list = (registry or {}).get(name, {}).get("contributed_by", []) if contrib_list: for cb in contrib_list: @@ -2101,8 +2154,8 @@ def generate_platform_page( pr = cb.get("pr") pr_link = f" ([#{pr}]({REPO_URL}/pull/{pr}))" if pr else "" lines.append( - f"| Contributed by | [@{username}](https://github.com/{username})" - f" - {contribution}{pr_link} |" + f"| Contributed by | [@{username}](https://github.com/{username}):" + f" {contribution}{pr_link} |" ) lines.extend( [ @@ -2794,10 +2847,13 @@ def generate_system_page( consoles: dict[str, list], consumers: FileConsumers, ) -> str: - manufacturer.lower().replace(" ", "-") + slug = manufacturer.lower().replace(" ", "-") lines = [ f"# {manufacturer}", "", + f"The files below by console, with every hash, are also published as " + f"[JSON](../api/v1/systems/{urllib.parse.quote(slug)}.json).", + "", ] anchored: set[str] = set() @@ -5439,6 +5495,11 @@ def _generate(args: argparse.Namespace) -> None: slug = mfr.lower().replace(" ", "-") page = generate_system_page(mfr, consoles, consumers) PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles) + write_single_item( + docs / "api" / "v1" / "systems" / f"{slug}.json", "systems", + system_export_item(slug, mfr, consoles), + str(db.get("generated_at") or _timestamp()), + ) write_if_changed(str(docs / "systems" / f"{slug}.md"), page) print("Generating emulator pages...") diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 2f9c3d52..9186078a 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -419,7 +419,9 @@ def main() -> int: api_dir / name for name in ("catalog.json", "platforms.json", "emulators.json", "gaps.json") if (api_dir / name).exists() - ] + ] + sorted((api_dir / "platforms").glob("*.json")) + sorted( + (api_dir / "systems").glob("*.json") + ) errors.extend( _validate_json_files(envelope_paths, "site-api-envelope.schema.json") ) diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 43693d08..91643f13 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -882,5 +882,42 @@ class NestedProfileBlocksRenderAsLists(unittest.TestCase): self.assertIn("name: a.bin, size: 2", html) +class EveryPageLinksItsOwnData(unittest.TestCase): + """An emulator page linked its YAML and JSON. A platform page linked + nothing, and a tool had to download platforms.json (2.8 MB) and filter + it. A system page linked nothing either.""" + + def test_platform_and_system_documents(self): + import generate_site # noqa: PLC0415 + from cross_reference import FileConsumers # noqa: PLC0415 + + cov = {"platform": "Batocera", "mode": "md5", "present": 0, "total": 0, + "verified": 0, "missing": 0, "details": [], "config": {"systems": {}}} + page = generate_site.generate_platform_page("batocera", cov) + self.assertIn("[JSON](../api/v1/platforms/batocera.json)", page) + self.assertIn("/blob/main/platforms/batocera.yml)", page) + + entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3, + "path": "bios/Sony/PlayStation/a.bin"} + page = generate_site.generate_system_page( + "Sony", {"PlayStation": [entry]}, FileConsumers({"files": {}})) + self.assertIn("[JSON](../api/v1/systems/sony.json)", page) + + item = generate_site.system_export_item("sony", "Sony", {"PlayStation": [entry]}) + self.assertEqual(item["consoles"]["PlayStation"][0]["sha1"], "a" * 40) + with tempfile.TemporaryDirectory(dir=Path(__file__).resolve().parents[1] / "tmp") as tmp: + path = Path(tmp) / "sony.json" + generate_site.write_single_item(path, "systems", item, "2026-10-21T00:00:00Z") + document = json.loads(path.read_text()) + self.assertEqual(document["count"], 1) + try: + import jsonschema # noqa: PLC0415 + except ImportError: + return + root = Path(__file__).resolve().parents[1] + envelope = json.loads((root / "schemas" / "site-api-envelope.schema.json").read_text()) + jsonschema.validate(document, envelope) + + if __name__ == "__main__": unittest.main() From b89383ba6819c38ed48d61263a1e86b830b670b8 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:16:14 +0200 Subject: [PATCH 10/18] fix: open the gap page on what is missing The page opened on a slogan and four tiles that read Verified (100%) for 5,437 of 5,438 files and Missing 0, above a table of red cells. It now opens on what it covers and one sentence that says every count of the platforms' own lists and points to the emulator files below. --- scripts/generate_site.py | 59 +++++++++++++++++++------------------- tests/test_site_exports.py | 17 +++++++++++ 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 13cefaea..e04b154b 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -4138,6 +4138,30 @@ def _render_problem_files( return lines +def _platform_lists_sentence(total: int, verified: int, untested: int, missing: int) -> str: + """The platforms' own lists in one sentence, every count said. + + Four tiles opened the page: "Verified (100%)" for 5,437 of 5,438 and + "Missing 0", above a table of red cells, on the page about what is + missing. The gaps are in the files the emulators load, and the sentence + says so. + """ + def _is(count: int) -> str: + return "is" if count == 1 else "are" + + parts = [f"{verified:,} {_is(verified)} held and pass the platform's own check"] + if untested: + parts.append(f"{untested:,} {_is(untested)} held but not verified " + "(listed under Untested below)") + parts.append(f"{missing:,} {_is(missing)} missing") + listed = ", ".join(parts[:-1]) + f" and {parts[-1]}" + return ( + f"Of the {total:,} files on the platforms' own BIOS lists, {listed}. " + "The files the emulators load beyond those lists are counted per " + "platform in the table below." + ) + + def generate_gap_analysis( profiles: dict, coverages: dict, @@ -4198,44 +4222,19 @@ def generate_gap_analysis( "path": d.get("path") or "", }) - pct_verified = ( - f"{total_verified / total_files_verify * 100:.0f}%" - if total_files_verify - else "0%" - ) - lines = [ "# Gap analysis", "", - "Unified view of BIOS verification, file provenance, and coverage gaps.", + "What each platform and its emulators still lack, and where the " + "collection's copies come from.", "", "[Download gaps CSV](downloads/gaps.csv){ .md-button } " "[Open gaps API](api/v1/gaps.json){ .md-button } " "[All data exports](data.md){ .md-button }", "", - '
', - "", - '
', - f'{total_files_verify:,}', - 'Total files (all platforms)', - "
", - "", - '
', - f'{total_verified:,}', - f'Verified ({pct_verified})', - "
", - "", - '
', - f'{total_untested:,}', - 'Untested', - "
", - "", - '
', - f'{total_missing_verify:,}', - 'Missing', - "
", - "", - "
", + _platform_lists_sentence( + total_files_verify, total_verified, total_untested, total_missing_verify + ), "", ] diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 91643f13..2285d234 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -882,6 +882,23 @@ class NestedProfileBlocksRenderAsLists(unittest.TestCase): self.assertIn("name: a.bin, size: 2", html) +class TheGapPageOpensOnCounts(unittest.TestCase): + """The page opened on a slogan and on tiles reading "Verified (100%)" + for 5,437 of 5,438 files and "Missing 0", above a table of red cells.""" + + def test_every_count_is_said(self): + import generate_site # noqa: PLC0415 + + text = generate_site._platform_lists_sentence(5438, 5437, 1, 0) + self.assertIn("Of the 5,438 files", text) + self.assertIn("5,437 are held", text) + self.assertIn("1 is held but not verified", text) + self.assertIn("0 are missing", text) + self.assertNotIn("%", text) + source = (Path(__file__).resolve().parents[1] / "scripts" / "generate_site.py").read_text() + self.assertNotIn("Unified view of BIOS verification", source) + + class EveryPageLinksItsOwnData(unittest.TestCase): """An emulator page linked its YAML and JSON. A platform page linked nothing, and a tool had to download platforms.json (2.8 MB) and filter From ccd1ff3b0bc6563ea14999b950c70a01f4a2e92d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:17:04 +0200 Subject: [PATCH 11/18] docs: describe each setup by what it is The which-pack page called RetroPie the largest community with the most guides, Batocera an easy setup and Recalbox good for a first build, claims nothing in the repository measures. Each row now says what the setup is, with board names taken from the platform target lists. --- scripts/generate_site.py | 20 ++++++++++---------- tests/test_site_exports.py | 22 ++++++++++++++++++++++ 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index e04b154b..8d4f455e 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -5106,7 +5106,7 @@ ZIP: extract them all into the same folder. | Setup | What it is | Pack | Extract to | |-------|-----------|------|-----------| | [EmuDeck](https://www.emudeck.com/) | Installs and configures multiple emulators, adds each game to the Steam library | [EmuDeck]({rel}) | `~/Emulation/bios/` | -| [RetroDECK](https://retrodeck.net/) | Single Flatpak app, all emulators bundled, one-click install from Discover | [RetroDECK]({rel}) | `~/retrodeck/` (the pack carries its own `bios/`) | +| [RetroDECK](https://retrodeck.net/) | Single Flatpak app with its emulators bundled, installed from Flathub or Discover | [RetroDECK]({rel}) | `~/retrodeck/` (the pack carries its own `bios/`) | | RetroArch standalone | Installed from Discover, Steam, or Flatpak | [RetroArch]({rel}) | Open RetroArch > Settings > Directory > System, that is the folder | ### Windows @@ -5115,7 +5115,7 @@ ZIP: extract them all into the same folder. |-------|-----------|------|-----------| | [RetroArch](https://www.retroarch.com/) | Multi-system emulator, loads different cores for each console | [RetroArch]({rel}) | The `system` folder next to `retroarch.exe` | | [RetroBat](https://www.retrobat.org/) | Windows frontend with EmulationStation, includes RetroArch and standalone emulators | [RetroBat]({rel}) | The `bios` folder inside the RetroBat installation | -| [BizHawk](https://tasvideos.org/BizHawk) | Accuracy-focused multi-system emulator, popular for speedruns and TAS | [BizHawk]({rel}) | The `Firmware` folder inside the BizHawk installation | +| [BizHawk](https://tasvideos.org/BizHawk) | Multi-system emulator with rerecording and debugging tools, hosted by TASVideos | [BizHawk]({rel}) | The `Firmware` folder inside the BizHawk installation | | [LaunchBox](https://www.launchbox-app.com/) | Game library manager and launcher, uses RetroArch or standalone emulators behind the scenes | [RetroArch]({rel}) | Open RetroArch (via LaunchBox) > Settings > Directory > System | ### Linux @@ -5137,24 +5137,24 @@ ZIP: extract them all into the same folder. | Setup | What it is | Pack | Extract to | |-------|-----------|------|-----------| -| [RetroPie](https://retropie.org.uk/) | The classic Pi emulation setup, largest community, most online guides | [RetroPie]({rel}) | `~/RetroPie/BIOS/` | -| [Lakka](https://www.lakka.tv/) | Lightweight RetroArch OS, minimal config, boots straight into the UI | [Lakka]({rel}) | `/storage/system/` | -| [Batocera](https://batocera.org/) | Easy setup, works on Pi 3/4/5 and many other boards (Odroid, etc.) | [Batocera]({rel}) | `/userdata/bios/` | -| [Recalbox](https://www.recalbox.com/) | Plug-and-play experience, good for a first build | [Recalbox]({rel}) | `/recalbox/share/bios/` | +| [RetroPie](https://retropie.org.uk/) | Scripts that install RetroArch and standalone emulators on Raspberry Pi OS | [RetroPie]({rel}) | `~/RetroPie/BIOS/` | +| [Lakka](https://www.lakka.tv/) | Operating system built around RetroArch, boots into its menu | [Lakka]({rel}) | `/storage/system/` | +| [Batocera](https://batocera.org/) | Bootable OS, with images for Raspberry Pi 3, 4 and 5 and other boards (Odroid XU4 among them) | [Batocera]({rel}) | `/userdata/bios/` | +| [Recalbox](https://www.recalbox.com/) | Bootable OS for retro gaming that configures its emulators itself | [Recalbox]({rel}) | `/recalbox/share/bios/` | ### Handhelds | Setup | What it is | Pack | Extract to | |-------|-----------|------|-----------| -| Android (Retroid Pocket, Odin, etc.) | Most Android handhelds run RetroArch | [RetroArch]({rel}) | `RetroArch/system/` on internal storage or SD card | -| [ROCKNIX](https://rocknix.org/) | Linux OS for ARM and x86 handhelds (RG35XX, RG552, Deck) | [ROCKNIX]({rel}) | `/storage/roms/bios/`, over SSH or the network share | -| [Batocera](https://batocera.org/) | Also images many handhelds | [Batocera]({rel}) | `/userdata/bios/` | +| Android (Retroid Pocket, Odin, etc.) | RetroArch for Android | [RetroArch]({rel}) | `RetroArch/system/` on internal storage or SD card | +| [ROCKNIX](https://rocknix.org/) | Linux OS for ARM and x86 handhelds, among them the RG35XX (H700) and the RG552 (RK3399) | [ROCKNIX]({rel}) | `/storage/roms/bios/`, over SSH or the network share | +| [Batocera](https://batocera.org/) | Also has images for handheld chips, among them H700 and RK3326 | [Batocera]({rel}) | `/userdata/bios/` | ### FPGA | Setup | What it is | Pack | Extract to | |-------|-----------|------|-----------| -| [MiSTer FPGA](https://mister-devel.github.io/MkDocs_MiSTer/) | Hardware-level recreation of consoles on a DE10-Nano board | [MiSTer FPGA]({rel}) | `/media/fat/games/`, one subfolder per core | +| [MiSTer FPGA](https://mister-devel.github.io/MkDocs_MiSTer/) | FPGA cores that reimplement console hardware on a DE10-Nano board | [MiSTer FPGA]({rel}) | `/media/fat/games/`, one subfolder per core | ### Self-hosted ROM manager diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index 2285d234..f6b19fa2 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -899,6 +899,28 @@ class TheGapPageOpensOnCounts(unittest.TestCase): self.assertNotIn("Unified view of BIOS verification", source) +class WhichPackDescribesNoPlatformByOpinion(unittest.TestCase): + """The page called RetroPie the largest community with the most guides, + Batocera an easy setup and Recalbox good for a first build: claims + nothing in the repository measures, ranking platforms against each + other.""" + + def test_no_ranking_words(self): + import re + + import generate_site # noqa: PLC0415 + + page = generate_site.generate_which_pack() + rows = [line for line in page.splitlines() if line.startswith("| ")] + found = [ + row for row in rows + if re.search(r"\b(largest|most|popular|easy|best|classic|lightweight|" + r"one-click|plug-and-play|first build|accuracy-focused)\b", + row, re.IGNORECASE) + ] + self.assertEqual(found, []) + + class EveryPageLinksItsOwnData(unittest.TestCase): """An emulator page linked its YAML and JSON. A platform page linked nothing, and a tool had to download platforms.json (2.8 MB) and filter From 4129536121775c219f3e747caa475332c45cdf0a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:18:07 +0200 Subject: [PATCH 12/18] chore: pin upstream revisions read by date or tag dosbox_pure takes the upstream tag its core_version names (1.0-preview7), triforce the Dolphin release its tree reports (2407). The others take the upstream commit at profiled_date, the revision profile_sync already compared them against: the standalone build for standalone + libretro profiles, and for cannonball and jl-mod the cited lines land there and not at the fork's merge base. Every upstream-role ref lands on its declared value at the pin. --- emulators/81.yml | 1 + emulators/azahar.yml | 1 + emulators/cannonball.yml | 1 + emulators/dolphin-mmjr.yml | 1 + emulators/dolphin-mmjr2.yml | 4 +++- emulators/dolphin.yml | 1 + emulators/dosbox_pure.yml | 1 + emulators/easyrpg.yml | 1 + emulators/fsuae.yml | 1 + emulators/hypseus-singe.yml | 1 + emulators/ishiiruka.yml | 1 + emulators/jl-mod.yml | 1 + emulators/syobonaction.yml | 1 + emulators/triforce.yml | 1 + emulators/uae4arm.yml | 1 + 15 files changed, 17 insertions(+), 1 deletion(-) diff --git a/emulators/81.yml b/emulators/81.yml index ca923ee7..90da855f 100644 --- a/emulators/81.yml +++ b/emulators/81.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/81-libretro" upstream: "https://github.com/charlierobson/EightyOne" profiled_date: "2026-03-22" source_commit: "fa7094910d040baa5fd8b11dbf6a1a618330ecd9" +upstream_commit: "4de3cdb8cff0060ad03e43ae2681b0628a9c5f5d" core_version: "1.0a" display_name: "Sinclair - ZX 81 (EightyOne)" cores: ["81"] diff --git a/emulators/azahar.yml b/emulators/azahar.yml index aed198ab..270b1eac 100644 --- a/emulators/azahar.yml +++ b/emulators/azahar.yml @@ -5,6 +5,7 @@ source: "https://github.com/johndoe6345789/azahar_libretro" upstream: "https://github.com/azahar-emu/azahar" profiled_date: "2026-03-23" source_commit: "9dd12da328c753806cbd4f2b0e048fced97ae97d" +upstream_commit: "9b045bf8370d035cacd99e5ad19a082df9e3b262" core_version: "Git" display_name: "Nintendo - 3DS (Azahar)" cores: [azahar] diff --git a/emulators/cannonball.yml b/emulators/cannonball.yml index 621b7496..9e0a1ebf 100644 --- a/emulators/cannonball.yml +++ b/emulators/cannonball.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/cannonball" upstream: "https://github.com/djyt/cannonball" profiled_date: "2026-03-23" source_commit: "0d83575d920ea4c0a280847a19d7802f673e6a68" +upstream_commit: "27493ebf62be3498dff93ed6a45e8e2db819bae1" core_version: "1.0" display_name: "Cannonball" cores: diff --git a/emulators/dolphin-mmjr.yml b/emulators/dolphin-mmjr.yml index d9c5a874..30434848 100644 --- a/emulators/dolphin-mmjr.yml +++ b/emulators/dolphin-mmjr.yml @@ -5,6 +5,7 @@ source: "https://github.com/acidtech/Dolphin-MMJR" upstream: "https://github.com/weihuoya/dolphin" profiled_date: "2026-08-11" source_commit: "1b095b64857c2eabb49a42cc56ffd9d7881bc263" +upstream_commit: "0f4c971326ae9389b3ad55b0fefacb708d148f4d" core_version: "11460 - bankaimaster999" display_name: "Nintendo - GameCube / Wii (Dolphin MMJR)" cores: diff --git a/emulators/dolphin-mmjr2.yml b/emulators/dolphin-mmjr2.yml index fcce0ee4..d5161f8a 100644 --- a/emulators/dolphin-mmjr2.yml +++ b/emulators/dolphin-mmjr2.yml @@ -5,6 +5,7 @@ source: "https://github.com/nachoverdon/Dolphin-MMJR2" upstream: "https://github.com/dolphin-emu/dolphin" profiled_date: "2026-08-11" source_commit: "4d0fdb8eea616320e967408cf5547c72a95196f3" +upstream_commit: "4ff214cdd26f135251d2266d676ed72dc1c2f8c9" core_version: "2.0-18025" display_name: "Nintendo - GameCube / Wii (Dolphin MMJR2)" cores: @@ -22,7 +23,8 @@ notes: | selective merge of Dolphin Official carrying that label rather than a checkout of it: at Dolphin 5.0-18025 the memory subsystem has already moved behind Core::System, and here it has not, so no single Dolphin revision - matches and no upstream pin is declared. + matches the tree. upstream_commit is the Dolphin revision the upstream + citations were read at, not a base of this tree. ref: Source/Android/app/build.gradle:32, Source/Android/app/build.gradle:135 Two roots. The user directory is mmjr-revamp at the root of external storage. diff --git a/emulators/dolphin.yml b/emulators/dolphin.yml index f7a1a468..3f520166 100644 --- a/emulators/dolphin.yml +++ b/emulators/dolphin.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/dolphin-emu/dolphin" logo: "https://raw.githubusercontent.com/dolphin-emu/dolphin/master/Data/dolphin-emu.svg" profiled_date: "2026-03-23" source_commit: "4d23cf151640eb810cb1b8e9d9fc922cf59c0b87" +upstream_commit: "f5dfb7e3d36ce24551ed0bb886b79f3f192dac9d" core_version: "Git" display_name: "Nintendo - GameCube / Wii (Dolphin)" cores: diff --git a/emulators/dosbox_pure.yml b/emulators/dosbox_pure.yml index eeaa8c51..528f0b63 100644 --- a/emulators/dosbox_pure.yml +++ b/emulators/dosbox_pure.yml @@ -6,6 +6,7 @@ upstream: "https://codeberg.org/schelling/dosbox-pure" logo: "https://raw.githubusercontent.com/schellingb/dosbox-pure/main/images/logo.png" profiled_date: "2026-03-23" source_commit: "cdaea10ce8cac231bc341ca05d4c5e4e5fe0fbd5" +upstream_commit: "cdaea10ce8cac231bc341ca05d4c5e4e5fe0fbd5" core_version: "1.0-preview7" display_name: "DOS (DOSBox-Pure)" cores: [dosbox_pure] diff --git a/emulators/easyrpg.yml b/emulators/easyrpg.yml index 4d5482fa..9a4231f8 100644 --- a/emulators/easyrpg.yml +++ b/emulators/easyrpg.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/easyrpg-libretro" upstream: "https://github.com/EasyRPG/Player" profiled_date: "2026-03-23" source_commit: "a3fbb7c24fd5a39fd95388d09135256a6b19f1e4" +upstream_commit: "8acd2e17c6a8a0a75cb7ce95625c6bdc89aebe1a" core_version: "0.8.1.0" display_name: "RPG Maker 2000/2003 (EasyRPG)" cores: diff --git a/emulators/fsuae.yml b/emulators/fsuae.yml index 2e4ab2db..a5fd235f 100644 --- a/emulators/fsuae.yml +++ b/emulators/fsuae.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/libretro-fsuae" upstream: "https://github.com/FrodeSolheim/fs-uae" profiled_date: "2026-03-26" source_commit: "089bb69207fe8fa5eec0006d1d759fb6426f4027" +upstream_commit: "f362278ccd4c60991caac3b4d240d4a3f751bea2" core_version: "v5.0.7" display_name: "Commodore - Amiga (FS-UAE)" cores: diff --git a/emulators/hypseus-singe.yml b/emulators/hypseus-singe.yml index 792b2947..eb611232 100644 --- a/emulators/hypseus-singe.yml +++ b/emulators/hypseus-singe.yml @@ -4,6 +4,7 @@ core_classification: enhanced_fork source: "https://github.com/DirtBagXon/hypseus-singe" upstream: "https://github.com/mirror/daphne-emu" source_commit: "cda87965fe0a031d027b2253c6715d37cca6b56a" +upstream_commit: "cd214c6046365047704f8a0563928970b30444e6" profiled_date: "2026-08-08" core_version: "v3.0.0" display_name: "Arcade (Hypseus Singe)" diff --git a/emulators/ishiiruka.yml b/emulators/ishiiruka.yml index fed1148c..9dbe31c7 100644 --- a/emulators/ishiiruka.yml +++ b/emulators/ishiiruka.yml @@ -5,6 +5,7 @@ source: "https://github.com/libretro/ishiiruka" upstream: "https://github.com/Tinob/Ishiiruka" profiled_date: "2026-03-24" source_commit: "f228bde5fbf4861ba749a229803addcd6238663f" +upstream_commit: "cba6d7279b44894297d8c1dde1effdbba3605c32" core_version: "Git" display_name: "Nintendo - GameCube / Wii (Ishiiruka)" cores: diff --git a/emulators/jl-mod.yml b/emulators/jl-mod.yml index ee2c5f5f..c1775e76 100644 --- a/emulators/jl-mod.yml +++ b/emulators/jl-mod.yml @@ -6,6 +6,7 @@ upstream: "https://github.com/nikita36078/J2ME-Loader" author: "Yury Kharchenko" profiled_date: "2026-08-11" source_commit: "f723a190c0bdb44b31c3bc0ead6f8665c7ea517d" +upstream_commit: "c2fdac77edba94709796b67ed5493915f31d620f" core_version: "0.87.1" display_name: "Mobile - J2ME (JL-Mod)" cores: diff --git a/emulators/syobonaction.yml b/emulators/syobonaction.yml index 9200ffed..5b64195e 100644 --- a/emulators/syobonaction.yml +++ b/emulators/syobonaction.yml @@ -5,6 +5,7 @@ source: "https://github.com/iyzsong/syobonaction-libretro" upstream: "https://github.com/akemin-dayo/OpenSyobonAction" profiled_date: "2026-03-25" source_commit: "792c40cb590bcc7286cf720b5fb85807082b534e" +upstream_commit: "77ca2c128784512eca10e24a453ca19c270a8af4" core_version: "0.3" display_name: "Syobon Action" cores: [syobonaction] diff --git a/emulators/triforce.yml b/emulators/triforce.yml index 675a5b44..eab95ad9 100644 --- a/emulators/triforce.yml +++ b/emulators/triforce.yml @@ -5,6 +5,7 @@ source: "https://github.com/REG-Linux/dolphin-triforce" upstream: "https://github.com/dolphin-emu/dolphin" profiled_date: "2026-08-08" source_commit: "5c456e8ff0da0235be2cf1d55ba7bb2115b0cae2" +upstream_commit: "b92e354389bb7c0bd114a8631b8af110d3cb3a14" core_version: "2407" display_name: "Namco / Sega / Nintendo - Triforce (Dolphin Triforce)" cores: diff --git a/emulators/uae4arm.yml b/emulators/uae4arm.yml index a2561b38..cbd2b97d 100644 --- a/emulators/uae4arm.yml +++ b/emulators/uae4arm.yml @@ -6,6 +6,7 @@ source: "https://github.com/libretro/uae4arm-libretro" upstream: "https://github.com/PandTomB/uae4arm" profiled_date: "2026-03-25" source_commit: "26f444f01771f0dfa2fa652906ff8222e92e476e" +upstream_commit: "161ddbaafe90a9747e9fb9c65ee4f38c163f335e" core_version: "v0.5" display_name: "Commodore - Amiga (UAE4ARM)" systems: From f15fa46d859d06eccf33bf2f40a0c431925d089c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:20:19 +0200 Subject: [PATCH 13/18] feat: offer the one-liner where users land The download page had no Android line and no Windows form with options, and platform and system pages offered only ZIP parts. One generator now writes the Linux, Windows and Android commands for the download page and the platform, emulator and system pages. A system section lists each platform's command under the system id its install manifest records, the only names --system accepts. --- scripts/generate_site.py | 148 ++++++++++++++++++++++++++++++------- tests/test_site_exports.py | 44 +++++++++++ 2 files changed, 167 insertions(+), 25 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 8d4f455e..1aed0642 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -20,6 +20,7 @@ import io import json import os import re +import shlex import shutil import sqlite3 import sys @@ -2157,6 +2158,7 @@ def generate_platform_page( f"| Contributed by | [@{username}](https://github.com/{username}):" f" {contribution}{pr_link} |" ) + lines.extend(["", *_platform_install_block(name, display)]) lines.extend( [ "", @@ -2842,11 +2844,46 @@ def _card_title(text: str, anchor: str) -> str: return f"#### {text} {{ {attributes} }}" +def system_install_choices(coverages: dict) -> dict[str, dict[str, list[tuple[str, str]]]]: + """slug -> console -> the (platform, system id) pairs linked to it. + + Read through the same table as every system link, so a command sits in + the section the platform's system links to. Only a system the + platform's install manifest records is offered: --system refuses any + other name. + """ + from siterender import _page_by_norm # noqa: PLC0415 + + choices: dict[str, dict[str, list[tuple[str, str]]]] = {} + for name, cov in sorted(coverages.items()): + manifest = _REPO_ROOT / "install" / f"{name}.json" + if not manifest.is_file(): + continue + document = json.loads(manifest.read_text(encoding="utf-8")) + recorded = { + str(system) + for entry in (document.get("files") or []) + (document.get("omitted_files") or []) + for system in entry.get("systems") or [] + } + for sys_id in sorted(set(cov["config"].get("systems") or {}) & recorded): + page = _system_page_map.get(sys_id) or _page_by_norm(sys_id) + if page: + slug, console = page + choices.setdefault(slug, {}).setdefault(console, []).append((name, sys_id)) + return choices + + def generate_system_page( manufacturer: str, consoles: dict[str, list], consumers: FileConsumers, + installs: dict[str, list[tuple[str, str]]] | None = None, ) -> str: + """One manufacturer's page, a section per console. + + ``installs`` gives, per console, the (platform, system id) pairs whose + install command the section offers. + """ slug = manufacturer.lower().replace(" ", "-") lines = [ f"# {manufacturer}", @@ -2862,6 +2899,7 @@ def generate_system_page( icon_md = system_icon_markdown(manufacturer, console_name) lines.append(f"## {icon_md}{console_name}") lines.append("") + lines.extend(_console_install_block(console_name, (installs or {}).get(console_name, []))) main_files = [f for f in files if "/.variants/" not in f["path"]] variant_files = [f for f in files if "/.variants/" in f["path"]] @@ -3477,6 +3515,37 @@ def _render_metadata_rows(profile: dict) -> list[str]: return lines +INSTALL_BASE = "https://raw.githubusercontent.com/Abdess/retrobios/main" + + +def install_command(shell: str, args: str = "") -> str: + """The one-liner for a shell, with installer options when given. + + PowerShell passes options only through a scriptblock: `irm ... | iex` + takes none. + """ + if shell == "sh": + return f"curl -fsSL {INSTALL_BASE}/install.sh | sh" + (f" -s -- {args}" if args else "") + if args: + return f"&([scriptblock]::Create((irm {INSTALL_BASE}/install.ps1))) {args}" + return f"irm {INSTALL_BASE}/install.ps1 | iex" + + +def install_commands(args: str = "", indent: str = "") -> list[str]: + """The same install for each place a user runs it: one generator for the + download page and the platform, system and emulator pages, so none of + them loses the Android or Windows form.""" + lines = [] + for label, shell in ( + ("Linux, macOS, Steam Deck:", "sh"), + ("Windows (PowerShell):", "powershell"), + ("Android (Termux, after `pkg install python` and `termux-setup-storage`):", "sh"), + ): + lines += [f"{indent}{label}", "", f"{indent}```{shell}", + f"{indent}{install_command(shell, args)}", f"{indent}```", ""] + return lines + + def _install_block(name: str, profile: dict) -> list[str]: """One command that installs this emulator's files where it reads them. @@ -3486,26 +3555,49 @@ def _install_block(name: str, profile: dict) -> list[str]: """ if profile.get("type") in ("alias", "launcher", "test"): return [] - base = "https://raw.githubusercontent.com/Abdess/retrobios/main" return [ '??? tip "Install this emulator\'s files"', "", " The installer finds your platform and its BIOS folder, then downloads " "only the files this emulator reads there and checks each one.", "", - " Linux, macOS, Steam Deck:", + *install_commands(f"--emulator {name}", " "), + ] + + +def _platform_install_block(name: str, display: str) -> list[str]: + """The platform's pack, installed file by file: what the ZIP holds, with + resume, verification and only what is missing.""" + return [ + '???+ tip "Install with one command"', + "", + f" The installer downloads the files of the {display} pack that are " + "missing or differ from it, checks each one, and resumes where it " + "stopped. Add `--system` with the ids of the systems overview below " + "to install fewer systems.", + "", + *install_commands(f"--platform {name}", " "), + ] + + +def _console_install_block(console: str, declared: list[tuple[str, str]]) -> list[str]: + """Each platform's command for one console, by the system id it gives it.""" + if not declared: + return [] + lines = [ + f'??? tip "Install the {console} files"', + "", + " Each platform names the system its own way. On Windows, run the " + "same options through `&([scriptblock]::Create((irm " + f"{INSTALL_BASE}/install.ps1)))`.", "", " ```sh", - f" curl -fsSL {base}/install.sh | sh -s -- --emulator {name}", - " ```", - "", - " Windows (PowerShell):", - "", - " ```powershell", - f" &([scriptblock]::Create((irm {base}/install.ps1))) --emulator {name}", - " ```", - "", ] + lines += [ + f" {install_command('sh', f'--platform {platform} --system {shlex.quote(system)}')}" + for platform, system in declared + ] + return lines + [" ```", ""] def _render_platform_details(profile: dict) -> list[str]: @@ -5045,6 +5137,10 @@ def _mkdocs_config(docs: Path, text: str) -> tuple[Path, str]: def generate_which_pack() -> str: """Generate the 'Which pack?' decision page.""" rel = RELEASE_URL + quick = "\n".join(install_commands()) + check_sh, check_ps = install_command("sh", "--check"), install_command("powershell", "--check") + sd_card = install_command( + "sh", "--platform retroarch --dest /path/to/sdcard/retroarch/cores/system") return _link_pack_cells(f"""\ # Download @@ -5063,18 +5159,7 @@ incorrect files, verifies their hashes, and installs them atomically. The small bootstrap checks the Python installer against an embedded SHA-256 before running it, and the installer reads its file list from that same revision. -**Linux / Mac / Steam Deck:** - -```sh -curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -``` - -**Windows (PowerShell):** - -```powershell -irm https://raw.githubusercontent.com/Abdess/retrobios/main/install.ps1 | iex -``` - +{quick} That is the complete default flow. Extra copies into detected standalone emulator directories are deliberately opt-in with `--standalone-copies`, so automatic setup never writes outside the selected platform tree unexpectedly. @@ -5082,7 +5167,19 @@ Adding `--check` reads the BIOS folder, hashes what is there and writes nothing: ```sh -curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --check +{check_sh} +``` + +```powershell +{check_ps} +``` + +An SD card mounted on a PC is installed with `--platform` and `--dest`, the +BIOS folder on the card (RetroArch on Switch: `retroarch/cores/system`, on +Android: `RetroArch/system`): + +```sh +{sd_card} ``` Options, environment overrides, how each platform is detected and what the @@ -5488,11 +5585,12 @@ def _generate(args: argparse.Namespace) -> None: write_if_changed( str(docs / "systems" / "index.md"), generate_systems_index(manufacturers) ) + console_installs = system_install_choices(coverages) (docs / "files").mkdir(parents=True, exist_ok=True) write_if_changed(str(docs / "files" / "index.md"), generate_files_page()) for mfr, consoles in manufacturers.items(): slug = mfr.lower().replace(" ", "-") - page = generate_system_page(mfr, consoles, consumers) + page = generate_system_page(mfr, consoles, consumers, console_installs.get(slug)) PAGE_DATA[f"systems/{slug}.md"] = system_structured_data(slug, consoles) write_single_item( docs / "api" / "v1" / "systems" / f"{slug}.json", "systems", diff --git a/tests/test_site_exports.py b/tests/test_site_exports.py index f6b19fa2..65948994 100644 --- a/tests/test_site_exports.py +++ b/tests/test_site_exports.py @@ -882,6 +882,50 @@ class NestedProfileBlocksRenderAsLists(unittest.TestCase): self.assertIn("name: a.bin, size: 2", html) +class TheOneLinerIsWhereUsersLand(unittest.TestCase): + """The download page had no Android line and no Windows form with + options, and the platform and system pages offered only ZIP parts.""" + + def test_download_page(self): + import generate_site # noqa: PLC0415 + + page = generate_site.generate_which_pack() + self.assertIn("Android (Termux", page) + self.assertIn("install.ps1))) --check", page) + self.assertIn("--platform retroarch --dest", page) + + def test_platform_and_system_pages(self): + import generate_site # noqa: PLC0415 + from cross_reference import FileConsumers # noqa: PLC0415 + + cov = {"platform": "Batocera", "mode": "md5", "present": 0, "total": 0, + "verified": 0, "missing": 0, "details": [], "config": {"systems": {}}} + page = generate_site.generate_platform_page("batocera", cov) + self.assertIn("install.sh | sh -s -- --platform batocera", page) + self.assertIn("install.ps1))) --platform batocera", page) + self.assertLess(page.index("--platform batocera"), page.index("md-button")) + + entry = {"name": "a.bin", "sha1": "a" * 40, "md5": "b" * 32, "size": 3, + "path": "bios/Recalbox/MSX/a.bin"} + page = generate_site.generate_system_page( + "Microsoft", {"MSX": [entry]}, FileConsumers({"files": {}}), + {"MSX": [("recalbox", "msx1,msx2,msxturbor"), ("batocera", "msx1")]}) + self.assertIn("--platform recalbox --system msx1,msx2,msxturbor\n", page) + self.assertIn("--platform batocera --system msx1", page) + + def test_a_system_is_offered_only_where_the_manifest_records_it(self): + import generate_site # noqa: PLC0415 + import siterender # noqa: PLC0415 + + saved = dict(siterender._system_page_map) + self.addCleanup(siterender.set_system_page_map, saved) + siterender.set_system_page_map({"sony-playstation": ("sony", "PlayStation"), + "nowhere": ("sony", "PlayStation")}) + cov = {"retroarch": {"config": {"systems": {"sony-playstation": {}, "nowhere": {}}}}} + choices = generate_site.system_install_choices(cov) + self.assertEqual(choices["sony"]["PlayStation"], [("retroarch", "sony-playstation")]) + + class TheGapPageOpensOnCounts(unittest.TestCase): """The page opened on a slogan and on tiles reading "Verified (100%)" for 5,437 of 5,438 files and "Missing 0", above a table of red cells.""" From a492e68045824ca3dd07fb8f1a3c1b2e8df86801 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:34:46 +0200 Subject: [PATCH 14/18] fix: read a tagged upstream at its tag only detect_pinned_tag matched core_version against every view, so an upstream_commit set to the release tag a port was cut from (triforce on Dolphin 2407, dosbox_pure on 1.0-preview7, GroovyMAME on mame0289) froze the whole profile: the port was no longer compared with its HEAD and bump_commit refused. Only the documented build's repository freezes the profile now. An upstream on its tag is self-checked at that tag, is not reported orphaned, and a bump leaves upstream_commit on it. --- scripts/profile_sync.py | 58 +++++++++++++++++++++++++++++++----- tests/test_profile_sync.py | 61 ++++++++++++++++++++++++++++++++++++++ wiki/profiling.md | 5 +++- 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/scripts/profile_sync.py b/scripts/profile_sync.py index b91627d1..743829d4 100644 --- a/scripts/profile_sync.py +++ b/scripts/profile_sync.py @@ -936,6 +936,9 @@ class ProfileReport: upstream_pin: str | None = None upstream_head: str | None = None upstream_origin: str | None = None + # The release tag the upstream view is pinned on, when it is one. That + # view is read at its tag, and a bump leaves upstream_commit there. + upstream_tag: str | None = None # Repositories whose pin is no ancestor of the branch the profile # tracks: history was rewritten, or the pin lies on another branch. orphaned: list[str] = field(default_factory=list) @@ -1291,7 +1294,23 @@ def detect_pinned_tag( The declared `core_version` names the tag, which is looked up by name so that a repository publishing nightly tags cannot hide an old release behind pagination. + + Only the repository of the documented build freezes the profile. An + `upstream` a port was cut from sits on a release tag as its base + (GroovyMAME on mame0289, triforce on Dolphin 2407) while the port itself + keeps moving: that view is judged against its tag by `tag_pinned_views`, + and the port is still compared with its HEAD and still bumped. """ + documented = [view for view in views if view.field != "upstream"] or views + tagged = tag_pinned_views(profile, documented, cache_dir, offline) + return next(iter(tagged.values()), None) + + +def tag_pinned_views( + profile: dict, views: list[RepoView], cache_dir: str, offline: bool +) -> dict[tuple[str, str], str]: + """Views whose pin is the tag `core_version` names, keyed by repository.""" + tagged: dict[tuple[str, str], str] = {} for view in views: if view.pin == view.head: continue @@ -1299,8 +1318,9 @@ def detect_pinned_tag( profile.get("core_version"), view.repo.name ): if upstream.tag_commit(view.repo, tag, cache_dir, offline) == view.pin: - return tag - return None + tagged[(view.repo.host, view.repo.slug)] = tag + break + return tagged def _resolve_project_words( @@ -1484,12 +1504,27 @@ def build_report( report.pinned_tag = frozen or detect_pinned_tag( profile, views, cache_dir, offline ) + # A repository pinned on its release tag is read at that tag, whichever + # role it plays: comparing mame0174 refs with mamedev HEAD says nothing. + tagged = tag_pinned_views(profile, views, cache_dir, offline) + report.upstream_tag = next( + ( + tagged[(view.repo.host, view.repo.slug)] + for view in views + if view.field == "upstream" and (view.repo.host, view.repo.slug) in tagged + ), + None, + ) + + def on_own_revision(view: RepoView) -> bool: + return view.pin == view.head or (view.repo.host, view.repo.slug) in tagged + if not report.pinned_tag: # A tag or a held pin is off the tracked branch by design. report.orphaned = [ f"{view.repo.slug}: pin {view.pin[:8]} is no ancestor of {view.head[:8]}" for view in views - if view.pin != view.head + if not on_own_revision(view) and upstream.compare( view.repo, view.pin, view.head, cache_dir, offline ).diverged @@ -1842,9 +1877,9 @@ def build_report( if report.pinned_tag: return True if not part.role and len(path_roles.get(part.path, ())) > 1: - return all(view.pin == view.head for view in views) + return all(on_own_revision(view) for view in views) view, _ = resolve_path(part.path, part.start, tuple(tokens), role_of(part)) - return view.pin == view.head + return on_own_revision(view) def judge( part: RefPart, flag: bool, tokens: Sequence[str], hashes: Sequence[str] @@ -1877,7 +1912,7 @@ def build_report( # port's 305-356 read past the end of the upstream's 345 # lines. As in a comparison, a ref fails only everywhere. for view in views: - if view is owner or view.pin != view.head: + if view is owner or not on_own_revision(view): continue lines = fetch_from(view.repo, view.pin, part.path) if lines is None: @@ -1994,6 +2029,11 @@ def format_report(report: ProfileReport, changed_only: bool = False) -> str: ) elif report.pin and report.pin == report.head: lines.append(" pin is HEAD: checked for self-consistency") + if report.upstream_tag and not report.pinned_tag: + lines.append( + f" upstream pinned to tag {report.upstream_tag}: checked against " + "its own revision, not against HEAD" + ) for url in report.unread or []: lines.append(f" not read (unsupported host): {url}") for line in report.orphaned: @@ -2837,8 +2877,12 @@ def bump_commit( file=sys.stderr, ) return False + # An upstream pinned on a release tag is the base the port was cut from: + # its HEAD is another program, so the bump leaves it on the tag. upstream_moves = bool( - report.upstream_head and report.upstream_pin != report.upstream_head + report.upstream_head + and report.upstream_pin != report.upstream_head + and not report.upstream_tag ) if document.get("source_commit") == report.head and not upstream_moves: # Rewriting the pin to the value it already holds is not a bump, and diff --git a/tests/test_profile_sync.py b/tests/test_profile_sync.py index dd69ddb3..d1be83f8 100644 --- a/tests/test_profile_sync.py +++ b/tests/test_profile_sync.py @@ -1691,6 +1691,49 @@ class TestBuildReport(unittest.TestCase): report = build_report("test", self._versioned("v0.78"), self.dir) self.assertEqual(report.pinned_tag, "0.78") + def test_an_upstream_on_its_release_tag_does_not_freeze_the_port(self): + """GroovyMAME declares 0.289 and its upstream_commit is the mame0289 + tag. Reading that as a frozen profile stopped every comparison and + bump of the port, which keeps moving. The upstream is read at its tag, + the port against its HEAD.""" + self.files[("portpin", "src/p.c")] = ["x", "load p.bin", "y"] + self.files[("porthead", "src/p.c")] = ["x", "load p.bin", "y"] + self.files[("uppin", "src/u.c")] = ["x", "ROM_LOAD(u.bin)", "y"] + self.files[("uphead", "src/u.c")] = ["x", "", "", "", "y"] + heads = {"o/port": "porthead", "o/up": "uphead"} + profile_sync.upstream.resolve_head = ( + lambda repo, cache, offline=False, branch=None: heads[repo.slug] + ) + # A tag off the tracked branch is not an orphaned pin. + profile_sync.upstream.compare = ( + lambda repo, base, head, cache_dir, offline=False: CompareResult( + [], False, repo.slug == "o/up" + ) + ) + self.tag_commits = {"0.289": "uppin"} + profile = { + "emulator": "Port", + "source": "https://github.com/o/port", + "upstream": "https://github.com/o/up", + "source_commit": "portpin", + "upstream_commit": "uppin", + "core_version": "0.289", + "files": [ + {"name": "p.bin", "source_ref": "source src/p.c:2"}, + {"name": "u.bin", "source_ref": "upstream src/u.c:2"}, + ], + } + report = build_report("port", profile, self.dir) + self.assertIsNone(report.pinned_tag) + self.assertEqual(report.upstream_tag, "0.289") + self.assertEqual(report.orphaned, []) + self.assertEqual( + [entry.status for entry in report.entries], ["ANCHORED", "ANCHORED"] + ) + self.assertIn( + "upstream pinned to tag 0.289", profile_sync.format_report(report) + ) + def test_a_tag_pointing_elsewhere_is_not_flagged(self): self.files[("pinsha", "a.c")] = ["x", "hit"] self.files[("headsha", "a.c")] = ["x", "hit"] @@ -2943,6 +2986,24 @@ class TestBumpCommit(unittest.TestCase): self.assertEqual(rebase_refs(self.path, report), ["a.c:10-12 -> a.c:30-32"]) self.assertTrue(bump_commit(self.path, report)) + def test_an_upstream_on_its_release_tag_stays_there(self): + """GroovyMAME's upstream_commit is the mame0289 tag it merged: a bump + of the port must not carry it to mamedev HEAD, another program.""" + part = PartResult( + RefPart("a.c", 10, 12, "a.c:10-12"), "ANCHORED", None, 10, 12, [] + ) + report = ProfileReport( + name="p", repo="o/n", pin="pin", head="newhead", + entries=[EntryReport("a.bin", "a.c:10-12", "ANCHORED", [part])], + counts={"ANCHORED": 1}, + upstream_pin="uptag", upstream_head="uphead", + upstream_origin="upstream_commit", upstream_tag="mame0289", + ) + self.assertTrue(bump_commit(self.path, report)) + written = yaml.safe_load(self.path.read_text()) + self.assertEqual(written["source_commit"], "newhead") + self.assertNotIn("upstream_commit", written) + def test_upstream_pin_moves_with_source_pin(self): """A recale onto upstream HEAD lines must advance upstream_commit too.""" part = PartResult( diff --git a/wiki/profiling.md b/wiki/profiling.md index 6684eb45..809d79f3 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -301,7 +301,10 @@ only hold at one revision, so a few rules keep them checkable. it. - **A frozen line is pinned to its tag.** A profile documenting a release that lives as a tag in a still active repository (`pcsx2-legacy` at PCSX2 v1.6.0) - is checked against its own revision and never rebased onto the head. + is checked against its own revision and never rebased onto the head. An + `upstream` whose pin is the tag `core_version` names (GroovyMAME's base, + `mame0289`) is read at that tag the same way, while the port itself is still + compared with its head, and a bump leaves `upstream_commit` on the tag. - **A fork's base comes from the forge.** `upstream_commit` of a fork is the merge base the compare API returns. A vendored `version.h` can name the project the fork once came from. When a package ships prebuilt binaries From 755030d1e9344187929f7951bc48dfe573f40a69 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:34:46 +0200 Subject: [PATCH 15/18] fix: record groovymame as mame 0.289 source_commit 953db38f is the tip of branch groovymame0289, its makefile:1513 writes BARE_BUILD_VERSION "0.289", and the compare with mamedev tag mame0289 is 83 ahead, 0 behind, with no file under src/mame or src/devices. The notes described the earlier pin e3fa1305 (branch groovymame0288). --- emulators/groovymame.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/emulators/groovymame.yml b/emulators/groovymame.yml index 604e1215..b650160c 100644 --- a/emulators/groovymame.yml +++ b/emulators/groovymame.yml @@ -7,7 +7,7 @@ source_commit: "953db38f4aaab1d75faa4ac79e1b45686214f987" upstream_commit: "f34f02505e32c1993c6a782b6814232cbfc74e36" logo: "https://raw.githubusercontent.com/mamedev/mame/master/docs/source/images/MAMElogo.svg" profiled_date: "2026-08-08" -core_version: "0.288" +core_version: "0.289" display_name: "Arcade (GroovyMAME)" cores: @@ -123,13 +123,14 @@ systems: notes: | MAME fork adding Switchres 2.2.2 for CRT modeline generation, frame delay - and low latency vsync. Branch groovymame0288, BARE_BUILD_VERSION 0.288 - (makefile:1508). TARGET := mame with no SUBTARGET override (makefile:274-282), + and low latency vsync. Branch groovymame0289, BARE_BUILD_VERSION 0.289 + (makefile:1513). TARGET := mame with no SUBTARGET override (makefile:274-282), so the build carries the full machine set. - src/mame and src/devices are byte identical to mamedev/mame tag mame0288, - so ROM requirements are MAME 0.288's. The fork's changes sit in src/osd and - 3rdparty/switchres and pull in no external data file: monitor presets are + src/mame and src/devices are byte identical to mamedev/mame tag mame0289, + so ROM requirements are MAME 0.289's. The fork's changes sit in src/osd, + src/emu, src/frontend, 3rdparty/switchres, 3rdparty/emusync and + 3rdparty/lz4, and pull in no external data file: monitor presets are compiled in (3rdparty/switchres/monitor.cpp:124-145), and switchres.ini plus display#.ini are optional user configuration read only when the switchres_ini option is set (src/osd/modules/switchres/switchres_module.cpp:166,173). From cc550c21e728899511585b94840e7eac4251e4b0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:34:54 +0200 Subject: [PATCH 16/18] fix: cite port refs in the port's repository These refs were labelled upstream but land in the port at source_commit: flycast sb_mem.cpp:55-59,196-231, reios.cpp:737 and naomi_roms.h:87-94 in libretro/flycast@bde196bf (filestream_open, get_rom_names), lrps2 BiosTools.cpp:189-229,339-340 and CDVD.cpp:196-245 in libretro/ps2@6eabf224 (RETRO_VFS reads absent from PCSX2), mesence bios.col at FirmwareTypeExtensions.cs:70 in MesenCE@20f497c9 (line 62 in Mesen2). --- emulators/flycast.yml | 10 +++++----- emulators/lrps2.yml | 8 ++++---- emulators/mesence.yml | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/emulators/flycast.yml b/emulators/flycast.yml index c1abd8ac..a764465a 100644 --- a/emulators/flycast.yml +++ b/emulators/flycast.yml @@ -68,7 +68,7 @@ files: hle_fallback: true size: 2097152 note: "Dreamcast boot ROM. HLE BIOS (reios) used as fallback. Code searches prefix + boot.bin, boot.bin.bin, bios.bin, bios.bin.bin sequentially." - source_ref: "upstream core/hw/holly/sb_mem.cpp:219, 55-57" + source_ref: "source core/hw/holly/sb_mem.cpp:219, 55-57" - name: "dc_nvmem.bin" path: "dc/dc_nvmem.bin" @@ -78,14 +78,14 @@ files: hle_fallback: true size: 131072 note: "Dreamcast flash memory / NVRAM. Created if not found. Code searches prefix + nvmem.bin, flash_wb.bin, flash.bin, flash.bin.bin sequentially." - source_ref: "upstream core/hw/holly/sb_mem.cpp:231, 196-213" + source_ref: "source core/hw/holly/sb_mem.cpp:231, 196-213" - name: "font.bin" path: "dc/font.bin" system: sega-dreamcast required: false note: "Dreamcast system font for HLE BIOS text rendering. Built-in font used as fallback." - source_ref: "upstream core/reios/reios.cpp:737-753" + source_ref: "source core/reios/reios.cpp:737-753" # ------------------------------------------------------- # Sega NAOMI @@ -97,7 +97,7 @@ files: required: false size: 2097152 note: "NAOMI boot ROM (flat binary). Alternative to naomi.zip. epr-21576d.bin is searched without prefix." - source_ref: "upstream core/hw/holly/sb_mem.cpp:219, 58-59" + source_ref: "source core/hw/holly/sb_mem.cpp:219, 58-59" - name: "naomi.zip" path: "dc/naomi.zip" @@ -169,7 +169,7 @@ files: system: sega-atomiswave required: true note: "Atomiswave system BIOS archive (MAME format). Contains bios0.ic23 (Sammy, 128 KB)." - source_ref: "upstream core/hw/naomi/naomi_roms.h:87-94" + source_ref: "source core/hw/naomi/naomi_roms.h:87-94" contents: - name: "bios0.ic23" size: 131072 diff --git a/emulators/lrps2.yml b/emulators/lrps2.yml index facc380a..8aad8b6b 100644 --- a/emulators/lrps2.yml +++ b/emulators/lrps2.yml @@ -57,7 +57,7 @@ files: required: false max_size: 4194304 description: "DVD player ROM" - source_ref: "upstream pcsx2/ps2/BiosTools.cpp:189-229, 339" + source_ref: "source pcsx2/ps2/BiosTools.cpp:189-229, 339" note: > DVD player ROM. Tries .rom1 (appended) then .rom1 (extension replaced). Silently skipped if not found. @@ -67,7 +67,7 @@ files: required: false max_size: 524288 description: "Chinese ROM extension" - source_ref: "upstream pcsx2/ps2/BiosTools.cpp:189-229, 340" + source_ref: "source pcsx2/ps2/BiosTools.cpp:189-229, 340" note: > Chinese region ROM extension. Same naming convention as rom1. Only present on Chinese region consoles. @@ -78,7 +78,7 @@ files: hle_fallback: true size: 1024 description: "NVRAM / EEPROM data" - source_ref: "upstream pcsx2/CDVD/CDVD.cpp:196-227" + source_ref: "source pcsx2/CDVD/CDVD.cpp:196-227" note: > Console EEPROM data (language, timezone, region, iLink ID, OSD settings). Path derived from BIOS path with extension replaced to .nvm. @@ -90,7 +90,7 @@ files: hle_fallback: true size: 4 description: "Mechacon version" - source_ref: "upstream pcsx2/CDVD/CDVD.cpp:229-245" + source_ref: "source pcsx2/CDVD/CDVD.cpp:229-245" note: > Mechacon (disc drive controller) version as u32. Auto-created with default 0x00020603 if missing. diff --git a/emulators/mesence.yml b/emulators/mesence.yml index 16b48963..0ea80bc6 100644 --- a/emulators/mesence.yml +++ b/emulators/mesence.yml @@ -621,7 +621,7 @@ files: validation: [size] sha256: "990bf1956f10207d8781b619eb74f89b00d921c8d45c95c334c16c8cceca09ad,fb4a898eb93b19b36773d87a6c70eb28f981b2686bebdd2d431b05dcdf9cffd4" note: "ColecoVision BIOS, NTSC or PAL console dump. The system boots through it, and games cannot start without it." - source_ref: "source Core/SMS/SmsConsole.cpp:71-74, source Core/Shared/FirmwareHelper.h:381-398, upstream UI/Interop/FirmwareTypeExtensions.cs:70" + source_ref: "source Core/SMS/SmsConsole.cpp:71-74, source Core/Shared/FirmwareHelper.h:381-398, source UI/Interop/FirmwareTypeExtensions.cs:70" # ------------------------------------------------------- # WonderSwan (standalone only: UseBootRom is never set by the libretro port) From 4dbf3babcb3119439ce99879692f6115793685fc Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:34:54 +0200 Subject: [PATCH 17/18] fix: cite eightyone files from the repo root EightyOne/Source/ROMs/... answers 404 at charlierobson/EightyOne@4de3cdb8. Source/ROMs/{FDCs/interface1.edition1,spectrum48.arabic,tk90,tk95}.rom hash to the md5 each entry declares. The notes refs drop the clone-directory prefix too and name their repository. --- emulators/81.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/emulators/81.yml b/emulators/81.yml index 90da855f..7860258d 100644 --- a/emulators/81.yml +++ b/emulators/81.yml @@ -24,10 +24,10 @@ systems: notes: | Libretro port embeds zx81.rom and dkchr.rom as C arrays (xxd -i). Core never accesses filesystem for ROMs. Hardcodes MACHINEZX81. - ref: 81-libretro/81/zx81/snap.c:487-508, src/libretro.cpp:393 + ref: source 81/zx81/snap.c:487-508, source src/libretro.cpp:393 Original EightyOne supports 15+ machines, loads ROMs from ROM/ directory. - ref: EightyOne/Source/zx81/zx81.cpp:309, EightyOne/Source/HW_.cpp:2734 + ref: upstream Source/zx81/zx81.cpp:309, upstream Source/HW_.cpp:2734 .info has no firmware declarations (firmware_count absent). @@ -271,7 +271,7 @@ files: mode: standalone size: 8192 md5: 5c11c61a2dd2ca4bf39328d9ff42d289 - source_ref: "upstream EightyOne/Source/ROMs/FDCs/interface1.edition1.rom" + source_ref: "upstream Source/ROMs/FDCs/interface1.edition1.rom" - name: "+d.g+dos.v1-a.rom" system: sinclair-zx-spectrum required: false @@ -483,7 +483,7 @@ files: mode: standalone size: 16384 md5: 9f6f56003307b9d55be96f406fdbf2b1 - source_ref: "upstream EightyOne/Source/ROMs/spectrum48.arabic.rom" + source_ref: "upstream Source/ROMs/spectrum48.arabic.rom" - name: spectrum48.arabic.version1.rom system: sinclair-zx-spectrum required: false @@ -600,7 +600,7 @@ files: size: 16384 md5: b7893ad25728fbf6410f08eddd4a1cad note: "Microdigital TK90X (Spectrum clone)." - source_ref: "upstream EightyOne/Source/ROMs/tk90.rom" + source_ref: "upstream Source/ROMs/tk90.rom" - name: tk95.rom system: sinclair-zx-spectrum required: false @@ -609,4 +609,4 @@ files: size: 16384 md5: b7893ad25728fbf6410f08eddd4a1cad note: "Microdigital TK95 (Spectrum clone)." - source_ref: "upstream EightyOne/Source/ROMs/tk95.rom" + source_ref: "upstream Source/ROMs/tk95.rom" From 0d3fde9b0d3c00edf7bd14793bec127dea9927ac Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Wed, 21 Oct 2026 10:34:54 +0200 Subject: [PATCH 18/18] fix: pin primehack refs to its dolphin base Seven CommonPaths.h refs were written in March against a 2025 layout (bfef65a9..8cbfee21, IPL.bin at 138), and SYSCONF was later set to 118 from the current file, so no revision held all eight. They now cite d8558142, the merge base of shiiion/dolphin@53f53e0f with dolphin-emu master, where each define sits on the cited line, and upstream_commit names it. --- emulators/primehack.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/emulators/primehack.yml b/emulators/primehack.yml index 7fe78122..860c93f6 100644 --- a/emulators/primehack.yml +++ b/emulators/primehack.yml @@ -5,6 +5,7 @@ source: "https://github.com/shiiion/dolphin" upstream: "https://github.com/dolphin-emu/dolphin" profiled_date: "2026-03-26" source_commit: "53f53e0f5bad27ad62a807cb93b136d84f68777f" +upstream_commit: "d8558142ead21b3fa39809c4aedb982fbfc85feb" core_version: "1.0.8" display_name: "PrimeHack (Dolphin fork for Metroid Prime)" cores: @@ -41,7 +42,7 @@ files: required: false hle_fallback: true note: "GameCube NTSC-U boot ROM. HLE available but real IPL needed for GC menu boot and accurate fonts" - source_ref: "upstream Source/Core/Common/CommonPaths.h:138, source Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp:109-110" + source_ref: "upstream Source/Core/Common/CommonPaths.h:139, source Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp:109-110" - name: "IPL.bin" path: "GC/EUR/IPL.bin" @@ -78,7 +79,7 @@ files: adler32_pairs_with: "dsp_coef.bin" adler32_pairs: [["0x66f334fe", "0xf3b93527"], ["0xe789b5a5", "0xa4a575f5"]] note: "DSP instruction ROM for LLE audio. Free replacement (v0.4) included" - source_ref: "upstream Source/Core/Common/CommonPaths.h:135, source Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-117, source Source/Core/Core/DSP/DSPCore.cpp:28-102" + source_ref: "upstream Source/Core/Common/CommonPaths.h:136, source Source/Core/Core/HW/DSPLLE/DSPLLE.cpp:87-117, source Source/Core/Core/DSP/DSPCore.cpp:28-102" - name: "dsp_coef.bin" path: "GC/dsp_coef.bin" @@ -106,7 +107,7 @@ files: sha1: "483a17ff25b68a8f18cae691da99f16beefbe370" md5: "9bef639b7426ef6948602a12b33522a8" note: "Windows-1252 font for GC/Wii text rendering. The free alternative ships in Sys/GC/ of the 1.0.8 Windows and macOS releases, and the real one from an IPL dump is preferred" - source_ref: "upstream Source/Core/Common/CommonPaths.h:132, source Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp:131, 203-218" + source_ref: "upstream Source/Core/Common/CommonPaths.h:133, source Source/Core/Core/HW/EXI/EXI_DeviceIPL.cpp:131, 203-218" - name: "font_japanese.bin" path: "GC/font_japanese.bin" @@ -127,7 +128,7 @@ files: required: false hle_fallback: true note: "GBA BIOS for GC-GBA link feature (uses integrated mGBA). Path configurable in settings" - source_ref: "upstream Source/Core/Common/CommonPaths.h:144, source Source/Core/Core/HW/GBACore.cpp:344-361" + source_ref: "upstream Source/Core/Common/CommonPaths.h:145, source Source/Core/Core/HW/GBACore.cpp:344-361" # -- Wii System Files -- - name: "SYSCONF" @@ -143,7 +144,7 @@ files: required: false hle_fallback: true note: "Wii region/language settings. Auto-generated during Wii boot emulation" - source_ref: "upstream Source/Core/Common/CommonPaths.h:152, source Source/Core/Core/Boot/Boot_BS2Emu.cpp:377-454" + source_ref: "upstream Source/Core/Common/CommonPaths.h:153, source Source/Core/Core/Boot/Boot_BS2Emu.cpp:377-454" # -- Wii NAND Backup (BootMii) -- - name: "nand.bin" @@ -167,7 +168,7 @@ files: required: false hle_fallback: true note: "Virtual SD card image for Wii homebrew. Auto-created, supports SD/SDHC up to 4GB" - source_ref: "upstream Source/Core/Common/CommonPaths.h:149" + source_ref: "upstream Source/Core/Common/CommonPaths.h:150" # -- Gecko Code Handler -- - name: "codehandler.bin" @@ -179,7 +180,7 @@ files: sha1: "e6c0050b3c67ee26198488fbbe4a5d019c15b628" md5: "1f118b8735fcf40c20b38c3184eebfd9" note: "Gecko/Ocarina cheat code handler. Shipped in Sys/ of the 1.0.8 Windows and macOS releases" - source_ref: "upstream Source/Core/Common/CommonPaths.h:154, source Source/Core/Core/GeckoCode.cpp:121" + source_ref: "upstream Source/Core/Common/CommonPaths.h:155, source Source/Core/Core/GeckoCode.cpp:121" # -- Wii System Menu (WAD) -- - name: "Wii System Menu"