From 77a1a7285a5e0d4c4f2261e8f10595cf9e802015 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 27 Oct 2026 18:59:30 +0100 Subject: [PATCH] feat: write the romset table from the profiles The FAQ table of arcade romsets was typed by hand and told readers to collect MAME 0.160 and FinalBurn Neo 1.0.0.2 sets for cores whose source states 0.159 and 1.0.0.03. generate_site now writes it between generated markers from the profiles, and a test holds the page to the generator. An emulator page names where its version is read (the file, the tag and the commits after it, the package), and says when the code states none or the version could not be confirmed. HBMAME no longer claims a MAME romset. tools.md and profiling.md document version_basis and the two modes. --- scripts/generate_site.py | 129 +++++++++++++++++++++++++++++--- tests/test_audit_regressions.py | 38 +++++----- tests/test_site_r2_emulators.py | 7 +- tests/test_site_r4_emulators.py | 2 +- wiki/profiling.md | 9 ++- wiki/technical-faq.md | 40 +++++----- wiki/tools.md | 15 ++++ 7 files changed, 183 insertions(+), 57 deletions(-) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index dd04f5697..f20054bf3 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2510,7 +2510,9 @@ def _emulator_description(markdown: str, key: str, title: str, intro: str) -> st row = {name: _plain_markdown(value) for name, value in re.findall( r"^\| (Type|Classification|Version) \| (.+?) \|$", markdown, re.M)} kind = row.get("Classification") or row.get("Type") or "emulator" - version = row.get("Version", "") + # The cell goes on to say where the version is read: the intro + # keeps the version alone. + version = row.get("Version", "").split(", ")[0] known = f", version {version}" if version and version[0].isdigit() else "" intro = f"{title}: {kind.lower()}{known}." elif title not in intro: @@ -7757,7 +7759,8 @@ def emulator_romset(profile: dict) -> str: version = str(profile.get("core_version") or "") if profile.get("mame_version"): return f"MAME {profile['mame_version']}" - if "MAME" in emulator and _MAME_RELEASE.match(version): + # HBMAME's sets are its own homebrew sets, whatever MAME it is based on. + if "MAME" in emulator and "HBMAME" not in emulator and _MAME_RELEASE.match(version): return f"MAME {version}" if emulator.startswith("FinalBurn Neo") and version: return f"FinalBurn Neo {version}" @@ -7766,6 +7769,113 @@ def emulator_romset(profile: dict) -> str: return "" +def _version_said(value: object, basis: dict) -> str: + """One build's version and where the profile reads it.""" + kind = basis.get("kind") + if kind == "none": + return "none stated in the source" + if value is None: + if kind == "unverified": + return f"unknown: {_UNVERIFIED.get(str(basis.get('why')), 'unread')}" + return "none published" + value = str(value) + if kind == "file": + return f"{value}, stated in `{basis.get('path')}`" + if kind == "tag": + distance = int(basis.get("distance") or 0) + if not distance: + return f"{value}, the tag at this revision" + return f"{value}, {distance} commit{'s' if distance != 1 else ''} after tag `{basis.get('tag')}`" + if kind == "package": + return f"{value}, read from {basis.get('artifact')}" + if kind == "unverified": + return f"{value}, not confirmed: {_UNVERIFIED.get(str(basis.get('why')), 'unread')}" + return value + + +# Why a profile's version could not be read, said to a reader. +_UNVERIFIED = { + "no_pin": "no revision recorded", + "host_unsupported": "its host cannot be read", + "host_gone": "its source is gone", + "host_unreachable": "its host did not answer", + "no_artifact": "no build recorded as read", + "no_source": "no source declared", +} + + +def version_markdown(profile: dict) -> str: + """The Version cell: core_version and where version_basis reads it. + + A null version said "none published" whether the code states none or + nobody looked. Keyed by build mode like source, each build is named, + where the page printed the dict's Python repr. + """ + if "core_version" not in profile and not profile.get("version_basis"): + return "unknown" + value = profile.get("core_version") + basis = profile.get("version_basis") if isinstance(profile.get("version_basis"), dict) else {} + if isinstance(value, dict) or (basis and "kind" not in basis): + values = value if isinstance(value, dict) else {} + modes = list(dict.fromkeys([*values, *basis])) + return ", ".join( + f"{_version_said(values.get(mode), basis.get(mode) or {})} ({mode})" + for mode in modes + ) + return _version_said(value, basis) + + +ROMSET_FAQ_PAGE = Path(WIKI_SRC_DIR) / "technical-faq.md" +_ROMSET_BLOCK = re.compile( + r"(\n)(.*?)()", re.S) + + +def _retroarch_label(profile: dict) -> str: + """"Arcade (MAME 2003)" gives "MAME 2003", "Arcade (FinalBurn Neo) + (Neo-Geo)" gives "FinalBurn Neo (Neo-Geo)".""" + inner = str(profile.get("display_name") or profile.get("emulator") or "").split("(", 1)[-1] + return inner.replace(")", "", 1).strip() + + +def romset_table(profiles: dict) -> list[str]: + """The FAQ's arcade table: each MAME and FinalBurn libretro core and the + romset its profile reads at its pin. + + Written by hand, it told readers to collect MAME 0.160 sets for a core + whose src/version.c states 0.159, and FinalBurn Neo 1.0.0.2 for a source + at 1.0.0.03. A core whose name gives its version (Recalbox's MAME 0.258) + needs no row. + """ + rows: dict[str, list[str]] = {} + for key, profile in sorted(profiles.items()): + if not re.match(r"(mame|fbneo|fbalpha)", key) or \ + "libretro" not in str(profile.get("type", "")): + continue + romset = emulator_romset(profile) + label = _retroarch_label(profile) + if not romset or romset.split(" ", 1)[1].lstrip("v") in label: + continue + rows.setdefault(romset, []).append(label) + + def order(romset: str) -> tuple: + family = 0 if romset.startswith("MAME") else 1 if romset.startswith("FB Alpha") else 2 + numbers = tuple(int(n) for n in re.findall(r"\d+", romset.split(" (", 1)[0])) + return family, numbers + + lines = ["| Emulator, as RetroArch names it | Sets from |", "|---|---|"] + for romset in sorted(rows, key=order): + lines.append(f"| {', '.join(sorted(rows[romset]))} | {romset} |") + return lines + + +def with_romset_table(page: str, profiles: dict) -> str: + """The FAQ page with its romset table read from the profiles.""" + table = "\n".join(romset_table(profiles)) + "\n" + if not _ROMSET_BLOCK.search(page): + raise ValueError(f"{ROMSET_FAQ_PAGE}: no romsets block to write the table in") + return _ROMSET_BLOCK.sub(lambda m: f"{m.group(1)}{table}{m.group(3)}", page, count=1) + + def _render_metadata_rows(profile: dict) -> list[str]: """Scalar profile fields, one table row each. @@ -8856,15 +8966,7 @@ def emulator_pages( upstream_raw = profile.get("upstream", "") upstream_none = "upstream" in profile and upstream_raw is None upstream = str(upstream_raw) if isinstance(upstream_raw, str) else "" - version_raw = profile.get("core_version", "unknown") - if version_raw is None: - version = "none published" - elif isinstance(version_raw, dict): - # Keyed by build mode like source: each build's version, named, - # where the page printed the dict's Python repr. - version = ", ".join(f"{v} ({mode})" for mode, v in version_raw.items() if v) - else: - version = version_raw + version = version_markdown(profile) profiled = profile.get("profiled_date", "unknown") systems = profile.get("systems", []) cores = profile.get("cores", [name]) @@ -11334,6 +11436,11 @@ def _generate(args: argparse.Namespace) -> None: wiki_dest = docs / "wiki" wiki_dest.mkdir(parents=True, exist_ok=True) wiki_src = Path(WIKI_SRC_DIR) + # The romset table is the profiles' versions, never typed: the source + # page is rewritten before it is copied, as the README is. + faq = _REPO_ROOT / ROMSET_FAQ_PAGE + if faq.is_file(): + write_if_changed(str(faq), with_romset_table(faq.read_text(encoding="utf-8"), profiles)) if wiki_src.is_dir(): for src_file in wiki_src.glob("*.md"): shutil.copy2(src_file, wiki_dest / src_file.name) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 9aad3b0e5..ead9e3b9c 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -790,16 +790,6 @@ class FaqRegressions(unittest.TestCase): existence platform. Readers act on all three. """ - _MAME_GENERATIONS = { - "MAME 2000": "mame2000", - "MAME 2003": "mame2003", - "MAME 2009": "mame2009", - "MAME 2010": "mame2010", - "MAME 2015": "mame2015", - "MAME 2016": "mame2016", - "current MAME": "mame", - } - @staticmethod def _faq() -> str: # The questions about hashes and checks moved to the technical FAQ. @@ -813,17 +803,23 @@ class FaqRegressions(unittest.TestCase): with (ROOT / "emulators" / f"{key}.yml").open(encoding="utf-8") as handle: return yaml.safe_load(handle) - def test_mame_versions_match_their_profiles(self): - # The list is wrapped, so a label and its version can straddle a line. - faq = " ".join(self._faq().split()) - for label, key in self._MAME_GENERATIONS.items(): - match = re.search(rf"{re.escape(label)} \(([^)]+)\)", faq) - self.assertIsNotNone(match, f"FAQ no longer states a version for {label}") - self.assertEqual( - match.group(1), - self._profile(key)["core_version"], - f"FAQ version for {label} has drifted from emulators/{key}.yml", - ) + def test_the_romset_table_is_written_from_the_profiles(self): + # Typed by hand, the table and the prose under it kept MAME 0.160 + # and FinalBurn Neo 1.0.0.2, versions no source at the pins states, + # and the test holding them to the profiles kept the errors in step. + sys.path.insert(0, str(ROOT / "scripts")) + import generate_site # noqa: PLC0415 + from common import load_emulator_profiles # noqa: PLC0415 + + profiles = load_emulator_profiles(str(ROOT / "emulators"), skip_aliases=False) + page = (ROOT / "wiki" / "technical-faq.md").read_text(encoding="utf-8") + self.assertEqual( + page, generate_site.with_romset_table(page, profiles), + "wiki/technical-faq.md romset table is stale: run scripts/generate_site.py", + ) + prose = page.split("### How this is known", 1)[1].split("\n## ", 1)[0] + self.assertIsNone(re.search(r"\(0\.\d", prose), + "the prose under the table types a version again") def test_adler32_is_attributed_to_the_files_that_carry_it(self): dolphin = self._profile("dolphin") diff --git a/tests/test_site_r2_emulators.py b/tests/test_site_r2_emulators.py index 0ca86ecf2..9a5c5916e 100644 --- a/tests/test_site_r2_emulators.py +++ b/tests/test_site_r2_emulators.py @@ -86,9 +86,10 @@ class TheHeaderReadsAsWords(unittest.TestCase): self.assertNotIn('rb-badge-muted">japan', self.page) def test_profile_version_names_the_pinned_build(self): - # DuckStation numbers builds v0.1-: the pin, - # 4122fed9, is 12,070 commits past it. - self.assertRegex(str(_profile("duckstation")["core_version"]), r"^v0\.1-\d+$") + # DuckStation tags builds v0.1-, and the + # pin sits past the last such tag: git describe's form names both. + self.assertRegex(str(_profile("duckstation")["core_version"]), + r"^v0\.1-\d+(-\d+-g[0-9a-f]{7})?$") def test_the_second_spelling_is_a_platform_name(self): # The lookup is case-sensitive and EmuELEC launches "Duckstation": diff --git a/tests/test_site_r4_emulators.py b/tests/test_site_r4_emulators.py index 75198dcbe..9ba99adb5 100644 --- a/tests/test_site_r4_emulators.py +++ b/tests/test_site_r4_emulators.py @@ -89,7 +89,7 @@ class AnAppThatReadsAnyBios(unittest.TestCase): class EveryArcadeCoreNamesItsRomset(unittest.TestCase): def test_finalburn_and_mame_2003_plus(self): - for name, romset in (("fbneo", "FinalBurn Neo v1.0.0.2"), + for name, romset in (("fbneo", f"FinalBurn Neo {_profile('fbneo')['core_version']}"), ("mame2003_plus", "MAME 0.78 (plus backports)")): page = _pages(name)[name] line = page.index(f"Its BIOS and game sets must come from the {romset} romset.") diff --git a/wiki/profiling.md b/wiki/profiling.md index aacb9123f..978b7665b 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -278,7 +278,11 @@ core_classification: community_fork source: https://github.com/libretro/dolphin upstream: https://github.com/dolphin-emu/dolphin profiled_date: 2026-03-25 -core_version: 5.0-21264 +core_version: "2609-437-g3aac3f0" +version_basis: + kind: tag + tag: "2609" + distance: 437 systems: - nintendo-gamecube - nintendo-wii @@ -509,7 +513,8 @@ reports the profile as uncheckable, and the CI job that runs | `source` | no | libretro core repository URL. A dict when the core ships under several repos | | `upstream` | no | original emulator repository URL | | `profiled_date` | see above | date of source analysis. Quote it, or YAML parses it as a date object | -| `core_version` | no | version analyzed | +| `core_version` | yes | the version of the code at the pin, read where `version_basis` says. null when the code states none, never `Git`, `SVN` or a branch name. A dict keyed by build mode when the builds differ | +| `version_basis` | yes | where `core_version` is read: `file` (`path`, `pattern`, optional `format`), `tag` (the nearest `tag` in the pin's history and its `distance`, `core_version` in git describe's form), `alias` (`of`), `package` (`artifact`, the APK or binary read) or `none`. `repo` names `upstream` when the version is the upstream tree's. `profile_sync --verify-version` reads it again, and a pin advance rewrites `core_version` from it | | `source_commit` | see above | commit SHA of `source` the code was read at. Anchors every `source_ref` line number | | `upstream_commit` | no | same, for `upstream` | | `source_branch` | no | branch `source` was read from when it is not the default one (a port living on a pull request branch). profile_sync follows it | diff --git a/wiki/technical-faq.md b/wiki/technical-faq.md index c03bd9393..27a018bb0 100644 --- a/wiki/technical-faq.md +++ b/wiki/technical-faq.md @@ -124,38 +124,40 @@ per catalog and the full list of catalogued dumps still missing. Your game sets and BIOS sets must come from the same version as your arcade emulator. + | Emulator, as RetroArch names it | Sets from | |---|---| | MAME 2000 | MAME 0.37b5 | | MAME 2003, MAME 2003 Midway | MAME 0.78 | -| MAME 2003-Plus | MAME 0.78, with later fixes | +| MAME 2003-Plus | MAME 0.78 (plus backports) | | MAME 2009 | MAME 0.135u4 | | MAME 2010 | MAME 0.139 | -| MAME 2014, MAME 2015 | MAME 0.160 | +| MAME 2014, MAME 2015 | MAME 0.159 | | MAME 2016 | MAME 0.174 | -| MAME/MESS | MAME 0.287 | -| MAME, MAME/Arcade | MAME 0.289 | -| FB Alpha 2012 CPS-1, FB Alpha 2012 CPS-2 | FB Alpha 0.2.97.28 | -| FB Alpha 2012, FB Alpha 2012 CPS-3, FB Alpha 2012 Neo Geo | FB Alpha 0.2.97.29 | -| FB Alpha 2019 | FB Alpha 0.2.97.44 | -| FinalBurn Neo, FinalBurn Neo (Neo-Geo), FinalBurn Neo (CPS-1 / CPS-2) | FinalBurn Neo 1.0.0.2 | +| MAME, MAME/Arcade, MAME/MESS | MAME 0.289 | +| FB Alpha 2012 CPS-1, FB Alpha 2012 CPS-2 | FB Alpha v0.2.97.28 | +| FB Alpha 2012 Neo Geo | FB Alpha v0.2.97.29 | +| FB Alpha 2012, FB Alpha 2012 CPS-3 | FB Alpha v0.2.97.30 | +| FB Alpha 2019 | FB Alpha v0.2.97.44 | +| FinalBurn Neo, FinalBurn Neo (CPS-1 / CPS-2), FinalBurn Neo (Neo-Geo) | FinalBurn Neo v1.0.0.03 | + Each emulator's page names its version too. ### How this is known Arcade BIOS sets are coupled to the romset version, so there is one -profile per MAME core generation: MAME 2000 (0.37b5), MAME 2003 (0.78), -MAME 2003-Plus, MAME 2009 (0.135u4), MAME 2010 (0.139), MAME 2015 -(0.160), MAME 2016 (0.174), and current MAME (0.289), each read from that -version's own source tree, because the BIOS root sets differ between -versions. Derivatives and ports carry their own profiles on top of those -generations: `mamearcade`, `mamemess`, `mame2003_midway`, the three -`mame4droid` drops, `advancemame`, `groovymame` and `hbmame`. Recalbox -compiles two MAME versions of its own, 0.258 and 0.278, profiled as -`mame0258` and `mame0278`. The pack -ships the sets matching the core you run, and a generic unversioned arcade -pack cannot do that. +profile per MAME core generation (MAME 2000, 2003, 2003-Plus, 2009, 2010, +2015, 2016 and current MAME), each read from that version's own source +tree, because the BIOS root sets differ between versions. The table is +written from the profiles, and each version is the one the core's source +states at the revision its profile was read at. Derivatives and ports +carry their own profiles on top of those generations: `mamearcade`, +`mamemess`, `mame2003_midway`, the three `mame4droid` drops, +`advancemame`, `groovymame` and `hbmame`. Recalbox compiles two MAME +versions of its own, 0.258 and 0.278, profiled as `mame0258` and +`mame0278`. The pack ships the sets matching the core you run, and a +generic unversioned arcade pack cannot do that. ## Why does my verification report say UNTESTED? diff --git a/wiki/tools.md b/wiki/tools.md index 80f7fa019..30417bf00 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -585,6 +585,21 @@ match no entry. Both land in the `--json` report (`new_files`, with since the pin or an undeclared hash makes the profile stale. `--tree-diff` shows added, removed and renamed files in the directories the refs point at. +`core_version` is the version of the code at the pin, and `version_basis` +says where it is read: a pattern in one file (`file`), the nearest tag and +the commits after it (`tag`, written in git describe's form), the profile an +alias follows (`alias`), an artefact read outside any source tree +(`package`), or `none` when the code states nothing, with `core_version` +null. `--verify-version` reads it again at the pin and exits 1 on a +difference or a missing basis. `--set-version` writes what the code states. +A pin advance reads the version at the new pin in the same write and is +refused when it cannot: a package version moves by hand. + +```bash +python scripts/profile_sync.py --all --verify-version --changed-only +python scripts/profile_sync.py --emulator snes9x --set-version +``` + Branch tips, tags and releases are cached for six hours. `--fresh` asks the forges again for them, which matters right after an upstream commit or a hand-set pin: a tip cached before the pin reads as behind it. It is refused