From 1937f0c50953346311d9641592d9570d250f5a98 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 6 Oct 2026 12:55:41 +0200 Subject: [PATCH] fix: name a file's readers by resolution --- scripts/cross_reference.py | 51 ++++++++++++++++++++ scripts/generate_site.py | 67 +++++++++++--------------- tests/test_file_consumers.py | 91 ++++++++++++++++++++++++++++++++++++ 3 files changed, 169 insertions(+), 40 deletions(-) create mode 100644 tests/test_file_consumers.py diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 30385718..72965143 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -29,6 +29,7 @@ from common import ( name_match_size_ok, parse_md5_list, require_yaml, + resolve_local_file, runs_standalone, ) from validation import read_from_system_dir @@ -200,6 +201,56 @@ def _data_hit( return any(name in data_names for name in looked_up) +class FileConsumers: + """Who reads each collected file, joined through the resolver. + + The site joined database files to emulators and platforms by bare name: + each of the four pak0.pk3 of four games was attributed to every engine + reading a pak0.pk3. A consumer here is an entry that resolves to the + file by the evidence the pack uses, so a homonym is nobody's file. + """ + + def __init__(self, db: dict, zip_contents: dict | None = None) -> None: + self._db = db + self._zip_contents = zip_contents + self._sha1_by_path = { + path: sha1 + for sha1, entry in db.get("files", {}).items() + for path in {entry.get("path", ""), *entry.get("paths", [])} + if path + } + self.platforms: dict[str, set[str]] = {} + self.emulators: dict[str, set[str]] = {} + + def sha1_of(self, entry: dict, dest: str = "") -> str | None: + """The collected file an entry resolves to, or None.""" + if not entry.get("name"): + return None + path, status = resolve_local_file( + entry, self._db, self._zip_contents, dest_hint=dest + ) + if not path or status in ("not_found", "hash_mismatch"): + return None + return self._sha1_by_path.get(path) + + def add_platforms(self, configs: dict[str, dict]) -> None: + for name, config in configs.items(): + for system in config.get("systems", {}).values(): + for entry in system.get("files", []): + sha1 = self.sha1_of(entry, entry.get("destination", "")) + if sha1: + self.platforms.setdefault(sha1, set()).add(name) + + def add_emulators(self, profiles: dict[str, dict]) -> None: + for name, profile in profiles.items(): + if profile.get("type") == "alias": + continue + for entry in profile.get("files", []): + sha1 = self.sha1_of(entry, entry.get("path") or "") + if sha1: + self.emulators.setdefault(sha1, set()).add(name) + + def entry_source(f: dict, index: dict) -> str | None: """Where the collection holds a profile entry, or None. diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 1bcd12cf..6e495ac9 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -31,6 +31,7 @@ from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from common import ( + build_zip_contents_index, compute_composition, count_catalog_matched, list_registered_platforms, @@ -43,6 +44,7 @@ from common import ( write_if_changed as _write_artifact, yaml_load, ) +from cross_reference import FileConsumers from nativemode import reads_file_contents @@ -1647,8 +1649,7 @@ def generate_provenance_page(db: dict, report: dict) -> str: def generate_system_page( manufacturer: str, consoles: dict[str, list], - platform_files: dict[str, set], - emulator_files: dict[str, dict], + consumers: FileConsumers, ) -> str: manufacturer.lower().replace(" ", "-") lines = [ @@ -1671,13 +1672,9 @@ def generate_system_page( md5_full = f.get("md5", "unknown") size = _fmt_size(f.get("size", 0)) - # Cross-reference - plats = sorted(p for p, names in platform_files.items() if name in names) - emus = sorted( - e - for e, data in emulator_files.items() - if name in data.get("files", set()) - ) + # Who reads this file, not who reads a file of this name + plats = sorted(consumers.platforms.get(sha1_full, ())) + emus = sorted(consumers.emulators.get(sha1_full, ())) # Truncated hashes for readability sha1_short = sha1_full[:12] if sha1_full != "unknown" else "-" @@ -1935,7 +1932,7 @@ def _file_badges(f: dict, in_repo: bool) -> list[str]: def _render_emulator_file( f: dict, profile: dict, - platform_files: dict | None, + consumers: FileConsumers | None, files: list, _file_available, ) -> list[str]: @@ -2049,10 +2046,9 @@ def _render_emulator_file( details.append( f"Source: {_source_ref_markdown(profile, source_ref)}" ) - if platform_files: - plats = sorted( - p for p, names in platform_files.items() if fname in names - ) + if consumers: + sha1 = consumers.sha1_of(f, f.get("path") or "") + plats = sorted(consumers.platforms.get(sha1, ())) if sha1 else [] if plats: plat_links = [_platform_link(p, p, "../") for p in plats] details.append(f"Platforms: {', '.join(plat_links)}") @@ -2255,7 +2251,7 @@ def generate_emulator_page( name: str, profile: dict, db: dict, - platform_files: dict | None = None, + consumers: FileConsumers | None = None, data_names: set[str] | None = None, ) -> str: if profile.get("type") == "alias": @@ -2404,7 +2400,7 @@ def generate_emulator_page( for f in files: lines.extend( _render_emulator_file( - f, profile, platform_files, files, _file_available + f, profile, consumers, files, _file_available ) ) @@ -3247,30 +3243,21 @@ def generate_wiki_data_model(db: dict, profiles: dict) -> str: # Build cross-reference indexes -def _build_platform_file_index(coverages: dict) -> dict[str, set]: - """Map platform_name -> set of declared file names.""" - index = {} - for name, cov in coverages.items(): - names = set() - config = cov["config"] - for system in config.get("systems", {}).values(): - for fe in system.get("files", []): - names.add(fe.get("name", "")) - index[name] = names - return index +def _build_file_consumers(db: dict, coverages: dict, profiles: dict) -> FileConsumers: + """Which platforms and emulators read each collected file.""" + consumers = FileConsumers(db, build_zip_contents_index(db)) + consumers.add_platforms({name: cov["config"] for name, cov in coverages.items()}) + consumers.add_emulators(profiles) + return consumers def _build_emulator_file_index(profiles: dict) -> dict[str, dict]: - """Map emulator_name -> {files: set, systems: set} for cross-reference.""" - index = {} - for name, profile in profiles.items(): - if profile.get("type") == "alias": - continue - index[name] = { - "files": {f.get("name", "") for f in profile.get("files", [])}, - "systems": set(profile.get("systems", [])), - } - return index + """Map emulator_name -> {systems: set}, for the platform pages.""" + return { + name: {"systems": set(profile.get("systems", []))} + for name, profile in profiles.items() + if profile.get("type") != "alias" + } # mkdocs.yml nav generator @@ -3614,7 +3601,7 @@ def main(): ) # Build cross-reference indexes - platform_files = _build_platform_file_index(coverages) + consumers = _build_file_consumers(db, coverages, profiles) emulator_files = _build_emulator_file_index(profiles) # Generate home @@ -3671,7 +3658,7 @@ def main(): ) for mfr, consoles in manufacturers.items(): slug = mfr.lower().replace(" ", "-") - page = generate_system_page(mfr, consoles, platform_files, emulator_files) + page = generate_system_page(mfr, consoles, consumers) write_if_changed(str(docs / "systems" / f"{slug}.md"), page) # Generate emulator pages @@ -3685,7 +3672,7 @@ def main(): if profile.get("type") not in ("alias", "test") } for name, profile in public_profiles.items(): - page = generate_emulator_page(name, profile, db, platform_files, suppl_names) + page = generate_emulator_page(name, profile, db, consumers, suppl_names) write_if_changed(str(docs / "emulators" / f"{name}.md"), page) # Generate cross-reference page diff --git a/tests/test_file_consumers.py b/tests/test_file_consumers.py new file mode 100644 index 00000000..6dce611b --- /dev/null +++ b/tests/test_file_consumers.py @@ -0,0 +1,91 @@ +"""A file's readers are the entries that resolve to it, not those sharing its name. + +The site joined database files to emulators and platforms by bare name. The +collection holds the pak0.pk3 of Quake III and of Enemy Territory; each page +named every engine that reads a pak0.pk3 as a reader of both, which is the +homonym tests/test_game_data_homonyms.py forbids the resolver to serve. +""" + +from __future__ import annotations + +import hashlib +import sys +import tempfile +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from cross_reference import FileConsumers +from generate_site import generate_system_page + +_TMP = tempfile.TemporaryDirectory() +_ROOT = Path(_TMP.name) + + +def _collected(rel: str, payload: bytes) -> tuple[str, dict]: + path = _ROOT / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + sha1 = hashlib.sha1(payload).hexdigest() + return sha1, {"name": path.name, "sha1": sha1, "path": str(path), + "md5": hashlib.md5(payload).hexdigest(), "size": len(payload)} + + +Q3, _q3 = _collected("bios/id/Quake III/baseq3/pak0.pk3", b"quake iii demo") +ET, _et = _collected("bios/id/ET/etmain/pak0.pk3", b"enemy territory") +DB = { + "files": {Q3: _q3, ET: _et}, + "indexes": { + "by_name": {"pak0.pk3": [Q3, ET]}, + "by_md5": {_q3["md5"]: Q3, _et["md5"]: ET}, + "by_path_suffix": { + "baseq3/pak0.pk3": [Q3], "Quake III/baseq3/pak0.pk3": [Q3], + "etmain/pak0.pk3": [ET], "ET/etmain/pak0.pk3": [ET], + }, + }, +} +PROFILES = { + "ioquake3": {"type": "standalone", "files": [ + {"name": "pak0.pk3", "path": "baseq3/pak0.pk3", "sha1": Q3}]}, + "etlegacy": {"type": "standalone", "files": [ + {"name": "pak0.pk3", "path": "etmain/pak0.pk3", "sha1": ET}]}, + "named_only": {"type": "standalone", "files": [ + {"name": "pak0.pk3", "path": "baseq3/pak0.pk3", "sha1": "3" * 40}]}, +} + + +class ReadersResolve(unittest.TestCase): + def setUp(self): + self.consumers = FileConsumers(DB) + self.consumers.add_emulators(PROFILES) + self.consumers.add_platforms({ + "plat": {"systems": {"s": {"files": [ + {"name": "pak0.pk3", "destination": "etmain/pak0.pk3", "sha1": ET}, + ]}}}, + }) + + def test_each_file_has_its_own_readers(self): + self.assertEqual(self.consumers.emulators[Q3], {"ioquake3"}) + self.assertEqual(self.consumers.emulators[ET], {"etlegacy"}) + self.assertNotIn(Q3, self.consumers.platforms) + self.assertEqual(self.consumers.platforms[ET], {"plat"}) + + def test_the_system_page_names_them(self): + consoles = {"Quake III": [DB["files"][Q3]], "ET": [DB["files"][ET]]} + page = generate_system_page("id", consoles, self.consumers) + q3 = page[page.index("## Quake III"):] + et = page[page.index("## ET"):page.index("## Quake III")] + self.assertIn("ioquake3", q3) + self.assertNotIn("etlegacy", q3) + self.assertIn("etlegacy", et) + self.assertNotIn("ioquake3", et) + + +def tearDownModule(): + _TMP.cleanup() + + +if __name__ == "__main__": + unittest.main()