From 0eec5ab4be92959e26d332d7af7af2e3014e9180 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 20 Oct 2026 16:14:55 +0200 Subject: [PATCH] fix: file a truth entry by content, not by name A name the platform declares filed an entry under that system before its own system:, and the last declaration of a name won. fmsx's 16 KB DISK.ROM went to nec-pc-88 for quasi88's 2 KB disk.rom and came out as a second DISK.ROM line under "NEC - PC-8801"; dosbox's SC-55 ROM1.BIN/ROM2.BIN went to galaksija. A declaration now files an entry only when it can be the same file: a shared hash, or no size both admit. The same content declared under two systems is filed under both, an entry's own system stands against a refuted homonym, and a systemless profile's placeholder takes only names the platform does not declare. Merging reads the same size rule, so a 24 KB ROM and an entry capped at 16 KB stay two files. --- scripts/truth.py | 169 ++++++++++++++++++++++++++++--------- tests/test_truth_layout.py | 110 ++++++++++++++++++++++++ 2 files changed, 237 insertions(+), 42 deletions(-) diff --git a/scripts/truth.py b/scripts/truth.py index d02f3022..01918d87 100644 --- a/scripts/truth.py +++ b/scripts/truth.py @@ -6,6 +6,7 @@ and diffs truth against scraped platform data to find divergences. from __future__ import annotations +import copy import sys from common import ( @@ -208,13 +209,12 @@ def _declared(entry: dict, field: str) -> set[str]: def _contents_disagree(a: dict, b: dict) -> bool: - """A hash both declare without a value in common, or two declared sizes.""" + """A hash both declare without a value in common, or no size both admit.""" for field in ("sha1", "md5", "sha256", "crc32"): ours, theirs = _declared(a, field), _declared(b, field) if ours and theirs and not ours & theirs: return True - size_a, size_b = a.get("size"), b.get("size") - return isinstance(size_a, int) and isinstance(size_b, int) and size_a != size_b + return _sizes_clash(a, b) def _fold_into(existing: dict, file_entry: dict, emu_name: str) -> None: @@ -388,6 +388,59 @@ def _archives_not_members(files: list[dict]) -> list[dict]: return out +def _sizes(entry: dict) -> tuple[set[int] | None, int | None, int | None]: + """The sizes an entry admits: exact values, then a min and a max.""" + raw = entry.get("size") + values = raw if isinstance(raw, list) else [raw] + exact = {int(v) for v in values if isinstance(v, int) and not isinstance(v, bool)} + low, high = entry.get("min_size"), entry.get("max_size") + return (exact or None, + low if isinstance(low, int) else None, + high if isinstance(high, int) else None) + + +def _sizes_clash(a: dict, b: dict) -> bool: + """No size both entries admit, lists and ranges included.""" + exact_a, low_a, high_a = _sizes(a) + exact_b, low_b, high_b = _sizes(b) + + def fits(value: int, low: int | None, high: int | None) -> bool: + return (low is None or value >= low) and (high is None or value <= high) + + if exact_a and exact_b: + return not exact_a & exact_b + if exact_a: + return not any(fits(v, low_b, high_b) for v in exact_a) + if exact_b: + return not any(fits(v, low_a, high_a) for v in exact_b) + return False + + +def _identity_values(entry: dict, db: dict | None) -> set[str]: + """Every hash an entry declares, with the siblings the database knows.""" + probe = {k: entry[k] for k in ("sha1", "md5", "crc32", "size") if entry.get(k)} + if db and probe and not any(isinstance(v, list) for v in probe.values()): + _enrich_hashes(probe, db) + values: set[str] = set() + for field in ("sha1", "md5", "sha256", "crc32"): + values |= _declared(entry, field) | _declared(probe, field) + return values + + +def _declaration_verdict(entry: dict, declaration: dict, db: dict | None) -> str: + """same, other or undecided: can a platform declaration be this file? + + A shared hash proves it. Sizes no value of which agrees refute it, as + name_match_size_ok refutes a candidate. Two hashes at one size are two + revisions of one slot, as slots.py reads them, so they refute nothing. + """ + if _identity_values(entry, db) & _identity_values(declaration, db): + return "same" + if _sizes_clash(entry, declaration): + return "other" + return "undecided" + + def generate_platform_truth( platform_name: str, config: dict, @@ -424,34 +477,58 @@ def generate_platform_truth( # 3. Normalized match: strip manufacturer prefix + separators platform_sys_ids = set(config.get("systems", {}).keys()) - # File->platform_system reverse index from scraped config - file_to_plat_sys: dict[str, str] = {} + # Every platform declaration of a name, in the platform's order. The + # last one used to win, whatever the content. + declared_by_name: dict[str, list[tuple[str, dict]]] = {} for psid, sys_data in config.get("systems", {}).items(): for fe in sys_data.get("files", []): fname = fe.get("name", "").lower() if fname: - file_to_plat_sys[fname] = psid - for alias in fe.get("aliases", []): - file_to_plat_sys[alias.lower()] = psid + declared_by_name.setdefault(fname, []).append((psid, fe)) # Normalized ID -> platform system ID norm_to_platform: dict[str, str] = {} for psid in platform_sys_ids: norm_to_platform[_norm_system_id(psid)] = psid - def _map_sys_id(profile_sid: str, file_name: str = "") -> str: + def _map_sys_id(profile_sid: str) -> str: """Map a profile system ID to the platform's system ID.""" - # 1. File-based lookup (handles composites and name mismatches) - if file_name: - plat_sys = file_to_plat_sys.get(file_name.lower()) - if plat_sys: - return plat_sys - # 2. Exact match if profile_sid in platform_sys_ids: return profile_sid - # 3. Normalized match - normed = _norm_system_id(profile_sid) - return norm_to_platform.get(normed, profile_sid) + return norm_to_platform.get(_norm_system_id(profile_sid), profile_sid) + + def _attribute(fe: dict, own: str | None, on_platform: set[str]) -> list[str]: + """The platform systems an entry belongs to, [] when nothing decides. + + A declaration of the same name settles the system only when it can + be the same file: a shared hash, or no contradicting size. fmsx's + 16 KB DISK.ROM is not quasi88's 2 KB disk.rom, and dosbox's SC-55 + ROM1.BIN is not galaksija's. The same content declared under two + systems is filed under both. + """ + verdicts = [ + (psid, _declaration_verdict(fe, decl, db)) + for psid, decl in declared_by_name.get(fe.get("name", "").lower(), []) + ] + same = sorted({psid for psid, v in verdicts if v == "same"}) + undecided = sorted({psid for psid, v in verdicts if v == "undecided"}) + if own is not None: + mapped = _map_sys_id(own) + if any(psid == mapped and v != "other" for psid, v in verdicts): + return [mapped] + if same: + return same + if len(undecided) == 1: + return undecided + return [mapped] + if same: + return same + if len(undecided) == 1: + return undecided + if undecided: + ours = [psid for psid in undecided if psid in on_platform] + return ours if len(ours) == 1 else [] + return [next(iter(on_platform))] if len(on_platform) == 1 else [] systems: dict[str, dict] = {} unattributed: dict[str, int] = {} @@ -480,34 +557,42 @@ def generate_platform_truth( profile_sid = fe.get("system", "") sys_ids = profile.get("systems", []) if profile_sid: - sys_id = _map_sys_id(profile_sid, fe.get("name", "")) - elif len(sys_ids) <= 1: - sys_id = _map_sys_id(sys_ids[0] if sys_ids else "unknown", fe.get("name", "")) + own: str | None = profile_sid + elif len(sys_ids) == 1: + own = sys_ids[0] + elif not sys_ids: + # A profile that names no system: a platform declaration of + # the same file files it, and the placeholder takes only a + # name the platform does not declare. mess2015's hiscore.dat + # beside the arcade one put an "unknown" section in System.dat. + own = None else: - # A multi-system profile's entry without a system: the - # platform's own declaration of the name decides, or the one + # A multi-system profile's entry without a system is filed by + # the platform's own declaration of the same file, or the one # profile system the platform has. Otherwise it is not filed: # the first system was a guess, and CLK's Apple, Mac, ZX and # Amiga ROMs went to RomM as Amstrad CPC firmware. - declared = file_to_plat_sys.get(fe.get("name", "").lower()) - if declared: - sys_id = declared - elif len(on_platform) == 1: - sys_id = next(iter(on_platform)) - else: - unattributed[emu_name] = unattributed.get(emu_name, 0) + 1 - continue - system = systems.setdefault(sys_id, {}) - _merge_file_into_system(system, fe, emu_name, db) - # Track core contribution per system - sys_cov = system_cores.setdefault( - sys_id, - { - "profiled": set(), - "unprofiled": set(), - }, - ) - sys_cov["profiled"].add(emu_name) + own = None + sys_targets = _attribute(fe, own, on_platform) + if not sys_ids and not profile_sid and not sys_targets \ + and fe.get("name", "").lower() not in declared_by_name: + sys_targets = [_map_sys_id("unknown")] + if not sys_targets: + unattributed[emu_name] = unattributed.get(emu_name, 0) + 1 + continue + for position, sys_id in enumerate(sys_targets): + system = systems.setdefault(sys_id, {}) + _merge_file_into_system( + system, fe if position == 0 else copy.deepcopy(fe), emu_name, db + ) + sys_cov = system_cores.setdefault( + sys_id, + { + "profiled": set(), + "unprofiled": set(), + }, + ) + sys_cov["profiled"].add(emu_name) # Ensure all systems of resolved cores have entries (even with 0 files). # This documents that the system is covered: the core was analyzed and diff --git a/tests/test_truth_layout.py b/tests/test_truth_layout.py index 50484680..1beaf92d 100644 --- a/tests/test_truth_layout.py +++ b/tests/test_truth_layout.py @@ -184,5 +184,115 @@ class EveryRevisionACoreAcceptsStays(unittest.TestCase): ) + +def _filed(truth: dict, name: str) -> list[str]: + """The systems an entry of that name was filed under.""" + return sorted( + sid for sid, system in truth.get("systems", {}).items() + for f in system.get("files", []) if f["name"] == name + ) + + +class ADeclarationOnlyAttributesTheSameFile(unittest.TestCase): + """A name the platform declares filed an entry under that system before + anything else, and the last declaration of a name won. fmsx's 16 KB + DISK.ROM went to nec-pc-88 for quasi88's 2 KB disk.rom, and dosbox's + SC-55 ROM1.BIN to galaksija, whatever their own system said.""" + + @staticmethod + def truth(platform_systems: dict, profiles: dict) -> dict: + config = {"cores": list(profiles), "systems": platform_systems} + return generate_platform_truth("p", config, {}, profiles) + + def test_a_same_named_rom_of_another_size_does_not_take_the_entry(self): + fmsx = {"type": "libretro", "systems": ["microsoft-msx", "microsoft-msx2"], + "files": [{"name": "DISK.ROM", "size": 16384}]} + truth = self.truth({ + "microsoft-msx": {"files": [{"name": "DISK.ROM", "size": 16384}]}, + "nec-pc-88": {"files": [{"name": "disk.rom", "size": 2048}]}, + }, {"fmsx": fmsx}) + self.assertEqual(_filed(truth, "DISK.ROM"), ["microsoft-msx"]) + + def test_a_refuted_declaration_leaves_the_profile_system(self): + dosbox = {"type": "libretro", "systems": ["dos"], + "files": [{"name": "ROM1.BIN", "size": 32768}]} + truth = self.truth({ + "dos": {"files": []}, + "galaksija": {"files": [{"name": "ROM1.BIN", "size": 4096}]}, + }, {"dosbox": dosbox}) + self.assertEqual(_filed(truth, "ROM1.BIN"), ["dos"]) + + def test_an_explicit_system_is_not_moved_by_a_homonym(self): + clk = {"type": "libretro", "systems": ["oric", "amstrad-cpc"], + "files": [{"name": "basic10.rom", "system": "oric", "size": 16384}]} + truth = self.truth({ + "elektronika-bk": {"files": [{"name": "BASIC10.ROM", "size": 24576}]}, + }, {"clk": clk}) + self.assertEqual(_filed(truth, "basic10.rom"), ["oric"]) + + def test_a_multi_system_entry_refuted_everywhere_is_unattributed(self): + oricutron = {"type": "libretro", "systems": ["oric", "oric-atmos"], + "files": [{"name": "basic10.rom", "max_size": 16384}]} + truth = self.truth({ + "elektronika-bk": {"files": [{"name": "BASIC10.ROM", "size": 24576}]}, + }, {"oricutron": oricutron}) + self.assertEqual(_filed(truth, "basic10.rom"), []) + self.assertEqual(truth["_coverage"]["unattributed"], {"oricutron": 1}) + + def test_a_rom_declared_under_two_systems_is_filed_under_both(self): + atari800 = {"type": "libretro", "systems": ["atari-400-800", "atari-5200"], + "files": [{"name": "ATARIXL.ROM", "md5": "a" * 32}]} + truth = self.truth({ + "atari-400-800": {"files": [{"name": "ATARIXL.ROM", "md5": "a" * 32}]}, + "atari-5200": {"files": [{"name": "ATARIXL.ROM", "md5": "a" * 32}]}, + }, {"atari800": atari800}) + self.assertEqual(_filed(truth, "ATARIXL.ROM"), ["atari-400-800", "atari-5200"]) + + def test_an_equal_size_revision_is_still_the_file(self): + """Two md5 at one size are two revisions of one slot, as slots.py + reads them: atari800's ATARIOSB.ROM against System.dat's.""" + atari800 = {"type": "libretro", "systems": ["atari-400-800", "atari-5200"], + "files": [{"name": "ATARIOSB.ROM", "md5": "a" * 32, "size": 10240}]} + truth = self.truth({ + "atari-400-800": {"files": [{"name": "ATARIOSB.ROM", "md5": "b" * 32, + "size": 10240}]}, + }, {"atari800": atari800}) + self.assertEqual(_filed(truth, "ATARIOSB.ROM"), ["atari-400-800"]) + + def test_a_systemless_profile_does_not_file_a_refuted_name_as_unknown(self): + """mess2015 declares no system. Its hiscore.dat is not the arcade + one the platform declares, and filing it under the placeholder put + an "unknown" section into System.dat.""" + mess = {"type": "libretro", "systems": [], + "files": [{"name": "hiscore.dat", "size": 216250}, + {"name": "only_mine.dat", "size": 10}]} + truth = self.truth({"arcade": {"files": [{"name": "hiscore.dat", "size": 324575}]}}, + {"mess2015": mess}) + self.assertEqual(_filed(truth, "hiscore.dat"), []) + self.assertEqual(_filed(truth, "only_mine.dat"), ["unknown"]) + + def test_a_bare_declaration_still_translates(self): + """The platform's own id for our slug, from a declaration that + states no content: nothing contradicts it.""" + pcsx = {"type": "libretro", "systems": ["sony-playstation"], + "files": [{"name": "scph1001.bin", "md5": "a" * 32}]} + truth = self.truth({"psx": {"files": [{"name": "scph1001.bin"}]}}, {"pcsx": pcsx}) + self.assertEqual(_filed(truth, "scph1001.bin"), ["psx"]) + + +class ASizeOutsideARangeIsAnotherFile(unittest.TestCase): + def test_two_cores_do_not_merge_across_a_declared_range(self): + """bk's 24 KB BASIC10.ROM and oricutron's basic10.rom (max_size + 16384) merged into one entry, which took the Oric md5.""" + from truth import _merge_file_into_system # noqa: PLC0415 + + system: dict = {} + _merge_file_into_system(system, {"name": "BASIC10.ROM", "size": 24576, + "md5": "a" * 32}, "bk", None) + _merge_file_into_system(system, {"name": "basic10.rom", "max_size": 16384}, + "oricutron", None) + self.assertEqual(len(system["files"]), 2) + + if __name__ == "__main__": unittest.main()