diff --git a/scripts/cross_reference.py b/scripts/cross_reference.py index 2967cfde..a0fc5241 100644 --- a/scripts/cross_reference.py +++ b/scripts/cross_reference.py @@ -427,6 +427,17 @@ def _resolve_archive_source( return result if result is not None else "missing" +def _first_hash(entry: dict) -> str: + """The first hash an entry declares, strongest first, or ''.""" + for field in ("sha1", "sha256", "md5", "crc32"): + value = entry.get(field) + if isinstance(value, list): + value = value[0] if value else "" + if value: + return f"{field}:{str(value).lower()}" + return "" + + def slot_key(f: dict) -> tuple: """The slot a profile entry fills: its entries on one slot are the versions its code accepts there, and holding any of them fills it.""" @@ -591,6 +602,8 @@ def _cross_reference_profile( "in_platform": False, "in_repo": in_repo, "source": source, + # What tells this file from a homonym another emulator wants. + "identity": _first_hash(f), } gaps.append(entry) diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 824b848a..2ef9e64e 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2979,25 +2979,44 @@ def generate_gap_analysis( for emu_name in matched: emu_to_platforms.setdefault(emu_name, set()).add(pname) - all_src_missing: set[str] = set() + # One row per file to find: a name two emulators share is one file + # only when both declare the same hash. Keyed by name alone, Fallout + # 2's master.dat vanished behind Fallout 1's, and the three Sonic + # decompilations' Data.rsdk showed as one. src_missing_details: list[dict] = [] + by_identity: dict[tuple[str, str], dict] = {} + entries = 0 for emu_name, data in emulator_gaps: for g in data["gap_details"]: - if g["source"] == "missing" and g["name"] not in all_src_missing: - all_src_missing.add(g["name"]) - src_missing_details.append({ + if g["source"] != "missing": + continue + entries += 1 + key = (g["name"], g.get("identity") or f"emulator:{emu_name}") + row = by_identity.get(key) + if row is None: + row = by_identity[key] = { "name": g["name"], - "emulator": data["emulator"], - "emu_key": emu_name, + "emulators": [data["emulator"]], + "emu_keys": [emu_name], "required": g["required"], "source_ref": g["source_ref"], - }) + } + src_missing_details.append(row) + elif emu_name not in row["emu_keys"]: + row["emulators"].append(data["emulator"]) + row["emu_keys"].append(emu_name) + row["required"] = row["required"] or g["required"] if src_missing_details: req_src = [m for m in src_missing_details if m["required"]] lines.extend([ - f"### Files to Source ({len(src_missing_details)} unique, " - f"{len(req_src)} required)", + f"### Files to Source ({len(src_missing_details)} files, " + f"{len(req_src)} required" + + ( + f"; {entries} entries, a file declared with the same hash " + "at several places is listed once)" + if entries != len(src_missing_details) else ")" + ), "", "| File | Emulator | Required | Affects platforms | Source ref |", "|------|----------|----------|------------------|-----------|", @@ -3006,7 +3025,9 @@ def generate_gap_analysis( src_missing_details, key=lambda x: (not x["required"], x["name"]), ): - plats = sorted(emu_to_platforms.get(m["emu_key"], set())) + plats = sorted(set().union( + *(emu_to_platforms.get(key, set()) for key in m["emu_keys"]) + )) plat_badges = ( " ".join( f'{p}' @@ -3017,9 +3038,9 @@ def generate_gap_analysis( ) req = "yes" if m["required"] else "no" lines.append( - f"| `{m['name']}` | {m['emulator']} | {req} | " + f"| `{m['name']}` | {', '.join(m['emulators'])} | {req} | " f"{plat_badges} | " - f"{_source_ref_markdown(profiles[m['emu_key']], m['source_ref'])} |" + f"{_source_ref_markdown(profiles[m['emu_keys'][0]], m['source_ref'])} |" ) lines.append("") diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 1861a92a..14fedc5e 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -3569,5 +3569,20 @@ class TheGapReportKeepsItsShape(unittest.TestCase): self.assertIn("across the emulators batocera runs", text) + +class AFileToSourceIsAFileNotAName(unittest.TestCase): + """The gaps page keyed acquisition targets by name: Fallout 2's + master.dat disappeared behind Fallout 1's.""" + + def test_rows_are_keyed_by_name_and_hash(self): + source = (ROOT / "scripts" / "generate_site.py").read_text() + self.assertIn('key = (g["name"], g.get("identity") or f"emulator:{emu_name}")', source) + from scripts.cross_reference import _first_hash + + self.assertEqual(_first_hash({"md5": ["AB", "cd"]}), "md5:ab") + self.assertEqual(_first_hash({"crc32": "1", "sha1": "F" * 40}), "sha1:" + "f" * 40) + self.assertEqual(_first_hash({}), "") + + if __name__ == "__main__": unittest.main()