diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 7a3e43d8..03ca6935 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -2091,6 +2091,9 @@ def load_derivable(path: str = "derivable.json") -> dict[str, dict]: return {} +_ROMSET_SOURCES = {"mame": "MAME", "fbneo": "FBNeo"} + + def generate_provenance_page( db: dict, report: dict, derivable: dict[str, dict] | None = None ) -> str: @@ -2228,6 +2231,12 @@ def generate_provenance_page( # A pipe in catalog text would split the markdown table row name = entry["name"].replace("|", "-") desc = (entry.get("description") or "").replace("|", "-") + # Where an emulator's romset already carries the dump. + for hit in entry.get("romsets", []): + desc += ( + f" Ships in {_ROMSET_SOURCES.get(hit['source'], hit['source'])} " + f"`{hit['archive']}` as `{hit['member']}`." + ) lines.append(f" | `{name}` | {desc} | `{sha1}` |") lines.append("") diff --git a/scripts/provenance_report.py b/scripts/provenance_report.py index 42c549f1..5ab713a5 100644 --- a/scripts/provenance_report.py +++ b/scripts/provenance_report.py @@ -20,6 +20,7 @@ import json import os import sys import zipfile +from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from common import ( @@ -78,6 +79,59 @@ def _held_in_archive(entry: dict, members: dict) -> bool: return False +RECIPES_DIR = Path(__file__).resolve().parent.parent / "recipes" + + +def romset_index(recipes_dir: Path = RECIPES_DIR) -> dict[tuple[str, int], list[dict]]: + """(crc32, size) of every member of an imported romset -> where it sits. + + A dump the collection lacks may still ship inside an emulator's romset: + 17 TOSEC ZX Spectrum ROMs are members of MAME's spectrum.zip. Naming the + set and the versions that carry it turns a dead end into a download. + """ + index: dict[tuple[str, int], list[dict]] = {} + for snapshot in sorted(recipes_dir.glob("*.json")): + with snapshot.open(encoding="utf-8") as handle: + data = json.load(handle) + source = data.get("source", snapshot.stem) + for entry in data.get("entries", []): + dats = entry.get("dats") or [entry.get("dat", "")] + for member in entry.get("members", []) or []: + crc, size = str(member.get("crc32") or "").lower(), member.get("size") + if not crc or size is None: + continue + index.setdefault((crc.zfill(8), int(size)), []).append({ + "source": source, + "archive": entry.get("name", ""), + "member": member.get("name", ""), + "sha1": member.get("sha1", ""), + "dats": dats, + }) + return index + + +def romsets_holding(entry: dict, index: dict) -> list[dict]: + """The imported romsets whose members are this dump. + + A member and an entry that both carry a sha1 must agree on it: crc32 + and size alone match by accident across 40 000 ROMs. + """ + crc, size = str(entry.get("crc32") or "").lower(), entry.get("size") + if not crc or size is None: + return [] + sha1 = str(entry.get("sha1") or "").lower() + # A set whose contents changed between versions is stored once per + # distinct content: one line per set and member, with every version. + merged: dict[tuple[str, str, str], dict] = {} + for hit in index.get((crc.zfill(8), int(size)), []): + if sha1 and hit["sha1"] and hit["sha1"].lower() != sha1: + continue + key = (hit["source"], hit["archive"], hit["member"]) + held = merged.setdefault(key, {**hit, "dats": []}) + held["dats"] = sorted(set(held["dats"]) | set(hit["dats"])) + return list(merged.values()) + + def dump_identity(entry: dict) -> tuple: """What makes two catalog listings one dump: the strongest hash stated.""" if entry.get("sha1"): @@ -94,7 +148,12 @@ def distinct_missing(report: dict) -> int: return len({dump_identity(e) for data in report.values() for e in data["missing"]}) -def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict: +def build_report( + db: dict, + snapshots: dict, + members: dict | None = None, + romsets: dict | None = None, +) -> dict: """Compare each snapshot against the collection. A DAT counts as covered when the collection holds at least one of @@ -109,6 +168,7 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict # its Wanted page without them and asked the public for the Astrocade # and Gamate BIOS that astrocdw.zip and gamate.zip already carry. members = archive_members(db) if members is None else members + romsets = romset_index() if romsets is None else romsets by_md5_size = { (entry.get("md5", ""), entry.get("size", 0)) for entry in by_sha1.values() } @@ -149,6 +209,10 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict elif entry.get("dat", "") not in held["dats"]: held["dats"].append(entry.get("dat", "")) missing = list(merged.values()) + for entry in missing: + holders = romsets_holding(entry, romsets) + if holders: + entry["romsets"] = holders out_of_scope = len({ dump_identity(e) for e in unmatched if e.get("dat", "") not in covered_dats }) @@ -243,6 +307,12 @@ def main() -> int: f"of {' + '.join(source['from'])}") else: print(f" MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}") + for hit in entry.get("romsets", []): + versions = hit["dats"][0] if len(hit["dats"]) == 1 else ( + f"{hit['dats'][0]} to {hit['dats'][-1]}" + ) + print(f" in {hit['source']} {hit['archive']} as " + f"{hit['member']} ({versions})") total_missing = distinct_missing(report) if total_missing: diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 8b33b6c6..ab0917ca 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -477,6 +477,26 @@ class TestProvenancePage(unittest.TestCase): self.assertEqual(row.count("|"), 4) self.assertIn("has - pipe", row) + def test_a_romset_carrying_the_dump_is_named(self): + extra = [{ + "dat": "Sony - PlayStation - BIOS Images", "name": "carried.bin", + "description": "kept in a set", "size": 4, "crc32": "0000beef", + "md5": "88888888888888888888888888888888", + "sha1": "8888888888888888888888888888888888888888", + }] + romsets = {("0000beef", 4): [{ + "source": "mame", "archive": "set.zip", "member": "c.rom", + "sha1": "", "dats": ["MAME mame0190"], + }]} + db = {"files": {"b05def971d8ec59f346f2d9ac21fb742e3eb6917": { + "name": "scph5500.bin", "size": 524288, + "md5": "8dd7d5296a650fac7319bce665a6a53c"}}} + snapshots = {"redump": {"source": "redump", "imported_at": "", "dats": {}, + "entries": _snapshot_entries() + extra}} + page = generate_provenance_page(db, build_report(db, snapshots, {}, romsets)) + row = next(line for line in page.split("\n") if "carried.bin" in line) + self.assertIn("Ships in MAME `set.zip` as `c.rom`.", row) + def test_page_without_any_match(self): page = self._page(provenance=False) self.assertIn("**0** of 2 system files", page) @@ -560,5 +580,41 @@ class TheWantedPageAsksOnlyForWhatCannotBeMade(unittest.TestCase): self.assertIn("byte swap", derived) +class MissingDumpsNameTheirRomsets(unittest.TestCase): + """17 TOSEC ZX Spectrum ROMs listed as missing ship in MAME's + spectrum.zip: the report said MISSING and nothing about where to look.""" + + def _index(self, entries: list[dict]) -> dict: + from scripts.provenance_report import romset_index + + with tempfile.TemporaryDirectory() as tmp: + Path(tmp, "mame.json").write_text( + json.dumps({"source": "mame", "entries": entries}), encoding="utf-8" + ) + return romset_index(Path(tmp)) + + def test_versions_of_one_set_merge(self): + from scripts.provenance_report import romsets_holding + + member = {"name": "1986es.rom", "size": 16384, "crc32": "aaaa0001", "sha1": "f9"} + index = self._index([ + {"name": "spectrum.zip", "members": [member], "dats": ["MAME mame0190", "MAME mame0220"]}, + {"name": "spectrum.zip", "members": [member, {**member, "name": "x.rom", "crc32": "b"}], + "dats": ["MAME mame0289"]}, + ]) + hits = romsets_holding({"crc32": "AAAA0001", "size": 16384, "sha1": "F9"}, index) + self.assertEqual(len(hits), 1) + self.assertEqual(hits[0]["archive"], "spectrum.zip") + self.assertEqual(hits[0]["dats"], ["MAME mame0190", "MAME mame0220", "MAME mame0289"]) + + def test_a_disagreeing_sha1_is_not_the_dump(self): + from scripts.provenance_report import romsets_holding + + member = {"name": "a.rom", "size": 8, "crc32": "00000001", "sha1": "aa"} + index = self._index([{"name": "set.zip", "members": [member], "dats": ["MAME mame0190"]}]) + self.assertEqual(romsets_holding({"crc32": "00000001", "size": 8, "sha1": "bb"}, index), []) + self.assertEqual(len(romsets_holding({"crc32": "00000001", "size": 8}, index)), 1) + + if __name__ == "__main__": unittest.main()