feat: name the romset that carries a wanted dump

17 TOSEC ZX Spectrum ROMs reported missing ship inside MAME's spectrum.zip from 0.190 to 0.289, while the collection holds only FBNeo's one-member archive of that name. provenance_report --missing and the site's wanted list now name the set, the member and the versions, matched by crc32 and size and by sha1 where both sides carry one.

scripts/generate_site.py

@@ -2091,6 +2091,9 @@ def load_derivable(path: str = "derivable.json") -> dict[str, dict]:
         return {}
 
 
+_ROMSET_SOURCES = {"mame": "MAME", "fbneo": "FBNeo"}
+
+
 def generate_provenance_page(
     db: dict, report: dict, derivable: dict[str, dict] | None = None
 ) -> str:
@@ -2228,6 +2231,12 @@ def generate_provenance_page(
                 # A pipe in catalog text would split the markdown table row
                 name = entry["name"].replace("|", "-")
                 desc = (entry.get("description") or "").replace("|", "-")
+                # Where an emulator's romset already carries the dump.
+                for hit in entry.get("romsets", []):
+                    desc += (
+                        f" Ships in {_ROMSET_SOURCES.get(hit['source'], hit['source'])} "
+                        f"`{hit['archive']}` as `{hit['member']}`."
+                    )
                 lines.append(f"    | `{name}` | {desc} | `{sha1}` |")
             lines.append("")
 

scripts/provenance_report.py

@@ -20,6 +20,7 @@ import json
 import os
 import sys
 import zipfile
+from pathlib import Path
 
 sys.path.insert(0, os.path.dirname(__file__))
 from common import (
@@ -78,6 +79,59 @@ def _held_in_archive(entry: dict, members: dict) -> bool:
     return False
 
 
+RECIPES_DIR = Path(__file__).resolve().parent.parent / "recipes"
+
+
+def romset_index(recipes_dir: Path = RECIPES_DIR) -> dict[tuple[str, int], list[dict]]:
+    """(crc32, size) of every member of an imported romset -> where it sits.
+
+    A dump the collection lacks may still ship inside an emulator's romset:
+    17 TOSEC ZX Spectrum ROMs are members of MAME's spectrum.zip. Naming the
+    set and the versions that carry it turns a dead end into a download.
+    """
+    index: dict[tuple[str, int], list[dict]] = {}
+    for snapshot in sorted(recipes_dir.glob("*.json")):
+        with snapshot.open(encoding="utf-8") as handle:
+            data = json.load(handle)
+        source = data.get("source", snapshot.stem)
+        for entry in data.get("entries", []):
+            dats = entry.get("dats") or [entry.get("dat", "")]
+            for member in entry.get("members", []) or []:
+                crc, size = str(member.get("crc32") or "").lower(), member.get("size")
+                if not crc or size is None:
+                    continue
+                index.setdefault((crc.zfill(8), int(size)), []).append({
+                    "source": source,
+                    "archive": entry.get("name", ""),
+                    "member": member.get("name", ""),
+                    "sha1": member.get("sha1", ""),
+                    "dats": dats,
+                })
+    return index
+
+
+def romsets_holding(entry: dict, index: dict) -> list[dict]:
+    """The imported romsets whose members are this dump.
+
+    A member and an entry that both carry a sha1 must agree on it: crc32
+    and size alone match by accident across 40 000 ROMs.
+    """
+    crc, size = str(entry.get("crc32") or "").lower(), entry.get("size")
+    if not crc or size is None:
+        return []
+    sha1 = str(entry.get("sha1") or "").lower()
+    # A set whose contents changed between versions is stored once per
+    # distinct content: one line per set and member, with every version.
+    merged: dict[tuple[str, str, str], dict] = {}
+    for hit in index.get((crc.zfill(8), int(size)), []):
+        if sha1 and hit["sha1"] and hit["sha1"].lower() != sha1:
+            continue
+        key = (hit["source"], hit["archive"], hit["member"])
+        held = merged.setdefault(key, {**hit, "dats": []})
+        held["dats"] = sorted(set(held["dats"]) | set(hit["dats"]))
+    return list(merged.values())
+
+
 def dump_identity(entry: dict) -> tuple:
     """What makes two catalog listings one dump: the strongest hash stated."""
     if entry.get("sha1"):
@@ -94,7 +148,12 @@ def distinct_missing(report: dict) -> int:
     return len({dump_identity(e) for data in report.values() for e in data["missing"]})
 
 
-def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict:
+def build_report(
+    db: dict,
+    snapshots: dict,
+    members: dict | None = None,
+    romsets: dict | None = None,
+) -> dict:
     """Compare each snapshot against the collection.
 
     A DAT counts as covered when the collection holds at least one of
@@ -109,6 +168,7 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict
     # its Wanted page without them and asked the public for the Astrocade
     # and Gamate BIOS that astrocdw.zip and gamate.zip already carry.
     members = archive_members(db) if members is None else members
+    romsets = romset_index() if romsets is None else romsets
     by_md5_size = {
         (entry.get("md5", ""), entry.get("size", 0)) for entry in by_sha1.values()
     }
@@ -149,6 +209,10 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict
             elif entry.get("dat", "") not in held["dats"]:
                 held["dats"].append(entry.get("dat", ""))
         missing = list(merged.values())
+        for entry in missing:
+            holders = romsets_holding(entry, romsets)
+            if holders:
+                entry["romsets"] = holders
         out_of_scope = len({
             dump_identity(e) for e in unmatched if e.get("dat", "") not in covered_dats
         })
@@ -243,6 +307,12 @@ def main() -> int:
                           f"of {' + '.join(source['from'])}")
                 else:
                     print(f"    MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}")
+                for hit in entry.get("romsets", []):
+                    versions = hit["dats"][0] if len(hit["dats"]) == 1 else (
+                        f"{hit['dats'][0]} to {hit['dats'][-1]}"
+                    )
+                    print(f"      in {hit['source']} {hit['archive']} as "
+                          f"{hit['member']} ({versions})")
 
     total_missing = distinct_missing(report)
     if total_missing:

tests/test_provenance.py

@@ -477,6 +477,26 @@ class TestProvenancePage(unittest.TestCase):
         self.assertEqual(row.count("|"), 4)
         self.assertIn("has - pipe", row)
 
+    def test_a_romset_carrying_the_dump_is_named(self):
+        extra = [{
+            "dat": "Sony - PlayStation - BIOS Images", "name": "carried.bin",
+            "description": "kept in a set", "size": 4, "crc32": "0000beef",
+            "md5": "88888888888888888888888888888888",
+            "sha1": "8888888888888888888888888888888888888888",
+        }]
+        romsets = {("0000beef", 4): [{
+            "source": "mame", "archive": "set.zip", "member": "c.rom",
+            "sha1": "", "dats": ["MAME mame0190"],
+        }]}
+        db = {"files": {"b05def971d8ec59f346f2d9ac21fb742e3eb6917": {
+            "name": "scph5500.bin", "size": 524288,
+            "md5": "8dd7d5296a650fac7319bce665a6a53c"}}}
+        snapshots = {"redump": {"source": "redump", "imported_at": "", "dats": {},
+                                "entries": _snapshot_entries() + extra}}
+        page = generate_provenance_page(db, build_report(db, snapshots, {}, romsets))
+        row = next(line for line in page.split("\n") if "carried.bin" in line)
+        self.assertIn("Ships in MAME `set.zip` as `c.rom`.", row)
+
     def test_page_without_any_match(self):
         page = self._page(provenance=False)
         self.assertIn("**0** of 2 system files", page)
@@ -560,5 +580,41 @@ class TheWantedPageAsksOnlyForWhatCannotBeMade(unittest.TestCase):
         self.assertIn("byte swap", derived)
 
 
+class MissingDumpsNameTheirRomsets(unittest.TestCase):
+    """17 TOSEC ZX Spectrum ROMs listed as missing ship in MAME's
+    spectrum.zip: the report said MISSING and nothing about where to look."""
+
+    def _index(self, entries: list[dict]) -> dict:
+        from scripts.provenance_report import romset_index
+
+        with tempfile.TemporaryDirectory() as tmp:
+            Path(tmp, "mame.json").write_text(
+                json.dumps({"source": "mame", "entries": entries}), encoding="utf-8"
+            )
+            return romset_index(Path(tmp))
+
+    def test_versions_of_one_set_merge(self):
+        from scripts.provenance_report import romsets_holding
+
+        member = {"name": "1986es.rom", "size": 16384, "crc32": "aaaa0001", "sha1": "f9"}
+        index = self._index([
+            {"name": "spectrum.zip", "members": [member], "dats": ["MAME mame0190", "MAME mame0220"]},
+            {"name": "spectrum.zip", "members": [member, {**member, "name": "x.rom", "crc32": "b"}],
+             "dats": ["MAME mame0289"]},
+        ])
+        hits = romsets_holding({"crc32": "AAAA0001", "size": 16384, "sha1": "F9"}, index)
+        self.assertEqual(len(hits), 1)
+        self.assertEqual(hits[0]["archive"], "spectrum.zip")
+        self.assertEqual(hits[0]["dats"], ["MAME mame0190", "MAME mame0220", "MAME mame0289"])
+
+    def test_a_disagreeing_sha1_is_not_the_dump(self):
+        from scripts.provenance_report import romsets_holding
+
+        member = {"name": "a.rom", "size": 8, "crc32": "00000001", "sha1": "aa"}
+        index = self._index([{"name": "set.zip", "members": [member], "dats": ["MAME mame0190"]}])
+        self.assertEqual(romsets_holding({"crc32": "00000001", "size": 8, "sha1": "bb"}, index), [])
+        self.assertEqual(len(romsets_holding({"crc32": "00000001", "size": 8}, index)), 1)
+
+
 if __name__ == "__main__":
     unittest.main()