fix: count a catalogued dump once across its DATs

Several DATs list the same dump, as No-Intro's three FM-7 sets do
for their system ROMs, and the provenance report counted it once per
listing: 2485 missing announced for 2284 distinct dumps, on the site
as well. Dumps are now counted by identity, each keeping the DATs
that list it.

scripts/generate_site.py

@@ -65,7 +65,7 @@ from generate_readme import (
     release_totals,
 )
 from profile_sync import source_ref_values, split_source_ref
-from provenance_report import build_report
+from provenance_report import build_report, distinct_missing
 import release_record
 import upstream
 
@@ -1774,7 +1774,7 @@ def generate_provenance_page(db: dict, report: dict) -> str:
             ]
         )
 
-    total_missing = sum(len(d["missing"]) for d in report.values())
+    total_missing = distinct_missing(report)
     lines.extend(
         [
             "## Wanted: catalogued dumps this project does not have",
@@ -1795,7 +1795,8 @@ def generate_provenance_page(db: dict, report: dict) -> str:
         label = _PROVENANCE_LABELS.get(source, source)
         by_dat: dict[str, list] = {}
         for entry in data["missing"]:
-            by_dat.setdefault(entry.get("dat", ""), []).append(entry)
+            for dat in entry.get("dats") or [entry.get("dat", "")]:
+                by_dat.setdefault(dat, []).append(entry)
         lines.extend([f"### {label}", ""])
         for dat in sorted(by_dat):
             entries = by_dat[dat]

scripts/provenance_report.py

@@ -73,6 +73,22 @@ def _held_in_archive(entry: dict, members: dict) -> bool:
     return False
 
 
+def dump_identity(entry: dict) -> tuple:
+    """What makes two catalog listings one dump: the strongest hash stated."""
+    if entry.get("sha1"):
+        return ("sha1", str(entry["sha1"]).lower())
+    if entry.get("md5"):
+        return ("md5", str(entry["md5"]).lower(), entry.get("size"))
+    if entry.get("crc32"):
+        return ("crc32", str(entry["crc32"]).lower(), entry.get("size"))
+    return ("name", entry.get("dat", ""), entry.get("name", ""))
+
+
+def distinct_missing(report: dict) -> int:
+    """Missing dumps across every catalog, each counted once."""
+    return len({dump_identity(e) for data in report.values() for e in data["missing"]})
+
+
 def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict:
     """Compare each snapshot against the collection.
 
@@ -101,7 +117,7 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict
 
     report = {}
     for source, snapshot in sorted(snapshots.items()):
-        matched = 0
+        matched: set = set()
         covered_dats = set()
         unmatched = []
         for entry in snapshot["entries"]:
@@ -111,12 +127,27 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict
                 or (str(entry.get("crc32") or "").lower(), entry.get("size")) in by_crc32_size
                 or _held_in_archive(entry, members)
             ):
-                matched += 1
+                matched.add(dump_identity(entry))
                 covered_dats.add(entry.get("dat", ""))
             else:
                 unmatched.append(entry)
-        missing = [e for e in unmatched if e.get("dat", "") in covered_dats]
-        out_of_scope = len(unmatched) - len(missing)
+        # One dump listed by several DATs (No-Intro's three FM-7 DATs share
+        # DICROM.rom, KANJI.rom...) is one dump: counted per listing, the
+        # report announced 2 485 missing for 2 284 distinct ones.
+        merged: dict = {}
+        for entry in unmatched:
+            if entry.get("dat", "") not in covered_dats:
+                continue
+            held = merged.get(dump_identity(entry))
+            if held is None:
+                merged[dump_identity(entry)] = {**entry, "dats": [entry.get("dat", "")]}
+            elif entry.get("dat", "") not in held["dats"]:
+                held["dats"].append(entry.get("dat", ""))
+        missing = list(merged.values())
+        out_of_scope = len({
+            dump_identity(e) for e in unmatched if e.get("dat", "") not in covered_dats
+        })
+        matched = len(matched)
         report[source] = {
             "imported_at": snapshot.get("imported_at", ""),
             "dats": snapshot.get("dats", {}),
@@ -192,9 +223,9 @@ def main() -> int:
                 else:
                     print(f"    MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}")
 
-    total_missing = sum(len(d["missing"]) for d in report.values())
+    total_missing = distinct_missing(report)
     if total_missing:
-        line = f"  {total_missing} catalog entries missing from collection"
+        line = f"  {total_missing} catalogued dumps missing from collection"
         if args.derive:
             line += f", {len(derived)} of them derivable from held dumps"
         print(line)

tests/test_provenance.py

@@ -510,5 +510,29 @@ class ArchiveMembersAreHeld(unittest.TestCase):
         self.assertEqual(report["no-intro"]["matched"], 1)
         self.assertEqual([e["name"] for e in report["no-intro"]["missing"]], ["other.bin"])
 
+class OneDumpIsCountedOnce(unittest.TestCase):
+    """No-Intro's three FM-7 DATs each list DICROM.rom and KANJI.rom: the
+    report said 2 485 missing where 2 284 distinct dumps were missing, and
+    the site's Wanted page published the larger number."""
+
+    def test_a_dump_in_several_dats_is_one_missing_entry(self):
+        from scripts import provenance_report  # noqa: PLC0415
+
+        rom = {"name": "DICROM.rom", "sha1": "a" * 40, "size": 32768}
+        held = {"name": "held.rom", "sha1": "b" * 40, "size": 16}
+        snapshots = {"no-intro": {"entries": [
+            {**rom, "dat": "FM-7 (Bitstream)"}, {**held, "dat": "FM-7 (Bitstream)"},
+            {**rom, "dat": "FM-7 (Flux)"}, {**held, "dat": "FM-7 (Flux)"},
+        ]}}
+        db = {"files": {"b" * 40: {"md5": "", "size": 16}}}
+        report = provenance_report.build_report(db, snapshots, {})
+        (entry,) = report["no-intro"]["missing"]
+        self.assertEqual(entry["dats"], ["FM-7 (Bitstream)", "FM-7 (Flux)"])
+        self.assertEqual(report["no-intro"]["matched"], 1)
+        self.assertEqual(provenance_report.distinct_missing(report), 1)
+        twice = {"redump": report["no-intro"], "no-intro": report["no-intro"]}
+        self.assertEqual(provenance_report.distinct_missing(twice), 1)
+
+
 if __name__ == "__main__":
     unittest.main()