diff --git a/scripts/generate_site.py b/scripts/generate_site.py index eaef8450..6a931ac7 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -65,7 +65,7 @@ from generate_readme import ( release_totals, ) from profile_sync import source_ref_values, split_source_ref -from provenance_report import build_report +from provenance_report import build_report, distinct_missing import release_record import upstream @@ -1774,7 +1774,7 @@ def generate_provenance_page(db: dict, report: dict) -> str: ] ) - total_missing = sum(len(d["missing"]) for d in report.values()) + total_missing = distinct_missing(report) lines.extend( [ "## Wanted: catalogued dumps this project does not have", @@ -1795,7 +1795,8 @@ def generate_provenance_page(db: dict, report: dict) -> str: label = _PROVENANCE_LABELS.get(source, source) by_dat: dict[str, list] = {} for entry in data["missing"]: - by_dat.setdefault(entry.get("dat", ""), []).append(entry) + for dat in entry.get("dats") or [entry.get("dat", "")]: + by_dat.setdefault(dat, []).append(entry) lines.extend([f"### {label}", ""]) for dat in sorted(by_dat): entries = by_dat[dat] diff --git a/scripts/provenance_report.py b/scripts/provenance_report.py index 83cb33d5..fd93a067 100644 --- a/scripts/provenance_report.py +++ b/scripts/provenance_report.py @@ -73,6 +73,22 @@ def _held_in_archive(entry: dict, members: dict) -> bool: return False +def dump_identity(entry: dict) -> tuple: + """What makes two catalog listings one dump: the strongest hash stated.""" + if entry.get("sha1"): + return ("sha1", str(entry["sha1"]).lower()) + if entry.get("md5"): + return ("md5", str(entry["md5"]).lower(), entry.get("size")) + if entry.get("crc32"): + return ("crc32", str(entry["crc32"]).lower(), entry.get("size")) + return ("name", entry.get("dat", ""), entry.get("name", "")) + + +def distinct_missing(report: dict) -> int: + """Missing dumps across every catalog, each counted once.""" + return len({dump_identity(e) for data in report.values() for e in data["missing"]}) + + def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict: """Compare each snapshot against the collection. @@ -101,7 +117,7 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict report = {} for source, snapshot in sorted(snapshots.items()): - matched = 0 + matched: set = set() covered_dats = set() unmatched = [] for entry in snapshot["entries"]: @@ -111,12 +127,27 @@ def build_report(db: dict, snapshots: dict, members: dict | None = None) -> dict or (str(entry.get("crc32") or "").lower(), entry.get("size")) in by_crc32_size or _held_in_archive(entry, members) ): - matched += 1 + matched.add(dump_identity(entry)) covered_dats.add(entry.get("dat", "")) else: unmatched.append(entry) - missing = [e for e in unmatched if e.get("dat", "") in covered_dats] - out_of_scope = len(unmatched) - len(missing) + # One dump listed by several DATs (No-Intro's three FM-7 DATs share + # DICROM.rom, KANJI.rom...) is one dump: counted per listing, the + # report announced 2 485 missing for 2 284 distinct ones. + merged: dict = {} + for entry in unmatched: + if entry.get("dat", "") not in covered_dats: + continue + held = merged.get(dump_identity(entry)) + if held is None: + merged[dump_identity(entry)] = {**entry, "dats": [entry.get("dat", "")]} + elif entry.get("dat", "") not in held["dats"]: + held["dats"].append(entry.get("dat", "")) + missing = list(merged.values()) + out_of_scope = len({ + dump_identity(e) for e in unmatched if e.get("dat", "") not in covered_dats + }) + matched = len(matched) report[source] = { "imported_at": snapshot.get("imported_at", ""), "dats": snapshot.get("dats", {}), @@ -192,9 +223,9 @@ def main() -> int: else: print(f" MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}") - total_missing = sum(len(d["missing"]) for d in report.values()) + total_missing = distinct_missing(report) if total_missing: - line = f" {total_missing} catalog entries missing from collection" + line = f" {total_missing} catalogued dumps missing from collection" if args.derive: line += f", {len(derived)} of them derivable from held dumps" print(line) diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 4ef006b8..7cdab183 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -510,5 +510,29 @@ class ArchiveMembersAreHeld(unittest.TestCase): self.assertEqual(report["no-intro"]["matched"], 1) self.assertEqual([e["name"] for e in report["no-intro"]["missing"]], ["other.bin"]) +class OneDumpIsCountedOnce(unittest.TestCase): + """No-Intro's three FM-7 DATs each list DICROM.rom and KANJI.rom: the + report said 2 485 missing where 2 284 distinct dumps were missing, and + the site's Wanted page published the larger number.""" + + def test_a_dump_in_several_dats_is_one_missing_entry(self): + from scripts import provenance_report # noqa: PLC0415 + + rom = {"name": "DICROM.rom", "sha1": "a" * 40, "size": 32768} + held = {"name": "held.rom", "sha1": "b" * 40, "size": 16} + snapshots = {"no-intro": {"entries": [ + {**rom, "dat": "FM-7 (Bitstream)"}, {**held, "dat": "FM-7 (Bitstream)"}, + {**rom, "dat": "FM-7 (Flux)"}, {**held, "dat": "FM-7 (Flux)"}, + ]}} + db = {"files": {"b" * 40: {"md5": "", "size": 16}}} + report = provenance_report.build_report(db, snapshots, {}) + (entry,) = report["no-intro"]["missing"] + self.assertEqual(entry["dats"], ["FM-7 (Bitstream)", "FM-7 (Flux)"]) + self.assertEqual(report["no-intro"]["matched"], 1) + self.assertEqual(provenance_report.distinct_missing(report), 1) + twice = {"redump": report["no-intro"], "no-intro": report["no-intro"]} + self.assertEqual(provenance_report.distinct_missing(twice), 1) + + if __name__ == "__main__": unittest.main()