feat: name the profile a missing dump would fill

The acquisition report never read a profile, so XM7's required DICROM.ROM
sat among 2 287 targets at the rank of dumps no core reads. A missing
entry that a profile reads and the collection lacks is now listed first
with the profile, matched by hash or, for a hashless entry with a size,
by name and size. 29 entries on the current snapshots.

scripts/provenance_report.py

@@ -10,6 +10,10 @@ Usage:
     python scripts/provenance_report.py
     python scripts/provenance_report.py --missing
     python scripts/provenance_report.py --json
+
+A missing dump that is a file an emulator profile reads and the collection
+lacks is named with that profile: by hash when the profile declares one, by
+name and size when it declares a size and no hash.
 """
 
 from __future__ import annotations
@@ -26,10 +30,117 @@ sys.path.insert(0, os.path.dirname(__file__))
 from common import (
     DEFAULT_PROVENANCE_DIR,
     load_database,
+    load_emulator_profiles,
     load_provenance_snapshots,
     write_if_changed,
 )
 
+HASH_FIELDS = ("sha1", "md5", "sha256")
+
+
+def _values(entry: dict, field: str) -> set[str]:
+    """Every value a profile or catalog entry gives for a hash field."""
+    value = entry.get(field)
+    values = value if isinstance(value, list) else str(value or "").split(",")
+    return {str(v).strip().lower() for v in values if str(v).strip()}
+
+
+def _crc_key(value: str) -> str:
+    return value.removeprefix("0x").zfill(8)
+
+
+def _fits(size: int | None, entry: dict) -> bool:
+    """Whether a size meets what a profile entry states about it."""
+    if size is None:
+        return False
+    declared = entry.get("size")
+    if declared is not None:
+        accepted = declared if isinstance(declared, list) else [declared]
+        return size in {int(value) for value in accepted}
+    low, high = entry.get("min_size"), entry.get("max_size")
+    return (low is None or size >= int(low)) and (high is None or size <= int(high))
+
+
+def profile_needs(profiles: dict, db: dict) -> dict:
+    """The files emulator profiles read that the collection does not hold.
+
+    Keyed by every hash a profile accepts, and, for an entry that declares
+    a size and no hash, by its names casefolded. An entry with neither
+    cannot be told apart from any same-named dump and is left out.
+    """
+    files = db.get("files", {})
+    indexes = db.get("indexes", {})
+    held_hashes = {
+        "sha1": set(files),
+        "md5": set(indexes.get("by_md5", {})),
+        "sha256": set(indexes.get("by_sha256", {})),
+    }
+    held_crc = {
+        (_crc_key(str(f.get("crc32", "")).lower()), f.get("size"))
+        for f in files.values() if f.get("crc32")
+    }
+    sizes_by_name: dict[str, set[int]] = {}
+    for record in files.values():
+        sizes_by_name.setdefault(str(record.get("name", "")).casefold(), set()).add(
+            record.get("size")
+        )
+
+    by_hash: dict[tuple, list[dict]] = {}
+    by_name: dict[str, list[tuple[dict, dict]]] = {}
+    for emulator, profile in sorted(profiles.items()):
+        for entry in profile.get("files", []) or []:
+            name = str(entry.get("name", ""))
+            if not name or name.endswith("/"):
+                continue
+            need = {"emulator": emulator, "name": name,
+                    "required": bool(entry.get("required"))}
+            hashes = {(field, v) for field in HASH_FIELDS for v in _values(entry, field)}
+            crcs = {_crc_key(v) for v in _values(entry, "crc32")}
+            declared = entry.get("size")
+            sizes = [] if declared is None else (
+                declared if isinstance(declared, list) else [declared]
+            )
+            crc_keys = {(crc, int(size)) for crc in crcs for size in sizes}
+            if hashes or crc_keys:
+                if any(v in held_hashes[field] for field, v in hashes) or (
+                    crc_keys & held_crc
+                ):
+                    continue
+                for key in hashes | {("crc32", crc, size) for crc, size in crc_keys}:
+                    by_hash.setdefault(key, []).append(need)
+                continue
+            if all(entry.get(k) is None for k in ("size", "min_size", "max_size")):
+                continue
+            names = [name, *(entry.get("aliases") or [])]
+            if any(
+                _fits(held, entry)
+                for alias in names
+                for held in sizes_by_name.get(str(alias).casefold(), ())
+            ):
+                continue
+            for alias in names:
+                by_name.setdefault(str(alias).casefold(), []).append((need, entry))
+    return {"by_hash": by_hash, "by_name": by_name}
+
+
+def needs_filled(entry: dict, needs: dict) -> list[dict]:
+    """The profile needs a catalog dump answers, required ones first."""
+    found: dict[tuple[str, str], dict] = {}
+    keys = [(field, v) for field in HASH_FIELDS for v in _values(entry, field)]
+    if entry.get("size") is not None:
+        keys += [("crc32", _crc_key(v), entry["size"]) for v in _values(entry, "crc32")]
+    for key in keys:
+        for need in needs["by_hash"].get(key, []):
+            found.setdefault((need["emulator"], need["name"]), {**need, "by": key[0]})
+    size = entry.get("size")
+    name = str(entry.get("name", "")).casefold()
+    for need, declared in needs["by_name"].get(name, []):
+        if _fits(size, declared):
+            found.setdefault(
+                (need["emulator"], need["name"]), {**need, "by": "name and size"}
+            )
+    return sorted(found.values(), key=lambda n: (not n["required"], n["emulator"], n["name"]))
+
 
 def archive_members(db: dict) -> dict[tuple[str, int], list[tuple[str, str]]]:
     """(crc32, size) of every member of the collection's ZIPs -> (zip, member).
@@ -153,6 +264,7 @@ def build_report(
     snapshots: dict,
     members: dict | None = None,
     romsets: dict | None = None,
+    needs: dict | None = None,
 ) -> dict:
     """Compare each snapshot against the collection.
 
@@ -213,6 +325,9 @@ def build_report(
             holders = romsets_holding(entry, romsets)
             if holders:
                 entry["romsets"] = holders
+            filled = needs_filled(entry, needs) if needs else []
+            if filled:
+                entry["needed_by"] = filled
         out_of_scope = len({
             dump_identity(e) for e in unmatched if e.get("dat", "") not in covered_dats
         })
@@ -245,6 +360,10 @@ def main() -> int:
         "byte swaps, concatenations). Reads the collection, about 35 s",
     )
     parser.add_argument("--json", action="store_true", help="Full report as JSON")
+    parser.add_argument(
+        "--emulators-dir", default="emulators",
+        help="profiles whose unheld files a missing dump answers (default: %(default)s)",
+    )
     parser.add_argument(
         "--write-derivable",
         metavar="PATH",
@@ -261,7 +380,8 @@ def main() -> int:
         print(f"No provenance snapshots in {args.provenance_dir}/")
         return 0
 
-    report = build_report(db, snapshots, archive_members(db))
+    needs = profile_needs(load_emulator_profiles(args.emulators_dir), db)
+    report = build_report(db, snapshots, archive_members(db), needs=needs)
     derived: dict[str, dict] = {}
     if args.derive:
         from derivations import derivable  # noqa: PLC0415
@@ -299,7 +419,8 @@ def main() -> int:
                 f"does not cover, not counted as targets"
             )
         if args.missing:
-            for entry in data["missing"]:
+            # A dump an emulator reads comes first: it fills a hole in a pack.
+            for entry in sorted(data["missing"], key=lambda e: not e.get("needed_by")):
                 label = entry.get("description") or entry["name"]
                 source = derived.get(entry.get("sha1", ""))
                 if source:
@@ -307,6 +428,10 @@ def main() -> int:
                           f"of {' + '.join(source['from'])}")
                 else:
                     print(f"    MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}")
+                for need in entry.get("needed_by", []):
+                    print(f"      needed by {need['emulator']} as {need['name']} "
+                          f"({'required' if need['required'] else 'optional'}, "
+                          f"matched by {need['by']})")
                 for hit in entry.get("romsets", []):
                     versions = hit["dats"][0] if len(hit["dats"]) == 1 else (
                         f"{hit['dats'][0]} to {hit['dats'][-1]}"
@@ -320,6 +445,13 @@ def main() -> int:
         if args.derive:
             line += f", {len(derived)} of them derivable from held dumps"
         print(line)
+        needed = {
+            dump_identity(e)
+            for data in report.values() for e in data["missing"] if e.get("needed_by")
+        }
+        if needed:
+            print(f"  {len(needed)} of them a profile reads and the collection lacks"
+                  + ("" if args.missing else " (--missing names them)"))
     return 0
 
 

tests/test_provenance.py

@@ -554,6 +554,55 @@ class OneDumpIsCountedOnce(unittest.TestCase):
         self.assertEqual(provenance_report.distinct_missing(twice), 1)
 
 
+class AMissingDumpNamesTheProfileItFills(unittest.TestCase):
+    """XM7 requires DICROM.ROM and the collection lacks it, yet the report
+    listed No-Intro's dump at the rank of a Kickstart beta no core reads."""
+
+    PROFILES = {
+        "xm7": {"files": [
+            {"name": "DICROM.ROM", "required": True, "min_size": 262144},
+            {"name": "held.rom", "required": True, "size": 16},
+        ]},
+        "hashed": {"files": [{"name": "x.bin", "sha1": "c" * 40}]},
+    }
+
+    def _report(self, db):
+        from scripts import provenance_report  # noqa: PLC0415
+
+        snapshots = {"no-intro": {"entries": [
+            {"dat": "FM-7", "name": "DICROM.rom", "sha1": "a" * 40, "size": 262144},
+            {"dat": "FM-7", "name": "held.rom", "sha1": "b" * 40, "size": 16},
+            {"dat": "FM-7", "name": "Other Name.rom", "sha1": "c" * 40, "size": 8},
+            {"dat": "FM-7", "name": "unread.rom", "sha1": "d" * 40, "size": 8},
+        ]}}
+        needs = provenance_report.profile_needs(self.PROFILES, db)
+        return provenance_report.build_report(db, snapshots, {}, {}, needs)
+
+    def test_a_dump_a_profile_reads_names_it(self):
+        db = {"files": {"b" * 40: {"name": "held.rom", "md5": "", "size": 16}}}
+        missing = {e["name"]: e.get("needed_by") for e in self._report(db)["no-intro"]["missing"]}
+        self.assertEqual(missing["DICROM.rom"], [
+            {"emulator": "xm7", "name": "DICROM.ROM", "required": True,
+             "by": "name and size"},
+        ])
+        self.assertEqual(missing["Other Name.rom"][0]["by"], "sha1")
+        self.assertIsNone(missing["unread.rom"])
+
+    def test_a_size_outside_the_profile_range_fills_nothing(self):
+        from scripts import provenance_report  # noqa: PLC0415
+
+        needs = provenance_report.profile_needs(self.PROFILES, {"files": {}})
+        small = {"name": "DICROM.rom", "size": 1024}
+        self.assertEqual(provenance_report.needs_filled(small, needs), [])
+
+    def test_a_need_the_collection_answers_is_not_one(self):
+        from scripts import provenance_report  # noqa: PLC0415
+
+        db = {"files": {"e" * 40: {"name": "dicrom.rom", "size": 262144}}}
+        needs = provenance_report.profile_needs(self.PROFILES, db)
+        self.assertNotIn("dicrom.rom", needs["by_name"])
+
+
 class TheWantedPageAsksOnlyForWhatCannotBeMade(unittest.TestCase):
     """The site's Wanted page asked the public for 79 dumps the collection
     already produces (a Kickstart's odd bytes, a character ROM's halves)."""

wiki/tools.md

@@ -897,6 +897,13 @@ The Amiga Kickstart chip pairs and TOSEC's split C64 and VES ROMs are found
 this way: 67 entries on the current snapshots, recorded in `derivable.json`
 by pipeline step 1b.
 
+A missing entry that is a file an emulator profile reads and the collection
+lacks is listed first, with `needed by <emulator> as <name>` and whether the
+profile requires it. The match is by hash when the profile declares one, by
+name and size when it declares a size or size range and no hash (XM7's
+`DICROM.ROM`, which No-Intro lists for the FM77AV40SX). `--emulators-dir`
+names the profiles read.
+
 | Catalog | Acquisition | Notes |
 |---------|------------|-------|
 | Redump | direct fetch from redump.info `/static/bios` | 4 BIOS DATs. redump.org serves stale DATs |