diff --git a/scripts/provenance_report.py b/scripts/provenance_report.py index 5ab713a5..44c7ca5b 100644 --- a/scripts/provenance_report.py +++ b/scripts/provenance_report.py @@ -10,6 +10,10 @@ Usage: python scripts/provenance_report.py python scripts/provenance_report.py --missing python scripts/provenance_report.py --json + +A missing dump that is a file an emulator profile reads and the collection +lacks is named with that profile: by hash when the profile declares one, by +name and size when it declares a size and no hash. """ from __future__ import annotations @@ -26,10 +30,117 @@ sys.path.insert(0, os.path.dirname(__file__)) from common import ( DEFAULT_PROVENANCE_DIR, load_database, + load_emulator_profiles, load_provenance_snapshots, write_if_changed, ) +HASH_FIELDS = ("sha1", "md5", "sha256") + + +def _values(entry: dict, field: str) -> set[str]: + """Every value a profile or catalog entry gives for a hash field.""" + value = entry.get(field) + values = value if isinstance(value, list) else str(value or "").split(",") + return {str(v).strip().lower() for v in values if str(v).strip()} + + +def _crc_key(value: str) -> str: + return value.removeprefix("0x").zfill(8) + + +def _fits(size: int | None, entry: dict) -> bool: + """Whether a size meets what a profile entry states about it.""" + if size is None: + return False + declared = entry.get("size") + if declared is not None: + accepted = declared if isinstance(declared, list) else [declared] + return size in {int(value) for value in accepted} + low, high = entry.get("min_size"), entry.get("max_size") + return (low is None or size >= int(low)) and (high is None or size <= int(high)) + + +def profile_needs(profiles: dict, db: dict) -> dict: + """The files emulator profiles read that the collection does not hold. + + Keyed by every hash a profile accepts, and, for an entry that declares + a size and no hash, by its names casefolded. An entry with neither + cannot be told apart from any same-named dump and is left out. + """ + files = db.get("files", {}) + indexes = db.get("indexes", {}) + held_hashes = { + "sha1": set(files), + "md5": set(indexes.get("by_md5", {})), + "sha256": set(indexes.get("by_sha256", {})), + } + held_crc = { + (_crc_key(str(f.get("crc32", "")).lower()), f.get("size")) + for f in files.values() if f.get("crc32") + } + sizes_by_name: dict[str, set[int]] = {} + for record in files.values(): + sizes_by_name.setdefault(str(record.get("name", "")).casefold(), set()).add( + record.get("size") + ) + + by_hash: dict[tuple, list[dict]] = {} + by_name: dict[str, list[tuple[dict, dict]]] = {} + for emulator, profile in sorted(profiles.items()): + for entry in profile.get("files", []) or []: + name = str(entry.get("name", "")) + if not name or name.endswith("/"): + continue + need = {"emulator": emulator, "name": name, + "required": bool(entry.get("required"))} + hashes = {(field, v) for field in HASH_FIELDS for v in _values(entry, field)} + crcs = {_crc_key(v) for v in _values(entry, "crc32")} + declared = entry.get("size") + sizes = [] if declared is None else ( + declared if isinstance(declared, list) else [declared] + ) + crc_keys = {(crc, int(size)) for crc in crcs for size in sizes} + if hashes or crc_keys: + if any(v in held_hashes[field] for field, v in hashes) or ( + crc_keys & held_crc + ): + continue + for key in hashes | {("crc32", crc, size) for crc, size in crc_keys}: + by_hash.setdefault(key, []).append(need) + continue + if all(entry.get(k) is None for k in ("size", "min_size", "max_size")): + continue + names = [name, *(entry.get("aliases") or [])] + if any( + _fits(held, entry) + for alias in names + for held in sizes_by_name.get(str(alias).casefold(), ()) + ): + continue + for alias in names: + by_name.setdefault(str(alias).casefold(), []).append((need, entry)) + return {"by_hash": by_hash, "by_name": by_name} + + +def needs_filled(entry: dict, needs: dict) -> list[dict]: + """The profile needs a catalog dump answers, required ones first.""" + found: dict[tuple[str, str], dict] = {} + keys = [(field, v) for field in HASH_FIELDS for v in _values(entry, field)] + if entry.get("size") is not None: + keys += [("crc32", _crc_key(v), entry["size"]) for v in _values(entry, "crc32")] + for key in keys: + for need in needs["by_hash"].get(key, []): + found.setdefault((need["emulator"], need["name"]), {**need, "by": key[0]}) + size = entry.get("size") + name = str(entry.get("name", "")).casefold() + for need, declared in needs["by_name"].get(name, []): + if _fits(size, declared): + found.setdefault( + (need["emulator"], need["name"]), {**need, "by": "name and size"} + ) + return sorted(found.values(), key=lambda n: (not n["required"], n["emulator"], n["name"])) + def archive_members(db: dict) -> dict[tuple[str, int], list[tuple[str, str]]]: """(crc32, size) of every member of the collection's ZIPs -> (zip, member). @@ -153,6 +264,7 @@ def build_report( snapshots: dict, members: dict | None = None, romsets: dict | None = None, + needs: dict | None = None, ) -> dict: """Compare each snapshot against the collection. @@ -213,6 +325,9 @@ def build_report( holders = romsets_holding(entry, romsets) if holders: entry["romsets"] = holders + filled = needs_filled(entry, needs) if needs else [] + if filled: + entry["needed_by"] = filled out_of_scope = len({ dump_identity(e) for e in unmatched if e.get("dat", "") not in covered_dats }) @@ -245,6 +360,10 @@ def main() -> int: "byte swaps, concatenations). Reads the collection, about 35 s", ) parser.add_argument("--json", action="store_true", help="Full report as JSON") + parser.add_argument( + "--emulators-dir", default="emulators", + help="profiles whose unheld files a missing dump answers (default: %(default)s)", + ) parser.add_argument( "--write-derivable", metavar="PATH", @@ -261,7 +380,8 @@ def main() -> int: print(f"No provenance snapshots in {args.provenance_dir}/") return 0 - report = build_report(db, snapshots, archive_members(db)) + needs = profile_needs(load_emulator_profiles(args.emulators_dir), db) + report = build_report(db, snapshots, archive_members(db), needs=needs) derived: dict[str, dict] = {} if args.derive: from derivations import derivable # noqa: PLC0415 @@ -299,7 +419,8 @@ def main() -> int: f"does not cover, not counted as targets" ) if args.missing: - for entry in data["missing"]: + # A dump an emulator reads comes first: it fills a hole in a pack. + for entry in sorted(data["missing"], key=lambda e: not e.get("needed_by")): label = entry.get("description") or entry["name"] source = derived.get(entry.get("sha1", "")) if source: @@ -307,6 +428,10 @@ def main() -> int: f"of {' + '.join(source['from'])}") else: print(f" MISSING {entry['name']} ({label}) sha1={entry.get('sha1')}") + for need in entry.get("needed_by", []): + print(f" needed by {need['emulator']} as {need['name']} " + f"({'required' if need['required'] else 'optional'}, " + f"matched by {need['by']})") for hit in entry.get("romsets", []): versions = hit["dats"][0] if len(hit["dats"]) == 1 else ( f"{hit['dats'][0]} to {hit['dats'][-1]}" @@ -320,6 +445,13 @@ def main() -> int: if args.derive: line += f", {len(derived)} of them derivable from held dumps" print(line) + needed = { + dump_identity(e) + for data in report.values() for e in data["missing"] if e.get("needed_by") + } + if needed: + print(f" {len(needed)} of them a profile reads and the collection lacks" + + ("" if args.missing else " (--missing names them)")) return 0 diff --git a/tests/test_provenance.py b/tests/test_provenance.py index ab0917ca..94dbe322 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -554,6 +554,55 @@ class OneDumpIsCountedOnce(unittest.TestCase): self.assertEqual(provenance_report.distinct_missing(twice), 1) +class AMissingDumpNamesTheProfileItFills(unittest.TestCase): + """XM7 requires DICROM.ROM and the collection lacks it, yet the report + listed No-Intro's dump at the rank of a Kickstart beta no core reads.""" + + PROFILES = { + "xm7": {"files": [ + {"name": "DICROM.ROM", "required": True, "min_size": 262144}, + {"name": "held.rom", "required": True, "size": 16}, + ]}, + "hashed": {"files": [{"name": "x.bin", "sha1": "c" * 40}]}, + } + + def _report(self, db): + from scripts import provenance_report # noqa: PLC0415 + + snapshots = {"no-intro": {"entries": [ + {"dat": "FM-7", "name": "DICROM.rom", "sha1": "a" * 40, "size": 262144}, + {"dat": "FM-7", "name": "held.rom", "sha1": "b" * 40, "size": 16}, + {"dat": "FM-7", "name": "Other Name.rom", "sha1": "c" * 40, "size": 8}, + {"dat": "FM-7", "name": "unread.rom", "sha1": "d" * 40, "size": 8}, + ]}} + needs = provenance_report.profile_needs(self.PROFILES, db) + return provenance_report.build_report(db, snapshots, {}, {}, needs) + + def test_a_dump_a_profile_reads_names_it(self): + db = {"files": {"b" * 40: {"name": "held.rom", "md5": "", "size": 16}}} + missing = {e["name"]: e.get("needed_by") for e in self._report(db)["no-intro"]["missing"]} + self.assertEqual(missing["DICROM.rom"], [ + {"emulator": "xm7", "name": "DICROM.ROM", "required": True, + "by": "name and size"}, + ]) + self.assertEqual(missing["Other Name.rom"][0]["by"], "sha1") + self.assertIsNone(missing["unread.rom"]) + + def test_a_size_outside_the_profile_range_fills_nothing(self): + from scripts import provenance_report # noqa: PLC0415 + + needs = provenance_report.profile_needs(self.PROFILES, {"files": {}}) + small = {"name": "DICROM.rom", "size": 1024} + self.assertEqual(provenance_report.needs_filled(small, needs), []) + + def test_a_need_the_collection_answers_is_not_one(self): + from scripts import provenance_report # noqa: PLC0415 + + db = {"files": {"e" * 40: {"name": "dicrom.rom", "size": 262144}}} + needs = provenance_report.profile_needs(self.PROFILES, db) + self.assertNotIn("dicrom.rom", needs["by_name"]) + + class TheWantedPageAsksOnlyForWhatCannotBeMade(unittest.TestCase): """The site's Wanted page asked the public for 79 dumps the collection already produces (a Kickstart's odd bytes, a character ROM's halves).""" diff --git a/wiki/tools.md b/wiki/tools.md index cbaaf6ce..6fc7679e 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -897,6 +897,13 @@ The Amiga Kickstart chip pairs and TOSEC's split C64 and VES ROMs are found this way: 67 entries on the current snapshots, recorded in `derivable.json` by pipeline step 1b. +A missing entry that is a file an emulator profile reads and the collection +lacks is listed first, with `needed by as ` and whether the +profile requires it. The match is by hash when the profile declares one, by +name and size when it declares a size or size range and no hash (XM7's +`DICROM.ROM`, which No-Intro lists for the FM77AV40SX). `--emulators-dir` +names the profiles read. + | Catalog | Acquisition | Notes | |---------|------------|-------| | Redump | direct fetch from redump.info `/static/bios` | 4 BIOS DATs. redump.org serves stale DATs |