From f66adfdcbc10331611ec8f77c1ff67bcafc3c952 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Thu, 22 Oct 2026 05:38:57 +0200 Subject: [PATCH] feat: categorize missing platform entries A platform list can name a game's data files or a file no core it runs opens. Verify gives each missing entry a category read from the profiles of the platform's cores: firmware, game data or unread. Only firmware gaps carry a severity and fail --strict; the others are counted on their own line, and the pack check no longer counts them as pack errors. Missing firmware stays a conformance error. --- scripts/exporter/candidates.py | 64 ++++++++++++++++++++++ scripts/packverify.py | 26 +++++---- scripts/verify.py | 53 ++++++++++++++++++- tests/test_audit_regressions.py | 6 ++- tests/test_e2e.py | 34 +++++++----- tests/test_entry_categories.py | 94 +++++++++++++++++++++++++++++++++ wiki/verification-modes.md | 19 +++++++ 7 files changed, 269 insertions(+), 27 deletions(-) create mode 100644 tests/test_entry_categories.py diff --git a/scripts/exporter/candidates.py b/scripts/exporter/candidates.py index d7f080d4..6bfe6bf6 100644 --- a/scripts/exporter/candidates.py +++ b/scripts/exporter/candidates.py @@ -224,3 +224,67 @@ def removal_candidates( "unresolved_cores": unresolved, }) return listed + + +# What a platform entry is, for the gates and the filters: firmware a core +# needs, a game's own data, or a name no examined core opens. +BIOS, GAME_DATA, UNREAD = "bios", "game_data", "unread" + + +class EntryCategories: + """Categorize a platform's entries by what the cores it runs read. + + An entry only game_data profile entries answer is the game's data, and + one nothing examined answers is unread. Unread needs the same certainty + as a removal candidate: every core the platform names is profiled, and + no core of the platform opens ROM sets by name when the entry is an + archive, and the platform declares the cores it runs. Anything else, + and every entry of a platform with no core to examine (BizHawk, MiSTer), + stays firmware. + """ + + def __init__(self, config: dict, profiles: dict[str, dict], + data_registry: dict | None = None) -> None: + examined = resolve_platform_cores(config, profiles) + # Without a declared core list, the examined cores are those sharing + # a system id: a guess about what runs, which cannot prove a negative. + self._open = ( + not examined or not config.get("cores") + or bool(unresolved_platform_cores(config, profiles)) + ) + self._romsets = any(_loads_romsets(profiles[key]) for key in examined) + self._firmware = ReadEvidence() + self._game = ReadEvidence() + for key in sorted(examined): + profile = profiles[key] + files = [f for f in profile.get("files") or [] if isinstance(f, dict)] + self._firmware.add_profile( + {**profile, "files": [f for f in files if f.get("category") != GAME_DATA]}, + data_registry or {}, + ) + self._game.add_profile( + {"files": [f for f in files if f.get("category") == GAME_DATA]}, + data_registry or {}, + ) + self._firmware.add_launchers(config.get("launchers") or []) + self._systems = { + _norm_system_id(str(sid)) + for key in examined + for sid in profiles[key].get("systems") or [] + } + + def category(self, entry: dict, destination: str, system: str) -> str: + """`system` is the platform's system id the entry is declared under. + + Unread also needs a core of that system among those examined: an + entry of a system no profiled core emulates is simply unexamined. + """ + if self._firmware.proof(entry, destination) is not None: + return BIOS + if self._game.proof(entry, destination) is not None: + return GAME_DATA + if self._open or _norm_system_id(system) not in self._systems: + return BIOS + if self._romsets and str(entry.get("name", "")).casefold().endswith(_ARCHIVES): + return BIOS + return UNREAD diff --git a/scripts/packverify.py b/scripts/packverify.py index 3ce9a5c2..d4aa29bf 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -31,6 +31,7 @@ from packpaths import _flat from nativemode import reads_file_contents import region as region_mod from packresolve import resolve_file +from exporter.candidates import BIOS, EntryCategories from common import resolve_platform_cores from common import sanitize_pack_path import zipfile @@ -454,12 +455,7 @@ def _held_nowhere( zip_contents: dict, data_dir_registry: dict | None, ) -> bool: - """Return whether no declaration for a destination resolves to a file. - - The builder cannot ship what the collection does not hold, so such a - destination is a data gap that verify.py reports as missing, not a pack - generation fault. A destination that resolves to anything stays checked. - """ + """Return whether no declaration for a destination resolves to a file.""" return bool(entries) and all( resolve_file( entry, @@ -660,7 +656,8 @@ def verify_pack_against_platform( Returns ``(all_ok, checked, present, errors, baseline_checked, baseline_present, core_checked, core_present, baseline_excluded, baseline_absent)``. ``baseline_absent`` counts destinations the collection - holds no file for: verify.py reports them missing. + holds no file for that are a game's data or that no core the platform + runs opens; verify.py counts them apart from firmware. """ config = load_platform_config(platform_name, platforms_dir) @@ -740,8 +737,10 @@ def verify_pack_against_platform( for i in range(1, len(parts)): zip_parents.add("/".join(parts[:i])) baseline_groups: dict[str, list[dict]] = {} - for _sys_id, system in (expected_systems if source != "truth" else {}).items(): + system_of: dict[int, str] = {} + for sys_id, system in (expected_systems if source != "truth" else {}).items(): for fe in system.get("files", []): + system_of[id(fe)] = sys_id if required_only and fe.get("required") is False: continue dest = sanitize_pack_path(fe.get("destination", fe.get("name", ""))) @@ -754,6 +753,7 @@ def verify_pack_against_platform( baseline_checked = len(baseline_groups) exclusion_index = build_zip_contents_index(db) if db is not None else {} + categories = EntryCategories(config, emu_profiles, data_registry) for expected, declarations in baseline_groups.items(): if expected in zip_set: member = expected @@ -773,9 +773,13 @@ def verify_pack_against_platform( ): baseline_excluded += 1 continue - elif db is not None and _held_nowhere( - declarations, db, exclusion_index, data_registry - ): + elif db is not None and all( + categories.category(fe, fe.get("destination", fe.get("name", "")), + system_of[id(fe)]) != BIOS + for fe in declarations + ) and _held_nowhere(declarations, db, exclusion_index, data_registry): + # A game's data or an entry no core opens that the collection + # lacks: verify counts it apart. Missing firmware stays an error. baseline_absent += 1 continue else: diff --git a/scripts/verify.py b/scripts/verify.py index 8481cb4c..26187b60 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -80,6 +80,7 @@ from common import ( ) yaml = require_yaml() +from exporter.candidates import BIOS, EntryCategories from launchers import LauncherLayer, declared_destinations from nativemode import ( SIZE_AND_ANY_HASH, @@ -102,7 +103,6 @@ from validation import ( build_ground_truth, check_file_validation, deliverable, - filter_files_by_mode, installed_with_emulator, unread_on, outside_gap_scope, @@ -1192,6 +1192,10 @@ def verify_platform( file_status: dict[str, str] = {} file_required: dict[str, bool] = {} file_severity: dict[str, str] = {} + # What a missing entry is: firmware, a game's data, or a name no core + # the platform runs opens. Only firmware gaps carry a severity. + file_category: dict[str, str] = {} + categories = EntryCategories(config, profiles, data_dir_registry) region_drops: set[str] = set() region_fallbacks: list[str] = [] @@ -1352,6 +1356,14 @@ def verify_platform( prev = file_status.get(dest) hle = hle_index.get(file_entry.get("name", ""), False) sev = compute_severity(cur, required, mode, hle) + category = ( + categories.category(file_entry, destination, sys_id) + if cur == Status.MISSING else BIOS + ) + result["category"] = category + if file_category.get(dest, category) == BIOS: + category = BIOS + file_category[dest] = category if prev is not None and (cur == Status.MISSING) != (prev == Status.MISSING): if cur == Status.MISSING: continue @@ -1439,7 +1451,12 @@ def verify_platform( Severity.WARNING: 0, Severity.CRITICAL: 0, } - for s in file_severity.values(): + not_firmware: dict[str, list[str]] = {} + for dest, s in file_severity.items(): + category = file_category.get(dest, BIOS) + if category != BIOS and file_status.get(dest) == Status.MISSING: + not_firmware.setdefault(category, []).append(dest) + continue counts[s] = counts.get(s, 0) + 1 # Held but refused: on a hash platform the builder leaves out a file whose @@ -1501,6 +1518,7 @@ def verify_platform( "pack_destinations": pack_destinations, "severity_counts": counts, "status_counts": status_counts, + "missing_not_firmware": {k: sorted(v) for k, v in sorted(not_firmware.items())}, "excluded_files": excluded, "packed_extras": packed_extras, "undeclared_files": undeclared, @@ -1583,6 +1601,13 @@ def _discrepancy_keys(details: list[dict]) -> set[str]: } +# A platform entry that is not firmware, as the report names it. +_NOT_FIRMWARE = { + "game_data": "game data", + "unread": "no core the platform runs opens it", +} + + def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) -> None: """Print UNTESTED, MISSING, and DISCREPANCY entries from verification details.""" for d in details: @@ -1605,6 +1630,13 @@ def _print_detail_entries(details: list[dict], seen: set[str], verbose: bool) -> key = f"{d['system']}/{d['name']}" if key in seen: continue + label = _NOT_FIRMWARE.get(d.get("category", BIOS)) + if label: + # Counted on their own line; named on request. + if verbose: + seen.add(key) + print(f" MISSING ({label}): {key}") + continue seen.add(key) req = "required" if d.get("required", True) else "optional" hle = ", HLE available" if d.get("hle_fallback") else "" @@ -1808,6 +1840,15 @@ def print_platform_result( print(f" No BIOS for the requested regions, all kept ({len(fallbacks)}): " + ", ".join(_group_label(g) for g in fallbacks)) + not_firmware = result.get("missing_not_firmware") or {} + if not_firmware: + counts = ", ".join( + f"{len(dests)} {_NOT_FIRMWARE[category]}" + for category, dests in not_firmware.items() + ) + listed = "" if verbose else " (--verbose lists them)" + print(f" Missing, not firmware: {counts}{listed}") + seen_details: set[str] = set() _print_detail_entries(result["details"], seen_details, verbose) _print_undeclared_section(result, verbose) @@ -2863,6 +2904,14 @@ def _overall_summary(group_results: list[tuple[dict, list[str]]]) -> str: detail = "; ".join(f"{group}: {', '.join(names)}" for group, names in missing.items() if names) lines.append(f"{total} core file(s) a core cannot run without are missing ({detail})") + for category, label in _NOT_FIRMWARE.items(): + counts = { + " / ".join(group): len((r.get("missing_not_firmware") or {}).get(category, [])) + for r, group in group_results + } + total = sum(counts.values()) + if total: + lines.append(f"{total} platform entries missing, not firmware, {label}{where(counts)}") return "\n".join(lines) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index b2ea55bc..f018ce78 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -4226,8 +4226,10 @@ class VerifyPacksSaysWhatItExcluded(unittest.TestCase): source = (ROOT / "scripts" / "generate_pack.py").read_text() body = source[source.index("def _run_verify_packs"):] body = body[:body.index("\ndef ")] - self.assertIn("bl_excluded) = result", body) - self.assertIn("unsafe excluded", body) + self.assertIn("bl_excluded, bl_absent) = result", body) + self.assertIn("_baseline_shortfall(bl_excluded, bl_absent)", body) + shortfall = source[source.index("def _baseline_shortfall"):] + self.assertIn("unsafe excluded", shortfall[:shortfall.index("\ndef ")]) self.assertNotIn("errors[:5]", body) self.assertNotRegex(body, r"result\[\d\]") diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 5d99fedc..566d9c13 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -5268,13 +5268,17 @@ struct BurnDriver BurnDrvneogeo = { ) self.assertEqual(result[3], [], f"unexpected errors: {result[3]}") - def test_213b_file_the_collection_lacks_is_no_pack_error(self): - """A declared file nothing resolves to is a data gap verify reports; - a held file absent from the pack stays a pack error.""" + def test_213b_absent_game_data_is_counted_apart(self): + """Game data the collection lacks is counted apart from the pack's + errors; firmware the collection lacks stays one.""" import tempfile from generate_pack import generate_pack, verify_pack_against_platform + profiles = {"eng": {"emulator": "Eng", "type": "game", "cores": ["eng"], + "systems": ["arcade-sys"], + "files": [{"name": "data.pak", "category": "game_data"}, + {"name": "firmware.bin"}]}} with tempfile.TemporaryDirectory() as tmpdir: plat_dir, bios_root, out_dir, db, _h, _hl = self._make_hash_zip_fixture( tmpdir, declare_md5=True @@ -5282,22 +5286,28 @@ struct BurnDriver BurnDrvneogeo = { config_path = os.path.join(plat_dir, "test_hashzip.yml") with open(config_path) as fh: config = yaml.safe_load(fh) + config["cores"] = ["eng"] config["systems"]["arcade-sys"]["files"].append( - {"name": "absent.bin", "destination": "dc/absent.bin"} + {"name": "data.pak", "destination": "dc/data.pak"} ) with open(config_path, "w") as fh: yaml.dump(config, fh) - zip_path = generate_pack("test_hashzip", plat_dir, db, bios_root, out_dir) - result = verify_pack_against_platform(zip_path, "test_hashzip", plat_dir, db=db) + zip_path = generate_pack("test_hashzip", plat_dir, db, bios_root, out_dir, + emu_profiles=profiles) + result = verify_pack_against_platform( + zip_path, "test_hashzip", plat_dir, db=db, emu_profiles=profiles) self.assertEqual(result[3], []) self.assertEqual((result[4], result[5], result[9]), (2, 1, 1)) - empty = os.path.join(out_dir, "Empty.zip") - with zipfile.ZipFile(empty, "w") as zf: - zf.writestr("README.txt", "") - result = verify_pack_against_platform(empty, "test_hashzip", plat_dir, db=db) - self.assertEqual(result[3], ["baseline missing: dc/testbios.zip"]) - self.assertEqual(result[9], 1) + config["systems"]["arcade-sys"]["files"][-1]["name"] = "firmware.bin" + config["systems"]["arcade-sys"]["files"][-1]["destination"] = "dc/firmware.bin" + with open(os.path.join(plat_dir, "test_hashzip_fw.yml"), "w") as fh: + yaml.dump(config, fh) + result = verify_pack_against_platform( + zip_path, "test_hashzip_fw", plat_dir, db=db, emu_profiles=profiles) + self.assertEqual([e.rsplit("/", 1)[-1] for e in result[3]], ["firmware.bin"]) + self.assertTrue(result[3][0].startswith("baseline missing: ")) + self.assertEqual(result[9], 0) def test_214_hash_matches_truncated_and_multi(self): """Declared-hash compare handles truncation, case, multi-values.""" diff --git a/tests/test_entry_categories.py b/tests/test_entry_categories.py new file mode 100644 index 00000000..73cc42f6 --- /dev/null +++ b/tests/test_entry_categories.py @@ -0,0 +1,94 @@ +"""What a platform entry is: firmware, a game's data, or unread. + +Strict verification fails on firmware gaps only, so the category has to come +from what the cores the platform runs open, never from the entry's name. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) + +import verify # noqa: E402 +from exporter.candidates import BIOS, GAME_DATA, UNREAD, EntryCategories # noqa: E402 + +PROFILES = { + "bluemsx": { + "emulator": "blueMSX", "type": "libretro", "cores": ["bluemsx"], + "systems": ["microsoft-msx"], + "files": [{"name": "MSX.ROM", "path": "Machines/Shared Roms/MSX.ROM"}], + }, + "devilutionx": { + "emulator": "DevilutionX", "type": "game", "cores": ["devilutionx"], + "systems": ["diablo"], + "files": [{"name": "diabdat.mpq", "category": "game_data"}], + }, +} + + +def _config(cores: list[str], files: list[dict], system: str = "s") -> dict: + return {"platform": "P", "verification_mode": "md5", "cores": cores, + "systems": {system: {"files": files}}} + + +class Category(unittest.TestCase): + def test_each_category(self): + cats = EntryCategories(_config(["bluemsx", "devilutionx"], []), PROFILES) + self.assertEqual(cats.category({"name": "MSX.ROM"}, "bios/Machines/MSX/MSX.ROM", "microsoft-msx"), BIOS) + self.assertEqual(cats.category({"name": "diabdat.mpq"}, "ports/diablo/diabdat.mpq", "diablo"), + GAME_DATA) + self.assertEqual( + cats.category({"name": "hardwareconfig.xml"}, "bios/Machines/MSX/hardwareconfig.xml", "microsoft-msx"), + UNREAD) + + def test_a_core_without_profile_leaves_unread_open(self): + cats = EntryCategories(_config(["bluemsx", "devilutionx", "mystery"], []), PROFILES) + self.assertEqual( + cats.category({"name": "hardwareconfig.xml"}, "bios/Machines/MSX/hardwareconfig.xml", "microsoft-msx"), + BIOS) + self.assertEqual(cats.category({"name": "diabdat.mpq"}, "ports/diablo/diabdat.mpq", "diablo"), + GAME_DATA) + + def test_a_system_no_examined_core_emulates_stays_firmware(self): + cats = EntryCategories(_config(["bluemsx", "devilutionx"], []), PROFILES) + self.assertEqual(cats.category({"name": "syscard3.pce"}, "syscard3.pce", "nec-pc-engine"), + BIOS) + + def test_cores_guessed_from_system_ids_prove_nothing(self): + config = _config([], []) + del config["cores"] + cats = EntryCategories(config, PROFILES) + self.assertEqual( + cats.category({"name": "hardwareconfig.xml"}, "bios/Machines/MSX/hardwareconfig.xml", + "microsoft-msx"), + BIOS) + + def test_a_platform_with_no_core_is_all_firmware(self): + cats = EntryCategories(_config([], []), PROFILES) + self.assertEqual(cats.category({"name": "anything.bin"}, "anything.bin", "s"), BIOS) + + +class StrictCountsFirmwareOnly(unittest.TestCase): + def test_missing_game_data_and_unread_carry_no_severity(self): + config = _config(["bluemsx", "devilutionx"], [ + {"name": "diabdat.mpq", "destination": "ports/diablo/diabdat.mpq", "required": True}, + {"name": "hardwareconfig.xml", + "destination": "bios/Machines/MSX/hardwareconfig.xml", "required": True}, + {"name": "MSX.ROM", "destination": "bios/Machines/MSX/MSX.ROM", "required": True}, + ], system="microsoft-msx") + db = {"files": {}, "indexes": {"by_md5": {}, "by_name": {}, "by_crc32": {}, + "by_path_suffix": {}}} + result = verify.verify_platform(config, db, emu_profiles=PROFILES) + self.assertEqual(result["severity_counts"][verify.Severity.CRITICAL], 1) + self.assertEqual(result["missing_not_firmware"], { + "game_data": ["ports/diablo/diabdat.mpq"], + "unread": ["bios/Machines/MSX/hardwareconfig.xml"], + }) + self.assertEqual(result["status_counts"][verify.Status.MISSING], 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/verification-modes.md b/wiki/verification-modes.md index 2af813f4..e81e3091 100644 --- a/wiki/verification-modes.md +++ b/wiki/verification-modes.md @@ -280,6 +280,25 @@ files are downgraded to INFO regardless of platform mode or required status. The is still included in packs (better accuracy with the real BIOS), but its absence is not actionable. +**Firmware, game data, unread**: a platform's list can name more than firmware. +EmuELEC lists the data files of the games its ports run (Diablo's `diabdat.mpq`, the +Sonic `Data.rsdk` files), and AmberELEC lists a `hardwareconfig.xml` in each blueMSX +machine folder, a file blueMSX never opens. Each missing entry gets one of three +categories, read from the profiles of the cores the platform runs: + +| Category | When | Severity | +|----------|------|----------| +| firmware | a core reads it as firmware, or nothing settles the question | from the matrix above | +| game data | only `category: game_data` entries of those profiles name it | none, counted apart | +| unread | the platform declares its cores, every one has a profile, at least one emulates the entry's system, and none opens the entry | none, counted apart | + +The report counts the last two on a line of their own (`Missing, not firmware: ...`) +and `--verbose` names them. They stay in the platform's file and in the totals, but +`--strict` fails on firmware gaps only, and the pack check does not count them as +pack errors. A core the platform names that has no profile yet makes nothing unread. +The same goes for a system none of the examined cores emulates: in both cases the +absence of a reader proves nothing. + ## File resolution chain