diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 43777c99..bb297cf5 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -94,7 +94,6 @@ from validation import ( agnostic_substitute, _build_validation_index, deliverable, - filter_files_by_mode, with_archive_members, extra_alternatives, ) @@ -2551,6 +2550,16 @@ def _pack_output_lock(output_dir: str, exclusive: bool = True) -> Iterator[None] sys.exit(1) +def _baseline_shortfall(excluded: int, absent: int) -> list[str]: + """Why a verified pack holds fewer baseline files than the platform declares.""" + parts = [] + if excluded: + parts.append(f"{excluded} unsafe excluded") + if absent: + parts.append(f"{absent} not in the collection") + return parts + + def _run_verify_packs(args: argparse.Namespace) -> None: """Verify each pack in the output directory against its platform. @@ -2634,10 +2643,12 @@ def _run_verify_packs(args: argparse.Namespace) -> None: required_only=required_only, ) (ok, _checked, _present, errors, bl_checked, bl_present, - core_checked, core_present, bl_excluded) = result - # The unsafe exclusions explain a baseline short of its total: a - # dump whose hash the frontend refuses is left out on purpose. - excluded = f" ({bl_excluded} unsafe excluded)" if bl_excluded else "" + core_checked, core_present, bl_excluded, bl_absent) = result + # These explain a baseline short of its total: a dump whose hash + # the frontend refuses is left out on purpose, and a file the + # collection does not hold cannot be packed (verify reports it). + shortfall = ", ".join(_baseline_shortfall(bl_excluded, bl_absent)) + excluded = f" ({shortfall})" if shortfall else "" counts = ( f"{bl_present}/{bl_checked} baseline{excluded}, " f"{core_present}/{core_checked} cores" @@ -4816,6 +4827,7 @@ def verify_and_finalize_packs( core_checked, core_present, bl_excluded, + bl_absent, ) = verify_pack_against_platform( zip_path, pname, @@ -4827,8 +4839,8 @@ def verify_and_finalize_packs( target_cores=_target_cores_for(pname, target_name, platforms_dir), ) status = "OK" if p_ok else "FAILED" - exclusion_note = ( - f", {bl_excluded} unsafe excluded" if bl_excluded else "" + exclusion_note = "".join( + f", {part}" for part in _baseline_shortfall(bl_excluded, bl_absent) ) print( f" platform {pname}: {bl_present}/{bl_checked} baseline present" diff --git a/scripts/packverify.py b/scripts/packverify.py index dfc7f806..3ce9a5c2 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -448,6 +448,31 @@ def _repo_holds_member(file_entry: dict, db: dict) -> bool: return False +def _held_nowhere( + entries: list[dict], + db: dict, + zip_contents: dict, + data_dir_registry: dict | None, +) -> bool: + """Return whether no declaration for a destination resolves to a file. + + The builder cannot ship what the collection does not hold, so such a + destination is a data gap that verify.py reports as missing, not a pack + generation fault. A destination that resolves to anything stays checked. + """ + return bool(entries) and all( + resolve_file( + entry, + db, + DEFAULT_BIOS_DIR, + zip_contents, + data_dir_registry=data_dir_registry, + offline=True, + )[1] == "not_found" + for entry in entries + ) + + def _intentional_hash_exclusion( entries: list[dict], db: dict, @@ -613,7 +638,7 @@ def verify_pack_against_platform( target_cores: set[str] | None = None, source: str = "full", required_only: bool = False, -) -> tuple[bool, int, int, list[str], int, int, int, int, int]: +) -> tuple[bool, int, int, list[str], int, int, int, int, int, int]: """Verify a pack ZIP against its platform config and core requirements. A region priority list narrows the expectation to what the builder would @@ -633,7 +658,9 @@ def verify_pack_against_platform( 5. No unexpected zero-byte BIOS files Returns ``(all_ok, checked, present, errors, baseline_checked, - baseline_present, core_checked, core_present, baseline_excluded)``. + baseline_present, core_checked, core_present, baseline_excluded, + baseline_absent)``. ``baseline_absent`` counts destinations the collection + holds no file for: verify.py reports them missing. """ config = load_platform_config(platform_name, platforms_dir) @@ -702,6 +729,7 @@ def verify_pack_against_platform( baseline_checked = 0 baseline_present = 0 baseline_excluded = 0 + baseline_absent = 0 decl_by_member: dict[str, list[dict]] = {} # Mirror the builder's path-conflict logic: a declaration whose path # collides file-vs-directory with a packed member was skipped by the @@ -745,6 +773,11 @@ def verify_pack_against_platform( ): baseline_excluded += 1 continue + elif db is not None and _held_nowhere( + declarations, db, exclusion_index, data_registry + ): + baseline_absent += 1 + continue else: errors.append(f"baseline missing: {expected}") continue @@ -835,4 +868,5 @@ def verify_pack_against_platform( core_checked, core_present, baseline_excluded, + baseline_absent, ) diff --git a/tests/test_e2e.py b/tests/test_e2e.py index 0059adad..5d99fedc 100644 --- a/tests/test_e2e.py +++ b/tests/test_e2e.py @@ -5268,6 +5268,37 @@ struct BurnDriver BurnDrvneogeo = { ) self.assertEqual(result[3], [], f"unexpected errors: {result[3]}") + def test_213b_file_the_collection_lacks_is_no_pack_error(self): + """A declared file nothing resolves to is a data gap verify reports; + a held file absent from the pack stays a pack error.""" + import tempfile + + from generate_pack import generate_pack, verify_pack_against_platform + + with tempfile.TemporaryDirectory() as tmpdir: + plat_dir, bios_root, out_dir, db, _h, _hl = self._make_hash_zip_fixture( + tmpdir, declare_md5=True + ) + config_path = os.path.join(plat_dir, "test_hashzip.yml") + with open(config_path) as fh: + config = yaml.safe_load(fh) + config["systems"]["arcade-sys"]["files"].append( + {"name": "absent.bin", "destination": "dc/absent.bin"} + ) + with open(config_path, "w") as fh: + yaml.dump(config, fh) + zip_path = generate_pack("test_hashzip", plat_dir, db, bios_root, out_dir) + result = verify_pack_against_platform(zip_path, "test_hashzip", plat_dir, db=db) + self.assertEqual(result[3], []) + self.assertEqual((result[4], result[5], result[9]), (2, 1, 1)) + + empty = os.path.join(out_dir, "Empty.zip") + with zipfile.ZipFile(empty, "w") as zf: + zf.writestr("README.txt", "") + result = verify_pack_against_platform(empty, "test_hashzip", plat_dir, db=db) + self.assertEqual(result[3], ["baseline missing: dc/testbios.zip"]) + self.assertEqual(result[9], 1) + def test_214_hash_matches_truncated_and_multi(self): """Declared-hash compare handles truncation, case, multi-values.""" from generate_pack import _hash_matches diff --git a/tests/test_pack_conformance_dirs.py b/tests/test_pack_conformance_dirs.py index cd81fbdb..f17d77eb 100644 --- a/tests/test_pack_conformance_dirs.py +++ b/tests/test_pack_conformance_dirs.py @@ -59,7 +59,7 @@ class ConformanceUsesTheRunDirectories(unittest.TestCase): def record(*args, **kwargs): seen.update(kwargs) - return True, 0, 0, [], 0, 0, 0, 0, 0 + return True, 0, 0, [], 0, 0, 0, 0, 0, 0 with mock.patch.object(gp, "list_registered_platforms", return_value=["fix"]), \ mock.patch.object(