diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index fa7b434a..fb17d4a9 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -936,19 +936,16 @@ def generate_pack( total_files += 1 continue - local_path, status = resolve_file( + local_path, status = resolve_declaration( file_entry, db, bios_dir, zip_contents, - data_dir_registry=data_registry, + data_registry, + slot_overrides.get(full_dest), offline=offline, ) - local_path, status = slots.apply_override( - slot_overrides.get(full_dest), local_path, status - ) - if status == "external": file_ext = os.path.splitext(file_entry["name"])[1] or "" with tempfile.NamedTemporaryFile( @@ -4243,14 +4240,15 @@ def generate_manifest( file_entry: dict, sys_id: str, dest: str, override: str | None ) -> tuple[str | None, str]: """The file served once the slot is settled, or None and why not.""" - local_path, status = slots.apply_override(override, *resolve_file( + local_path, status = resolve_declaration( file_entry, db, bios_dir, zip_contents, - data_dir_registry=data_registry, + data_registry, + override, offline=offline, - )) + ) if status == "not_found" and not reads_file_contents(verification_mode): found = agnostic_substitute(file_entry, sys_id, db, manifest_profiles) if found: @@ -4914,6 +4912,7 @@ from packresolve import ( # noqa: E402,F401 parse_hash_input, parse_hash_file, lookup_hashes, + resolve_declaration, resolve_file, ) diff --git a/scripts/packextras.py b/scripts/packextras.py index cf046dd9..ad31cf17 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -1229,14 +1229,18 @@ def _kept(entries: list[dict], required_only: bool) -> list[dict]: return [e for e in entries if e.get("required") is not False] -def _ships(config: dict, file_entry: dict, db: dict, dest: str) -> bool: +def _ships(config: dict, file_entry: dict, db: dict) -> bool: """Whether the builder packs this declaration: resolved, and not refused by a hash frontend. A dump whose md5 Recalbox would reject is left out of the pack, so it cannot win a regional slot over a file that ships. """ + from packresolve import resolve_declaration from validation import baseline_verdict - path, status = resolve_local_file(file_entry, db, dest_hint=dest) + # The builder's resolution: no destination hint, release assets included. + path, status = resolve_declaration( + file_entry, db, "bios", None, None, None, offline=True, + ) if not path: return False return baseline_verdict( @@ -1288,7 +1292,7 @@ def platform_region_groups( if dest: members.append((dest, file_entry.get("name", ""))) if held_out is not None and db is not None and _ships( - config, file_entry, db, dest + config, file_entry, db ): held_out.add(dest) diff --git a/scripts/packresolve.py b/scripts/packresolve.py index 1d54413b..a851291f 100644 --- a/scripts/packresolve.py +++ b/scripts/packresolve.py @@ -286,6 +286,46 @@ def resolve_file( return None, "not_found" +def apply_override( + override: str | None, local_path: str | None, status: str +) -> tuple[str | None, str]: + """The file a pack serves at a destination once its slot is settled. + + An entry the platform stores outside the collection, or a release asset + fetched for its declared hash, keeps its own answer. + """ + if override and status not in ("external", "release_asset"): + return override, "slot_arbitrated" + return local_path, status + + +def resolve_declaration( + file_entry: dict, + db: dict, + bios_dir: str, + zip_contents: dict | None, + data_dir_registry: dict | None, + override: str | None, + *, + offline: bool | None = None, +) -> tuple[str | None, str]: + """The file a platform pack serves for one of the platform's declarations. + + Read by the builder, the install manifest, verify and the region pass. + verify resolved with common.resolve_local_file and put the slot override + first: without the release-asset tier a gitignored file absent from bios/ + read as missing there while the pack shipped the cached asset. + """ + return apply_override(override, *resolve_file( + file_entry, + db, + bios_dir, + zip_contents, + data_dir_registry=data_dir_registry, + offline=offline, + )) + + def preferred_entries( pack_systems: dict, db: dict, diff --git a/scripts/slots.py b/scripts/slots.py index 6a5668bc..854718fc 100644 --- a/scripts/slots.py +++ b/scripts/slots.py @@ -578,20 +578,6 @@ def slot_choices( return choices -def apply_override( - override: str | None, local_path: str | None, status: str -) -> tuple[str | None, str]: - """The file a pack serves at a destination once its slot is settled. - - An entry the platform stores outside the collection, or a release asset - fetched for its declared hash, keeps its own answer. The builder and the - install manifest both read this. - """ - if override and status not in ("external", "release_asset"): - return override, "slot_arbitrated" - return local_path, status - - def _strongest(claims: list[Claim]) -> Claim: """The profile claim with the strongest proof, the first emulator breaking ties. diff --git a/scripts/verify.py b/scripts/verify.py index 09947723..8ece9575 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1047,18 +1047,20 @@ def _resolve_for_platform( ) -> tuple[str | None, str]: """Resolve one platform entry the way the pack builder does. - A slot the arbitration decided wins over the resolver, and a frontend that + The builder's own resolution (packresolve.resolve_declaration), read from + disk and the large-file cache: verify never downloads. A frontend that never reads the bytes takes the filename-free core's own image. """ + from packresolve import resolve_declaration + override = slot_overrides.get( sanitize_pack_path(file_entry.get("destination", file_entry.get("name", ""))) ) - if override: - return override, "slot_arbitrated" - local_path, status = resolve_local_file( - file_entry, db, zip_contents, data_dir_registry=data_dir_registry + local_path, status = resolve_declaration( + file_entry, db, DEFAULT_BIOS_DIR, zip_contents, data_dir_registry, + override, offline=True, ) - if local_path is None and not reads_file_contents(mode): + if status == "not_found" and not reads_file_contents(mode): found = agnostic_substitute(file_entry, sys_id, db, platform_profiles) if found: return found[0], "agnostic_fallback" diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index d12efd68..c9d8c848 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -1207,7 +1207,7 @@ class OneBaselineVerdict(unittest.TestCase): ) def test_a_release_asset_keeps_its_answer(self): - from scripts.slots import apply_override + from scripts.packresolve import apply_override self.assertEqual( apply_override("bios/x.bin", "/cache/big.bin", "release_asset"), @@ -1229,9 +1229,49 @@ class OneBaselineVerdict(unittest.TestCase): body = ast.get_source_segment(source, function) or "" with self.subTest(function=name): self.assertIn("baseline_verdict(", body) - self.assertIn("slots.apply_override(", body) + self.assertIn("resolve_declaration(", body) self.assertNotIn('"slot_arbitrated"', body) + def test_verify_and_the_region_pass_resolve_as_the_builder(self): + """verify called common.resolve_local_file and put the slot override + first, the region pass passed a destination hint the builder never + gives: a gitignored file absent from bios/ read as missing in verify + while the pack shipped its cached release asset.""" + for module, name in (("verify.py", "_resolve_for_platform"), + ("packextras.py", "_ships")): + source = (ROOT / "scripts" / module).read_text(encoding="utf-8") + function = next( + node for node in ast.parse(source).body + if isinstance(node, ast.FunctionDef) and node.name == name + ) + body = ast.get_source_segment(source, function) or "" + with self.subTest(function=name): + self.assertIn("resolve_declaration(", body) + self.assertNotIn("resolve_local_file(", body) + self.assertNotIn('"slot_arbitrated"', body) + + def test_verify_finds_the_cached_release_asset(self): + # verify imports the scripts directory's modules by their bare names. + import packresolve + from scripts.verify import _resolve_for_platform + + with tempfile.TemporaryDirectory(dir=TMP_ROOT) as directory: + cached = Path(directory) / "BIG.PUP" + cached.write_bytes(b"large firmware") + sha1 = hashlib.sha1(cached.read_bytes()).hexdigest() + db = {"files": {sha1: {"path": f"{directory}/absent/BIG.PUP", + "name": "BIG.PUP", "sha1": sha1}}, + "indexes": {"by_name": {"BIG.PUP": [sha1]}}} + entry = {"name": "BIG.PUP", "sha1": sha1, "destination": "BIG.PUP"} + with mock.patch.object( + packresolve, "fetch_large_file", return_value=str(cached) + ): + found = _resolve_for_platform( + entry, "sys", db, {}, None, {"BIG.PUP": "bios/other.bin"}, + "md5", {}, + ) + self.assertEqual(found, (str(cached), "release_asset")) + class PackExclusionRegressions(unittest.TestCase): def test_slug_platform_core_requirement_uses_the_generated_destination(self): diff --git a/tests/test_region_held.py b/tests/test_region_held.py index 158471de..cb384cb4 100644 --- a/tests/test_region_held.py +++ b/tests/test_region_held.py @@ -41,9 +41,9 @@ class ARefusedDumpHoldsNothing(unittest.TestCase): "by_path_suffix": {}}, } entry = {"name": "us.bin", "md5": "0" * 32} - self.assertFalse(_ships({"verification_mode": "md5"}, entry, db, "us.bin")) - self.assertTrue(_ships({"verification_mode": "existence"}, entry, db, "us.bin")) - self.assertTrue(_ships({"verification_mode": "md5"}, {"name": "us.bin"}, db, "us.bin")) + self.assertFalse(_ships({"verification_mode": "md5"}, entry, db)) + self.assertTrue(_ships({"verification_mode": "existence"}, entry, db)) + self.assertTrue(_ships({"verification_mode": "md5"}, {"name": "us.bin"}, db)) if __name__ == "__main__":