diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 03ef16791..36e15ba85 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -678,6 +678,9 @@ def _select_variants( one_per_slot: bool, required_only: bool = False, undeclared: list[dict] | None = None, + zip_contents: dict | None = None, + data_registry: dict | None = None, + slot_overrides: dict[str, str] | None = None, ) -> tuple[set, list, list]: """Which regional and slot alternatives this pack leaves out. @@ -707,6 +710,9 @@ def _select_variants( required_only=required_only, held_out=held, undeclared=undeclared, + zip_contents=zip_contents, + data_registry=data_registry, + slot_overrides=slot_overrides, ) if regions: region_index = region_mod.platform_region_index( @@ -961,7 +967,8 @@ def generate_pack( region_drops, region_fallbacks, slot_undecidable = _select_variants( config, pack_systems, emulators_dir, db, base_dest, emu_profiles, target_cores, source, regions, one_per_slot, required_only, - undeclared=undeclared, + undeclared=undeclared, zip_contents=zip_contents, + data_registry=data_registry, slot_overrides=slot_overrides, ) with ( @@ -1671,8 +1678,14 @@ def generate_emulator_pack( if record is None: os.makedirs(output_dir, exist_ok=True) + data_registry = load_data_dir_registry( + platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms") + ) region_drops: set[str] = ( - emulator_region_drops(selected, standalone, regions, required_only, db) + emulator_region_drops( + selected, standalone, regions, required_only, db, + zip_contents=zip_contents, data_registry=data_registry, + ) if regions else set() ) @@ -1703,9 +1716,6 @@ def generate_emulator_pack( missing_files.append(name) data_dir_notices: list[str] = [] - data_registry = load_data_dir_registry( - platforms_dir or os.path.join(os.path.dirname(__file__), "..", "platforms") - ) with ( contextlib.nullcontext(record) if record is not None @@ -4353,6 +4363,9 @@ def _manifest_selection( required_only: bool, regions: list[str] | None, undeclared: list[dict] | None = None, + zip_contents: dict | None = None, + data_registry: dict | None = None, + slot_overrides: dict[str, str] | None = None, ) -> tuple[set[str], SelectionIndex, set[str]]: """Destinations a region list removes, the groups the installer replays, and the destinations the builder holds. @@ -4376,6 +4389,9 @@ def _manifest_selection( extras_out=extras, held_out=held, undeclared=undeclared, + zip_contents=zip_contents, + data_registry=data_registry, + slot_overrides=slot_overrides, ) region_index = region_mod.platform_region_index( config, emu_profiles, target_cores @@ -4655,6 +4671,8 @@ def generate_manifest( region_drops, selection, builder_held = _manifest_selection( config, pack_systems, emulators_dir, db, base_dest, emu_profiles, target_cores, source, required_only, regions, undeclared=undeclared, + zip_contents=zip_contents, data_registry=data_registry, + slot_overrides=slot_overrides, ) # Phase 1: baseline files diff --git a/scripts/packextras.py b/scripts/packextras.py index 1fa563273..90ce9b7dd 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -1633,17 +1633,26 @@ def report_owned_by_systems(report: list[dict], system_ids: list[str]) -> list[d ] -def _ships(config: dict, file_entry: dict, db: dict) -> bool: +def _ships( + config: dict, + file_entry: dict, + db: dict, + zip_contents: dict | None, + data_registry: dict | None, + override: str | None, +) -> bool: """Whether the builder packs this declaration: resolved, and not refused by a hash frontend. A dump whose md5 Recalbox would reject is left out of the pack, so it cannot win a regional slot over a file that ships. + + Resolved with the builder's own inputs: without the data directories, + AmberELEC's bluemsx config.ini read as missing while the pack held it. """ from packresolve import resolve_declaration from validation import baseline_verdict - # The builder's resolution: no destination hint, release assets included. path, status = resolve_declaration( - file_entry, db, "bios", None, None, None, offline=True, + file_entry, db, "bios", zip_contents, data_registry, override, offline=True, ) if not path: return False @@ -1668,6 +1677,9 @@ def platform_region_groups( extras_out: list[dict] | None = None, held_out: set[str] | None = None, undeclared: list[dict] | None = None, + zip_contents: dict | None = None, + data_registry: dict | None = None, + slot_overrides: dict[str, str] | None = None, ) -> tuple[dict[str, list[tuple[str, str]]], dict[tuple[str, str, str], str]]: """Group a platform's pack candidates the way region filtering reads them. @@ -1682,7 +1694,9 @@ def platform_region_groups( two answer differently on the same request. `extras_out` receives the core extras that were grouped, for the install manifest. ``undeclared`` is the gap report when the caller holds it (undeclared_report, with the - same target_cores and include_all). + same target_cores and include_all). ``zip_contents``, ``data_registry`` + and ``slot_overrides`` (keyed by full pack destination) are the ones the + builder resolves with, so a held declaration is the one the pack ships. """ groups: dict[str, list[tuple[str, str]]] = {} for sys_id, system in systems.items(): @@ -1695,8 +1709,10 @@ def platform_region_groups( ) if dest: members.append((dest, file_entry.get("name", ""))) + full_dest = f"{base_dest}/{dest}" if base_dest else dest if held_out is not None and db is not None and _ships( - config, file_entry, db + config, file_entry, db, zip_contents, data_registry, + (slot_overrides or {}).get(full_dest), ): held_out.add(dest) @@ -1799,6 +1815,8 @@ def emulator_region_drops( regions: list[str], required_only: bool = False, db: dict | None = None, + zip_contents: dict | None = None, + data_registry: dict | None = None, ) -> set[str]: """Destinations an emulator-mode `--region` withdraws. @@ -1810,7 +1828,8 @@ def emulator_region_drops( With `db`, a requested-region file the collection lacks never displaces a held one, as in platform mode: a held scph5501.bin was withdrawn for a Japanese image nobody had collected, and the pack shipped no PlayStation - BIOS at all. + BIOS at all. ``zip_contents`` and ``data_registry`` are the ones the + emulator pack resolves with. """ import region as region_mod from packpaths import _resolve_destination @@ -1832,27 +1851,36 @@ def emulator_region_drops( (dest, fe.get("name", "")) ) if held is not None and _emulator_ships( - fe, emu_name, profile, db, dest, validation_index + fe, emu_name, profile, db, validation_index, + zip_contents, data_registry, ): held.add(dest) return region_mod.resolve_region_drops(region_groups, region_index, regions, held) def _emulator_ships( - file_entry: dict, emu_name: str, profile: dict, db: dict, dest: str, - validation_index: dict, + file_entry: dict, emu_name: str, profile: dict, db: dict, + validation_index: dict, zip_contents: dict | None, data_registry: dict | None, ) -> bool: """Whether the emulator pack ships this entry: held, and not refused by the core's own check, the test _ships applies on the platform side. + + Resolved as generate_emulator_pack resolves: without the data + directories, dolphin-mmjr2's nwc24msg.cfg read as missing while the pack + held it. """ + from packresolve import resolve_file from validation import validated_choice, with_archive_members - local, _status = resolve_local_file( + local, status = resolve_file( {**with_archive_members(file_entry, profile.get("files", [])), "source_profile": emu_name}, - db, dest_hint=dest, + db, "bios", zip_contents, + dest_hint=file_entry.get("path", ""), + data_dir_registry=data_registry, + offline=True, ) - if not local: + if status in ("not_found", "user_provided") or not local: return False _chosen, disagreement = validated_choice( file_entry, local, db, validation_index, "bios", None, emulators={emu_name}, diff --git a/scripts/packverify.py b/scripts/packverify.py index 60486be1a..2f6d825d1 100644 --- a/scripts/packverify.py +++ b/scripts/packverify.py @@ -30,6 +30,7 @@ from packextras import platform_region_groups from packpaths import _flat from nativemode import reads_file_contents import region as region_mod +import slots from packresolve import resolve_declaration, resolve_file from exporter.candidates import BIOS, EntryCategories from gamedata import without_game_data @@ -706,6 +707,7 @@ def verify_pack_against_platform( config, emu_profiles, target_cores ) held: set[str] = set() + held_zip_contents = build_zip_contents_index(db) if db is not None else {} region_groups, _extra_dests = platform_region_groups( config, expected_systems, @@ -718,6 +720,12 @@ def verify_pack_against_platform( include_all=(source == "truth"), required_only=required_only, held_out=held, + zip_contents=held_zip_contents, + data_registry=data_registry, + slot_overrides=slots.pack_overrides( + config, emu_profiles, db, held_zip_contents, data_registry, + target_cores=target_cores, source=source, + ) if db is not None else None, ) region_drops = region_mod.resolve_region_drops( region_groups, region_index, regions, held diff --git a/scripts/verify.py b/scripts/verify.py index 5f909d155..bd6c7069a 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1210,12 +1210,13 @@ def verify_platform( else load_emulator_profiles(emulators_dir) ) base_dest = config.get("base_destination", "") + pack_overrides = slots.pack_overrides( + config, profiles, db, zip_contents, data_dir_registry, + target_cores=target_cores, + ) slot_overrides = { (key[len(base_dest) + 1:] if base_dest and key.startswith(f"{base_dest}/") else key): path - for key, path in slots.pack_overrides( - config, profiles, db, zip_contents, data_dir_registry, - target_cores=target_cores, - ).items() + for key, path in pack_overrides.items() } # Build HLE + validation indexes from emulator profiles @@ -1279,6 +1280,9 @@ def verify_platform( target_cores=target_cores, required_only=required_only, held_out=held, + zip_contents=zip_contents, + data_registry=data_dir_registry, + slot_overrides=pack_overrides, ) region_index = region_mod.platform_region_index(config, profiles, target_cores) region_drops = region_mod.resolve_region_drops( @@ -2153,7 +2157,10 @@ def verify_emulator( if regions: from packextras import emulator_region_drops - region_drops = emulator_region_drops(selected, standalone, regions, required_only, db=db) + region_drops = emulator_region_drops( + selected, standalone, regions, required_only, db=db, + zip_contents=zip_contents, data_registry=data_registry, + ) # The emulator pack walks profiles in name order and keeps the first # file at a path, folding case: a later profile's different file there diff --git a/tests/test_region_held.py b/tests/test_region_held.py index cb384cb44..36fb524f7 100644 --- a/tests/test_region_held.py +++ b/tests/test_region_held.py @@ -41,9 +41,66 @@ class ARefusedDumpHoldsNothing(unittest.TestCase): "by_path_suffix": {}}, } entry = {"name": "us.bin", "md5": "0" * 32} - self.assertFalse(_ships({"verification_mode": "md5"}, entry, db)) - self.assertTrue(_ships({"verification_mode": "existence"}, entry, db)) - self.assertTrue(_ships({"verification_mode": "md5"}, {"name": "us.bin"}, db)) + def ships(config: dict, fe: dict) -> bool: + return _ships(config, fe, db, None, None, None) + + self.assertFalse(ships({"verification_mode": "md5"}, entry)) + self.assertTrue(ships({"verification_mode": "existence"}, entry)) + self.assertTrue(ships({"verification_mode": "md5"}, {"name": "us.bin"})) + + +class ADataDirectoryCopyIsHeld(unittest.TestCase): + """The region pass resolves with the builder's inputs. + + Without the data directories, AmberELEC's config.ini (served from the + bluemsx tree) and dolphin-mmjr2's nwc24msg.cfg (served from the Dolphin + Sys tree) read as missing while both packs held them. + """ + + @classmethod + def setUpClass(cls): + if not (REPO_ROOT / "data" / "bluemsx").is_dir() or not ( + REPO_ROOT / "data" / "dolphin-sys" + ).is_dir(): + raise unittest.SkipTest("data/ not refreshed") + previous = os.getcwd() + os.chdir(REPO_ROOT) + try: + from common import load_data_dir_registry, load_database + from common import load_emulator_profiles, load_platform_config + cls.db = load_database("database.json") + cls.registry = load_data_dir_registry("platforms") + cls.profiles = load_emulator_profiles("emulators") + cls.amberelec = load_platform_config("amberelec", "platforms") + finally: + os.chdir(previous) + + def setUp(self): + previous = os.getcwd() + os.chdir(REPO_ROOT) + self.addCleanup(os.chdir, previous) + + def test_platform_declaration_from_a_data_tree(self): + entry = next( + fe for fe in self.amberelec["systems"]["coleco-colecovision"]["files"] + if fe["destination"].endswith("Dina/config.ini") + ) + self.assertTrue( + _ships(self.amberelec, entry, self.db, {}, self.registry, None) + ) + + def test_emulator_entry_from_a_data_tree(self): + from packextras import _emulator_ships + from validation import _build_validation_index + + profile = self.profiles["dolphin-mmjr2"] + entry = next( + fe for fe in profile["files"] if fe.get("name") == "nwc24msg.cfg" + ) + index = _build_validation_index({"dolphin-mmjr2": profile}) + self.assertTrue(_emulator_ships( + entry, "dolphin-mmjr2", profile, self.db, index, {}, self.registry, + )) if __name__ == "__main__":