diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 2bd8cdf4..2d529757 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -1456,10 +1456,13 @@ def generate_emulator_pack( with _writing_pack(zip_path) as zf: for emu_name, profile in sorted(selected): pack_structure = profile.get("pack_structure") - files = expand_directory_entries( - filter_files_by_mode(profile.get("files", []), standalone), - db, - standalone, + files = expand_agnostic_entries( + expand_directory_entries( + filter_files_by_mode(profile.get("files", []), standalone), + db, + standalone, + ), + emu_name, profile, db, standalone, all_profiles, ) if system_ids: files = [ @@ -4648,6 +4651,7 @@ from packextras import ( # noqa: E402,F401 platform_region_groups, emulator_region_drops, _emulator_region_group, + expand_agnostic_entries, ) diff --git a/scripts/packextras.py b/scripts/packextras.py index d144c172..9b2d89e6 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -400,6 +400,117 @@ def _agnostic_scan_extras( return extras +def _placeholder_scan( + files: list[dict], emu_name: str, profile: dict, db: dict, standalone: bool, + all_profiles: dict | None = None, +) -> list[dict]: + """The dumps a profile made of placeholders accepts, found by evidence. + + `.bin` resolves to nothing, so it anchors no directory. The folder + where other profiles of the same system resolve their hashed files of + that shape is one: PCSX2 proves bios/Sony/PlayStation 2/ by content, and + LRPS2 reads any 4-8 MB image. Only that folder, not the ones below it + (its DVD firmware). Nothing proven, nothing scanned. + """ + files_db = db.get("files", {}) + systems = set(profile.get("systems", [])) + extras: list[dict] = [] + for seed in files: + name = str(seed.get("name", "")) + if "<" not in name or seed.get("category", "bios") != "bios": + continue + if not (seed.get("size") or seed.get("min_size") or seed.get("max_size")): + continue + directories: set[str] = set() + for other_name, other in (all_profiles or load_emulator_profiles("emulators")).items(): + if other_name == emu_name or not systems & set(other.get("systems", [])): + continue + for f in other.get("files", []): + if not any(f.get(h) for h in ("sha1", "md5", "sha256", "crc32")): + continue + if f.get("size") and not size_fits(seed, f["size"]): + continue + local, status = resolve_local_file({**f, "source_profile": other_name}, db) + if local and "/" in local and resolution_is_hash_exact(status): + directories.add(local.rsplit("/", 1)[0]) + layout = (seed.get("standalone_path") if standalone else None) or seed.get("path") or "" + folder = layout.rsplit("/", 1)[0] if "/" in layout else "" + for sha1, entry in sorted(files_db.items(), key=lambda kv: kv[1].get("path", "")): + path = entry.get("path", "") + if "/" not in path or path.rsplit("/", 1)[0] not in directories: + continue + if "/.variants/" in path or not size_fits(seed, entry.get("size", 0)): + continue + scan_name = entry.get("name", "") + dest = f"{folder}/{scan_name}" if folder else scan_name + extras.append({ + "name": scan_name, + "destination": dest, + "sha1": sha1, + "source_system": seed.get("system"), + "region": seed.get("region"), + }) + extras.extend(_companion_extras( + _companions(path, scan_name, _companion_extensions(profile, name), files_db), + folder, "", set(), profile, emu_name, seed, + )) + return extras + + +def expand_agnostic_entries( + files: list[dict], emu_name: str, profile: dict, db: dict, standalone: bool, + all_profiles: dict | None = None, +) -> list[dict]: + """A filename-agnostic profile's placeholders, as the files they accept. + + `.bin` names no file: the core takes any dump of the right shape + in its BIOS folder, which is what the platform pack's scan carries. The + emulator pack and verify --emulator looked the placeholder up by name, + found nothing and shipped LRPS2 without a PS2 BIOS. The same scan + replaces the placeholders here; with nothing found they stay, a gap. + """ + if profile.get("bios_mode") != "agnostic" and not any(f.get("agnostic") for f in files): + return files + def folder(entry: dict) -> str: + layout = (entry.get("standalone_path") if standalone else None) or entry.get("path") or "" + return layout.rsplit("/", 1)[0] if "/" in layout else "" + + scanned = _agnostic_scan_extras( + {emu_name: {**profile, "files": files}}, {emu_name}, db, + db.get("indexes", {}).get("by_name", {}), set(), "", + {emu_name} if standalone else set(), + ) + # A placeholder is answered by files found in its own folder: PCEE2's + # eeprom.dat, scanned at the root, said nothing about pcsx2/bios/. + covered = {e["destination"].rsplit("/", 1)[0] if "/" in e["destination"] else "" for e in scanned} + open_seeds = [ + f for f in files if "<" in str(f.get("name", "")) and folder(f) not in covered + ] + scanned += _placeholder_scan( + open_seeds, emu_name, profile, db, standalone, all_profiles + ) + covered = {e["destination"].rsplit("/", 1)[0] if "/" in e["destination"] else "" for e in scanned} + if not scanned: + return files + concrete = [ + { + "name": extra["name"], + "path": extra["destination"], + **({"standalone_path": extra["destination"]} if standalone else {}), + "sha1": extra["sha1"], + "required": False, + **({"system": extra["source_system"]} if extra.get("source_system") else {}), + **({"region": extra["region"]} if extra.get("region") else {}), + "agnostic_scan": True, + } + for extra in scanned + ] + return [ + f for f in files + if "<" not in str(f.get("name", "")) or folder(f) not in covered + ] + concrete + + def _companion_extras( companions: list[tuple[str, dict]], folder: str, diff --git a/scripts/verify.py b/scripts/verify.py index ee67b23b..7f47241e 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -1556,6 +1556,7 @@ def verify_emulator( same list would be narrowed, through the emulator pack's own drop set. `system_ids` keeps the entries of those systems, as the system pack does. """ + from packextras import expand_agnostic_entries from packpaths import ( _archive_destination, _register_path, @@ -1623,10 +1624,13 @@ def verify_emulator( placed[dest.lower()] = (path, owner, dest) _register_path(dest, placed_files, placed_parents) for emu_name, profile in sorted(selected): - files = expand_directory_entries( - filter_files_by_mode(profile.get("files", []), standalone), - db, - standalone, + files = expand_agnostic_entries( + expand_directory_entries( + filter_files_by_mode(profile.get("files", []), standalone), + db, + standalone, + ), + emu_name, profile, db, standalone, all_profiles, ) if system_ids: files = [fe for fe in files if entry_serves_systems(fe, profile, system_ids)] diff --git a/tests/test_emulator_pack_counts.py b/tests/test_emulator_pack_counts.py index 13dfd616..3b43f7cf 100644 --- a/tests/test_emulator_pack_counts.py +++ b/tests/test_emulator_pack_counts.py @@ -62,5 +62,72 @@ class PackAndReportAgree(unittest.TestCase): self.assertEqual(_count(summary, "refused"), _count(verdict, "untested")) +class APlaceholderIsAnsweredByEvidence(unittest.TestCase): + """LRPS2 declares only .bin: no name to look up, so its emulator + pack shipped no PS2 BIOS and verify said missing. Another profile of the + system proves the folder by hash; the images there of the declared + size are what the core reads.""" + + def test_pack_and_report_carry_the_images(self): + import hashlib + import io + import os + import shutil + import zipfile + from contextlib import redirect_stdout + + import yaml + + sys.path.insert(0, str(REPO_ROOT / "scripts")) + from common import _emulator_profiles_cache # noqa: PLC0415 + from generate_pack import generate_emulator_pack # noqa: PLC0415 + from verify import verify_emulator # noqa: PLC0415 + + _emulator_profiles_cache.clear() + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + previous = os.getcwd() + os.chdir(root) + self.addCleanup(os.chdir, previous) + db = {"files": {}, "indexes": {"by_md5": {}, "by_name": {}, "by_crc32": {}, + "by_path_suffix": {}}} + hashes = {} + for name, data in (("proven.bin", b"P" * 64), ("other.bin", b"O" * 64), + ("small.bin", b"S" * 8)): + rel = f"bios/Sony/PS2/{name}" + Path(rel).parent.mkdir(parents=True, exist_ok=True) + Path(rel).write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + hashes[name] = sha1 + db["files"][sha1] = {"path": rel, "name": name, "size": len(data), + "md5": hashlib.md5(data).hexdigest(), "crc32": ""} + db["indexes"]["by_name"][name] = [sha1] + Path("emulators").mkdir() + Path("emulators/named.yml").write_text(yaml.safe_dump({ + "emulator": "Named", "type": "libretro", "systems": ["sony-playstation-2"], + "files": [{"name": "proven.bin", "sha1": hashes["proven.bin"], "size": 64}], + })) + Path("emulators/free.yml").write_text(yaml.safe_dump({ + "emulator": "Free", "type": "libretro", "bios_mode": "agnostic", + "systems": ["sony-playstation-2"], + "files": [{"name": ".bin", "path": "pcsx2/bios/.bin", + "min_size": 32, "max_size": 128, "required": True}], + })) + report = verify_emulator(["free"], "emulators", db, platforms_dir="platforms") + with redirect_stdout(io.StringIO()): + zip_path = generate_emulator_pack( + ["free"], "emulators", db, "bios", "out", offline=True, + platforms_dir="platforms", + ) + with zipfile.ZipFile(zip_path) as zf: + packed = sorted(zf.namelist()) + self.assertEqual( + [n for n in packed if n != "README.txt"], + ["pcsx2/bios/other.bin", "pcsx2/bios/proven.bin"], + ) + self.assertEqual(report["total_files"], 2) + + if __name__ == "__main__": unittest.main()