diff --git a/scripts/packextras.py b/scripts/packextras.py index 056eaedd0..df9d22557 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -513,6 +513,7 @@ def _agnostic_scan_extras( extras_prefix: str, standalone_set: set[str] | None = None, taken: dict[str, dict] | None = None, + all_profiles: dict | None = None, ) -> list[dict]: """Every interchangeable candidate a filename-agnostic core accepts. @@ -521,6 +522,8 @@ def _agnostic_scan_extras( The directory is the delicate part: anchoring it on a single filename walked into another emulator's tree, so the profile's own files have to agree before the scan follows an ambiguous name. + ``all_profiles`` are the profiles a placeholder's directory is proven + from, ``profiles`` when not given. """ extras: list[dict] = [] files_db = db.get("files", {}) @@ -617,7 +620,21 @@ def _agnostic_scan_extras( resolved_dirs.get(id(image), "") if image else resolved_dirs.get(id(f), "") ) - if not path_prefix: + # A placeholder (.bin) resolves to nothing and anchors no + # directory. The folders where the system's other profiles + # resolve hashed files of that shape do, those folders only: + # LRPS2 and PCEE2 are made of placeholders, and on Batocera, + # where no other libretro PS2 core anchors pcsx2/bios/, the + # pack held nothing there. A companion (.nvm) looks + # where its image is proven: RetroBat's pcsx2/bios/ had + # scph39001.bin and its .nvm only at the BIOS root. + exact_dirs: frozenset[str] = frozenset() + anchor = image if image else f + if not path_prefix and "<" in str(anchor.get("name", "")): + exact_dirs = placeholder_directories( + anchor, emu_name, profile, db, all_profiles or profiles + ) + if not path_prefix and not exact_dirs: continue # A name matching one file identifies it. A name matching several # identifies nothing on its own, so the profile's other files have @@ -629,7 +646,11 @@ def _agnostic_scan_extras( # Scan DB for all files under this prefix matching size for sha1, entry in files_db.items(): path = entry.get("path", "") - if not path.startswith(path_prefix): + if exact_dirs: + if ("/" not in path or path.rsplit("/", 1)[0] not in exact_dirs + or "/.variants/" in path): + continue + elif not path.startswith(path_prefix): continue if not companion_ext and not size_fits(f, entry.get("size", 0)): continue @@ -686,6 +707,31 @@ def _agnostic_scan_extras( return extras +def placeholder_directories( + seed: dict, emu_name: str, profile: dict, db: dict, all_profiles: dict, +) -> frozenset[str]: + """The folders that hold the dumps a placeholder entry accepts. + + Each folder where another profile of the same system resolves a hashed + file of the seed's shape by content: PCSX2 proves bios/Sony/PlayStation 2/. + Only that folder, not the ones below it (its DVD firmware). + """ + systems = set(profile.get("systems", [])) + directories: set[str] = set() + for other_name, other in all_profiles.items(): + if other_name == emu_name or not systems & set(other.get("systems", [])): + continue + for f in other.get("files", []): + if not isinstance(f, dict) or not any(f.get(h) for h in ("sha1", "md5", "sha256", "crc32")): + continue + if f.get("size") and not size_fits(seed, f["size"]): + continue + local, status = resolve_local_file({**f, "source_profile": other_name}, db) + if local and "/" in local and resolution_is_hash_exact(status): + directories.add(local.rsplit("/", 1)[0]) + return frozenset(directories) + + def _placeholder_scan( files: list[dict], emu_name: str, profile: dict, db: dict, standalone: bool, all_profiles: dict | None = None, @@ -699,7 +745,6 @@ def _placeholder_scan( (its DVD firmware). Nothing proven, nothing scanned. """ files_db = db.get("files", {}) - systems = set(profile.get("systems", [])) extras: list[dict] = [] for seed in files: name = str(seed.get("name", "")) @@ -707,18 +752,10 @@ def _placeholder_scan( continue if not (seed.get("size") or seed.get("min_size") or seed.get("max_size")): continue - directories: set[str] = set() - for other_name, other in (all_profiles or load_emulator_profiles("emulators")).items(): - if other_name == emu_name or not systems & set(other.get("systems", [])): - continue - for f in other.get("files", []): - if not any(f.get(h) for h in ("sha1", "md5", "sha256", "crc32")): - continue - if f.get("size") and not size_fits(seed, f["size"]): - continue - local, status = resolve_local_file({**f, "source_profile": other_name}, db) - if local and "/" in local and resolution_is_hash_exact(status): - directories.add(local.rsplit("/", 1)[0]) + directories = placeholder_directories( + seed, emu_name, profile, db, + all_profiles or load_emulator_profiles("emulators"), + ) layout = (seed.get("standalone_path") if standalone else None) or seed.get("path") or "" folder = layout.rsplit("/", 1)[0] if "/" in layout else "" for sha1, entry in sorted(files_db.items(), key=lambda kv: kv[1].get("path", "")): diff --git a/tests/test_agnostic_scope.py b/tests/test_agnostic_scope.py index 3a1d3d27a..1326f8242 100644 --- a/tests/test_agnostic_scope.py +++ b/tests/test_agnostic_scope.py @@ -55,6 +55,66 @@ class AgnosticScan(unittest.TestCase): ["pcsx2/bios/scph1.bin", "pcsx2/bios/scph2.bin"]) + def test_a_placeholder_scans_the_folder_another_profile_proves(self): + """LRPS2 and PCEE2 name their BIOS .bin, which resolves nowhere. + + On RetroArch yaps2 anchored pcsx2/bios/ for them; on Batocera no + other libretro PS2 core runs, and the pack held nothing there. The + folder where another profile of the system resolves a hashed dump of + that shape is the evidence, that folder only. + """ + from packextras import _agnostic_scan_extras + + with tempfile.TemporaryDirectory() as tmp: + previous = os.getcwd() + os.chdir(tmp) + self.addCleanup(os.chdir, previous) + files, by_name = {}, {} + for path, data in ( + ("bios/PS2/scph1.bin", b"a" * 64), + ("bios/PS2/scph2.bin", b"b" * 64), + ("bios/PS2/scph2.bin.rom1", b"r" * 8), + ("bios/PS2/scph1.nvm", b"n" * 8), + ("bios/PS2/.variants/scph3.bin", b"e" * 64), + ("bios/PS2/DVD/dvd.bin", b"f" * 64), + ): + Path(path).parent.mkdir(parents=True, exist_ok=True) + Path(path).write_bytes(data) + sha1 = hashlib.sha1(data).hexdigest() + name = path.rsplit("/", 1)[1] + files[sha1] = {"path": path, "name": name, "size": len(data), "sha1": sha1} + by_name.setdefault(name, []).append(sha1) + db = {"files": files, "indexes": {"by_name": by_name, "by_md5": {}, "by_crc32": {}, + "by_path_suffix": {}}} + placeholders = {"emulator": "L", "type": "libretro", "bios_mode": "agnostic", + "bios_directory": "pcsx2/bios/", "systems": ["ps2"], "files": [ + {"name": ".bin", "found_by": "content", + "path": "pcsx2/bios/.bin", "min_size": 32, "max_size": 64}, + {"name": ".rom1", "path": "pcsx2/bios/.rom1"}, + {"name": ".nvm", "path": "pcsx2/bios/.nvm", "size": 8}, + ]} + pinned = {"emulator": "S", "type": "standalone", "systems": ["ps2"], "files": [ + {"name": "scph1.bin", "size": 64, + "sha1": hashlib.sha1(b"a" * 64).hexdigest()}, + ]} + extras = _agnostic_scan_extras( + {"l": placeholders, "s": pinned}, {"l"}, db, by_name, set(), "") + self.assertEqual(sorted(e["destination"] for e in extras), [ + "pcsx2/bios/scph1.bin", "pcsx2/bios/scph1.nvm", "pcsx2/bios/scph2.bin", + "pcsx2/bios/scph2.bin.rom1", + ]) + self.assertTrue(all(e["source_profile"] == "l" for e in extras)) + # An image the platform already ships there still gets its + # companion: RetroBat lists pcsx2/bios/scph39001.bin itself. + shipped = {"pcsx2/bios/scph1.bin"} + extras = _agnostic_scan_extras( + {"l": placeholders, "s": pinned}, {"l"}, db, by_name, shipped, "") + self.assertIn("pcsx2/bios/scph1.nvm", {e["destination"] for e in extras}) + # Nothing proves the folder: nothing is scanned. + self.assertEqual( + _agnostic_scan_extras({"l": placeholders}, {"l"}, db, by_name, set(), ""), []) + + class BundledHashIsProof(unittest.TestCase): """A bundled file with a declared hash is served by those bytes or flagged.