From 55983f6b013b360372b9dd69d17080a01e112b7c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Tue, 6 Oct 2026 09:53:09 +0200 Subject: [PATCH] fix: pack undistributed core extras in a split part --- scripts/generate_pack.py | 46 ++++++++++++++++-------- tests/test_split_extras.py | 74 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 14 deletions(-) create mode 100644 tests/test_split_extras.py diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 979dea12..d38a91cc 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -591,6 +591,7 @@ def generate_pack( required_only: bool = False, system_filter: list[str] | None = None, precomputed_extras: list[dict] | None = None, + extras_only: bool = False, source: str = "full", flatten: bool = True, regions: list[str] | None = None, @@ -690,6 +691,10 @@ def generate_pack( ) return None pack_systems = filtered + if extras_only: + # The part of a split that carries the core extras no platform system + # owns: the platform's own files and trees went to the other parts. + pack_systems = {} preferred_entries: dict[str, int] = {} if source != "truth": @@ -1707,6 +1712,14 @@ def generate_split_packs( ) else: all_extras = [] + version = config.get("version", config.get("dat_version", "")) + ver_tag = f"_{version.replace(' ', '')}" if version else "" + narrow_tags = "".join( + tag + for tag, _label in _narrowings( + source, regions, target_name, one_per_slot, required_only + ) + ) results = [] for group_name, group_system_ids in sorted(groups.items()): group_extras = _extras_for_systems(all_extras, group_system_ids) @@ -1731,14 +1744,6 @@ def generate_split_packs( offline=offline, ) if zip_path: - version = config.get("version", config.get("dat_version", "")) - ver_tag = f"_{version.replace(' ', '')}" if version else "" - narrow_tags = "".join( - tag - for tag, _label in _narrowings( - source, regions, target_name, one_per_slot, required_only - ) - ) safe_group = _name_part(group_name, "_") new_name = f"{platform_display.replace(' ', '_')}{ver_tag}{narrow_tags}_{safe_group}_BIOS_Pack.zip" new_path = os.path.join(split_dir, new_name) @@ -1747,7 +1752,9 @@ def generate_split_packs( zip_path = new_path results.append(zip_path) - # Warn about extras that couldn't be distributed (emulators without systems: field) + # Extras whose system the platform does not declare, or that name none, + # belong to no group: they go in a part of their own, or the parts would + # not add up to the full pack (4507 of RetroArch's 5430 were left out). distributed = { id(fe) for fe in _extras_for_systems( @@ -1756,12 +1763,23 @@ def generate_split_packs( } undistributed = [fe for fe in all_extras if id(fe) not in distributed] if undistributed: - emus = sorted({fe.get("source_emulator", "?") for fe in undistributed}) - print( - f" NOTE: {len(undistributed)} core extras from {len(emus)} emulators " - f"not in split packs (missing systems: field in profiles: " - f"{', '.join(emus[:5])}{'...' if len(emus) > 5 else ''})" + zip_path = generate_pack( + platform_name, platforms_dir, db, bios_dir, split_dir, + emulators_dir=emulators_dir, zip_contents=zip_contents, + data_registry=data_registry, emu_profiles=emu_profiles, + target_cores=target_cores, required_only=required_only, + precomputed_extras=undistributed, extras_only=True, source=source, + regions=regions, target_name=target_name, one_per_slot=one_per_slot, + offline=offline, ) + if zip_path: + new_path = os.path.join( + split_dir, + f"{platform_display.replace(' ', '_')}{ver_tag}{narrow_tags}" + "_Other_Cores_BIOS_Pack.zip", + ) + os.replace(zip_path, new_path) + results.append(new_path) return results diff --git a/tests/test_split_extras.py b/tests/test_split_extras.py new file mode 100644 index 00000000..568cd34b --- /dev/null +++ b/tests/test_split_extras.py @@ -0,0 +1,74 @@ +"""Split packs add up to the full pack. + +Core extras went only to the part of a system the platform declares: an +extra for a system it does not declare, or with none, was in the full pack +and in no part (4507 of RetroArch's 5430). +""" + +from __future__ import annotations + +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from common import build_zip_contents_index, compute_hashes, load_emulator_profiles # noqa: E402 +from generate_pack import generate_split_packs # noqa: E402 + + +class SplitPartsCoverTheExtras(unittest.TestCase): + def test_an_extra_of_an_undeclared_system_has_a_part(self): + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + root = Path(tmp) + (root / "platforms").mkdir() + (root / "emulators").mkdir() + files = {} + for name in ("bios_a.bin", "core_c.bin"): + path = root / "bios" / name + path.parent.mkdir(exist_ok=True) + path.write_bytes(name.encode()) + files[name] = (str(path), compute_hashes(str(path))) + db = { + "files": { + h["sha1"]: {"name": n, "md5": h["md5"], "sha1": h["sha1"], + "sha256": h["sha256"], "path": p, "size": len(n)} + for n, (p, h) in files.items() + }, + "indexes": { + "by_md5": {h["md5"]: h["sha1"] for _p, h in files.values()}, + "by_name": {n: [h["sha1"]] for n, (_p, h) in files.items()}, + "by_crc32": {}, "by_path_suffix": {}, + }, + } + (root / "platforms" / "_registry.yml").write_text( + yaml.dump({"platforms": {"plat": {"status": "active"}}}) + ) + (root / "platforms" / "plat.yml").write_text(yaml.dump({ + "platform": "Plat", "verification_mode": "existence", + "cores": ["corec"], + "systems": {"sys-a": {"files": [ + {"name": "bios_a.bin", "sha1": files["bios_a.bin"][1]["sha1"]} + ]}}, + })) + (root / "emulators" / "corec.yml").write_text(yaml.dump({ + "emulator": "CoreC", "type": "libretro", "systems": ["sys-other"], + "files": [{"name": "core_c.bin", "required": True}], + })) + parts = generate_split_packs( + "plat", str(root / "platforms"), db, str(root / "bios"), str(root / "dist"), + emulators_dir=str(root / "emulators"), + zip_contents=build_zip_contents_index(db), + emu_profiles=load_emulator_profiles(str(root / "emulators")), + ) + members = {n for p in parts for n in zipfile.ZipFile(p).namelist()} + self.assertIn("core_c.bin", members) + + +if __name__ == "__main__": + unittest.main()