fix: pack undistributed core extras in a split part

scripts/generate_pack.py

@@ -591,6 +591,7 @@ def generate_pack(
     required_only: bool = False,
     system_filter: list[str] | None = None,
     precomputed_extras: list[dict] | None = None,
+    extras_only: bool = False,
     source: str = "full",
     flatten: bool = True,
     regions: list[str] | None = None,
@@ -690,6 +691,10 @@ def generate_pack(
             )
             return None
         pack_systems = filtered
+    if extras_only:
+        # The part of a split that carries the core extras no platform system
+        # owns: the platform's own files and trees went to the other parts.
+        pack_systems = {}
 
     preferred_entries: dict[str, int] = {}
     if source != "truth":
@@ -1707,6 +1712,14 @@ def generate_split_packs(
         )
     else:
         all_extras = []
+    version = config.get("version", config.get("dat_version", ""))
+    ver_tag = f"_{version.replace(' ', '')}" if version else ""
+    narrow_tags = "".join(
+        tag
+        for tag, _label in _narrowings(
+            source, regions, target_name, one_per_slot, required_only
+        )
+    )
     results = []
     for group_name, group_system_ids in sorted(groups.items()):
         group_extras = _extras_for_systems(all_extras, group_system_ids)
@@ -1731,14 +1744,6 @@ def generate_split_packs(
             offline=offline,
         )
         if zip_path:
-            version = config.get("version", config.get("dat_version", ""))
-            ver_tag = f"_{version.replace(' ', '')}" if version else ""
-            narrow_tags = "".join(
-                tag
-                for tag, _label in _narrowings(
-                    source, regions, target_name, one_per_slot, required_only
-                )
-            )
             safe_group = _name_part(group_name, "_")
             new_name = f"{platform_display.replace(' ', '_')}{ver_tag}{narrow_tags}_{safe_group}_BIOS_Pack.zip"
             new_path = os.path.join(split_dir, new_name)
@@ -1747,7 +1752,9 @@ def generate_split_packs(
                 zip_path = new_path
             results.append(zip_path)
 
-    # Warn about extras that couldn't be distributed (emulators without systems: field)
+    # Extras whose system the platform does not declare, or that name none,
+    # belong to no group: they go in a part of their own, or the parts would
+    # not add up to the full pack (4507 of RetroArch's 5430 were left out).
     distributed = {
         id(fe)
         for fe in _extras_for_systems(
@@ -1756,12 +1763,23 @@ def generate_split_packs(
     }
     undistributed = [fe for fe in all_extras if id(fe) not in distributed]
     if undistributed:
-        emus = sorted({fe.get("source_emulator", "?") for fe in undistributed})
+        zip_path = generate_pack(
-        print(
+            platform_name, platforms_dir, db, bios_dir, split_dir,
-            f"  NOTE: {len(undistributed)} core extras from {len(emus)} emulators "
+            emulators_dir=emulators_dir, zip_contents=zip_contents,
-            f"not in split packs (missing systems: field in profiles: "
+            data_registry=data_registry, emu_profiles=emu_profiles,
-            f"{', '.join(emus[:5])}{'...' if len(emus) > 5 else ''})"
+            target_cores=target_cores, required_only=required_only,
+            precomputed_extras=undistributed, extras_only=True, source=source,
+            regions=regions, target_name=target_name, one_per_slot=one_per_slot,
+            offline=offline,
         )
+        if zip_path:
+            new_path = os.path.join(
+                split_dir,
+                f"{platform_display.replace(' ', '_')}{ver_tag}{narrow_tags}"
+                "_Other_Cores_BIOS_Pack.zip",
+            )
+            os.replace(zip_path, new_path)
+            results.append(new_path)
 
     return results
 

tests/test_split_extras.py

@@ -0,0 +1,74 @@
+"""Split packs add up to the full pack.
+
+Core extras went only to the part of a system the platform declares: an
+extra for a system it does not declare, or with none, was in the full pack
+and in no part (4507 of RetroArch's 5430).
+"""
+
+from __future__ import annotations
+
+import sys
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+from common import build_zip_contents_index, compute_hashes, load_emulator_profiles  # noqa: E402
+from generate_pack import generate_split_packs  # noqa: E402
+
+
+class SplitPartsCoverTheExtras(unittest.TestCase):
+    def test_an_extra_of_an_undeclared_system_has_a_part(self):
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+            root = Path(tmp)
+            (root / "platforms").mkdir()
+            (root / "emulators").mkdir()
+            files = {}
+            for name in ("bios_a.bin", "core_c.bin"):
+                path = root / "bios" / name
+                path.parent.mkdir(exist_ok=True)
+                path.write_bytes(name.encode())
+                files[name] = (str(path), compute_hashes(str(path)))
+            db = {
+                "files": {
+                    h["sha1"]: {"name": n, "md5": h["md5"], "sha1": h["sha1"],
+                                "sha256": h["sha256"], "path": p, "size": len(n)}
+                    for n, (p, h) in files.items()
+                },
+                "indexes": {
+                    "by_md5": {h["md5"]: h["sha1"] for _p, h in files.values()},
+                    "by_name": {n: [h["sha1"]] for n, (_p, h) in files.items()},
+                    "by_crc32": {}, "by_path_suffix": {},
+                },
+            }
+            (root / "platforms" / "_registry.yml").write_text(
+                yaml.dump({"platforms": {"plat": {"status": "active"}}})
+            )
+            (root / "platforms" / "plat.yml").write_text(yaml.dump({
+                "platform": "Plat", "verification_mode": "existence",
+                "cores": ["corec"],
+                "systems": {"sys-a": {"files": [
+                    {"name": "bios_a.bin", "sha1": files["bios_a.bin"][1]["sha1"]}
+                ]}},
+            }))
+            (root / "emulators" / "corec.yml").write_text(yaml.dump({
+                "emulator": "CoreC", "type": "libretro", "systems": ["sys-other"],
+                "files": [{"name": "core_c.bin", "required": True}],
+            }))
+            parts = generate_split_packs(
+                "plat", str(root / "platforms"), db, str(root / "bios"), str(root / "dist"),
+                emulators_dir=str(root / "emulators"),
+                zip_contents=build_zip_contents_index(db),
+                emu_profiles=load_emulator_profiles(str(root / "emulators")),
+            )
+            members = {n for p in parts for n in zipfile.ZipFile(p).namelist()}
+            self.assertIn("core_c.bin", members)
+
+
+if __name__ == "__main__":
+    unittest.main()