fix: report the extras a platform cannot place

scripts/packextras.py

@@ -134,6 +134,42 @@ def _map_emulator_to_slug(
             return sys_to_slug.get(target, "")
     return ""
 
+def _slug_for(
+    u: dict, profiles: dict, platform_systems: set, norm_map: dict, sys_to_slug: dict
+) -> str:
+    """The platform slug a report entry's emulator files under, or empty."""
+    emu_name = u.get("profile") or u.get("emulator", "")
+    profile = profiles.get(emu_name, {})
+    if not profile:
+        # The report names the display name where the key failed.
+        profile = next(
+            (pp for pp in profiles.values() if pp.get("emulator") == emu_name), {}
+        )
+    return _map_emulator_to_slug(profile, platform_systems, norm_map, sys_to_slug)
+
+
+def unplaceable_extras(config: dict, undeclared: list[dict], profiles: dict) -> set:
+    """Keys of the report entries a slug-based platform cannot place.
+
+    RomM files every BIOS under the slug of its system and knows no slug
+    for DOS or for the arcade sets MAME 2003 reads; the builder skipped
+    those entries and the report counted them in the pack.
+    """
+    from common import _norm_system_id
+
+    is_slug_based, sys_to_slug = _detect_slug_structure(config)
+    if not is_slug_based:
+        return set()
+    platform_systems = set(config.get("systems", {}).keys())
+    norm_map = {_norm_system_id(sid): sid for sid in platform_systems}
+    return {
+        (u.get("emulator", ""), u.get("name", ""), u.get("path") or "")
+        for u in undeclared
+        if u.get("in_repo")
+        and not _slug_for(u, profiles, platform_systems, norm_map, sys_to_slug)
+    }
+
+
 def _agnostic_scan_extras(
     profiles: dict,
     relevant: set,
@@ -412,9 +448,18 @@ def _collect_emulator_extras(
     emu_profiles: dict | None = None,
     target_cores: set[str] | None = None,
     include_all: bool = False,
+    claimants: dict[tuple[str, str, str], str] | None = None,
+    unplaceable: list[dict] | None = None,
 ) -> list[dict]:
     """Collect core requirement files from emulator profiles not in the platform pack.
 
+    ``claimants`` receives the destination of every report entry that
+    asked for one, the ones a sibling already took included: the region
+    report withdraws by that key, and an entry without one stayed listed
+    after the builder had withdrawn its destination. ``unplaceable``
+    receives the entries a slug-based platform cannot place, so the
+    manifest and the report can say so instead of counting them packed.
+
     Uses the same system-overlap matching as verify.py cross-reference:
     - Matches emulators by shared system IDs with the platform
     - Filters mode: standalone, type: launcher, type: alias
@@ -472,25 +517,18 @@ def _collect_emulator_extras(
 
         # Slug-based platforms: prefix dest with system slug
         if is_slug_based:
-            emu_name = u.get("profile") or u.get("emulator", "")
-            profile = profiles.get(emu_name, {})
-            # Try finding profile by display name if key lookup failed
-            if not profile:
-                for pn, pp in profiles.items():
-                    if pp.get("emulator") == emu_name:
-                        profile = pp
-                        break
-            slug = _map_emulator_to_slug(
-                profile,
-                platform_systems,
-                norm_map,
-                sys_to_slug,
-            )
+            slug = _slug_for(u, profiles, platform_systems, norm_map, sys_to_slug)
             if not slug:
-                continue  # can't place without slug
+                if unplaceable is not None:
+                    unplaceable.append(u)
+                continue
             dest = f"{slug}/{dest}"
 
         full_dest = f"{extras_prefix}/{dest}" if extras_prefix else dest
+        if claimants is not None:
+            claimants[(u.get("emulator", ""), u.get("name", ""), u.get("path") or "")] = (
+                sanitize_pack_path(dest)
+            )
         if full_dest in seen_dests:
             continue
         seen_dests.add(full_dest)
@@ -681,6 +719,7 @@ def platform_region_groups(
         emu_profiles,
         target_cores=target_cores,
         include_all=include_all,
+        claimants=extra_dests,
     )
     for extra in _kept(extras, required_only):
         dest = sanitize_pack_path(extra.get("destination", extra.get("name", "")))

tests/test_unplaced_extras.py

@@ -0,0 +1,141 @@
+"""A core file the platform cannot place is not in the pack.
+
+RomM files every BIOS under the slug of its system and knows no slug for
+DOS or for the arcade sets MAME 2003 reads. The builder skipped those
+entries without a word; the report counted them "in pack" and the manifest
+listed them neither in files nor in omitted_files.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import common  # noqa: E402
+import generate_db  # noqa: E402
+import generate_pack as builder  # noqa: E402
+from packextras import platform_region_groups  # noqa: E402
+from verify import verify_platform  # noqa: E402
+
+PROFILE = """\
+emulator: Dosbox
+type: libretro
+systems: [dos]
+cores: [dosbox]
+files:
+  - name: MT32_CONTROL.ROM
+    required: false
+"""
+
+
+class SlugPlatform(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self._cwd = os.getcwd()
+        os.chdir(self._tmp.name)
+        self.emulators = Path("emulators")
+        self.platforms = Path("platforms")
+        for d in (self.emulators, self.platforms, Path("bios/Roland"), Path("bios/Sony")):
+            d.mkdir(parents=True)
+        files = {}
+        for rel, payload in (("bios/Roland/MT32_CONTROL.ROM", b"mt32"), ("bios/Sony/scph5501.bin", b"psx")):
+            Path(rel).write_bytes(payload)
+            sha1 = hashlib.sha1(payload).hexdigest()
+            files[sha1] = {
+                "path": rel, "name": Path(rel).name, "size": len(payload), "sha1": sha1,
+                "md5": hashlib.md5(payload).hexdigest(),
+                "sha256": hashlib.sha256(payload).hexdigest(), "crc32": "00000001",
+            }
+        self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+        self.config = {
+            "platform": "Slugs",
+            "verification_mode": "md5",
+            "base_destination": "bios",
+            "cores": ["dosbox"],
+            "systems": {
+                "psx": {"files": [{"name": "scph5501.bin", "destination": "psx/scph5501.bin",
+                                   "md5": hashlib.md5(b"psx").hexdigest()}]},
+                "ps2": {"files": [{"name": "scph.bin", "destination": "ps2/scph.bin",
+                                   "md5": "0" * 32}]},
+                "snes": {"files": [{"name": "s.bin", "destination": "snes/s.bin", "md5": "1" * 32}]},
+            },
+        }
+        (self.platforms / "slugs.yml").write_text(yaml.dump(self.config))
+        (self.platforms / "_registry.yml").write_text(
+            yaml.dump({"platforms": {"slugs": {"status": "active"}}})
+        )
+        (self.emulators / "dosbox.yml").write_text(PROFILE)
+        common._platform_config_cache.clear()
+        common._emulator_profiles_cache.clear()
+        self.profiles = common.load_emulator_profiles("emulators")
+
+    def tearDown(self):
+        common._platform_config_cache.clear()
+        common._emulator_profiles_cache.clear()
+        os.chdir(self._cwd)
+        self._tmp.cleanup()
+
+    def test_the_report_does_not_count_it_packed(self):
+        report = verify_platform(
+            self.config, self.db, "emulators", self.profiles, supplemental_names=set()
+        )
+        entry = next(u for u in report["undeclared_files"] if u["name"] == "MT32_CONTROL.ROM")
+        self.assertTrue(entry["in_repo"])
+        self.assertFalse(entry.get("in_pack", True))
+
+    def test_the_manifest_says_why_it_is_absent(self):
+        manifest = builder.generate_manifest(
+            "slugs", "platforms", self.db, "bios", "platforms/_registry.yml",
+            emulators_dir="emulators", emu_profiles=self.profiles, offline=True,
+        )
+        names = {entry["dest"] for entry in manifest["files"]}
+        self.assertFalse(any("MT32_CONTROL" in n for n in names))
+        omitted = [o for o in manifest["omitted_files"] if o["name"] == "MT32_CONTROL.ROM"]
+        self.assertEqual([o["reason"] for o in omitted], ["no_platform_slug"])
+
+
+class RegionKeysForEveryClaimant(unittest.TestCase):
+    """Two profiles claiming one destination: the second, deduplicated by
+    the collector, had no key in the region map, and the report kept it
+    after the builder withdrew the destination."""
+
+    def test_both_claimants_are_keyed(self):
+        with tempfile.TemporaryDirectory() as tmp:
+            previous = os.getcwd()
+            os.chdir(tmp)
+            self.addCleanup(os.chdir, previous)
+            Path("emulators").mkdir()
+            Path("bios").mkdir()
+            Path("bios/jp.bin").write_bytes(b"jp")
+            sha1 = hashlib.sha1(b"jp").hexdigest()
+            files = {sha1: {"path": "bios/jp.bin", "name": "jp.bin", "size": 2, "sha1": sha1,
+                            "md5": hashlib.md5(b"jp").hexdigest(),
+                            "sha256": hashlib.sha256(b"jp").hexdigest(), "crc32": "00000001"}}
+            db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+            for name, extra in (("one", "    system: sys\n"), ("two", "")):
+                Path(f"emulators/{name}.yml").write_text(
+                    f"emulator: {name}\ntype: libretro\nsystems: [sys]\ncores: [{name}]\n"
+                    f"files:\n  - name: jp.bin\n    region: [japan]\n{extra}"
+                )
+            common._emulator_profiles_cache.clear()
+            profiles = common.load_emulator_profiles("emulators")
+            config = {"platform": "P", "verification_mode": "existence", "cores": ["one", "two"],
+                      "systems": {"sys": {"files": []}}}
+            _groups, extra_dests = platform_region_groups(
+                config, config["systems"], "emulators", db, "", profiles
+            )
+            claimants = {key[0] for key, dest in extra_dests.items() if dest == "jp.bin"}
+            self.assertEqual(claimants, {"one", "two"}, extra_dests)
+
+
+if __name__ == "__main__":
+    unittest.main()