feat: keep one bios per slot on declared order

emulators/picodrive.yml

@@ -65,6 +65,7 @@ files:
   - name: "us_scd2_9306.bin"
     system: sega-segacd
     region: [north-america]
+    search_rank: 1
     required: true
     size: 131072  # 128 KB (0x20000)
     note: "US Sega CD Model 2 BIOS (September 1993). First in US search order."
@@ -73,6 +74,7 @@ files:
   - name: "SegaCDBIOS9303.bin"
     system: sega-segacd
     region: [north-america]
+    search_rank: 2
     required: false
     size: 131072
     note: "US Sega CD BIOS (March 1993). Second in US search order."
@@ -81,6 +83,7 @@ files:
   - name: "us_scd1_9210.bin"
     system: sega-segacd
     region: [north-america]
+    search_rank: 3
     required: false
     size: 131072
     note: "US Sega CD Model 1 BIOS (October 1992). Third in US search order."
@@ -89,6 +92,7 @@ files:
   - name: "bios_CD_U.bin"
     system: sega-segacd
     region: [north-america]
+    search_rank: 4
     required: false
     size: 131072
     note: "US Sega CD BIOS (generic name). Last in US search order."
@@ -100,6 +104,7 @@ files:
   - name: "eu_mcd2_9306.bin"
     system: sega-megacd
     region: [europe]
+    search_rank: 1
     required: true
     size: 131072
     note: "EU Mega CD Model 2 BIOS (June 1993). First in EU search order."
@@ -108,6 +113,7 @@ files:
   - name: "eu_mcd2_9303.bin"
     system: sega-megacd
     region: [europe]
+    search_rank: 2
     required: false
     size: 131072
     note: "EU Mega CD Model 2 BIOS (March 1993). Second in EU search order."
@@ -116,6 +122,7 @@ files:
   - name: "eu_mcd1_9210.bin"
     system: sega-megacd
     region: [europe]
+    search_rank: 3
     required: false
     size: 131072
     note: "EU Mega CD Model 1 BIOS (October 1992). Third in EU search order."
@@ -124,6 +131,7 @@ files:
   - name: "bios_CD_E.bin"
     system: sega-megacd
     region: [europe]
+    search_rank: 4
     required: false
     size: 131072
     note: "EU Mega CD BIOS (generic name). Last in EU search order."
@@ -135,6 +143,7 @@ files:
   - name: "jp_mcd2_921222.bin"
     system: sega-megacd
     region: [japan]
+    search_rank: 1
     required: true
     size: 131072
     note: "JP Mega CD Model 2 BIOS (December 1992). First in JP search order."
@@ -143,6 +152,7 @@ files:
   - name: "jp_mcd1_9112.bin"
     system: sega-megacd
     region: [japan]
+    search_rank: 2
     required: false
     size: 131072
     note: "JP Mega CD Model 1 BIOS (December 1991). Second in JP search order."
@@ -151,6 +161,7 @@ files:
   - name: "jp_mcd1_9111.bin"
     system: sega-megacd
     region: [japan]
+    search_rank: 3
     required: false
     size: 131072
     note: "JP Mega CD Model 1 BIOS (November 1991). Third in JP search order."
@@ -159,6 +170,7 @@ files:
   - name: "bios_CD_J.bin"
     system: sega-megacd
     region: [japan]
+    search_rank: 4
     required: false
     size: 131072
     note: "JP Mega CD BIOS (generic name). Last in JP search order."

schemas/emulator.schema.json

@@ -396,6 +396,11 @@
           "load_from": {
             "type": "string"
           },
+          "search_rank": {
+            "description": "Position in an ordered search list the code walks, 1 = tried first. Only set it when the source declares such a list.",
+            "type": "integer",
+            "minimum": 1
+          },
           "priority": {
             "type": "integer"
           },

scripts/generate_pack.py

@@ -53,6 +53,7 @@ from common import (
     yaml_load,
 )
 import region as region_mod
+import slot as slot_mod
 from deterministic_zip import _FIXED_DATE_TIME, rebuild_zip_deterministic
 from nativemode import hash_mismatch_excludes_file
 from validation import (
@@ -251,6 +252,45 @@ def _find_candidate_satisfying_both(
     return None
 
 
+def _pack_member_groups(
+    config: dict,
+    pack_systems: dict,
+    emulators_dir: str,
+    db: dict,
+    base_dest: str,
+    emu_profiles: dict | None,
+    target_cores: set[str] | None,
+    source: str,
+) -> dict[str, list[tuple[str, str]]]:
+    """Group every candidate of a pack by system, as (destination, name).
+
+    Covers the platform baseline and the core extras, because a narrowing pass
+    that saw only one of the two would compare an incomplete set.
+    """
+    groups: dict[str, list[tuple[str, str]]] = {}
+    for sys_id, system in pack_systems.items():
+        members = groups.setdefault(sys_id, [])
+        for fe in system.get("files", []):
+            dest = _sanitize_path(fe.get("destination", fe.get("name", "")))
+            if dest:
+                members.append((dest, fe.get("name", "")))
+    if source == "platform":
+        return groups
+    emu_systems = {
+        n: list(p.get("systems", [])) for n, p in (emu_profiles or {}).items()
+    }
+    for fe in _collect_emulator_extras(
+        config, emulators_dir, db, set(), base_dest, emu_profiles,
+        target_cores=target_cores, include_all=(source == "truth"),
+    ):
+        dest = _sanitize_path(fe.get("destination", fe.get("name", "")))
+        if not dest:
+            continue
+        for sys_id in emu_systems.get(fe.get("source_emulator", ""), ["_extras"]):
+            groups.setdefault(sys_id, []).append((dest, fe.get("name", "")))
+    return groups
+
+
 def _target_tag(target_name: str) -> str:
     """Filename tag for a hardware target.
 
@@ -1161,6 +1201,7 @@ def generate_pack(
     flatten: bool = True,
     regions: list[str] | None = None,
     target_name: str | None = None,
+    one_per_slot: bool = False,
     offline: bool | None = None,
 ) -> str | None:
     """Generate a ZIP pack for a platform.
@@ -1350,6 +1391,30 @@ def generate_pack(
             region_groups, region_index, regions
         )
 
+    slot_undecidable: list[str] = []
+    if one_per_slot:
+        idx = region_mod.build_region_index(emu_profiles or {})
+        members_by_system = _pack_member_groups(
+            config, pack_systems, emulators_dir, db, base_dest,
+            emu_profiles, target_cores, source,
+        )
+        slot_groups: dict[str, list[tuple[str, str]]] = {}
+        for sys_id, members in members_by_system.items():
+            for dest, name in members:
+                if dest in region_drops:
+                    continue
+                tier = ",".join(sorted(region_mod.lookup_regions(idx, dest, name)))
+                slot_groups.setdefault(f"{sys_id}|{tier}", []).append((dest, name))
+        slot_drops, slot_undecidable = slot_mod.resolve_slot_drops(
+            slot_groups, slot_mod.build_slot_index(emu_profiles or {})
+        )
+        region_drops = region_drops | slot_drops
+        if slot_undecidable:
+            print(
+                f"  {len(slot_undecidable)} slot(s) with no declared order: "
+                f"every candidate kept"
+            )
+
     with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
       if source != "truth":
         for sys_id, system in sorted(pack_systems.items()):
@@ -2371,6 +2436,7 @@ def generate_split_packs(
     source: str = "full",
     regions: list[str] | None = None,
     target_name: str | None = None,
+    one_per_slot: bool = False,
     offline: bool | None = None,
 ) -> list[str]:
     """Generate split packs (one ZIP per system or manufacturer)."""
@@ -2438,6 +2504,7 @@ def generate_split_packs(
             source=source,
             regions=regions,
             target_name=target_name,
+            one_per_slot=one_per_slot,
             offline=offline,
         )
         if zip_path:
@@ -2701,6 +2768,13 @@ def _validate_args(args, parser):
         parser.error("--manifest is incompatible with --split")
     if getattr(args, "region", None) and has_from_md5:
         parser.error("--region and --from-md5 are mutually exclusive")
+    if getattr(args, "one_per_slot", False):
+        if has_from_md5:
+            parser.error("--one-per-slot and --from-md5 are mutually exclusive")
+        if args.manifest:
+            parser.error("--one-per-slot is incompatible with --manifest")
+        if has_emulator or has_system:
+            parser.error("--one-per-slot requires --platform or --all")
 
 
 def _write_manifest_if_changed(path: str, manifest: dict) -> None:
@@ -2941,6 +3015,7 @@ def _run_platform_packs(
                         source=source,
                         regions=getattr(args, "regions", None),
                         target_name=args.target,
+                        one_per_slot=args.one_per_slot,
                         offline=args.offline,
                     )
                     print(f"  Split into {len(zip_paths)} packs")
@@ -2962,6 +3037,7 @@ def _run_platform_packs(
                         source=source,
                         regions=getattr(args, "regions", None),
                         target_name=args.target,
+                        one_per_slot=args.one_per_slot,
                         offline=args.offline,
                     )
                 if not args.split and zip_path and aliases:
@@ -3090,6 +3166,11 @@ def main():
         "--region",
         help="Region priority list, best first (e.g. us,eu,jp)",
     )
+    parser.add_argument(
+        "--one-per-slot",
+        action="store_true",
+        help="Keep one BIOS per system and region where the core declares an order",
+    )
     parser.add_argument(
         "--list-targets",
         action="store_true",

scripts/slot.py

@@ -0,0 +1,116 @@
+"""One file per slot: keep the BIOS a core would actually load.
+
+A slot is a (system, region) pair holding several interchangeable BIOS. Region
+filtering narrows a pack to one territory; a system can still ship three US
+PlayStation BIOS that differ only by revision, which is what leaves the choice
+open in the frontend.
+
+The winner is only ever taken from an ordered search list the core's code
+actually walks, recorded as `search_rank:` on the file entry: rank 1 is tried
+first, and the first file found wins. PicoDrive's biosfiles_us/eu/jp arrays are
+the shape this describes.
+
+`priority:` is deliberately not read. Its meaning is disputed: the field
+reference calls it a tie-breaker where higher wins, while DuckStation's own
+selection compares the numbers the other way, and its values rank PS2 images
+above the plain PlayStation BIOS. Selecting on it dropped scph5501, the very
+file most US setups load.
+
+Where no ordered list is declared, the group is reported undecidable and every
+candidate is kept: picking one would be the arbitrary selection this exists to
+remove, and it could discard the file the core would have loaded.
+"""
+
+from __future__ import annotations
+
+BIOS_CATEGORY = "bios"
+
+
+def build_slot_index(profiles: dict) -> dict[str, dict]:
+    """Map a file key to its slot metadata, keyed by path then name."""
+    index: dict[str, dict] = {}
+    for emu_name, profile in sorted(profiles.items()):
+        if profile.get("type") in ("launcher", "alias"):
+            continue
+        for f in profile.get("files") or []:
+            if not isinstance(f, dict):
+                continue
+            if f.get("category", BIOS_CATEGORY) != BIOS_CATEGORY:
+                continue
+            name = f.get("name", "")
+            path = f.get("path") or ""
+            for key in {path, name} - {""}:
+                entry = index.setdefault(
+                    key, {"rank": None, "emulators": []}
+                )
+                rank = f.get("search_rank")
+                if rank is not None:
+                    current = entry["rank"]
+                    entry["rank"] = (
+                        rank if current is None else min(current, rank)
+                    )
+                if emu_name not in entry["emulators"]:
+                    entry["emulators"].append(emu_name)
+    return index
+
+
+def lookup_slot(index: dict[str, dict], destination: str, name: str) -> dict | None:
+    """Look a candidate up, trying the destination then its suffixes then name."""
+    if destination:
+        entry = index.get(destination)
+        if entry:
+            return entry
+        parts = destination.split("/")
+        for i in range(1, len(parts)):
+            entry = index.get("/".join(parts[i:]))
+            if entry:
+                return entry
+    return index.get(name)
+
+
+def resolve_slot_drops(
+    groups: dict[str, list[tuple[str, str]]],
+    index: dict[str, dict],
+) -> tuple[set[str], list[str]]:
+    """Destinations to skip, and the groups no declared order can decide.
+
+    Returns (drops, undecidable). A group is decided only when every candidate
+    declares a search rank and exactly one holds the first rank; anything else
+    keeps every candidate.
+    """
+    keep: set[str] = set()
+    drop: set[str] = set()
+    undecidable: list[str] = []
+
+    for group_id, members in groups.items():
+        candidates: list[tuple[str, int | None]] = []
+        for destination, name in members:
+            entry = lookup_slot(index, destination, name)
+            if entry is None:
+                keep.add(destination)
+                continue
+            candidates.append((destination, entry["rank"]))
+
+        if len(candidates) < 2:
+            keep.update(dest for dest, _p in candidates)
+            continue
+
+        # Every candidate must carry a rank: a set where some members are
+        # unranked cannot be ordered, and dropping the unranked ones would
+        # discard exactly the file the core may load.
+        if any(p is None for _d, p in candidates):
+            undecidable.append(group_id)
+            keep.update(dest for dest, _p in candidates)
+            continue
+
+        best = min(p for _d, p in candidates)
+        winners = [dest for dest, p in candidates if p == best]
+        if len(winners) != 1:
+            undecidable.append(group_id)
+            keep.update(dest for dest, _p in candidates)
+            continue
+
+        keep.add(winners[0])
+        drop.update(dest for dest, _p in candidates if dest != winners[0])
+
+    return drop - keep, sorted(undecidable)

tests/test_slot.py

@@ -0,0 +1,180 @@
+"""Slot resolution: keep one BIOS per slot only where the code declares an order."""
+
+from __future__ import annotations
+
+import os
+import sys
+import unittest
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts"))
+
+import slot
+
+
+def _pairs(*names: str) -> list[tuple[str, str]]:
+    return [(n, n) for n in names]
+
+
+class TestSlotIndex(unittest.TestCase):
+    def setUp(self):
+        self.index = slot.build_slot_index(
+            {
+                "picodrive": {
+                    "files": [
+                        {"name": "us_scd2_9306.bin", "search_rank": 1},
+                        {"name": "SegaCDBIOS9303.bin", "search_rank": 2},
+                        {"name": "us_scd1_9210.bin", "search_rank": 3},
+                        {"name": "bios_CD_U.bin", "search_rank": 4},
+                    ]
+                },
+                "duckstation": {
+                    "files": [
+                        {"name": "scph5501.bin", "priority": 5},
+                        {"name": "scph1001.bin", "priority": 10},
+                    ]
+                },
+                "launcher_profile": {
+                    "type": "launcher",
+                    "files": [{"name": "ignored.bin", "search_rank": 1}],
+                },
+                "engine": {
+                    "files": [
+                        {"name": "assets.pk3", "category": "game_data",
+                         "search_rank": 1},
+                    ]
+                },
+            }
+        )
+
+    def test_search_rank_is_indexed(self):
+        self.assertEqual(self.index["us_scd2_9306.bin"]["rank"], 1)
+
+    def test_priority_is_not_read(self):
+        self.assertIsNone(self.index["scph5501.bin"]["rank"])
+
+    def test_launcher_profiles_are_skipped(self):
+        self.assertNotIn("ignored.bin", self.index)
+
+    def test_non_bios_categories_are_skipped(self):
+        self.assertNotIn("assets.pk3", self.index)
+
+    def test_lookup_falls_back_from_destination_to_name(self):
+        entry = slot.lookup_slot(
+            self.index, "picodrive/us_scd2_9306.bin", "us_scd2_9306.bin"
+        )
+        self.assertIsNotNone(entry)
+        self.assertEqual(entry["rank"], 1)
+
+
+class TestResolveSlotDrops(unittest.TestCase):
+    def setUp(self):
+        self.index = slot.build_slot_index(
+            {
+                "picodrive": {
+                    "files": [
+                        {"name": "a.bin", "search_rank": 1},
+                        {"name": "b.bin", "search_rank": 2},
+                        {"name": "c.bin", "search_rank": 3},
+                    ]
+                },
+                "other": {
+                    "files": [
+                        {"name": "x.bin"},
+                        {"name": "y.bin"},
+                        {"name": "tie1.bin", "search_rank": 1},
+                        {"name": "tie2.bin", "search_rank": 1},
+                    ]
+                },
+            }
+        )
+
+    def test_first_rank_wins_and_the_rest_drop(self):
+        drops, undecidable = slot.resolve_slot_drops(
+            {"segacd|us": _pairs("a.bin", "b.bin", "c.bin")}, self.index
+        )
+        self.assertEqual(drops, {"b.bin", "c.bin"})
+        self.assertEqual(undecidable, [])
+
+    def test_an_unranked_candidate_makes_the_group_undecidable(self):
+        drops, undecidable = slot.resolve_slot_drops(
+            {"psx|us": _pairs("a.bin", "x.bin")}, self.index
+        )
+        self.assertEqual(drops, set())
+        self.assertEqual(undecidable, ["psx|us"])
+
+    def test_no_rank_at_all_is_undecidable(self):
+        drops, undecidable = slot.resolve_slot_drops(
+            {"psx|us": _pairs("x.bin", "y.bin")}, self.index
+        )
+        self.assertEqual(drops, set())
+        self.assertEqual(undecidable, ["psx|us"])
+
+    def test_a_tie_on_the_first_rank_is_undecidable(self):
+        drops, undecidable = slot.resolve_slot_drops(
+            {"t|us": _pairs("tie1.bin", "tie2.bin")}, self.index
+        )
+        self.assertEqual(drops, set())
+        self.assertEqual(undecidable, ["t|us"])
+
+    def test_a_lone_candidate_is_never_dropped(self):
+        drops, undecidable = slot.resolve_slot_drops(
+            {"solo|us": _pairs("b.bin")}, self.index
+        )
+        self.assertEqual(drops, set())
+        self.assertEqual(undecidable, [])
+
+    def test_a_destination_kept_by_another_group_survives(self):
+        drops, _u = slot.resolve_slot_drops(
+            {
+                "segacd|us": _pairs("a.bin", "b.bin", "c.bin"),
+                "segacd|world": _pairs("b.bin"),
+            },
+            self.index,
+        )
+        self.assertNotIn("b.bin", drops)
+
+    def test_unknown_files_are_never_dropped(self):
+        drops, undecidable = slot.resolve_slot_drops(
+            {"u|us": _pairs("a.bin", "b.bin", "nowhere.bin")}, self.index
+        )
+        self.assertNotIn("nowhere.bin", drops)
+
+
+class TestRepoProfiles(unittest.TestCase):
+    def test_declared_ranks_are_positive_and_unique_per_group(self):
+        import collections
+        import glob
+
+        import yaml
+
+        paths = sorted(
+            glob.glob(
+                os.path.join(os.path.dirname(__file__), "..", "emulators", "*.yml")
+            )
+        )
+        if not paths:
+            self.skipTest("emulators/ not present")
+        offenders: list[str] = []
+        for path in paths:
+            with open(path, encoding="utf-8") as fh:
+                profile = yaml.safe_load(fh) or {}
+            groups: dict[tuple, list[int]] = collections.defaultdict(list)
+            for f in profile.get("files") or []:
+                if not isinstance(f, dict) or f.get("search_rank") is None:
+                    continue
+                rank = f["search_rank"]
+                if not isinstance(rank, int) or rank < 1:
+                    offenders.append(f"{os.path.basename(path)}: {f.get('name')}")
+                    continue
+                key = (f.get("system", ""), tuple(f.get("region") or []))
+                groups[key].append(rank)
+            for key, ranks in groups.items():
+                if len(ranks) != len(set(ranks)):
+                    offenders.append(
+                        f"{os.path.basename(path)}: duplicate rank in {key}"
+                    )
+        self.assertEqual(offenders, [], "\n".join(offenders))
+
+
+if __name__ == "__main__":
+    unittest.main()

wiki/advanced-usage.md

@@ -109,6 +109,40 @@ same selection function, so the two never disagree: `--emulator duckstation`
 reports 105 files, `--emulator duckstation --region us` reports 34, and a pack
 built with the same flags carries exactly those 34.
 
+### One file per slot
+
+Region filtering leaves a system with every revision of the same BIOS:
+`scph1001` (v2.2), `scph5501` (v3.0) and `scph7001` (v4.1) are all North
+American. `--one-per-slot` keeps a single one per system and region:
+
+```bash
+python scripts/generate_pack.py --platform retroarch --region us --one-per-slot
+```
+
+It only acts on evidence. The winner comes from an ordered search list the
+core's code actually walks, recorded as `search_rank:` on the file entry, rank
+1 being tried first. PicoDrive declares three such lists, one per region
+(`biosfiles_us/eu/jp` in `platform/libretro/libretro.c`), and the pack keeps
+`us_scd2_9306.bin` over the three later candidates.
+
+Where no such list is declared, the group is left untouched and counted:
+
+```
+  63 slot(s) with no declared order: every candidate kept
+```
+
+That number is the remaining work, not a failure. Picking a file without a
+declared order would be the arbitrary selection this exists to remove, and it
+could drop the one the core would have loaded. `priority:` is deliberately not
+used for this: its meaning is disputed between the field reference and
+DuckStation's own comparison, and its values rank PS2 images above the plain
+PlayStation BIOS.
+
+`--one-per-slot` requires `--platform` or `--all`, and is refused with
+`--manifest`, `--emulator`, `--system` and `--from-md5` rather than silently
+ignored.
+
+
 `--region` composes with `--split`, `--target`, `--required-only`, `--source`,
 `--emulator` and `--system`. It is mutually exclusive with `--from-md5`, which
 selects by hash. `pipeline.py` never passes it, so the released packs stay