diff --git a/emulators/picodrive.yml b/emulators/picodrive.yml index 43cfbae1..13d7d8b7 100644 --- a/emulators/picodrive.yml +++ b/emulators/picodrive.yml @@ -65,6 +65,7 @@ files: - name: "us_scd2_9306.bin" system: sega-segacd region: [north-america] + search_rank: 1 required: true size: 131072 # 128 KB (0x20000) note: "US Sega CD Model 2 BIOS (September 1993). First in US search order." @@ -73,6 +74,7 @@ files: - name: "SegaCDBIOS9303.bin" system: sega-segacd region: [north-america] + search_rank: 2 required: false size: 131072 note: "US Sega CD BIOS (March 1993). Second in US search order." @@ -81,6 +83,7 @@ files: - name: "us_scd1_9210.bin" system: sega-segacd region: [north-america] + search_rank: 3 required: false size: 131072 note: "US Sega CD Model 1 BIOS (October 1992). Third in US search order." @@ -89,6 +92,7 @@ files: - name: "bios_CD_U.bin" system: sega-segacd region: [north-america] + search_rank: 4 required: false size: 131072 note: "US Sega CD BIOS (generic name). Last in US search order." @@ -100,6 +104,7 @@ files: - name: "eu_mcd2_9306.bin" system: sega-megacd region: [europe] + search_rank: 1 required: true size: 131072 note: "EU Mega CD Model 2 BIOS (June 1993). First in EU search order." @@ -108,6 +113,7 @@ files: - name: "eu_mcd2_9303.bin" system: sega-megacd region: [europe] + search_rank: 2 required: false size: 131072 note: "EU Mega CD Model 2 BIOS (March 1993). Second in EU search order." @@ -116,6 +122,7 @@ files: - name: "eu_mcd1_9210.bin" system: sega-megacd region: [europe] + search_rank: 3 required: false size: 131072 note: "EU Mega CD Model 1 BIOS (October 1992). Third in EU search order." @@ -124,6 +131,7 @@ files: - name: "bios_CD_E.bin" system: sega-megacd region: [europe] + search_rank: 4 required: false size: 131072 note: "EU Mega CD BIOS (generic name). Last in EU search order." @@ -135,6 +143,7 @@ files: - name: "jp_mcd2_921222.bin" system: sega-megacd region: [japan] + search_rank: 1 required: true size: 131072 note: "JP Mega CD Model 2 BIOS (December 1992). First in JP search order." @@ -143,6 +152,7 @@ files: - name: "jp_mcd1_9112.bin" system: sega-megacd region: [japan] + search_rank: 2 required: false size: 131072 note: "JP Mega CD Model 1 BIOS (December 1991). Second in JP search order." @@ -151,6 +161,7 @@ files: - name: "jp_mcd1_9111.bin" system: sega-megacd region: [japan] + search_rank: 3 required: false size: 131072 note: "JP Mega CD Model 1 BIOS (November 1991). Third in JP search order." @@ -159,6 +170,7 @@ files: - name: "bios_CD_J.bin" system: sega-megacd region: [japan] + search_rank: 4 required: false size: 131072 note: "JP Mega CD BIOS (generic name). Last in JP search order." diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index f05d3a27..4b7c7602 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -396,6 +396,11 @@ "load_from": { "type": "string" }, + "search_rank": { + "description": "Position in an ordered search list the code walks, 1 = tried first. Only set it when the source declares such a list.", + "type": "integer", + "minimum": 1 + }, "priority": { "type": "integer" }, diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 5dcaa774..5d1c44b2 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -53,6 +53,7 @@ from common import ( yaml_load, ) import region as region_mod +import slot as slot_mod from deterministic_zip import _FIXED_DATE_TIME, rebuild_zip_deterministic from nativemode import hash_mismatch_excludes_file from validation import ( @@ -251,6 +252,45 @@ def _find_candidate_satisfying_both( return None +def _pack_member_groups( + config: dict, + pack_systems: dict, + emulators_dir: str, + db: dict, + base_dest: str, + emu_profiles: dict | None, + target_cores: set[str] | None, + source: str, +) -> dict[str, list[tuple[str, str]]]: + """Group every candidate of a pack by system, as (destination, name). + + Covers the platform baseline and the core extras, because a narrowing pass + that saw only one of the two would compare an incomplete set. + """ + groups: dict[str, list[tuple[str, str]]] = {} + for sys_id, system in pack_systems.items(): + members = groups.setdefault(sys_id, []) + for fe in system.get("files", []): + dest = _sanitize_path(fe.get("destination", fe.get("name", ""))) + if dest: + members.append((dest, fe.get("name", ""))) + if source == "platform": + return groups + emu_systems = { + n: list(p.get("systems", [])) for n, p in (emu_profiles or {}).items() + } + for fe in _collect_emulator_extras( + config, emulators_dir, db, set(), base_dest, emu_profiles, + target_cores=target_cores, include_all=(source == "truth"), + ): + dest = _sanitize_path(fe.get("destination", fe.get("name", ""))) + if not dest: + continue + for sys_id in emu_systems.get(fe.get("source_emulator", ""), ["_extras"]): + groups.setdefault(sys_id, []).append((dest, fe.get("name", ""))) + return groups + + def _target_tag(target_name: str) -> str: """Filename tag for a hardware target. @@ -1161,6 +1201,7 @@ def generate_pack( flatten: bool = True, regions: list[str] | None = None, target_name: str | None = None, + one_per_slot: bool = False, offline: bool | None = None, ) -> str | None: """Generate a ZIP pack for a platform. @@ -1350,6 +1391,30 @@ def generate_pack( region_groups, region_index, regions ) + slot_undecidable: list[str] = [] + if one_per_slot: + idx = region_mod.build_region_index(emu_profiles or {}) + members_by_system = _pack_member_groups( + config, pack_systems, emulators_dir, db, base_dest, + emu_profiles, target_cores, source, + ) + slot_groups: dict[str, list[tuple[str, str]]] = {} + for sys_id, members in members_by_system.items(): + for dest, name in members: + if dest in region_drops: + continue + tier = ",".join(sorted(region_mod.lookup_regions(idx, dest, name))) + slot_groups.setdefault(f"{sys_id}|{tier}", []).append((dest, name)) + slot_drops, slot_undecidable = slot_mod.resolve_slot_drops( + slot_groups, slot_mod.build_slot_index(emu_profiles or {}) + ) + region_drops = region_drops | slot_drops + if slot_undecidable: + print( + f" {len(slot_undecidable)} slot(s) with no declared order: " + f"every candidate kept" + ) + with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: if source != "truth": for sys_id, system in sorted(pack_systems.items()): @@ -2371,6 +2436,7 @@ def generate_split_packs( source: str = "full", regions: list[str] | None = None, target_name: str | None = None, + one_per_slot: bool = False, offline: bool | None = None, ) -> list[str]: """Generate split packs (one ZIP per system or manufacturer).""" @@ -2438,6 +2504,7 @@ def generate_split_packs( source=source, regions=regions, target_name=target_name, + one_per_slot=one_per_slot, offline=offline, ) if zip_path: @@ -2701,6 +2768,13 @@ def _validate_args(args, parser): parser.error("--manifest is incompatible with --split") if getattr(args, "region", None) and has_from_md5: parser.error("--region and --from-md5 are mutually exclusive") + if getattr(args, "one_per_slot", False): + if has_from_md5: + parser.error("--one-per-slot and --from-md5 are mutually exclusive") + if args.manifest: + parser.error("--one-per-slot is incompatible with --manifest") + if has_emulator or has_system: + parser.error("--one-per-slot requires --platform or --all") def _write_manifest_if_changed(path: str, manifest: dict) -> None: @@ -2941,6 +3015,7 @@ def _run_platform_packs( source=source, regions=getattr(args, "regions", None), target_name=args.target, + one_per_slot=args.one_per_slot, offline=args.offline, ) print(f" Split into {len(zip_paths)} packs") @@ -2962,6 +3037,7 @@ def _run_platform_packs( source=source, regions=getattr(args, "regions", None), target_name=args.target, + one_per_slot=args.one_per_slot, offline=args.offline, ) if not args.split and zip_path and aliases: @@ -3090,6 +3166,11 @@ def main(): "--region", help="Region priority list, best first (e.g. us,eu,jp)", ) + parser.add_argument( + "--one-per-slot", + action="store_true", + help="Keep one BIOS per system and region where the core declares an order", + ) parser.add_argument( "--list-targets", action="store_true", diff --git a/scripts/slot.py b/scripts/slot.py new file mode 100644 index 00000000..26367082 --- /dev/null +++ b/scripts/slot.py @@ -0,0 +1,116 @@ +"""One file per slot: keep the BIOS a core would actually load. + +A slot is a (system, region) pair holding several interchangeable BIOS. Region +filtering narrows a pack to one territory; a system can still ship three US +PlayStation BIOS that differ only by revision, which is what leaves the choice +open in the frontend. + +The winner is only ever taken from an ordered search list the core's code +actually walks, recorded as `search_rank:` on the file entry: rank 1 is tried +first, and the first file found wins. PicoDrive's biosfiles_us/eu/jp arrays are +the shape this describes. + +`priority:` is deliberately not read. Its meaning is disputed: the field +reference calls it a tie-breaker where higher wins, while DuckStation's own +selection compares the numbers the other way, and its values rank PS2 images +above the plain PlayStation BIOS. Selecting on it dropped scph5501, the very +file most US setups load. + +Where no ordered list is declared, the group is reported undecidable and every +candidate is kept: picking one would be the arbitrary selection this exists to +remove, and it could discard the file the core would have loaded. +""" + +from __future__ import annotations + +BIOS_CATEGORY = "bios" + + +def build_slot_index(profiles: dict) -> dict[str, dict]: + """Map a file key to its slot metadata, keyed by path then name.""" + index: dict[str, dict] = {} + for emu_name, profile in sorted(profiles.items()): + if profile.get("type") in ("launcher", "alias"): + continue + for f in profile.get("files") or []: + if not isinstance(f, dict): + continue + if f.get("category", BIOS_CATEGORY) != BIOS_CATEGORY: + continue + name = f.get("name", "") + path = f.get("path") or "" + for key in {path, name} - {""}: + entry = index.setdefault( + key, {"rank": None, "emulators": []} + ) + rank = f.get("search_rank") + if rank is not None: + current = entry["rank"] + entry["rank"] = ( + rank if current is None else min(current, rank) + ) + if emu_name not in entry["emulators"]: + entry["emulators"].append(emu_name) + return index + + +def lookup_slot(index: dict[str, dict], destination: str, name: str) -> dict | None: + """Look a candidate up, trying the destination then its suffixes then name.""" + if destination: + entry = index.get(destination) + if entry: + return entry + parts = destination.split("/") + for i in range(1, len(parts)): + entry = index.get("/".join(parts[i:])) + if entry: + return entry + return index.get(name) + + +def resolve_slot_drops( + groups: dict[str, list[tuple[str, str]]], + index: dict[str, dict], +) -> tuple[set[str], list[str]]: + """Destinations to skip, and the groups no declared order can decide. + + Returns (drops, undecidable). A group is decided only when every candidate + declares a search rank and exactly one holds the first rank; anything else + keeps every candidate. + """ + keep: set[str] = set() + drop: set[str] = set() + undecidable: list[str] = [] + + for group_id, members in groups.items(): + candidates: list[tuple[str, int | None]] = [] + for destination, name in members: + entry = lookup_slot(index, destination, name) + if entry is None: + keep.add(destination) + continue + candidates.append((destination, entry["rank"])) + + if len(candidates) < 2: + keep.update(dest for dest, _p in candidates) + continue + + # Every candidate must carry a rank: a set where some members are + # unranked cannot be ordered, and dropping the unranked ones would + # discard exactly the file the core may load. + if any(p is None for _d, p in candidates): + undecidable.append(group_id) + keep.update(dest for dest, _p in candidates) + continue + + best = min(p for _d, p in candidates) + winners = [dest for dest, p in candidates if p == best] + if len(winners) != 1: + undecidable.append(group_id) + keep.update(dest for dest, _p in candidates) + continue + + keep.add(winners[0]) + drop.update(dest for dest, _p in candidates if dest != winners[0]) + + return drop - keep, sorted(undecidable) diff --git a/tests/test_slot.py b/tests/test_slot.py new file mode 100644 index 00000000..00686631 --- /dev/null +++ b/tests/test_slot.py @@ -0,0 +1,180 @@ +"""Slot resolution: keep one BIOS per slot only where the code declares an order.""" + +from __future__ import annotations + +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "scripts")) + +import slot + + +def _pairs(*names: str) -> list[tuple[str, str]]: + return [(n, n) for n in names] + + +class TestSlotIndex(unittest.TestCase): + def setUp(self): + self.index = slot.build_slot_index( + { + "picodrive": { + "files": [ + {"name": "us_scd2_9306.bin", "search_rank": 1}, + {"name": "SegaCDBIOS9303.bin", "search_rank": 2}, + {"name": "us_scd1_9210.bin", "search_rank": 3}, + {"name": "bios_CD_U.bin", "search_rank": 4}, + ] + }, + "duckstation": { + "files": [ + {"name": "scph5501.bin", "priority": 5}, + {"name": "scph1001.bin", "priority": 10}, + ] + }, + "launcher_profile": { + "type": "launcher", + "files": [{"name": "ignored.bin", "search_rank": 1}], + }, + "engine": { + "files": [ + {"name": "assets.pk3", "category": "game_data", + "search_rank": 1}, + ] + }, + } + ) + + def test_search_rank_is_indexed(self): + self.assertEqual(self.index["us_scd2_9306.bin"]["rank"], 1) + + def test_priority_is_not_read(self): + self.assertIsNone(self.index["scph5501.bin"]["rank"]) + + def test_launcher_profiles_are_skipped(self): + self.assertNotIn("ignored.bin", self.index) + + def test_non_bios_categories_are_skipped(self): + self.assertNotIn("assets.pk3", self.index) + + def test_lookup_falls_back_from_destination_to_name(self): + entry = slot.lookup_slot( + self.index, "picodrive/us_scd2_9306.bin", "us_scd2_9306.bin" + ) + self.assertIsNotNone(entry) + self.assertEqual(entry["rank"], 1) + + +class TestResolveSlotDrops(unittest.TestCase): + def setUp(self): + self.index = slot.build_slot_index( + { + "picodrive": { + "files": [ + {"name": "a.bin", "search_rank": 1}, + {"name": "b.bin", "search_rank": 2}, + {"name": "c.bin", "search_rank": 3}, + ] + }, + "other": { + "files": [ + {"name": "x.bin"}, + {"name": "y.bin"}, + {"name": "tie1.bin", "search_rank": 1}, + {"name": "tie2.bin", "search_rank": 1}, + ] + }, + } + ) + + def test_first_rank_wins_and_the_rest_drop(self): + drops, undecidable = slot.resolve_slot_drops( + {"segacd|us": _pairs("a.bin", "b.bin", "c.bin")}, self.index + ) + self.assertEqual(drops, {"b.bin", "c.bin"}) + self.assertEqual(undecidable, []) + + def test_an_unranked_candidate_makes_the_group_undecidable(self): + drops, undecidable = slot.resolve_slot_drops( + {"psx|us": _pairs("a.bin", "x.bin")}, self.index + ) + self.assertEqual(drops, set()) + self.assertEqual(undecidable, ["psx|us"]) + + def test_no_rank_at_all_is_undecidable(self): + drops, undecidable = slot.resolve_slot_drops( + {"psx|us": _pairs("x.bin", "y.bin")}, self.index + ) + self.assertEqual(drops, set()) + self.assertEqual(undecidable, ["psx|us"]) + + def test_a_tie_on_the_first_rank_is_undecidable(self): + drops, undecidable = slot.resolve_slot_drops( + {"t|us": _pairs("tie1.bin", "tie2.bin")}, self.index + ) + self.assertEqual(drops, set()) + self.assertEqual(undecidable, ["t|us"]) + + def test_a_lone_candidate_is_never_dropped(self): + drops, undecidable = slot.resolve_slot_drops( + {"solo|us": _pairs("b.bin")}, self.index + ) + self.assertEqual(drops, set()) + self.assertEqual(undecidable, []) + + def test_a_destination_kept_by_another_group_survives(self): + drops, _u = slot.resolve_slot_drops( + { + "segacd|us": _pairs("a.bin", "b.bin", "c.bin"), + "segacd|world": _pairs("b.bin"), + }, + self.index, + ) + self.assertNotIn("b.bin", drops) + + def test_unknown_files_are_never_dropped(self): + drops, undecidable = slot.resolve_slot_drops( + {"u|us": _pairs("a.bin", "b.bin", "nowhere.bin")}, self.index + ) + self.assertNotIn("nowhere.bin", drops) + + +class TestRepoProfiles(unittest.TestCase): + def test_declared_ranks_are_positive_and_unique_per_group(self): + import collections + import glob + + import yaml + + paths = sorted( + glob.glob( + os.path.join(os.path.dirname(__file__), "..", "emulators", "*.yml") + ) + ) + if not paths: + self.skipTest("emulators/ not present") + offenders: list[str] = [] + for path in paths: + with open(path, encoding="utf-8") as fh: + profile = yaml.safe_load(fh) or {} + groups: dict[tuple, list[int]] = collections.defaultdict(list) + for f in profile.get("files") or []: + if not isinstance(f, dict) or f.get("search_rank") is None: + continue + rank = f["search_rank"] + if not isinstance(rank, int) or rank < 1: + offenders.append(f"{os.path.basename(path)}: {f.get('name')}") + continue + key = (f.get("system", ""), tuple(f.get("region") or [])) + groups[key].append(rank) + for key, ranks in groups.items(): + if len(ranks) != len(set(ranks)): + offenders.append( + f"{os.path.basename(path)}: duplicate rank in {key}" + ) + self.assertEqual(offenders, [], "\n".join(offenders)) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md index 95c87b4f..e0f8fa8a 100644 --- a/wiki/advanced-usage.md +++ b/wiki/advanced-usage.md @@ -109,6 +109,40 @@ same selection function, so the two never disagree: `--emulator duckstation` reports 105 files, `--emulator duckstation --region us` reports 34, and a pack built with the same flags carries exactly those 34. +### One file per slot + +Region filtering leaves a system with every revision of the same BIOS: +`scph1001` (v2.2), `scph5501` (v3.0) and `scph7001` (v4.1) are all North +American. `--one-per-slot` keeps a single one per system and region: + +```bash +python scripts/generate_pack.py --platform retroarch --region us --one-per-slot +``` + +It only acts on evidence. The winner comes from an ordered search list the +core's code actually walks, recorded as `search_rank:` on the file entry, rank +1 being tried first. PicoDrive declares three such lists, one per region +(`biosfiles_us/eu/jp` in `platform/libretro/libretro.c`), and the pack keeps +`us_scd2_9306.bin` over the three later candidates. + +Where no such list is declared, the group is left untouched and counted: + +``` + 63 slot(s) with no declared order: every candidate kept +``` + +That number is the remaining work, not a failure. Picking a file without a +declared order would be the arbitrary selection this exists to remove, and it +could drop the one the core would have loaded. `priority:` is deliberately not +used for this: its meaning is disputed between the field reference and +DuckStation's own comparison, and its values rank PS2 images above the plain +PlayStation BIOS. + +`--one-per-slot` requires `--platform` or `--all`, and is refused with +`--manifest`, `--emulator`, `--system` and `--from-md5` rather than silently +ignored. + + `--region` composes with `--split`, `--target`, `--required-only`, `--source`, `--emulator` and `--system`. It is mutually exclusive with `--from-md5`, which selects by hash. `pipeline.py` never passes it, so the released packs stay