diff --git a/scripts/common.py b/scripts/common.py index bd1b6e7b..fc42d5f1 100644 --- a/scripts/common.py +++ b/scripts/common.py @@ -2499,6 +2499,106 @@ def _norm_system_id(sid: str) -> str: return s.replace("-", "") +def _name_key(name: str) -> str: + """Letters and digits only, as the installer compares names.""" + return re.sub(r"[^a-z0-9]", "", name.lower()) + + +def platform_system_names(config: dict, systems: list[str]) -> dict[str, list[str]]: + """Per platform system, the names the platform's own file gives it. + + A Batocera user knows PlayStation as `psx`, the folder under roms/, and + the platform YAML records that name as `native_system` on each file it + transcribes. Only the platform's own entries count (`native_layer`), a + name spelled like the system id adds nothing, and a name the platform + gives to two systems, or that reads as another system's id, names + neither. The install manifest's `system_names` and the `--system` of + generate_pack and verify read this one mapping. + """ + native = native_layer(config) or {} + by_name: dict[str, set[str]] = {} + for sys_id, system in native.get("systems", {}).items(): + if sys_id not in systems: + continue + for entry in system.get("files", []): + name = entry.get("native_system") + if isinstance(name, str) and name.strip(): + by_name.setdefault(name.strip(), set()).add(sys_id) + ids = {_name_key(sys_id) for sys_id in systems} + out: dict[str, list[str]] = {} + for name, owners in by_name.items(): + if len(owners) != 1 or _name_key(name) in ids: + continue + out.setdefault(next(iter(owners)), []).append(name) + return {sys_id: sorted(names) for sys_id, names in sorted(out.items())} + + +def select_systems(systems: dict, wanted: list[str]) -> dict: + """The systems a --system list keeps, by id or normalized id.""" + norm = {_norm_system_id(s) for s in wanted} + return { + sid: data for sid, data in systems.items() + if sid in wanted or _norm_system_id(sid) in norm + } + + +def resolve_system_arg( + value: str, + systems: list[str], + config: dict | None = None, + listing: str = "--list-systems", +) -> list[str]: + """The system ids a --system value names, or ValueError naming the rest. + + With a platform config, the platform's own names count as well (Batocera's + `psx` is sony-playstation), the same mapping the install manifest carries + as `system_names`. Without one, only our ids do. A whole value that is a + name wins over splitting it on commas: Recalbox names one system + `msx1,msx2`. An id wins over another system's native name, and a name + nothing else matches falls back to the normalized id (`playstation`). + An unknown name is refused, never ignored. + """ + names = platform_system_names(config, systems) if config else {} + exact: dict[str, list[str]] = {} + by_key: dict[str, list[str]] = {} + for sid in systems: + exact.setdefault(sid.lower(), [sid]) + by_key.setdefault(_name_key(sid), []).append(sid) + for sid, aliases in names.items(): + for alias in aliases: + exact.setdefault(alias.lower(), [sid]) + if _name_key(alias) not in by_key: + by_key[_name_key(alias)] = [sid] + by_norm: dict[str, list[str]] = {} + for sid in systems: + by_norm.setdefault(_norm_system_id(sid), []).append(sid) + + value = value.strip() + tokens = [value] if value.lower() in exact else value.split(",") + chosen: list[str] = [] + unknown: list[str] = [] + for token in (t.strip() for t in tokens): + if not token: + continue + keyed = by_key.get(_name_key(token), []) + found = ( + exact.get(token.lower()) + or (keyed if len(keyed) == 1 else None) + or by_norm.get(_norm_system_id(token)) + ) + if not found: + unknown.append(token) + continue + chosen += [sid for sid in found if sid not in chosen] + if unknown: + known = sorted({*systems, *(a for v in names.values() for a in v)}) + hint = suggest_names(unknown, known, listing, listed=len(systems)) + raise ValueError(f"unknown system {', '.join(unknown)}. {hint}") + if not chosen: + raise ValueError("--system names no system") + return chosen + + def designated_profiles(profiles: dict[str, dict], names: set[str]) -> set[str]: """The profiles a list of upstream core names designates. @@ -2759,15 +2859,18 @@ def list_system_ids(emulators_dir: str) -> None: def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None: - """Print system IDs from a platform's YAML config.""" + """Print system IDs from a platform's YAML config, with the platform's + own names for them, which --system also accepts.""" config = load_platform_config(platform_name, platforms_dir) systems = config.get("systems", {}) + names = platform_system_names(config, list(systems)) for sys_id in sorted(systems): file_count = len(systems[sys_id].get("files", [])) mfr = systems[sys_id].get("manufacturer", "") mfr_display = f" [{mfr.split('|')[0]}]" if mfr else "" count = f"({file_count} file{'s' if file_count != 1 else ''})" - print(f" {sys_id:35s} {count:12s}{mfr_display}".rstrip()) + also = f" also: {', '.join(names[sys_id])}" if sys_id in names else "" + print(f" {sys_id:35s} {count:12s}{mfr_display}{also}".rstrip()) def build_target_cores_cache( diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index fbc3f18f..8517709f 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -51,6 +51,9 @@ from common import ( group_identical_platforms, list_emulator_profiles, list_platform_system_ids, + platform_system_names, + resolve_system_arg, + select_systems, canonical_target_name, platform_files_target, list_registered_platforms, @@ -72,7 +75,7 @@ from common import ( import packresolve from refresh_data_dirs import cache_lock, hold_data_dirs import region as region_mod -from manifestselect import SelectionIndex, native_system_names +from manifestselect import SelectionIndex import slot as slot_mod import slots import split_pack @@ -718,14 +721,7 @@ def generate_pack( ) if system_filter: - from common import _norm_system_id - - norm_filter = {_norm_system_id(s) for s in system_filter} - filtered = { - sid: sys_data - for sid, sys_data in pack_systems.items() - if sid in system_filter or _norm_system_id(sid) in norm_filter - } + filtered = select_systems(pack_systems, system_filter) if not filtered: hint = suggest_names( system_filter, @@ -1876,24 +1872,6 @@ def _group_systems_by_manufacturer( return groups -def _extras_for_systems(extras: list[dict], system_ids: list[str]) -> list[dict]: - """Core extras owned by one of the given systems. - - Extras carry their profile/system identity directly. Display labels are - presentation only and must never drive routing. - """ - from common import _norm_system_id - - wanted = set(system_ids) | {_norm_system_id(s) for s in system_ids} - return [ - fe for fe in extras - if ( - set(_extra_system_ids(fe)) - | {_norm_system_id(s) for s in _extra_system_ids(fe)} - ) & wanted - ] - - def generate_split_packs( platform_name: str, platforms_dir: str, @@ -2309,37 +2287,36 @@ def generate_target_manifests(targets_dir: str, output_dir: str) -> None: print(f"Generated {count} target manifest(s) in {output_dir}") -def _refuse_unknown_systems( - system_filter: list[str], - platforms: list[str], - platforms_dir: str, - parser: argparse.ArgumentParser, -) -> None: - """Refuse a --system no selected platform declares, before any work. +def _resolve_systems(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: + """Turn --system into the platform's system ids, before any work. - The name is known once the config is loaded. Checked inside the build, a - typo cost the lock and a refresh of every data directory first, then - ended on a WARNING and exit 1. + With --platform the platform's own names count too (Batocera's psx), the + mapping the install manifest carries. With --all each name must be one + of our ids some platform declares. Checked inside the build, a typo cost + the lock and a refresh of every data directory first, then ended on a + WARNING and exit 1, and a list with one known name ignored the others. """ - from common import _norm_system_id - - wanted = {_norm_system_id(s) for s in system_filter} - declared: set[str] = set() - for name in platforms: - systems = load_platform_config(name, platforms_dir).get("systems", {}) - if any(sid in system_filter or _norm_system_id(sid) in wanted for sid in systems): - return - declared.update(systems) - where = f"--platform {platforms[0]}" if len(platforms) == 1 else "--all" - hint = suggest_names( - system_filter, - sorted(declared), - f"--platform {platforms[0]} --list-systems" if len(platforms) == 1 - else "--platform --list-systems", - ) - parser.error( - f"--system {','.join(system_filter)} matches no system of {where}. {hint}" - ) + if not args.system or not (args.platform or args.all): + return + if args.platform: + config = load_platform_config(args.platform, args.platforms_dir) + systems = list(config.get("systems", {})) + listing = f"--platform {args.platform} --list-systems" + else: + config = None + systems = sorted({ + sid + for name in list_registered_platforms( + args.platforms_dir, + include_archived=args.include_archived or args.manifest, + ) + for sid in load_platform_config(name, args.platforms_dir).get("systems", {}) + }) + listing = "--platform --list-systems" + try: + args.system_ids = resolve_system_arg(args.system, systems, config, listing) + except ValueError as exc: + parser.error(f"--system {args.system}: {exc}") def _refuse_for_verify_packs(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None: @@ -3241,6 +3218,7 @@ def _main(held: contextlib.ExitStack) -> None: return _validate_args(args, parser) + _resolve_systems(args, parser) # Unset until every refusal has run, so a typed default value is seen. args.source = args.source or "full" args.group_by = args.group_by or "system" @@ -3338,11 +3316,7 @@ def _main(held: contextlib.ExitStack) -> None: sys.exit(1) return - system_filter = ( - [s.strip() for s in args.system.split(",") if s.strip()] - if args.system - else None - ) + system_filter = getattr(args, "system_ids", None) # Platform mode if args.all: @@ -3360,9 +3334,6 @@ def _main(held: contextlib.ExitStack) -> None: parser.error("Specify --platform or --all") return - if system_filter: - _refuse_unknown_systems(system_filter, platforms, args.platforms_dir, parser) - data_registry = load_data_dir_registry(args.platforms_dir) # The output directory is claimed before the data directory refresh: a # refresh waits for every run that holds a cache, so a second build @@ -4457,7 +4428,7 @@ def generate_manifest( "system_cores": system_cores, # The names the platform itself gives its systems (Batocera's psx), # which --system accepts beside the ids. - "system_names": native_system_names(config, list(pack_systems)), + "system_names": platform_system_names(config, list(pack_systems)), "files": manifest_files, } if data_directories: @@ -4874,6 +4845,7 @@ from packextras import ( # noqa: E402,F401 _map_emulator_to_slug, _collect_emulator_extras, _extra_system_ids, + extras_for_systems as _extras_for_systems, platform_region_groups, emulator_region_drops, _emulator_region_group, diff --git a/scripts/manifestselect.py b/scripts/manifestselect.py index 1784efe9..fad640c7 100644 --- a/scripts/manifestselect.py +++ b/scripts/manifestselect.py @@ -17,10 +17,8 @@ from __future__ import annotations from dataclasses import dataclass -import re - import region as region_mod -from common import _norm_system_id, native_layer, sanitize_pack_path +from common import _norm_system_id, sanitize_pack_path from packextras import _extra_system_ids, _kept, extra_region_groups, system_spellings @@ -120,36 +118,3 @@ class SelectionIndex: if priority is not None: out["priority"] = priority return out - - -def _name_key(name: str) -> str: - """Letters and digits only, as the installer compares names.""" - return re.sub(r"[^a-z0-9]", "", name.lower()) - - -def native_system_names(config: dict, systems: list[str]) -> dict[str, list[str]]: - """Per platform system, the names the platform's own file gives it. - - A Batocera user knows PlayStation as `psx`, the folder under roms/, and - the platform YAML records that name as `native_system` on each file it - transcribes. Only the platform's own entries count (`native_layer`), a - name spelled like the system id adds nothing, and a name the platform - gives to two systems, or that reads as another system's id, names - neither. - """ - native = native_layer(config) or {} - by_name: dict[str, set[str]] = {} - for sys_id, system in native.get("systems", {}).items(): - if sys_id not in systems: - continue - for entry in system.get("files", []): - name = entry.get("native_system") - if isinstance(name, str) and name.strip(): - by_name.setdefault(name.strip(), set()).add(sys_id) - ids = {_name_key(sys_id) for sys_id in systems} - out: dict[str, list[str]] = {} - for name, owners in by_name.items(): - if len(owners) != 1 or _name_key(name) in ids: - continue - out.setdefault(next(iter(owners)), []).append(name) - return {sys_id: sorted(names) for sys_id, names in sorted(out.items())} diff --git a/scripts/packextras.py b/scripts/packextras.py index 66f15b63..a125e533 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -217,6 +217,7 @@ def hidden_extras( resolver: ExtraResolver | None = None, placed_kind: dict[str, str] | None = None, placed_local: dict[str, str] | None = None, + system_ids: list[str] | None = None, ) -> dict[tuple[str, str, str], str]: """Report entries the builder holds and still leaves out, with the reason. @@ -237,6 +238,8 @@ def hidden_extras( pass, the archive-prefix pass and the agnostic scan copy files no gap report entry names. ``placed_local`` receives the local copy the builder writes at each of those destinations, for the pack integrity check. + ``system_ids`` replays a pack built with --system: the extras collected + without the platform's paths, then those the systems own. ``resolver`` is the builder's lookup. An extra it cannot resolve is skipped and takes no path, as the builder skips it. """ @@ -256,10 +259,13 @@ def hidden_extras( lower = set(content) if case_insensitive else set() prefix = _detect_extras_prefix(config, base_dest) hidden: dict[tuple[str, str, str], str] = {} - for fe in _collect_emulator_extras( - config, emulators_dir, db, set(shipped), base_dest, profiles, - target_cores=target_cores, undeclared=report, - ): + extras = _collect_emulator_extras( + config, emulators_dir, db, set() if system_ids else set(shipped), + base_dest, profiles, target_cores=target_cores, undeclared=report, + ) + if system_ids: + extras = extras_for_systems(extras, system_ids) + for fe in extras: dest = sanitize_pack_path(fe.get("destination", fe.get("name", ""))) if not dest or (region_drops and dest in region_drops): continue @@ -1248,6 +1254,33 @@ def _kept(entries: list[dict], required_only: bool) -> list[dict]: return [e for e in entries if e.get("required") is not False] +def extras_for_systems(extras: list[dict], system_ids: list[str]) -> list[dict]: + """Core extras owned by one of the given systems. + + Extras carry their profile/system identity directly. Display labels are + presentation only and must never drive routing. + """ + wanted = set(system_ids) | {_norm_system_id(s) for s in system_ids} + return [ + fe for fe in extras + if ( + set(_extra_system_ids(fe)) + | {_norm_system_id(s) for s in _extra_system_ids(fe)} + ) & wanted + ] + + +def report_owned_by_systems(report: list[dict], system_ids: list[str]) -> list[dict]: + """Gap report entries whose extra a pack built with --system carries.""" + return [ + u for u in report + if extras_for_systems( + [{"source_system": u.get("system"), "source_systems": u.get("systems", [])}], + system_ids, + ) + ] + + def _ships(config: dict, file_entry: dict, db: dict) -> bool: """Whether the builder packs this declaration: resolved, and not refused by a hash frontend. A dump whose md5 Recalbox would reject is left out diff --git a/scripts/verify.py b/scripts/verify.py index cf5ae83c..5679340c 100644 --- a/scripts/verify.py +++ b/scripts/verify.py @@ -62,6 +62,8 @@ from common import ( load_database, load_emulator_profiles, load_platform_config, + resolve_system_arg, + select_systems, md5sum, require_yaml, resolve_local_file, @@ -897,6 +899,7 @@ def _mark_hidden( report: list[dict] | None = None, resolver: object | None = None, placed_kind: dict[str, str] | None = None, + system_ids: list[str] | None = None, ) -> set[str]: """What the builder holds but cannot place beside the files it packed. @@ -912,6 +915,7 @@ def _mark_hidden( emu_profiles if emu_profiles is not None else load_emulator_profiles(emulators_dir), shipped, target_cores, region_drops, placed, report, resolver, placed_kind, + system_ids=system_ids, ) for u in undeclared: reason = hidden.get( @@ -1086,11 +1090,14 @@ def verify_platform( target_cores: set[str] | None = None, data_dir_registry: dict | None = None, regions: list[str] | None = None, + system_filter: list[str] | None = None, ) -> dict: """Verify all BIOS files for a platform, including cross-reference gaps. A region priority list narrows the report to the files a pack built with the same list would carry, using the same selection function as the builder. + ``system_filter`` does the same for a pack built with --system: the + systems it keeps and the core extras those systems own. """ # Normalized once: a typo in a platform YAML must not verify with one # mode and be scored with another. @@ -1148,6 +1155,8 @@ def verify_platform( target_cores, platform_cores=plat_cores, ) + if system_filter: + verify_systems = select_systems(verify_systems, system_filter) details = [] file_status: dict[str, str] = {} @@ -1332,6 +1341,11 @@ def verify_platform( # The builder's own pass reads the same report: computed once here and # handed over as it stands now, before the markings below. report_for_builder = [dict(u) for u in undeclared] + if system_filter: + # A system pack carries the extras its systems own. + from packextras import report_owned_by_systems + + undeclared = report_owned_by_systems(undeclared, system_filter) if region_drops: undeclared = [ u @@ -1353,6 +1367,7 @@ def verify_platform( validation_owners, ), placed_kind, + system_filter, ) # What the builder places from the cores, one per pack path, by category: # several entries claim one path (rom.key from three Amiga cores) and the @@ -1434,6 +1449,7 @@ def verify_platform( return { "platform": platform, + **({"systems": sorted(verify_systems)} if system_filter else {}), "verification_mode": mode, "total_files": total, "pack_destinations": pack_destinations, @@ -1707,6 +1723,8 @@ def print_platform_result( total = result["total_files"] c = result["severity_counts"] label = " / ".join(group) + if result.get("systems"): + label += f" ({', '.join(result['systems'])})" ok_count = c[Severity.OK] problems = total - ok_count @@ -2437,11 +2455,15 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser for refused, message in ( (args.standalone and not (args.emulator or args.system), "--standalone requires --emulator or --system"), + (args.standalone and args.platform, + "--standalone is incompatible with --platform"), (args.include_archived and not args.all, "--include-archived requires --all"), (args.target and not (args.platform or args.all), "--target requires --platform or --all"), - (args.target and (args.emulator or args.system), + (args.target and (args.emulator or (args.system and not args.platform)), "--target is incompatible with --emulator and --system"), + (args.destinations and args.system, + "--destinations describes the full pack and is incompatible with --system"), (args.destinations and not (args.platform or args.all), "--destinations requires --platform or --all"), (args.destinations and getattr(args, "region", None), @@ -2554,7 +2576,9 @@ def _main(held: contextlib.ExitStack) -> None: _run_listing(args, parser) return - modes = sum(1 for x in (args.platform, args.all, args.emulator, args.system) if x) + # --system beside --platform narrows that platform, as in generate_pack. + system_mode = bool(args.system) and not args.platform + modes = sum(1 for x in (args.platform, args.all, args.emulator, system_mode) if x) if modes == 0: parser.error("Specify --platform, --all, --emulator, or --system") if modes > 1: @@ -2562,6 +2586,16 @@ def _main(held: contextlib.ExitStack) -> None: "--platform, --all, --emulator, and --system are mutually exclusive" ) _refuse_mode_flags(args, parser) + platform_systems: list[str] | None = None + if args.platform and args.system: + config = load_platform_config(args.platform, args.platforms_dir) + try: + platform_systems = resolve_system_arg( + args.system, list(config.get("systems", {})), config, + f"--platform {args.platform} --list-systems", + ) + except ValueError as exc: + parser.error(f"--system {args.system}: {exc}") # The trees stay put while names resolve against them and files are read. from refresh_data_dirs import hold_data_dirs @@ -2583,7 +2617,7 @@ def _main(held: contextlib.ExitStack) -> None: _exit_if_strict_emulator(args.strict, result) return - if args.system: + if system_mode: system_ids = [s.strip() for s in args.system.split(",") if s.strip()] result = verify_system( system_ids, args.emulators_dir, db, args.standalone, @@ -2643,6 +2677,7 @@ def _main(held: contextlib.ExitStack) -> None: target_cores=tc, data_dir_registry=data_registry, regions=requested_regions, + system_filter=platform_systems, ) names = [ load_platform_config(p, args.platforms_dir).get("platform", p) diff --git a/tests/test_audit_regressions.py b/tests/test_audit_regressions.py index 2ee3df6b..b2ea55bc 100644 --- a/tests/test_audit_regressions.py +++ b/tests/test_audit_regressions.py @@ -6624,7 +6624,10 @@ class AnUnknownSystemIsRefusedBeforeAnyWork(unittest.TestCase): cwd=ROOT, capture_output=True, text=True, timeout=120, ) self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) - self.assertIn("error: --system playstaton-nothing matches no system", proc.stderr) + self.assertIn( + "error: --system playstaton-nothing: unknown system playstaton-nothing", + proc.stderr, + ) self.assertFalse(out.exists()) def test_a_known_system_passes_the_early_check(self): @@ -6633,9 +6636,13 @@ class AnUnknownSystemIsRefusedBeforeAnyWork(unittest.TestCase): import generate_pack parser = argparse.ArgumentParser() - generate_pack._refuse_unknown_systems( - ["sony-playstation"], ["retroarch"], str(ROOT / "platforms"), parser + args = argparse.Namespace( + system="sony-playstation", platform="retroarch", all=False, + platforms_dir=str(ROOT / "platforms"), include_archived=False, + manifest=False, ) + generate_pack._resolve_systems(args, parser) + self.assertEqual(args.system_ids, ["sony-playstation"]) class NoArchiveRepeatsAMemberName(unittest.TestCase): diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 2dcfb9d5..a9d8fe95 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -753,10 +753,10 @@ class ThePlatformsOwnSystemNamesAreAccepted(unittest.TestCase): def test_the_names_come_from_the_platform_file(self): from common import load_platform_config # noqa: PLC0415 - from manifestselect import native_system_names # noqa: PLC0415 + from common import platform_system_names # noqa: PLC0415 config = load_platform_config("batocera", str(REPO_ROOT / "platforms")) - names = native_system_names(config, list(config["systems"])) + names = platform_system_names(config, list(config["systems"])) self.assertEqual(names["sony-playstation"], ["psx"]) self.assertEqual(names["nintendo-gba"], ["gba"]) # A name spelled like the system id adds nothing. @@ -767,7 +767,7 @@ class ThePlatformsOwnSystemNamesAreAccepted(unittest.TestCase): def test_the_builder_writes_them_into_the_manifest(self): source = (REPO_ROOT / "scripts" / "generate_pack.py").read_text(encoding="utf-8") - self.assertIn('"system_names": native_system_names(config, list(pack_systems))', + self.assertIn('"system_names": platform_system_names(config, list(pack_systems))', source) def test_a_native_name_narrows_and_lists(self): @@ -788,5 +788,177 @@ class ThePlatformsOwnSystemNamesAreAccepted(unittest.TestCase): self.assertIn("Did you mean: sony-playstation", typo.stderr) + +class ANativeNameSelectsOneSetEverywhere(unittest.TestCase): + """`--system psx` was accepted by the installer and refused by + generate_pack and verify. The platform's own name now resolves through + common.resolve_system_arg in both tools, from the mapping the manifest's + system_names is built from, and selects what the system id selects: + the same pack, the same verify report, the same installed files.""" + + CASES = ( + ("batocera", "psx", "sony-playstation"), + ("recalbox", "msx1,msx2", "microsoft-msx"), + ("romm", "acpc", "amstrad-cpc"), + ) + + @classmethod + def setUpClass(cls): + db_path = REPO_ROOT / "database.json" + if not db_path.exists(): + raise unittest.SkipTest("database.json is not built") + import generate_pack as gp + import pipeline + import verify + from common import load_emulator_profiles + + cls.gp, cls.pipeline, cls.verify = gp, pipeline, verify + cls.db = json.loads(db_path.read_text(encoding="utf-8")) + cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + + def _packed(self, platform: str, systems: list[str]) -> set[str]: + import contextlib + import io + import zipfile + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out, \ + contextlib.redirect_stdout(io.StringIO()): + zip_path = self.gp.generate_pack( + platform, str(REPO_ROOT / "platforms"), self.db, + str(REPO_ROOT / "bios"), out, + emulators_dir=str(REPO_ROOT / "emulators"), + emu_profiles=self.profiles, system_filter=systems, offline=True, + ) + with zipfile.ZipFile(zip_path) as zf: + infos = [i for i in zf.infolist() if not i.is_dir()] + return {i.filename: i.file_size for i in infos} + + def test_pack_verify_and_installer_agree(self): + from common import load_platform_config, resolve_system_arg + + for platform, native, system_id in self.CASES: + with self.subTest(platform=platform, name=native): + config = load_platform_config(platform, str(REPO_ROOT / "platforms")) + systems = list(config["systems"]) + chosen = resolve_system_arg(native, systems, config) + self.assertEqual(chosen, resolve_system_arg(system_id, systems, config)) + self.assertEqual(chosen, [system_id]) + + report = self.verify.verify_platform( + config, self.db, str(REPO_ROOT / "emulators"), self.profiles, + system_filter=chosen, + ) + self.assertEqual(report["systems"], [system_id]) + credited = set(report["pack_destinations"]) + trees = self.pipeline._tree_members(platform, str(REPO_ROOT / "platforms")) + members = self._packed(platform, chosen) + written = { + name for name, size in members.items() + if not self.pipeline._PACK_NOTES.search(name) + and not (trees.get(name) == size and name not in credited) + } + self.assertTrue(written) + self.assertEqual(credited, written) + + manifest = self.gp.generate_manifest( + platform, str(REPO_ROOT / "platforms"), self.db, + str(REPO_ROOT / "bios"), + str(REPO_ROOT / "platforms" / "_registry.yml"), + emulators_dir=str(REPO_ROOT / "emulators"), + emu_profiles=self.profiles, offline=True, + ) + picked = install.resolve_choices( + [native], + install.available_choices( + manifest["files"], manifest["omitted_files"] + )["systems"], + "system", install.system_aliases(manifest), + ) + self.assertEqual(picked, [system_id]) + files, _omitted = install.narrow( + manifest["files"], manifest["omitted_files"], picked, [], [], + ) + base = config.get("base_destination", "") + prefix = f"{base}/" if base else "" + packed = { + n[len(prefix):] if prefix and n.startswith(prefix) else n + for n in members + } + known = {f["dest"] for f in manifest["files"]} + self.assertEqual({f["dest"] for f in files}, packed & known) + + +class SystemNamesResolveOneWay(unittest.TestCase): + """common.resolve_system_arg, the one reading of --system.""" + + CONFIG = {"systems": { + "sony-playstation": {"files": [{"name": "a.bin", "native_system": "psx"}]}, + "microsoft-msx": {"files": [{"name": "b.rom", "native_system": "msx1,msx2"}]}, + "nintendo-gba": {"files": [{"name": "c.bin", "native_system": "gba"}]}, + "gb": {"files": [{"name": "d.bin", "native_system": "nintendo-gba"}]}, + }} + + def _resolve(self, value, config=CONFIG): + from common import resolve_system_arg + + return resolve_system_arg(value, list(self.CONFIG["systems"]), config) + + def test_native_names_and_ids(self): + self.assertEqual(self._resolve("psx"), ["sony-playstation"]) + self.assertEqual(self._resolve("PSX"), ["sony-playstation"]) + self.assertEqual(self._resolve("psx,gb"), ["sony-playstation", "gb"]) + self.assertEqual(self._resolve("Sony PlayStation"), ["sony-playstation"]) + self.assertEqual(self._resolve("playstation"), ["sony-playstation"]) + + def test_a_name_with_commas_is_one_name(self): + self.assertEqual(self._resolve("msx1,msx2"), ["microsoft-msx"]) + + def test_an_id_wins_over_another_systems_native_name(self): + self.assertEqual(self._resolve("nintendo-gba"), ["nintendo-gba"]) + + def test_without_a_platform_only_ids_count(self): + with self.assertRaises(ValueError): + self._resolve("psx", config=None) + + def test_an_unknown_name_is_refused_with_a_suggestion(self): + with self.assertRaises(ValueError) as caught: + self._resolve("psx,psxx") + self.assertIn("unknown system psxx", str(caught.exception)) + self.assertIn("Did you mean: psx?", str(caught.exception)) + + +class AnUnknownNativeNameIsRefusedBeforeAnyWork(unittest.TestCase): + """Both tools answer a typo with the platform's names before reading the + database: a missing --db would otherwise be the first error.""" + + def test_both_tools(self): + missing = str(REPO_ROOT / "tmp" / "no-such-database.json") + for script, extra in ( + ("verify.py", []), + ("generate_pack.py", ["--output-dir", str(REPO_ROOT / "tmp" / "never")]), + ): + with self.subTest(script=script): + proc = subprocess.run( + [sys.executable, f"scripts/{script}", "--platform", "batocera", + "--system", "psxx", "--db", missing, *extra], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=300, + ) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("unknown system psxx. Did you mean: psx?", proc.stderr) + self.assertNotIn("Traceback", proc.stderr) + self.assertFalse((REPO_ROOT / "tmp" / "never").exists()) + + def test_verify_takes_a_native_name_beside_platform(self): + proc = subprocess.run( + [sys.executable, "scripts/verify.py", "--platform", "batocera", + "--system", "psx", "--json"], + cwd=REPO_ROOT, capture_output=True, text=True, timeout=600, + ) + if "database" in proc.stderr and proc.returncode: + self.skipTest("database.json is not built") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(json.loads(proc.stdout)["batocera"]["systems"], ["sony-playstation"]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/advanced-usage.md b/wiki/advanced-usage.md index d96da477..f9374e5e 100644 --- a/wiki/advanced-usage.md +++ b/wiki/advanced-usage.md @@ -46,8 +46,19 @@ Extract only the files for a single system within a platform: ```bash python scripts/generate_pack.py --platform retroarch --system sony-playstation +python scripts/generate_pack.py --platform batocera --system psx +python scripts/verify.py --platform batocera --system psx ``` +Beside `--platform`, `--system` accepts the platform's own name for a system +as well as our id: `psx` is what Batocera calls `sony-playstation`, and both +build the same pack. The names come from `native_system` in the platform +YAML, the mapping the installer reads as `system_names`, and +`--platform batocera --list-systems` shows them after each id. A comma-separated +value that is itself one name stays one name (Recalbox's `msx1,msx2`). A name +the platform does not know is refused with the closest matches before the +database is read. + ### Required only Exclude optional files from the pack: diff --git a/wiki/tools.md b/wiki/tools.md index e3a61246..28980225 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -122,6 +122,7 @@ python scripts/verify.py --emulator dolphin # single emulator python scripts/verify.py --emulator dolphin --standalone # standalone mode only python scripts/verify.py --emulator handy --strict # exit 1 on a required file missing with no HLE python scripts/verify.py --system atari-lynx # single system +python scripts/verify.py --platform batocera --system psx # one system of a platform pack python scripts/verify.py --platform retroarch --target switch # filter by hardware python scripts/verify.py --platform recalbox --region us # regional priority list python scripts/verify.py --all --destinations tmp/paths.json # also write the paths each pack holds @@ -165,6 +166,7 @@ python scripts/generate_pack.py --system atari-lynx # Granular options python scripts/generate_pack.py --platform retroarch --system sony-playstation +python scripts/generate_pack.py --platform batocera --system psx python scripts/generate_pack.py --platform batocera --required-only python scripts/generate_pack.py --platform retroarch --split python scripts/generate_pack.py --platform retroarch --split --group-by manufacturer @@ -215,7 +217,14 @@ own verification mode ([discrepancy detection](verification-modes.md#discrepancy ### Granular options -- `--system` with `--platform`: filter to specific systems within a platform pack +- `--system` with `--platform`: filter to specific systems within a platform pack. + It takes our system ids (`sony-playstation`) and the platform's own names, + read from `native_system` in its YAML (Batocera `psx`, Recalbox `msx1,msx2`). + `--platform --list-systems` prints both. `verify.py --platform --system` + reports on the same narrowed pack, and the installer's `--system` reads the + same mapping from the manifest's `system_names`. An unknown name is refused + with the closest matches before any work. Without `--platform`, `--system` + takes our ids only. - `--required-only`: exclude optional files and the data directories declared `required: false`, keep only required - `--split`: generate one ZIP per system instead of one big pack