chore: merge audit fixes (sysnames)

scripts/common.py

@@ -2499,6 +2499,106 @@ def _norm_system_id(sid: str) -> str:
     return s.replace("-", "")
 
 
+def _name_key(name: str) -> str:
+    """Letters and digits only, as the installer compares names."""
+    return re.sub(r"[^a-z0-9]", "", name.lower())
+
+
+def platform_system_names(config: dict, systems: list[str]) -> dict[str, list[str]]:
+    """Per platform system, the names the platform's own file gives it.
+
+    A Batocera user knows PlayStation as `psx`, the folder under roms/, and
+    the platform YAML records that name as `native_system` on each file it
+    transcribes. Only the platform's own entries count (`native_layer`), a
+    name spelled like the system id adds nothing, and a name the platform
+    gives to two systems, or that reads as another system's id, names
+    neither. The install manifest's `system_names` and the `--system` of
+    generate_pack and verify read this one mapping.
+    """
+    native = native_layer(config) or {}
+    by_name: dict[str, set[str]] = {}
+    for sys_id, system in native.get("systems", {}).items():
+        if sys_id not in systems:
+            continue
+        for entry in system.get("files", []):
+            name = entry.get("native_system")
+            if isinstance(name, str) and name.strip():
+                by_name.setdefault(name.strip(), set()).add(sys_id)
+    ids = {_name_key(sys_id) for sys_id in systems}
+    out: dict[str, list[str]] = {}
+    for name, owners in by_name.items():
+        if len(owners) != 1 or _name_key(name) in ids:
+            continue
+        out.setdefault(next(iter(owners)), []).append(name)
+    return {sys_id: sorted(names) for sys_id, names in sorted(out.items())}
+
+
+def select_systems(systems: dict, wanted: list[str]) -> dict:
+    """The systems a --system list keeps, by id or normalized id."""
+    norm = {_norm_system_id(s) for s in wanted}
+    return {
+        sid: data for sid, data in systems.items()
+        if sid in wanted or _norm_system_id(sid) in norm
+    }
+
+
+def resolve_system_arg(
+    value: str,
+    systems: list[str],
+    config: dict | None = None,
+    listing: str = "--list-systems",
+) -> list[str]:
+    """The system ids a --system value names, or ValueError naming the rest.
+
+    With a platform config, the platform's own names count as well (Batocera's
+    `psx` is sony-playstation), the same mapping the install manifest carries
+    as `system_names`. Without one, only our ids do. A whole value that is a
+    name wins over splitting it on commas: Recalbox names one system
+    `msx1,msx2`. An id wins over another system's native name, and a name
+    nothing else matches falls back to the normalized id (`playstation`).
+    An unknown name is refused, never ignored.
+    """
+    names = platform_system_names(config, systems) if config else {}
+    exact: dict[str, list[str]] = {}
+    by_key: dict[str, list[str]] = {}
+    for sid in systems:
+        exact.setdefault(sid.lower(), [sid])
+        by_key.setdefault(_name_key(sid), []).append(sid)
+    for sid, aliases in names.items():
+        for alias in aliases:
+            exact.setdefault(alias.lower(), [sid])
+            if _name_key(alias) not in by_key:
+                by_key[_name_key(alias)] = [sid]
+    by_norm: dict[str, list[str]] = {}
+    for sid in systems:
+        by_norm.setdefault(_norm_system_id(sid), []).append(sid)
+
+    value = value.strip()
+    tokens = [value] if value.lower() in exact else value.split(",")
+    chosen: list[str] = []
+    unknown: list[str] = []
+    for token in (t.strip() for t in tokens):
+        if not token:
+            continue
+        keyed = by_key.get(_name_key(token), [])
+        found = (
+            exact.get(token.lower())
+            or (keyed if len(keyed) == 1 else None)
+            or by_norm.get(_norm_system_id(token))
+        )
+        if not found:
+            unknown.append(token)
+            continue
+        chosen += [sid for sid in found if sid not in chosen]
+    if unknown:
+        known = sorted({*systems, *(a for v in names.values() for a in v)})
+        hint = suggest_names(unknown, known, listing, listed=len(systems))
+        raise ValueError(f"unknown system {', '.join(unknown)}. {hint}")
+    if not chosen:
+        raise ValueError("--system names no system")
+    return chosen
+
+
 def designated_profiles(profiles: dict[str, dict], names: set[str]) -> set[str]:
     """The profiles a list of upstream core names designates.
 
@@ -2759,15 +2859,18 @@ def list_system_ids(emulators_dir: str) -> None:
 
 
 def list_platform_system_ids(platform_name: str, platforms_dir: str) -> None:
-    """Print system IDs from a platform's YAML config."""
+    """Print system IDs from a platform's YAML config, with the platform's
+    own names for them, which --system also accepts."""
     config = load_platform_config(platform_name, platforms_dir)
     systems = config.get("systems", {})
+    names = platform_system_names(config, list(systems))
     for sys_id in sorted(systems):
         file_count = len(systems[sys_id].get("files", []))
         mfr = systems[sys_id].get("manufacturer", "")
         mfr_display = f"  [{mfr.split('|')[0]}]" if mfr else ""
         count = f"({file_count} file{'s' if file_count != 1 else ''})"
-        print(f"  {sys_id:35s} {count:12s}{mfr_display}".rstrip())
+        also = f"  also: {', '.join(names[sys_id])}" if sys_id in names else ""
+        print(f"  {sys_id:35s} {count:12s}{mfr_display}{also}".rstrip())
 
 
 def build_target_cores_cache(

scripts/generate_pack.py

@@ -51,6 +51,9 @@ from common import (
     group_identical_platforms,
     list_emulator_profiles,
     list_platform_system_ids,
+    platform_system_names,
+    resolve_system_arg,
+    select_systems,
     canonical_target_name,
     platform_files_target,
     list_registered_platforms,
@@ -72,7 +75,7 @@ from common import (
 import packresolve
 from refresh_data_dirs import cache_lock, hold_data_dirs
 import region as region_mod
-from manifestselect import SelectionIndex, native_system_names
+from manifestselect import SelectionIndex
 import slot as slot_mod
 import slots
 import split_pack
@@ -718,14 +721,7 @@ def generate_pack(
     )
 
     if system_filter:
-        from common import _norm_system_id
-
-        norm_filter = {_norm_system_id(s) for s in system_filter}
-        filtered = {
-            sid: sys_data
-            for sid, sys_data in pack_systems.items()
-            if sid in system_filter or _norm_system_id(sid) in norm_filter
-        }
+        filtered = select_systems(pack_systems, system_filter)
         if not filtered:
             hint = suggest_names(
                 system_filter,
@@ -1876,24 +1872,6 @@ def _group_systems_by_manufacturer(
     return groups
 
 
-def _extras_for_systems(extras: list[dict], system_ids: list[str]) -> list[dict]:
-    """Core extras owned by one of the given systems.
-
-    Extras carry their profile/system identity directly. Display labels are
-    presentation only and must never drive routing.
-    """
-    from common import _norm_system_id
-
-    wanted = set(system_ids) | {_norm_system_id(s) for s in system_ids}
-    return [
-        fe for fe in extras
-        if (
-            set(_extra_system_ids(fe))
-            | {_norm_system_id(s) for s in _extra_system_ids(fe)}
-        ) & wanted
-    ]
-
-
 def generate_split_packs(
     platform_name: str,
     platforms_dir: str,
@@ -2309,37 +2287,36 @@ def generate_target_manifests(targets_dir: str, output_dir: str) -> None:
     print(f"Generated {count} target manifest(s) in {output_dir}")
 
 
-def _refuse_unknown_systems(
-    system_filter: list[str],
-    platforms: list[str],
-    platforms_dir: str,
-    parser: argparse.ArgumentParser,
-) -> None:
-    """Refuse a --system no selected platform declares, before any work.
+def _resolve_systems(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
+    """Turn --system into the platform's system ids, before any work.
 
-    The name is known once the config is loaded. Checked inside the build, a
-    typo cost the lock and a refresh of every data directory first, then
-    ended on a WARNING and exit 1.
+    With --platform the platform's own names count too (Batocera's psx), the
+    mapping the install manifest carries. With --all each name must be one
+    of our ids some platform declares. Checked inside the build, a typo cost
+    the lock and a refresh of every data directory first, then ended on a
+    WARNING and exit 1, and a list with one known name ignored the others.
     """
-    from common import _norm_system_id
-
-    wanted = {_norm_system_id(s) for s in system_filter}
-    declared: set[str] = set()
-    for name in platforms:
-        systems = load_platform_config(name, platforms_dir).get("systems", {})
-        if any(sid in system_filter or _norm_system_id(sid) in wanted for sid in systems):
-            return
-        declared.update(systems)
-    where = f"--platform {platforms[0]}" if len(platforms) == 1 else "--all"
-    hint = suggest_names(
-        system_filter,
-        sorted(declared),
-        f"--platform {platforms[0]} --list-systems" if len(platforms) == 1
-        else "--platform <name> --list-systems",
-    )
-    parser.error(
-        f"--system {','.join(system_filter)} matches no system of {where}. {hint}"
-    )
+    if not args.system or not (args.platform or args.all):
+        return
+    if args.platform:
+        config = load_platform_config(args.platform, args.platforms_dir)
+        systems = list(config.get("systems", {}))
+        listing = f"--platform {args.platform} --list-systems"
+    else:
+        config = None
+        systems = sorted({
+            sid
+            for name in list_registered_platforms(
+                args.platforms_dir,
+                include_archived=args.include_archived or args.manifest,
+            )
+            for sid in load_platform_config(name, args.platforms_dir).get("systems", {})
+        })
+        listing = "--platform <name> --list-systems"
+    try:
+        args.system_ids = resolve_system_arg(args.system, systems, config, listing)
+    except ValueError as exc:
+        parser.error(f"--system {args.system}: {exc}")
 
 
 def _refuse_for_verify_packs(args: argparse.Namespace, parser: argparse.ArgumentParser) -> None:
@@ -3241,6 +3218,7 @@ def _main(held: contextlib.ExitStack) -> None:
         return
 
     _validate_args(args, parser)
+    _resolve_systems(args, parser)
     # Unset until every refusal has run, so a typed default value is seen.
     args.source = args.source or "full"
     args.group_by = args.group_by or "system"
@@ -3338,11 +3316,7 @@ def _main(held: contextlib.ExitStack) -> None:
             sys.exit(1)
         return
 
-    system_filter = (
-        [s.strip() for s in args.system.split(",") if s.strip()]
-        if args.system
-        else None
-    )
+    system_filter = getattr(args, "system_ids", None)
 
     # Platform mode
     if args.all:
@@ -3360,9 +3334,6 @@ def _main(held: contextlib.ExitStack) -> None:
         parser.error("Specify --platform or --all")
         return
 
-    if system_filter:
-        _refuse_unknown_systems(system_filter, platforms, args.platforms_dir, parser)
-
     data_registry = load_data_dir_registry(args.platforms_dir)
     # The output directory is claimed before the data directory refresh: a
     # refresh waits for every run that holds a cache, so a second build
@@ -4457,7 +4428,7 @@ def generate_manifest(
         "system_cores": system_cores,
         # The names the platform itself gives its systems (Batocera's psx),
         # which --system accepts beside the ids.
-        "system_names": native_system_names(config, list(pack_systems)),
+        "system_names": platform_system_names(config, list(pack_systems)),
         "files": manifest_files,
     }
     if data_directories:
@@ -4874,6 +4845,7 @@ from packextras import (  # noqa: E402,F401
     _map_emulator_to_slug,
     _collect_emulator_extras,
     _extra_system_ids,
+    extras_for_systems as _extras_for_systems,
     platform_region_groups,
     emulator_region_drops,
     _emulator_region_group,

scripts/manifestselect.py

@@ -17,10 +17,8 @@ from __future__ import annotations
 
 from dataclasses import dataclass
 
-import re
-
 import region as region_mod
-from common import _norm_system_id, native_layer, sanitize_pack_path
+from common import _norm_system_id, sanitize_pack_path
 from packextras import _extra_system_ids, _kept, extra_region_groups, system_spellings
 
 
@@ -120,36 +118,3 @@ class SelectionIndex:
         if priority is not None:
             out["priority"] = priority
         return out
-
-
-def _name_key(name: str) -> str:
-    """Letters and digits only, as the installer compares names."""
-    return re.sub(r"[^a-z0-9]", "", name.lower())
-
-
-def native_system_names(config: dict, systems: list[str]) -> dict[str, list[str]]:
-    """Per platform system, the names the platform's own file gives it.
-
-    A Batocera user knows PlayStation as `psx`, the folder under roms/, and
-    the platform YAML records that name as `native_system` on each file it
-    transcribes. Only the platform's own entries count (`native_layer`), a
-    name spelled like the system id adds nothing, and a name the platform
-    gives to two systems, or that reads as another system's id, names
-    neither.
-    """
-    native = native_layer(config) or {}
-    by_name: dict[str, set[str]] = {}
-    for sys_id, system in native.get("systems", {}).items():
-        if sys_id not in systems:
-            continue
-        for entry in system.get("files", []):
-            name = entry.get("native_system")
-            if isinstance(name, str) and name.strip():
-                by_name.setdefault(name.strip(), set()).add(sys_id)
-    ids = {_name_key(sys_id) for sys_id in systems}
-    out: dict[str, list[str]] = {}
-    for name, owners in by_name.items():
-        if len(owners) != 1 or _name_key(name) in ids:
-            continue
-        out.setdefault(next(iter(owners)), []).append(name)
-    return {sys_id: sorted(names) for sys_id, names in sorted(out.items())}

scripts/packextras.py

@@ -217,6 +217,7 @@ def hidden_extras(
     resolver: ExtraResolver | None = None,
     placed_kind: dict[str, str] | None = None,
     placed_local: dict[str, str] | None = None,
+    system_ids: list[str] | None = None,
 ) -> dict[tuple[str, str, str], str]:
     """Report entries the builder holds and still leaves out, with the reason.
 
@@ -237,6 +238,8 @@ def hidden_extras(
     pass, the archive-prefix pass and the agnostic scan copy files no gap
     report entry names. ``placed_local`` receives the local copy the builder
     writes at each of those destinations, for the pack integrity check.
+    ``system_ids`` replays a pack built with --system: the extras collected
+    without the platform's paths, then those the systems own.
     ``resolver`` is the builder's lookup. An extra it cannot resolve is
     skipped and takes no path, as the builder skips it.
     """
@@ -256,10 +259,13 @@ def hidden_extras(
     lower = set(content) if case_insensitive else set()
     prefix = _detect_extras_prefix(config, base_dest)
     hidden: dict[tuple[str, str, str], str] = {}
-    for fe in _collect_emulator_extras(
-        config, emulators_dir, db, set(shipped), base_dest, profiles,
-        target_cores=target_cores, undeclared=report,
-    ):
+    extras = _collect_emulator_extras(
+        config, emulators_dir, db, set() if system_ids else set(shipped),
+        base_dest, profiles, target_cores=target_cores, undeclared=report,
+    )
+    if system_ids:
+        extras = extras_for_systems(extras, system_ids)
+    for fe in extras:
         dest = sanitize_pack_path(fe.get("destination", fe.get("name", "")))
         if not dest or (region_drops and dest in region_drops):
             continue
@@ -1248,6 +1254,33 @@ def _kept(entries: list[dict], required_only: bool) -> list[dict]:
     return [e for e in entries if e.get("required") is not False]
 
 
+def extras_for_systems(extras: list[dict], system_ids: list[str]) -> list[dict]:
+    """Core extras owned by one of the given systems.
+
+    Extras carry their profile/system identity directly. Display labels are
+    presentation only and must never drive routing.
+    """
+    wanted = set(system_ids) | {_norm_system_id(s) for s in system_ids}
+    return [
+        fe for fe in extras
+        if (
+            set(_extra_system_ids(fe))
+            | {_norm_system_id(s) for s in _extra_system_ids(fe)}
+        ) & wanted
+    ]
+
+
+def report_owned_by_systems(report: list[dict], system_ids: list[str]) -> list[dict]:
+    """Gap report entries whose extra a pack built with --system carries."""
+    return [
+        u for u in report
+        if extras_for_systems(
+            [{"source_system": u.get("system"), "source_systems": u.get("systems", [])}],
+            system_ids,
+        )
+    ]
+
+
 def _ships(config: dict, file_entry: dict, db: dict) -> bool:
     """Whether the builder packs this declaration: resolved, and not refused
     by a hash frontend. A dump whose md5 Recalbox would reject is left out

scripts/verify.py

@@ -62,6 +62,8 @@ from common import (
     load_database,
     load_emulator_profiles,
     load_platform_config,
+    resolve_system_arg,
+    select_systems,
     md5sum,
     require_yaml,
     resolve_local_file,
@@ -896,6 +898,7 @@ def _mark_hidden(
     report: list[dict] | None = None,
     resolver: object | None = None,
     placed_kind: dict[str, str] | None = None,
+    system_ids: list[str] | None = None,
 ) -> set[str]:
     """What the builder holds but cannot place beside the files it packed.
 
@@ -911,6 +914,7 @@ def _mark_hidden(
         emu_profiles if emu_profiles is not None
         else load_emulator_profiles(emulators_dir),
         shipped, target_cores, region_drops, placed, report, resolver, placed_kind,
+        system_ids=system_ids,
     )
     for u in undeclared:
         reason = hidden.get(
@@ -1085,11 +1089,14 @@ def verify_platform(
     target_cores: set[str] | None = None,
     data_dir_registry: dict | None = None,
     regions: list[str] | None = None,
+    system_filter: list[str] | None = None,
 ) -> dict:
     """Verify all BIOS files for a platform, including cross-reference gaps.
 
     A region priority list narrows the report to the files a pack built with the
     same list would carry, using the same selection function as the builder.
+    ``system_filter`` does the same for a pack built with --system: the
+    systems it keeps and the core extras those systems own.
     """
     # Normalized once: a typo in a platform YAML must not verify with one
     # mode and be scored with another.
@@ -1147,6 +1154,8 @@ def verify_platform(
         target_cores,
         platform_cores=plat_cores,
     )
+    if system_filter:
+        verify_systems = select_systems(verify_systems, system_filter)
 
     details = []
     file_status: dict[str, str] = {}
@@ -1331,6 +1340,11 @@ def verify_platform(
     # The builder's own pass reads the same report: computed once here and
     # handed over as it stands now, before the markings below.
     report_for_builder = [dict(u) for u in undeclared]
+    if system_filter:
+        # A system pack carries the extras its systems own.
+        from packextras import report_owned_by_systems
+
+        undeclared = report_owned_by_systems(undeclared, system_filter)
     if region_drops:
         undeclared = [
             u
@@ -1352,6 +1366,7 @@ def verify_platform(
             validation_owners,
         ),
         placed_kind,
+        system_filter,
     )
     # What the builder places from the cores, one per pack path, by category:
     # several entries claim one path (rom.key from three Amiga cores) and the
@@ -1433,6 +1448,7 @@ def verify_platform(
 
     return {
         "platform": platform,
+        **({"systems": sorted(verify_systems)} if system_filter else {}),
         "verification_mode": mode,
         "total_files": total,
         "pack_destinations": pack_destinations,
@@ -1706,6 +1722,8 @@ def print_platform_result(
     total = result["total_files"]
     c = result["severity_counts"]
     label = " / ".join(group)
+    if result.get("systems"):
+        label += f" ({', '.join(result['systems'])})"
     ok_count = c[Severity.OK]
     problems = total - ok_count
 
@@ -2436,11 +2454,15 @@ def _refuse_mode_flags(args: argparse.Namespace, parser: argparse.ArgumentParser
     for refused, message in (
         (args.standalone and not (args.emulator or args.system),
          "--standalone requires --emulator or --system"),
+        (args.standalone and args.platform,
+         "--standalone is incompatible with --platform"),
         (args.include_archived and not args.all, "--include-archived requires --all"),
         (args.target and not (args.platform or args.all),
          "--target requires --platform or --all"),
-        (args.target and (args.emulator or args.system),
+        (args.target and (args.emulator or (args.system and not args.platform)),
          "--target is incompatible with --emulator and --system"),
+        (args.destinations and args.system,
+         "--destinations describes the full pack and is incompatible with --system"),
         (args.destinations and not (args.platform or args.all),
          "--destinations requires --platform or --all"),
         (args.destinations and getattr(args, "region", None),
@@ -2553,7 +2575,9 @@ def _main(held: contextlib.ExitStack) -> None:
         _run_listing(args, parser)
         return
 
-    modes = sum(1 for x in (args.platform, args.all, args.emulator, args.system) if x)
+    # --system beside --platform narrows that platform, as in generate_pack.
+    system_mode = bool(args.system) and not args.platform
+    modes = sum(1 for x in (args.platform, args.all, args.emulator, system_mode) if x)
     if modes == 0:
         parser.error("Specify --platform, --all, --emulator, or --system")
     if modes > 1:
@@ -2561,6 +2585,16 @@ def _main(held: contextlib.ExitStack) -> None:
             "--platform, --all, --emulator, and --system are mutually exclusive"
         )
     _refuse_mode_flags(args, parser)
+    platform_systems: list[str] | None = None
+    if args.platform and args.system:
+        config = load_platform_config(args.platform, args.platforms_dir)
+        try:
+            platform_systems = resolve_system_arg(
+                args.system, list(config.get("systems", {})), config,
+                f"--platform {args.platform} --list-systems",
+            )
+        except ValueError as exc:
+            parser.error(f"--system {args.system}: {exc}")
     # The trees stay put while names resolve against them and files are read.
     from refresh_data_dirs import hold_data_dirs
 
@@ -2582,7 +2616,7 @@ def _main(held: contextlib.ExitStack) -> None:
         _exit_if_strict_emulator(args.strict, result)
         return
 
-    if args.system:
+    if system_mode:
         system_ids = [s.strip() for s in args.system.split(",") if s.strip()]
         result = verify_system(
             system_ids, args.emulators_dir, db, args.standalone,
@@ -2642,6 +2676,7 @@ def _main(held: contextlib.ExitStack) -> None:
             target_cores=tc,
             data_dir_registry=data_registry,
             regions=requested_regions,
+            system_filter=platform_systems,
         )
         names = [
             load_platform_config(p, args.platforms_dir).get("platform", p)