diff --git a/install.ps1 b/install.ps1 index a63580a9..bd38670a 100644 --- a/install.ps1 +++ b/install.ps1 @@ -8,7 +8,7 @@ param( $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -$defaultInstallSha256 = "f23eee534cfbf88187f39e47065165c8cc897efb06206a7daadc461de241f8d4" +$defaultInstallSha256 = "a1bae5feab3e85224526eaab7ed99ae0d711081c925aa520d3768532cd11a797" $maximumInstallerBytes = 2MB $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } $temporary = $null diff --git a/install.py b/install.py index 644fbdbe..14e9c173 100644 --- a/install.py +++ b/install.py @@ -10,6 +10,7 @@ Usage: python install.py --platform retroarch --dest ~/custom/bios python install.py --check python install.py --list-platforms + python install.py --platform retroarch --system sony-playstation --region us """ from __future__ import annotations @@ -93,6 +94,43 @@ OMISSION_REASONS = { _SHA1_RE = re.compile(r"^[0-9a-fA-F]{40}$") _SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") +# The region vocabulary of scripts/region.py, which this file cannot import. +# A test holds the two copies equal. +WORLD_REGION = "world" +REGION_TREE: dict[str, frozenset] = { + "north-america": frozenset({"canada"}), + "latin-america": frozenset({"brazil", "mexico", "argentina"}), + "europe": frozenset({ + "uk", "france", "germany", "italy", "spain", "netherlands", "portugal", + "greece", "poland", "russia", "sweden", "norway", "denmark", "finland", + }), + "asia": frozenset({ + "japan", "south-korea", "china", "taiwan", "hong-kong", "singapore", + "india", "asia-ntsc", "asia-pal", + }), + "oceania": frozenset({"australia", "new-zealand"}), +} +REGION_ALIASES: dict[str, str] = { + "jp": "japan", + "ntsc-j": "japan", + "us": "north-america", + "usa": "north-america", + "na": "north-america", + "ntsc-u": "north-america", + "eu": "europe", + "pal": "europe", + "kr": "south-korea", + "korea": "south-korea", + "auto": WORLD_REGION, + "gb": "uk", +} +_REGION_PARENT = { + member: parent for parent, members in REGION_TREE.items() for member in members +} +REGIONS = frozenset({WORLD_REGION} | set(REGION_TREE) | set(_REGION_PARENT)) +MAX_SELECTION_TAGS = 1000 +SELECTION_FIELDS = ("systems", "regions", "region_groups", "region_system_groups") + # Platforms with a manifest in install/. Manifest URLs are case sensitive, # so user input is normalized against this list before any fetch. AVAILABLE_PLATFORMS = ( @@ -813,6 +851,19 @@ def _destination_path(root: Path, value: object) -> Path: return root.resolve() / Path(*relative.parts) +def _check_tags(entry: dict, field: str, where: str) -> None: + """A selection field is absent or a short list of short strings.""" + value = entry.get(field) + if value is None: + return + if ( + not isinstance(value, list) + or len(value) > MAX_SELECTION_TAGS + or not all(isinstance(tag, str) and 0 < len(tag) <= 256 for tag in value) + ): + raise ValueError(f"invalid {field} for {where}") + + def _validate_manifest(data: object, plat: str) -> dict: """Validate the untrusted install-manifest boundary using stdlib only.""" if not isinstance(data, dict): @@ -871,6 +922,8 @@ def _validate_manifest(data: object, plat: str) -> dict: not isinstance(cores, list) or not all(isinstance(core, str) for core in cores) ): raise ValueError(f"invalid cores list for {dest}") + for field in SELECTION_FIELDS: + _check_tags(entry, field, dest) declared_total_files = data.get("total_files") if declared_total_files is not None and declared_total_files != len(files): @@ -908,6 +961,8 @@ def _validate_manifest(data: object, plat: str) -> dict: or not all(isinstance(core, str) for core in cores) ): raise ValueError(f"invalid omitted cores list for {dest}") + for field in SELECTION_FIELDS: + _check_tags(entry, field, dest) declared_total_omitted = data.get("total_omitted") if ( declared_total_omitted is not None @@ -1024,6 +1079,206 @@ def _filter_by_target( return result +def canonical_region(raw: str) -> str: + """One region name or alias, spelled the way manifests spell it.""" + key = raw.strip().lower() + key = REGION_ALIASES.get(key, key) + if key not in REGIONS: + raise ValueError(f"unknown region: {raw.strip()}") + return key + + +def _region_comparable(a: str, b: str) -> bool: + return a == b or _REGION_PARENT.get(a) == b or _REGION_PARENT.get(b) == a + + +def _region_rank(file_regions: set[str], requested: list[str]) -> int: + for index, wanted in enumerate(requested): + if any(_region_comparable(wanted, region) for region in file_regions): + return index + return len(requested) + + +def region_drops( + entries: list[dict], requested: list[str], systems: "list[str] | None" = None +) -> set[str]: + """Destinations a region priority list withdraws. + + The pack builder's rule (region.resolve_region_drops), replayed over the + groups the manifest records. Per group, the best-ranked regional file + stays; a world file beats unmatched regional ones; when nothing matches + and no world file exists, every regional file stays, so a group is never + emptied. A file with no region never competes and always stays. + + A core extra competes whatever systems are chosen; a platform's own + declaration competes only when its system is kept, as in the pack. + """ + if not requested: + return set() + kept_systems = set(systems) if systems else None + groups: dict[str, list[tuple[set, str]]] = {} + for entry in entries: + regions = entry.get("regions") + if not regions: + continue + joined = list(entry.get("region_groups") or []) + joined += [ + group for group in entry.get("region_system_groups") or [] + if kept_systems is None or group in kept_systems + ] + for group in joined: + groups.setdefault(group, []).append((set(regions), entry["dest"])) + keep: set[str] = set() + drop: set[str] = set() + for members in groups.values(): + world = {dest for regions, dest in members if WORLD_REGION in regions} + regional = [ + (_region_rank(regions, requested), dest) + for regions, dest in members + if WORLD_REGION not in regions + ] + keep |= world + if not regional: + continue + matched = [(rank, dest) for rank, dest in regional if rank < len(requested)] + if matched: + best = min(rank for rank, _dest in matched) + keep |= {dest for rank, dest in matched if rank == best} + drop |= {dest for _rank, dest in regional} + elif world: + drop |= {dest for _rank, dest in regional} + else: + keep |= {dest for _rank, dest in regional} + return drop - keep + + +def entry_systems(entry: dict) -> list[str]: + """The systems a file serves: a list on files, one name on omissions.""" + systems = entry.get("systems") + if systems: + return list(systems) + return [entry["system"]] if entry.get("system") else [] + + +def narrow( + files: list[dict], + omitted: list[dict], + systems: list[str], + cores: list[str], + regions: list[str], +) -> tuple[list[dict], list[dict]]: + """Keep what the chosen systems, cores and regions need. + + A core choice keeps the platform's own list and narrows only the files + cores load, the rule --target already follows. Regions are then decided + the way the pack builder decides them under --system: over the chosen + systems' declarations and every core extra. The system filter comes last. + """ + if cores: + files = _filter_by_target(files, cores) + omitted = _filter_by_target(omitted, cores) + if regions: + dropped = region_drops(files + omitted, regions, systems) + files = [f for f in files if f["dest"] not in dropped] + omitted = [o for o in omitted if o["dest"] not in dropped] + if systems: + wanted = set(systems) + files = [f for f in files if wanted & set(entry_systems(f))] + omitted = [o for o in omitted if wanted & set(entry_systems(o))] + return files, omitted + + +def records_selection(files: list[dict]) -> bool: + """Whether a manifest carries what narrowing reads; older ones do not.""" + return any("systems" in entry for entry in files) + + +def available_choices(files: list[dict], omitted: list[dict]) -> dict[str, list[str]]: + """The systems, cores and regions a manifest lets one narrow to.""" + systems = {s for e in files + omitted for s in entry_systems(e)} + cores = {c for f in files for c in (f.get("cores") or [])} + regions = { + r for e in files + omitted for r in (e.get("regions") or []) + if r != WORLD_REGION + } + return { + "systems": sorted(systems), + "cores": sorted(cores), + "regions": sorted(regions), + } + + +def resolve_choices(values: list[str], known: list[str], label: str) -> list[str]: + """Names given on the command line, matched without regard to case. + + Each value is one name or a comma-separated list. A whole value that is + itself a name wins over splitting it: Recalbox names one system + `msx1,msx2,msxturbor`. An unknown name is refused, never ignored, since + carrying on would install something other than what was asked. + """ + by_lower = {name.lower(): name for name in known} + chosen: list[str] = [] + unknown: list[str] = [] + for value in values: + value = value.strip() + tokens = [value] if value.lower() in by_lower else value.split(",") + for token in (t.strip() for t in tokens): + if not token: + continue + name = by_lower.get(token.lower()) + if name is None: + unknown.append(token) + elif name not in chosen: + chosen.append(name) + if unknown: + available = ", ".join(known) if known else "none" + raise ValueError( + f"unknown {label}: {', '.join(unknown)} (available: {available})" + ) + if values and not chosen: + raise ValueError(f"--{label} needs at least one name") + return chosen + + +def resolve_regions(values: list[str]) -> list[str]: + """A region priority list, best first, from names or aliases.""" + chosen: list[str] = [] + for value in values: + for token in value.split(","): + if token.strip(): + region = canonical_region(token) + if region not in chosen: + chosen.append(region) + if values and not chosen: + raise ValueError("--region needs at least one region") + return chosen + + +def parse_selection(text: str, count: int, ordered: bool = False) -> list[int]: + """Numbers typed at a prompt: `3`, `1,4`, `2-5`, or `all`, 1-based. + + Returned sorted, or in the typed order when the order is a priority. + """ + if text.strip().lower() in ("all", "*"): + return list(range(1, count + 1)) + chosen: list[int] = [] + for token in (t.strip() for t in text.split(",")): + if not token: + continue + low, sep, high = token.partition("-") + try: + start = int(low) + end = int(high) if sep else start + except ValueError: + raise ValueError(f"not a number or range: {token}") from None + if start > end or start < 1 or end > count: + raise ValueError(f"out of range: {token} (1-{count})") + for number in range(start, end + 1): + if number not in chosen: + chosen.append(number) + return chosen if ordered else sorted(chosen) + + def _digest_file(path: Path, algorithms: tuple[str, ...]) -> dict[str, str]: """Compute several digests of a file in a single read. @@ -1498,6 +1753,152 @@ def _prompt_platform_choice( return platforms +def _ask(prompt: str) -> str: + try: + return input(prompt).strip() + except (EOFError, KeyboardInterrupt): + print() + sys.exit(0) + + +def _sized(files: list[dict]) -> str: + return f"{_plural(len(files), 'file')}, {format_size(sum(f.get('size', 0) for f in files))}" + + +def _pick(title: str, rows: list[tuple[str, list[dict]]], ordered: bool = False) -> list[str]: + """Numbered list of names; an empty answer keeps every one.""" + if not rows: + return [] + print(f"\n{title}") + width = len(str(len(rows))) + name_width = min(max(len(name) for name, _files in rows), 32) + for index, (name, files) in enumerate(rows, 1): + print(f" {index:>{width}}) {name:<{name_width}} {_sized(files)}") + hint = "order of preference, e.g. 2,1" if ordered else "e.g. 1,3-5" + while True: + text = _ask(f"Numbers ({hint}), Enter for all, q to quit: ") + if not text: + return [] + if text.lower() == "q": + sys.exit(0) + try: + chosen = parse_selection(text, len(rows), ordered=ordered) + except ValueError as exc: + print(f" {exc}") + continue + if chosen: + return [rows[number - 1][0] for number in chosen] + + +def _prompt_custom_selection( + plat: str, files: list[dict], omitted: list[dict] +) -> "tuple[list[str], list[str], list[str]] | None": + """Offer to narrow the install by system, core and region. + + Enter installs everything, the default most runs want. Returns None + then, or the three choices; each empty list keeps everything on its axis. + """ + print(f"\n{plat.capitalize()}: {_sized(files)}.") + print(" Enter) install everything") + print(" c) choose systems, cores and regions") + print(" q) quit") + while True: + answer = _ask("> ").lower() + if answer == "": + return None + if answer == "q": + sys.exit(0) + if answer == "c": + break + + choices = available_choices(files, omitted) + systems = _pick("Systems:", [ + (name, [f for f in files if name in entry_systems(f)]) + for name in choices["systems"] + ]) + kept, kept_omitted = narrow(files, omitted, systems, [], []) + + cores: list[str] = [] + core_names = available_choices(kept, kept_omitted)["cores"] + if core_names: + print("\nThe platform's own files for these systems are always kept;") + print("the extra files each core loads can be narrowed.") + cores = _pick("Cores:", [ + (name, [f for f in kept if name in (f.get("cores") or [])]) + for name in core_names + ]) + kept, kept_omitted = narrow(kept, kept_omitted, [], cores, []) + + regions: list[str] = [] + region_names = available_choices(kept, kept_omitted)["regions"] + if region_names: + print("\nSome systems have one BIOS per region. Keeping only the") + print("preferred region removes the others where a match exists.") + regions = _pick("Regions:", [ + (name, [f for f in kept if name in (f.get("regions") or [])]) + for name in region_names + ], ordered=True) + return systems, cores, regions + + +def _same_choice_hint(plat: str, systems: list[str], cores: list[str], regions: list[str]) -> str: + """The options that repeat a choice made at the prompts.""" + def quoted(value: str) -> str: + return f'"{value}"' if any(c in value for c in " ;&|") else value + + parts = [f"--platform {plat}"] + parts += [f"--system {quoted(name)}" for name in systems] + parts += [f"--core {quoted(name)}" for name in cores] + if regions: + parts.append(f"--region {','.join(regions)}") + return " ".join(parts) + + +def _print_choices( + plat: str, files: list[dict], omitted: list[dict], systems: bool +) -> None: + """What --system or --core accepts for a platform, with what each holds.""" + choices = available_choices(files, omitted) + names = choices["systems"] if systems else choices["cores"] + if not names: + print(f" {plat} records no {'systems' if systems else 'cores'}") + return + width = min(max(len(name) for name in names), 40) + for name in names: + held = ( + [f for f in files if name in entry_systems(f)] + if systems + else [f for f in files if name in (f.get("cores") or [])] + ) + print(f" {name:<{width}} {_sized(held)}") + + +def _requested_narrowing( + args: argparse.Namespace, plat: str, files: list[dict], omitted: list[dict] +) -> "tuple[list[str], list[str], list[str]] | None": + """The systems, cores and regions named on the command line, checked.""" + if not (args.system or args.core or args.region): + return None + if not records_selection(files): + # Without the fields, a region filter would match nothing and install + # everything: refused rather than ignored. + print( + f"Error: the {plat} file list records no systems or regions; " + "narrowing needs a newer one", + file=sys.stderr, + ) + sys.exit(1) + choices = available_choices(files, omitted) + try: + systems = resolve_choices(args.system or [], choices["systems"], "system") + cores = resolve_choices(args.core or [], choices["cores"], "core") + regions = resolve_regions(args.region or []) + except ValueError as exc: + print(f"Error: {plat}: {exc}", file=sys.stderr) + sys.exit(1) + return systems, cores, regions + + def main() -> None: """Entry point.""" parser = argparse.ArgumentParser( @@ -1547,10 +1948,56 @@ def main() -> None: action="store_true", help="opt in to copies into detected standalone-emulator directories", ) + parser.add_argument( + "--system", + action="append", + metavar="NAME", + help="install only these systems (repeat or comma-separate; see --list-systems)", + ) + parser.add_argument( + "--core", + action="append", + metavar="NAME", + help="keep only the extra files these cores load (see --list-cores)", + ) + parser.add_argument( + "--region", + action="append", + metavar="LIST", + help="preferred regions, best first, e.g. us,eu,jp: keep one BIOS per " + "region where a system has several", + ) + parser.add_argument( + "--list-systems", + action="store_true", + help="list a platform's systems with their file counts and exit", + ) + parser.add_argument( + "--list-cores", + action="store_true", + help="list the cores whose extra files a platform installs and exit", + ) + parser.add_argument( + "--no-input", + action="store_true", + help="never ask a question; fail where an answer is needed", + ) args = parser.parse_args() if not 1 <= args.jobs <= 32: parser.error("--jobs must be between 1 and 32") + if args.region: + try: + resolve_regions(args.region) + except ValueError as exc: + parser.error(f"{exc} (known: {', '.join(sorted(REGIONS))})") + for option, values in (("--system", args.system), ("--core", args.core)): + if values is not None and not any( + token.strip() for value in values for token in value.split(",") + ): + parser.error(f"{option} needs at least one name") + interactive = sys.stdin.isatty() and not args.no_input + listing = args.list_targets or args.list_systems or args.list_cores print("RetroBIOS\n") os_type = detect_os() @@ -1598,7 +2045,7 @@ def main() -> None: platforms = detect_platforms(os_type) if not platforms: print(" No supported platform detected.") - if not sys.stdin.isatty() or args.list_targets: + if not interactive or listing: for line in _manual_usage_hint(os_type): print(line) sys.exit(1) @@ -1606,7 +2053,7 @@ def main() -> None: for name, path in platforms: print(f" Found {name.capitalize()} at {path}") - if len(platforms) > 1 and not args.list_targets and sys.stdin.isatty(): + if len(platforms) > 1 and not listing and interactive: platforms = _prompt_platform_choice(platforms) total_downloaded = 0 @@ -1672,6 +2119,27 @@ def main() -> None: omitted_files = _filter_by_target(omitted_files, target_cores) print(f" Filtered {before} -> {len(files)} files for target {args.target}") + selection = _requested_narrowing(args, plat_name, files, omitted_files) + if ( + selection is None + and interactive + and not args.check + and not listing + and records_selection(files) + ): + selection = _prompt_custom_selection(plat_name, files, omitted_files) + if selection is not None: + print("\n Same choice without questions: " + + _same_choice_hint(plat_name, *selection)) + if selection is not None: + before = len(files) + files, omitted_files = narrow(files, omitted_files, *selection) + print(f" Narrowed {before} -> {len(files)} files") + + if args.list_systems or args.list_cores: + _print_choices(plat_name, files, omitted_files, args.list_systems) + continue + total_size = sum(f.get("size", 0) for f in files) print(f" {len(files)} files ({format_size(total_size)})") if omitted_files: @@ -1764,7 +2232,7 @@ def main() -> None: "(use --standalone-copies to opt in)." ) - if not args.check and not args.list_targets: + if not args.check and not listing: _report_outcome( installed_paths, total_downloaded, diff --git a/install.sh b/install.sh index 0c43bd10..ea8b993b 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="f23eee534cfbf88187f39e47065165c8cc897efb06206a7daadc461de241f8d4" +DEFAULT_INSTALL_SHA256="a1bae5feab3e85224526eaab7ed99ae0d711081c925aa520d3768532cd11a797" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index 9b043a84..472b5e73 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -146,6 +146,89 @@ "uniqueItems": true } ] + }, + "regions": { + "description": "The regions the emulator code selects the file for. Absent when it never competes.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "argentina", + "asia", + "asia-ntsc", + "asia-pal", + "australia", + "brazil", + "canada", + "china", + "denmark", + "europe", + "finland", + "france", + "germany", + "greece", + "hong-kong", + "india", + "italy", + "japan", + "latin-america", + "mexico", + "netherlands", + "new-zealand", + "north-america", + "norway", + "oceania", + "poland", + "portugal", + "russia", + "singapore", + "south-korea", + "spain", + "sweden", + "taiwan", + "uk", + "world" + ] + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "region_groups": { + "description": "Groups the file competes in as a core extra, whatever systems are chosen.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "region_system_groups": { + "description": "Groups the file competes in as the platform's own declaration: present only when that system is kept.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "systems": { + "description": "Every platform system that declares the file, for the installer's --system.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true } } } @@ -203,6 +286,89 @@ "release_asset": { "type": "string", "pattern": "^[^/\\\\]+$" + }, + "systems": { + "description": "Every platform system that declares the file, for the installer's --system.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "regions": { + "description": "The regions the emulator code selects the file for. Absent when it never competes.", + "type": "array", + "items": { + "type": "string", + "enum": [ + "argentina", + "asia", + "asia-ntsc", + "asia-pal", + "australia", + "brazil", + "canada", + "china", + "denmark", + "europe", + "finland", + "france", + "germany", + "greece", + "hong-kong", + "india", + "italy", + "japan", + "latin-america", + "mexico", + "netherlands", + "new-zealand", + "north-america", + "norway", + "oceania", + "poland", + "portugal", + "russia", + "singapore", + "south-korea", + "spain", + "sweden", + "taiwan", + "uk", + "world" + ] + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "region_groups": { + "description": "Groups the file competes in as a core extra, whatever systems are chosen.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true + }, + "region_system_groups": { + "description": "Groups the file competes in as the platform's own declaration: present only when that system is kept.", + "type": "array", + "items": { + "type": "string", + "minLength": 1, + "maxLength": 256 + }, + "minItems": 1, + "maxItems": 1000, + "uniqueItems": true } }, "oneOf": [ diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index aa8b3cd3..148791e4 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -64,6 +64,7 @@ from common import ( import packresolve from refresh_data_dirs import cache_lock import region as region_mod +from manifestselect import SelectionIndex import slot as slot_mod import slots import split_pack @@ -2952,6 +2953,7 @@ def _manifest_entry( cores: list[str] | None, db: dict, repo_root: str, + selection_fields: dict, ) -> tuple[dict | None, int]: """The download record of a resolved file, and the size it occupies. @@ -2962,6 +2964,7 @@ def _manifest_entry( scan, a hash computed wrong) has neither a repo path nor a release asset. None then, so the caller records an omission instead of a dead entry that makes the installer refuse the whole manifest. + `selection_fields` carries what the installer narrows on. """ sha1 = "" sha256 = "" @@ -2982,6 +2985,7 @@ def _manifest_entry( "size": file_size, "repo_path": repo_path, "cores": cores, + **selection_fields, } if is_release_asset: entry["storage"] = "release" @@ -3007,6 +3011,7 @@ def _manifest_core_entries( omitted_by_destination: dict, record_omission, pack_only_sizes: list[int], + selection: SelectionIndex, required_only: bool = False, ) -> int: """Add the files a platform's cores need but its list does not name. @@ -3073,6 +3078,7 @@ def _manifest_core_entries( entry, file_size = _manifest_entry( local_path, manifest_dest, [source_emu] if source_emu else [], db, repo_root, + selection.fields(dest, fe.get("name", "")), ) if entry is None: systems = _extra_system_ids(fe) @@ -3095,7 +3101,7 @@ def _manifest_core_entries( return total_size -def _manifest_region_drops( +def _manifest_selection( config: dict, pack_systems: dict, emulators_dir: str, @@ -3106,10 +3112,13 @@ def _manifest_region_drops( source: str, required_only: bool, regions: list[str] | None, -) -> set[str]: - """Destinations a region list removes, grouped as the pack groups them.""" - if not regions: - return set() +) -> tuple[set[str], SelectionIndex]: + """Destinations a region list removes, and the groups the installer replays. + + Grouped as the pack groups them, so a region the installer applies to + the full manifest withdraws what `--region` withdraws from the pack. + """ + extras: list[dict] = [] region_groups, _extra_dests = platform_region_groups( config, pack_systems, @@ -3121,9 +3130,16 @@ def _manifest_region_drops( include_extras=(source != "platform"), include_all=(source == "truth"), required_only=required_only, + extras_out=extras, ) - return region_mod.resolve_region_drops( - region_groups, region_mod.build_region_index(emu_profiles), regions + region_index = region_mod.build_region_index(emu_profiles) + drops = ( + region_mod.resolve_region_drops(region_groups, region_index, regions) + if regions + else set() + ) + return drops, SelectionIndex.build( + region_groups, region_index, pack_systems, extras, required_only ) @@ -3234,9 +3250,17 @@ def generate_manifest( "required": bool(file_entry.get("required", True)), "reason": reason, "cores": cores, + # The builder counts an unavailable regional file in its groups: + # the installer must too, or it keeps a fallback the pack drops. + **selection.fields( + sanitize_pack_path( + file_entry.get("destination", file_entry.get("name", "")) + ), + str(file_entry.get("name") or ""), + ), } - region_drops = _manifest_region_drops( + region_drops, selection = _manifest_selection( config, pack_systems, emulators_dir, db, base_dest, emu_profiles, target_cores, source, required_only, regions, ) @@ -3325,7 +3349,8 @@ def generate_manifest( ) entry, file_size = _manifest_entry( - local_path, dest, None, db, repo_root + local_path, dest, None, db, repo_root, + selection.fields(dest, file_entry["name"]), ) if entry is None: record_omission(full_dest, file_entry, sys_id, "not_found", None) @@ -3366,7 +3391,8 @@ def generate_manifest( core_files, config, db, bios_dir, base_dest, repo_root, zip_contents, offline, region_drops, case_insensitive, seen_destinations, seen_lower, seen_parents, manifest_files, - omitted_by_destination, record_omission, pack_only_sizes, required_only, + omitted_by_destination, record_omission, pack_only_sizes, selection, + required_only, ) # Phase 3: data directories. The installer does not fetch them, so they diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index 2620a3ff..7bd488e1 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -372,7 +372,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str: "", "## Download BIOS packs", "", - "One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file, and `--target` narrows it to one machine; for a region or a bare minimum, build your own pack below.", + "One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file. At a terminal it asks whether to install everything or to choose systems, cores and regions, and `--system`, `--core`, `--region` and `--target` make the same choice without questions; for a bare minimum, build your own pack below.", "Files is how many the pack holds once extracted, the figure a file" " manager shows for the folder. It differs from one platform to the" " next because a pack carries only what that platform's emulators" diff --git a/scripts/generate_site.py b/scripts/generate_site.py index 269053ce..c5b111ad 100644 --- a/scripts/generate_site.py +++ b/scripts/generate_site.py @@ -3405,11 +3405,13 @@ guarantee either: source profiles can document files nobody has dumped or that only the user can provide, all of them visible in the [gap analysis](gaps.md). -Want less than everything? The installer takes `--target switch` to install -only what one machine's cores need. From a clone of the repository, -`python scripts/generate_pack.py --platform retroarch --region us` keeps one -BIOS per region and `--required-only` the bare minimum each core needs to -start; `--help` lists every way to build your own. +Want less than everything? At a terminal the installer asks whether to install +everything or to choose systems, cores and regions, and `--system`, `--core`, +`--region` and `--target` make the same choice without questions (see the +[installer reference](wiki/installer.md#choosing-what-to-install)). From a +clone of the repository, `python scripts/generate_pack.py --platform retroarch +--region us` builds the same narrowing as a pack and `--required-only` the bare +minimum each core needs to start; `--help` lists every way to build your own. --- diff --git a/scripts/manifestselect.py b/scripts/manifestselect.py new file mode 100644 index 00000000..0a2ddf91 --- /dev/null +++ b/scripts/manifestselect.py @@ -0,0 +1,112 @@ +"""What an install manifest records about each file so the installer can narrow it. + +The installer runs without the profiles or the database, yet a narrowed +install must hold the same files as a pack built with the same narrowing. +Each entry therefore carries the platform systems it belongs to and, when +the code selects it by territory, its regions and the groups in which it +competes. The installer replays region.resolve_region_drops over them. + +The pack's `--system X` keeps the platform's declarations under X and every +core extra owned by X, then groups regions over the declarations it kept and +over every core extra of the platform. A group membership is therefore either +unconditional (a core extra) or present only when the declaring system is +kept (a platform declaration), and the manifest records the two apart. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import region as region_mod +from common import _norm_system_id, sanitize_pack_path +from packextras import _extra_system_ids, _kept, extra_region_groups + + +def _dest(entry: dict) -> str: + return sanitize_pack_path(entry.get("destination", entry.get("name", ""))) + + +@dataclass(frozen=True) +class SelectionIndex: + """System and region facts for one platform, as the pack builder reads them.""" + + region_index: dict[str, dict] + competing: frozenset[str] + systems_by_dest: dict[str, frozenset[str]] + declared_by: dict[str, frozenset[str]] + extra_groups: dict[str, frozenset[str]] + + @classmethod + def build( + cls, + region_groups: dict[str, list[tuple[str, str]]], + region_index: dict[str, dict], + pack_systems: dict[str, dict], + extras: list[dict], + required_only: bool = False, + ) -> SelectionIndex: + by_norm: dict[str, set[str]] = {} + for sys_id in pack_systems: + by_norm.setdefault(_norm_system_id(sys_id), set()).add(sys_id) + + declared_by: dict[str, set[str]] = {} + for sys_id, system in pack_systems.items(): + for file_entry in _kept(system.get("files", []), required_only): + dest = _dest(file_entry) + if dest: + declared_by.setdefault(dest, set()).add(sys_id) + + owned_by: dict[str, set[str]] = {} + extra_groups: dict[str, set[str]] = {} + for extra in extras: + dest = _dest(extra) + if not dest: + continue + extra_groups.setdefault(dest, set()).update(extra_region_groups(extra)) + # A profile system the platform does not have is no system the + # pack's --system accepts: it would offer what the pack refuses. + for sys_id in _extra_system_ids(extra): + owned_by.setdefault(dest, set()).update( + by_norm.get(_norm_system_id(sys_id), ()) + ) + + systems_by_dest = { + dest: frozenset(declared_by.get(dest, set()) | owned_by.get(dest, set())) + for dest in set(declared_by) | set(owned_by) + } + return cls( + region_index, + frozenset( + dest for members in region_groups.values() for dest, _name in members + ), + systems_by_dest, + {dest: frozenset(ids) for dest, ids in declared_by.items()}, + {dest: frozenset(ids) for dest, ids in extra_groups.items()}, + ) + + def fields(self, dest: str, name: str) -> dict: + """What an entry carries for the installer's selection. + + `systems` drives --system. A regional entry adds its regions, + `region_groups` it always competes in as a core extra, and + `region_system_groups` it competes in only when that platform system + is kept. An entry that never competes carries no region field: an + untagged file always survives a region filter. + """ + out: dict = {} + systems = self.systems_by_dest.get(dest) + if systems: + out["systems"] = sorted(systems) + if dest not in self.competing: + return out + regions = region_mod.lookup_regions(self.region_index, dest, name) + if not regions: + return out + out["regions"] = sorted(regions) + always = self.extra_groups.get(dest) + if always: + out["region_groups"] = sorted(always) + declared = self.declared_by.get(dest) + if declared: + out["region_system_groups"] = sorted(declared) + return out diff --git a/scripts/packextras.py b/scripts/packextras.py index 96a12434..a74b3058 100644 --- a/scripts/packextras.py +++ b/scripts/packextras.py @@ -685,6 +685,7 @@ def platform_region_groups( include_extras: bool = True, include_all: bool = False, required_only: bool = False, + extras_out: list[dict] | None = None, ) -> tuple[dict[str, list[tuple[str, str]]], dict[tuple[str, str, str], str]]: """Group a platform's pack candidates the way region filtering reads them. @@ -692,7 +693,8 @@ def platform_region_groups( grouped under, keyed by (emulator, name). verify.py needs that mapping to withdraw from its report exactly what the builder withdraws from the pack: grouping the declared files here and the core extras there would let the - two answer differently on the same request. + two answer differently on the same request. `extras_out` receives the + core extras that were grouped, for the install manifest. """ groups: dict[str, list[tuple[str, str]]] = {} for sys_id, system in systems.items(): @@ -733,19 +735,24 @@ def platform_region_groups( extra.get("source_path", ""), ) ] = dest - variant = extra.get("variant_group") - if variant: - # A variant group is one slot of one core, whatever system each - # member is filed under: PicoDrive files its US Mega CD BIOS under - # sega-segacd and the EU/JP ones under sega-megacd, and find_bios - # picks one of the twelve. Keyed as emulator mode keys it. - group_ids = [f"{extra.get('source_emulator', '')}:variant:{variant}"] - else: - group_ids = _extra_system_ids(extra) or ["_extras"] - for group_id in group_ids: + for group_id in extra_region_groups(extra): groups.setdefault(group_id, []).append((dest, name)) + if extras_out is not None: + extras_out.append(extra) return groups, extra_dests + +def extra_region_groups(extra: dict) -> list[str]: + """The region groups a core extra competes in.""" + variant = extra.get("variant_group") + if variant: + # A variant group is one slot of one core, whatever system each + # member is filed under: PicoDrive files its US Mega CD BIOS under + # sega-segacd and the EU/JP ones under sega-megacd, and find_bios + # picks one of the twelve. Keyed as emulator mode keys it. + return [f"{extra.get('source_emulator', '')}:variant:{variant}"] + return _extra_system_ids(extra) or ["_extras"] + def emulator_region_drops( selected: list[tuple[str, dict]], standalone: bool, diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py new file mode 100644 index 00000000..eea4439a --- /dev/null +++ b/tests/test_install_selection.py @@ -0,0 +1,540 @@ +"""The installer narrows a platform the way the pack builder does. + +The one-liner can install a chosen set of systems, cores and regions. It runs +without the profiles or the database, from the manifest alone, so the +manifest records each file's systems and regional competition and the +installer replays the builder's region rule. These tests hold the two ends +to one answer. +""" + +from __future__ import annotations + +import functools +import hashlib +import http.server +import importlib.util +import json +import os +import random +import shlex +import shutil +import subprocess +import sys +import tempfile +import threading +import unittest +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT / "scripts")) + +import region # noqa: E402 + +_spec = importlib.util.spec_from_file_location("install", REPO_ROOT / "install.py") +install = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(install) + + +class OneRegionVocabulary(unittest.TestCase): + """install.py cannot import region.py, so it carries a copy.""" + + def test_the_copy_matches(self): + self.assertEqual(install.REGION_TREE, dict(region.REGION_TREE)) + self.assertEqual(install.REGION_ALIASES, region.ALIASES) + self.assertEqual(install.WORLD_REGION, region.WORLD) + self.assertEqual(install.REGIONS, region.REGIONS) + + +class TheRegionRuleIsTheBuilders(unittest.TestCase): + """Random groups, both implementations, one answer.""" + + POOL = ["north-america", "europe", "japan", "uk", "canada", "brazil", + "south-korea", "world", "asia", "oceania"] + + def _case(self, rng: random.Random): + groups: dict[str, list[tuple[str, str]]] = {} + index: dict[str, dict] = {} + entries: dict[str, dict] = {} + for g in range(rng.randint(1, 4)): + for m in range(rng.randint(1, 5)): + dest = f"d{g}_{m}" if rng.random() < 0.8 else f"shared{m}" + if dest not in entries: + regions = ( + set() + if rng.random() < 0.2 + else set(rng.sample(self.POOL, rng.randint(1, 2))) + ) + entries[dest] = {"dest": dest, "regions": sorted(regions), + "region_groups": []} + index[dest] = {"regions": regions, + "has_untagged": not regions, "emulators": []} + groups.setdefault(f"g{g}", []).append((dest, dest)) + if f"g{g}" not in entries[dest]["region_groups"]: + entries[dest]["region_groups"].append(f"g{g}") + for entry in entries.values(): + if not entry["regions"]: + del entry["regions"], entry["region_groups"] + requested = rng.sample(["north-america", "europe", "japan", "uk", "canada"], + rng.randint(1, 3)) + return groups, index, list(entries.values()), requested + + def test_two_thousand_random_platforms(self): + rng = random.Random(20261010) + for trial in range(2000): + groups, index, entries, requested = self._case(rng) + with self.subTest(trial=trial): + self.assertEqual( + install.region_drops(entries, requested), + region.resolve_region_drops(groups, index, requested), + ) + + +class Narrowing(unittest.TestCase): + FILES = [ + {"dest": "scph5500.bin", "size": 1, "cores": None, + "systems": ["sony-playstation"], "regions": ["japan"], + "region_system_groups": ["sony-playstation"]}, + {"dest": "scph5501.bin", "size": 1, "cores": None, + "systems": ["sony-playstation"], "regions": ["north-america"], + "region_system_groups": ["sony-playstation"]}, + {"dest": "gba_bios.bin", "size": 1, "cores": None, + "systems": ["nintendo-gba"]}, + {"dest": "pcsx/extra.bin", "size": 1, "cores": ["pcsx_rearmed"], + "systems": ["sony-playstation"]}, + {"dest": "beetle/extra.bin", "size": 1, "cores": ["beetle_psx"], + "systems": ["sony-playstation"]}, + ] + OMITTED = [ + {"dest": "scph5502.bin", "name": "scph5502.bin", "system": "sony-playstation", + "required": True, "reason": "not_found", "cores": None, + "regions": ["europe"], "region_system_groups": ["sony-playstation"]}, + ] + + def _dests(self, *args): + files, omitted = install.narrow(list(self.FILES), list(self.OMITTED), *args) + return sorted(f["dest"] for f in files), sorted(o["dest"] for o in omitted) + + def test_nothing_chosen_keeps_everything(self): + self.assertEqual(len(self._dests([], [], [])[0]), len(self.FILES)) + + def test_a_system(self): + files, omitted = self._dests(["nintendo-gba"], [], []) + self.assertEqual(files, ["gba_bios.bin"]) + self.assertEqual(omitted, []) + + def test_a_core_keeps_the_platform_list(self): + files, _ = self._dests(["sony-playstation"], ["pcsx_rearmed"], []) + self.assertEqual(files, ["pcsx/extra.bin", "scph5500.bin", "scph5501.bin"]) + + def test_a_region_keeps_one_bios(self): + files, omitted = self._dests([], [], ["north-america"]) + self.assertNotIn("scph5500.bin", files) + self.assertIn("scph5501.bin", files) + self.assertEqual(omitted, []) + + def test_an_unavailable_match_still_wins_as_the_builder_counts_it(self): + files, omitted = self._dests([], [], ["europe"]) + self.assertNotIn("scph5500.bin", files) + self.assertNotIn("scph5501.bin", files) + self.assertEqual(omitted, ["scph5502.bin"]) + + def test_no_match_keeps_every_regional_file(self): + files, _ = self._dests([], [], ["brazil"]) + self.assertIn("scph5500.bin", files) + self.assertIn("scph5501.bin", files) + + def test_a_core_variant_competes_across_systems(self): + """PicoDrive files its US Mega CD BIOS under sega-segacd and the EU + one under sega-mega-cd, one variant group. The pack groups every core + extra before keeping a system, so under --system sega-mega-cd + --region us the US member still wins and the EU one is dropped.""" + files = [ + {"dest": "us_scd.bin", "size": 1, "cores": ["picodrive"], + "systems": ["sega-segacd"], "regions": ["north-america"], + "region_groups": ["picodrive:variant:mcd"]}, + {"dest": "eu_mcd.bin", "size": 1, "cores": ["picodrive"], + "systems": ["sega-mega-cd"], "regions": ["europe"], + "region_groups": ["picodrive:variant:mcd"]}, + {"dest": "bios_CD_U.bin", "size": 1, "cores": None, + "systems": ["sega-mega-cd"]}, + ] + kept, _ = install.narrow(files, [], ["sega-mega-cd"], [], ["north-america"]) + self.assertEqual([f["dest"] for f in kept], ["bios_CD_U.bin"]) + + def test_an_unkept_systems_declaration_does_not_compete(self): + """Group `a` holds system a's own US file and a core extra filed + under a but owned by `shared`. Under --system shared the pack builds + group a from the extra alone: nothing matches, so the extra stays. + Counting a's own file would make it win and drop the extra.""" + files = [ + {"dest": "a/us.bin", "size": 1, "cores": None, "systems": ["a"], + "regions": ["north-america"], "region_system_groups": ["a"]}, + {"dest": "core/jp.bin", "size": 1, "cores": ["core"], + "systems": ["shared"], "regions": ["japan"], "region_groups": ["a"]}, + ] + kept, _ = install.narrow(files, [], ["shared"], [], ["north-america"]) + self.assertEqual([f["dest"] for f in kept], ["core/jp.bin"]) + + +class CommandLineChoices(unittest.TestCase): + KNOWN = ["msx1,msx2,msxturbor", "nintendo-gba", "sony-playstation"] + + def test_comma_lists_and_repeats(self): + self.assertEqual( + install.resolve_choices(["Sony-PlayStation,nintendo-gba"], self.KNOWN, "system"), + ["sony-playstation", "nintendo-gba"], + ) + + def test_a_name_with_commas_is_one_name(self): + self.assertEqual( + install.resolve_choices(["msx1,msx2,msxturbor"], self.KNOWN, "system"), + ["msx1,msx2,msxturbor"], + ) + + def test_an_unknown_name_is_refused(self): + with self.assertRaises(ValueError) as ctx: + install.resolve_choices(["sega-saturn"], self.KNOWN, "system") + self.assertIn("sega-saturn", str(ctx.exception)) + self.assertIn("nintendo-gba", str(ctx.exception)) + + def test_an_empty_list_is_refused(self): + with self.assertRaises(ValueError): + install.resolve_regions([" , "]) + with self.assertRaises(ValueError): + install.resolve_choices([","], self.KNOWN, "system") + + def test_region_aliases_keep_their_order(self): + self.assertEqual( + install.resolve_regions(["jp,us", "eu,jp"]), + ["japan", "north-america", "europe"], + ) + with self.assertRaises(ValueError): + install.resolve_regions(["mars"]) + + def test_prompt_numbers(self): + self.assertEqual(install.parse_selection("3,1-2", 5), [1, 2, 3]) + self.assertEqual(install.parse_selection("3,1", 5, ordered=True), [3, 1]) + self.assertEqual(install.parse_selection("all", 3), [1, 2, 3]) + for bad in ("0", "6", "4-2", "x", "1-"): + with self.subTest(text=bad), self.assertRaises(ValueError): + install.parse_selection(bad, 5) + + +class TheBoundaryChecksTheNewFields(unittest.TestCase): + def _manifest(self, **extra): + entry = {"dest": "a.bin", "sha1": "a" * 40, "size": 1, + "repo_path": "bios/a.bin", **extra} + return {"manifest_version": 2, "platform": "retroarch", "files": [entry]} + + def test_well_formed(self): + install._validate_manifest( + self._manifest(systems=["x"], regions=["japan"], region_groups=["x"]), + "retroarch", + ) + + def test_malformed(self): + for bad in ({"systems": "x"}, {"regions": [1]}, {"region_groups": [""]}, + {"region_system_groups": "x"}): + with self.subTest(field=bad), self.assertRaises(ValueError): + install._validate_manifest(self._manifest(**bad), "retroarch") + + +class _QuietHandler(http.server.SimpleHTTPRequestHandler): + def log_message(self, format, *args): # noqa: A002 - stdlib signature + return + + +class FakeRepository: + """A loopback copy of the repository: one manifest and its payloads.""" + + PAYLOADS = { + "scph5500.bin": (b"jp bios", ["sony-playstation"], ["japan"], None), + "scph5501.bin": (b"us bios", ["sony-playstation"], ["north-america"], None), + "gba_bios.bin": (b"gba bios", ["nintendo-gba"], None, None), + "pcsx/extra.bin": (b"pcsx", ["sony-playstation"], None, ["pcsx_rearmed"]), + "mgba/extra.bin": (b"mgba", ["nintendo-gba"], None, ["mgba"]), + } + + def __init__(self, selection: bool = True): + self.selection = selection + + def __enter__(self) -> FakeRepository: + self.root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + (self.root / "install").mkdir() + (self.root / "bios").mkdir() + files = [] + for name, (payload, systems, regions, cores) in self.PAYLOADS.items(): + stored = name.replace("/", "_") + (self.root / "bios" / stored).write_bytes(payload) + entry = { + "dest": name, + "sha1": hashlib.sha1(payload).hexdigest(), + "sha256": hashlib.sha256(payload).hexdigest(), + "size": len(payload), + "repo_path": f"bios/{stored}", + "cores": cores, + } + if self.selection: + entry["systems"] = systems + if regions: + entry.update(regions=regions, region_system_groups=systems) + files.append(entry) + manifest = {"manifest_version": 2, "platform": "retroarch", "files": files} + (self.root / "install" / "retroarch.json").write_text(json.dumps(manifest)) + handler = functools.partial(_QuietHandler, directory=str(self.root)) + self.httpd = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=self.httpd.serve_forever, daemon=True).start() + self.dest = self.root / "dest" + return self + + def __exit__(self, *exc) -> None: + self.httpd.shutdown() + self.httpd.server_close() + shutil.rmtree(self.root, ignore_errors=True) + + def env(self) -> dict: + env = dict(os.environ) + env["RETROBIOS_BASE_URL"] = f"http://127.0.0.1:{self.httpd.server_address[1]}" + return env + + def command(self, *options: str) -> list[str]: + return [sys.executable, str(REPO_ROOT / "install.py"), "--platform", + "retroarch", "--dest", str(self.dest), *options] + + def run(self, *options: str) -> subprocess.CompletedProcess: + return subprocess.run( + self.command(*options), env=self.env(), capture_output=True, + text=True, timeout=60, check=False, stdin=subprocess.DEVNULL, + ) + + def installed(self) -> list[str]: + if not self.dest.exists(): + return [] + return sorted( + p.relative_to(self.dest).as_posix() + for p in self.dest.rglob("*") if p.is_file() + ) + + +class TheOneLinerNarrows(unittest.TestCase): + def test_options_install_only_the_choice(self): + with FakeRepository() as repo: + proc = repo.run("--system", "sony-playstation", "--region", "us") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("Narrowed 5 -> 2 files", proc.stdout) + self.assertEqual(repo.installed(), ["pcsx/extra.bin", "scph5501.bin"]) + + def test_a_core_choice_keeps_the_platform_list(self): + with FakeRepository() as repo: + proc = repo.run("--core", "mgba") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual( + repo.installed(), + ["gba_bios.bin", "mgba/extra.bin", "scph5500.bin", "scph5501.bin"], + ) + + def test_an_older_file_list_is_refused_not_ignored(self): + """Without the fields, a region filter would match nothing and the + run would install everything while the user asked for one region.""" + with FakeRepository(selection=False) as repo: + proc = repo.run("--region", "us") + self.assertEqual(proc.returncode, 1) + self.assertIn("records no systems or regions", proc.stderr) + self.assertEqual(repo.installed(), []) + + def test_empty_names_are_a_usage_error(self): + with FakeRepository() as repo: + for option in (["--region", " , "], ["--system", ""], ["--core", ","]): + with self.subTest(option=option): + proc = repo.run(*option) + self.assertEqual(proc.returncode, 2, proc.stderr) + self.assertIn("needs at least one", proc.stderr) + + def test_listing_follows_the_narrowing(self): + with FakeRepository() as repo: + proc = repo.run("--list-cores", "--system", "nintendo-gba") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("mgba", proc.stdout) + self.assertNotIn("pcsx_rearmed", proc.stdout) + + def test_an_unknown_system_is_refused_before_any_download(self): + with FakeRepository() as repo: + proc = repo.run("--system", "sega-saturn") + self.assertEqual(proc.returncode, 1) + self.assertIn("unknown system: sega-saturn", proc.stderr) + self.assertIn("nintendo-gba", proc.stderr) + self.assertEqual(repo.installed(), []) + + def test_an_unknown_region_is_a_usage_error(self): + with FakeRepository() as repo: + proc = repo.run("--region", "mars") + self.assertEqual(proc.returncode, 2) + self.assertIn("unknown region: mars", proc.stderr) + + def test_listing_systems_installs_nothing(self): + with FakeRepository() as repo: + proc = repo.run("--list-systems") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertIn("nintendo-gba", proc.stdout) + self.assertIn("3 files", proc.stdout) + self.assertEqual(repo.installed(), []) + + def test_no_terminal_asks_nothing_and_installs_everything(self): + with FakeRepository() as repo: + proc = repo.run() + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertNotIn("choose systems", proc.stdout) + self.assertEqual(len(repo.installed()), 5) + + @unittest.skipUnless(shutil.which("script"), "script(1) provides the terminal") + def test_the_custom_mode_at_a_terminal(self): + """Typed answers: customise, PlayStation only, no core, US first.""" + with FakeRepository() as repo: + command = " ".join(shlex.quote(part) for part in repo.command()) + proc = subprocess.run( + ["script", "-qec", command, "/dev/null"], + input="c\n2\n\n2\n", env=repo.env(), capture_output=True, + text=True, timeout=60, check=False, + ) + out = proc.stdout + self.assertEqual(proc.returncode, 0, out + proc.stderr) + self.assertIn("choose systems, cores and regions", out) + self.assertIn("pcsx_rearmed", out) + self.assertIn("--system sony-playstation --region north-america", out) + self.assertEqual(repo.installed(), ["pcsx/extra.bin", "scph5501.bin"]) + + @unittest.skipUnless(shutil.which("script"), "script(1) provides the terminal") + def test_enter_installs_everything_at_a_terminal(self): + with FakeRepository() as repo: + command = " ".join(shlex.quote(part) for part in repo.command()) + proc = subprocess.run( + ["script", "-qec", command, "/dev/null"], + input="\n", env=repo.env(), capture_output=True, + text=True, timeout=60, check=False, + ) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(len(repo.installed()), 5) + + @unittest.skipUnless(shutil.which("script"), "script(1) provides the terminal") + def test_no_input_asks_nothing_at_a_terminal(self): + with FakeRepository() as repo: + command = " ".join( + shlex.quote(part) for part in repo.command("--no-input") + ) + proc = subprocess.run( + ["script", "-qec", command, "/dev/null"], + input="", env=repo.env(), capture_output=True, + text=True, timeout=60, check=False, + ) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertNotIn("choose systems", proc.stdout) + self.assertEqual(len(repo.installed()), 5) + + +class InstallMatchesThePack(unittest.TestCase): + """Narrowing the full manifest gives what the narrowed build carries.""" + + PLATFORM = "recalbox" + + @classmethod + def setUpClass(cls): + db_path = REPO_ROOT / "database.json" + if not db_path.exists(): + raise unittest.SkipTest("database.json is not built") + import generate_pack as gp + from common import load_emulator_profiles + + cls.gp = gp + cls.db = json.loads(db_path.read_text(encoding="utf-8")) + cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + cls.full = cls._manifest(None) + + @classmethod + def _manifest(cls, regions): + return cls.gp.generate_manifest( + cls.PLATFORM, str(REPO_ROOT / "platforms"), cls.db, + str(REPO_ROOT / "bios"), str(REPO_ROOT / "platforms" / "_registry.yml"), + emulators_dir=str(REPO_ROOT / "emulators"), emu_profiles=cls.profiles, + regions=regions, offline=True, + ) + + def _narrowed(self, systems, regions): + files, omitted = install.narrow( + self.full["files"], self.full["omitted_files"], systems, [], regions + ) + return {f["dest"] for f in files}, {o["dest"] for o in omitted} + + def test_regions(self): + for regions in (["north-america"], ["japan", "europe"]): + with self.subTest(regions=regions): + built = self._manifest(regions) + files, omitted = self._narrowed([], regions) + self.assertEqual(files, {f["dest"] for f in built["files"]}) + self.assertEqual(omitted, {o["dest"] for o in built["omitted_files"]}) + + +class InstallMatchesASystemPack(unittest.TestCase): + """`install.py --system X --region R` against a real `generate_pack.py + --system X --region R` build. The cases are the ones a review reproduced: + ColecoVision's extras filed at destinations another system declares, and + PicoDrive's Mega CD variant group spread over two systems.""" + + PLATFORM = "retroarch" + CASES = ( + (["coleco-colecovision"], []), + (["sega-mega-cd"], ["north-america"]), + (["sony-playstation"], ["japan"]), + ) + + @classmethod + def setUpClass(cls): + db_path = REPO_ROOT / "database.json" + if not db_path.exists(): + raise unittest.SkipTest("database.json is not built") + import generate_pack as gp + from common import load_emulator_profiles, load_platform_config + + cls.gp = gp + cls.db = json.loads(db_path.read_text(encoding="utf-8")) + cls.profiles = load_emulator_profiles(str(REPO_ROOT / "emulators")) + cls.base = load_platform_config( + cls.PLATFORM, str(REPO_ROOT / "platforms") + ).get("base_destination", "") + cls.full = gp.generate_manifest( + cls.PLATFORM, str(REPO_ROOT / "platforms"), cls.db, + str(REPO_ROOT / "bios"), str(REPO_ROOT / "platforms" / "_registry.yml"), + emulators_dir=str(REPO_ROOT / "emulators"), emu_profiles=cls.profiles, + offline=True, + ) + + def _packed(self, systems, regions) -> set[str]: + import zipfile + + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as out: + zip_path = self.gp.generate_pack( + self.PLATFORM, str(REPO_ROOT / "platforms"), self.db, + str(REPO_ROOT / "bios"), out, + emulators_dir=str(REPO_ROOT / "emulators"), + emu_profiles=self.profiles, system_filter=systems, + regions=regions or None, offline=True, + ) + with zipfile.ZipFile(zip_path) as zf: + names = set(zf.namelist()) + prefix = f"{self.base}/" if self.base else "" + return {n[len(prefix):] if prefix and n.startswith(prefix) else n for n in names} + + def test_each_case(self): + known = {f["dest"] for f in self.full["files"]} + for systems, regions in self.CASES: + with self.subTest(systems=systems, regions=regions): + files, _omitted = install.narrow( + self.full["files"], self.full["omitted_files"], + systems, [], regions, + ) + self.assertEqual( + {f["dest"] for f in files}, self._packed(systems, regions) & known + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_manifest_entry.py b/tests/test_manifest_entry.py index d248793c..0aa949fb 100644 --- a/tests/test_manifest_entry.py +++ b/tests/test_manifest_entry.py @@ -20,6 +20,7 @@ REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT / "scripts")) import generate_pack as gp # noqa: E402 +from manifestselect import SelectionIndex # noqa: E402 def _held_file(db: dict) -> tuple[str, dict] | None: @@ -60,7 +61,7 @@ class AnEntryWithoutSourceIsAnOmission(unittest.TestCase): gp._manifest_core_entries( [extra], {}, self.db, str(REPO_ROOT / "bios"), "", str(REPO_ROOT), {}, True, set(), False, set(), set(), set(), files, {}, - record, pack_only, + record, pack_only, SelectionIndex.build({}, {}, {}, []), ) return files, omitted, pack_only diff --git a/wiki/getting-started.md b/wiki/getting-started.md index 13b75449..8a5dc0c6 100644 --- a/wiki/getting-started.md +++ b/wiki/getting-started.md @@ -41,6 +41,10 @@ Override detection when needed: ```bash python install.py --platform retroarch --dest ~/custom/bios python install.py --target switch # keep only files for that hardware +python install.py --list-systems # systems a platform holds, with sizes +python install.py --system sony-playstation,nintendo-gba +python install.py --region us,eu,jp # one BIOS per region, best first +python install.py --no-input # ask nothing, even at a terminal python install.py --check # verify existing files, download nothing python install.py --list-platforms # supported platforms and what was detected python install.py --list-targets # hardware targets for a platform diff --git a/wiki/installer.md b/wiki/installer.md index 425eeeeb..25549109 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -67,6 +67,11 @@ Run from a clone, `install.sh` and `install.ps1` reuse the `install.py` sitting next to them and skip the download entirely; piped from stdin, `install.sh` never treats the working directory as a trusted location for it. +Piped from `curl`, the shell's standard input is the script itself, so +`install.sh` hands the installer the terminal (`/dev/tty`) when there is one. +Without that, every question the installer can ask was skipped on Linux and +macOS. + ## What the installer does 1. Detects the host OS and the platforms installed on it. @@ -75,7 +80,8 @@ never treats the working directory as a trusted location for it. always come from one commit. 3. Prints the file count and total size, then a safety notice counting the entries the collection cannot serve, how many of those the platform marks - required, and why. + required, and why. At a terminal, it then offers to install everything or + to choose systems, cores and regions (see below). 4. Hashes what is already in the destination and reports how many entries are present, verified, or present with the wrong contents. 5. Downloads what is missing or wrong, up to 8 files at a time. @@ -97,6 +103,60 @@ Checking existing files... 1872 files need downloading. ``` +## Choosing what to install + +By default the installer fetches everything the platform runs. At a terminal, +after detection, it asks: + +``` +Retroarch: 5861 files, 6.0 GB. + Enter) install everything + c) choose systems, cores and regions + q) quit +``` + +`c` walks through three numbered lists, each answered with numbers such as +`1,4,7-9`, or Enter to keep everything on that axis. + +- **Systems.** The consoles and computers the platform declares. +- **Cores.** The platform's own BIOS list for the chosen systems is always + kept; this narrows only the extra files each emulator core loads beyond it. + Asked only when the chosen systems have such files. +- **Regions.** Asked only when a chosen system has one BIOS per region. The + numbers give an order of preference: `2,1` keeps the second region where a + file for it exists, and the first one elsewhere. + +The run then prints the options that repeat the same choice without +questions, for instance +`--platform retroarch --system sony-playstation --region north-america`. + +The same choices on the command line: + +```bash +python install.py --platform retroarch --list-systems +python install.py --platform retroarch --system sony-playstation,nintendo-gba +python install.py --platform retroarch --system sony-playstation --core pcsx_rearmed +python install.py --platform recalbox --region us,eu,jp +``` + +A name the platform does not have is refused with the list of the ones it +has, before anything is downloaded: carrying on would install something other +than what was asked. A system name that itself contains commas, as Recalbox's +`msx1,msx2,msxturbor` does, is read as one name. + +The region rule is the one `generate_pack.py --region` applies, and a test +holds the two to the same file list. In each group of regional alternatives +the best-ranked match stays; a file declared for every region beats regional +files that match nothing asked; when nothing matches, every regional file +stays, so no system is left without its BIOS. A file the emulator does not +select by region always stays. The manifest records, for each file, the +systems that declare it, and for a regional file, its regions and the groups +it competes in; the installer reads nothing else. + +The questions are asked only when standard input is a terminal. Piped, +scripted or run with `--no-input`, the installer asks nothing and installs +what the options name, everything by default. + ## Options | Option | Effect | @@ -104,6 +164,12 @@ Checking existing files... | `--platform NAME` | Install for this platform instead of the detected one. Unknown names are refused with the available list | | `--dest PATH` | Destination directory, overriding detection. With `--dest` alone the file list is RetroArch's | | `--target NAME` | Keep only the files the cores of that hardware target need. An unknown target is refused rather than ignored, since carrying on would install everything | +| `--system NAMES` | Install only these systems. Comma-separated or repeated; `--list-systems` prints the names | +| `--core NAMES` | Keep the platform's own list and only the extra files these cores load; `--list-cores` prints the names | +| `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | +| `--list-systems` | Print a platform's systems with the files and size each holds | +| `--list-cores` | Print the cores whose extra files a platform installs | +| `--no-input` | Never ask a question, even at a terminal | | `--check` | Report and exit without writing | | `--list-platforms` | Print the supported platforms and what was detected here | | `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries |