From 35e7300465437f0e5f584b668e23b85861dc78f4 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:07:18 +0200 Subject: [PATCH 1/3] feat: add --declared-only and the events schema --declared-only keeps the entries the platform's own list declares (cores: null), each slot settled to the platform's copy with no core alternative left behind. Install, --check, --dry-run, --events, --list-systems, --system, --region, --emulator, --target, --installed-cores, --manifest and --from-pack apply it. --core, --list-cores, --list-emulators, --list-platforms, --list-targets, --standalone-copies, --restore and --tui refuse it, and the interface does not open by itself with it. On Batocera it gives 353 files of 14531, the list batocera-systems checks. Event decision: no schema bump. A `declared` key on `file` would add a key to an existing kind, which schema 1 forbids. A new kind, `declared`, follows `check` with the same counts over the platform's own files and their dests in file-event order. Readers already skip a kind they do not know (rbevents.py and the interface reader were checked), so EVENT_SCHEMA stays 1. An unknown --system is answered with the systems the list files its key file under: --platform retroarch --system snk-neogeo now suggests arcade, which holds neogeo.zip, instead of snk-neogeo-cd. neogeo, mvs and aes follow the same route. Never taken, only suggested. A system title the manifest carries (SNK Neo Geo CD) is taken for its id when no other system shares it and no platform name reads the same. schemas/install-events.schema.json is written from EVENT_KEYS by scripts/generate_event_schema.py. validate_schemas.py reports a stale file, and tests compare it with EVENT_KEYS and validate a real run. Bootstrap pins recomputed. --- install.ps1 | 2 +- install.py | 156 ++++++- install.sh | 2 +- schemas/install-events.schema.json | 704 +++++++++++++++++++++++++++++ scripts/generate_event_schema.py | 114 +++++ scripts/validate_schemas.py | 18 + tests/test_install.py | 76 ++++ tests/test_install_selection.py | 192 ++++++++ wiki/installer.md | 11 +- wiki/tools.md | 1 + 10 files changed, 1257 insertions(+), 19 deletions(-) create mode 100644 schemas/install-events.schema.json create mode 100644 scripts/generate_event_schema.py diff --git a/install.ps1 b/install.ps1 index 96a8a487..f7a8d2f5 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "c90676dcc433061bd97aca94930cd7acd74295624840cb5c85a22897f9c1dd12" + $defaultInstallSha256 = "7bc94c44f3696a0b777e5c280754633105a06756932ee52547f0a9c775a895b9" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index d7d51353..36fde413 100644 --- a/install.py +++ b/install.py @@ -526,6 +526,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { "checking": ("files_done", "files_total"), "check": ("platform", "name", "bios_dir", "files", "present", "missing", "wrong_hash", "bytes_needed", "data_missing", "not_installable"), + # Right after check: the same counts over the files the platform's own + # list declares (cores: null), and their dests in file-event order. A + # kind of its own, since schema 1 never adds a key to the file event. + "declared": ("platform", "files", "present", "missing", "wrong_hash", + "bytes_needed", "dests"), # A file to install that this machine already holds: misplaced in the # BIOS folder (folder is its bios_dir, reason says why in words) or in # a folder named with --from (reason empty). member names the ZIP member @@ -616,7 +621,8 @@ def _emit_check( mismatched: list[dict], data_missing: int, not_installable: int, chosen: list[str] | None = None, ) -> None: - """One event per file, per system and per platform after a check. + """One event per file, per system and per platform after a check, then + one for the files of the platform's own list. With systems chosen, only those are described: a file of SkyEmu's carries four systems, and --system nintendo-gba described gb, gbc and @@ -654,6 +660,15 @@ def _emit_check( bytes_needed=sum(entry.get("size", 0) for entry in needed), data_missing=data_missing, not_installable=not_installable, ) + own = [entry for entry in files if entry.get("cores") is None] + own_needed = [entry for entry in own if id(entry) not in present] + own_wrong = sum(1 for entry in own if id(entry) in wrong) + EVENTS.emit( + "declared", platform=plat, files=len(own), present=len(own) - len(own_needed), + missing=len(own_needed) - own_wrong, wrong_hash=own_wrong, + bytes_needed=sum(entry.get("size", 0) for entry in own_needed), + dests=[entry["dest"] for entry in own], + ) def leave_out_game_data( @@ -672,6 +687,22 @@ def leave_out_game_data( ) +def keep_declared(files: list[dict], omitted: list[dict]) -> tuple[list[dict], list[dict]]: + """The files and omissions the platform's own list declares (cores: null). + + A slot a core contradicts gets the platform's copy, and no core copy + stays behind as an alternative: narrowing by cores afterwards would + otherwise put a core's file back where a dropped system's was. + """ + def own(entries: list[dict]) -> list[dict]: + return [ + {k: v for k, v in _settle_slot(entry, set()).items() if k != "alternatives"} + for entry in entries if entry.get("cores") is None + ] + + return own(files), own(omitted) + + def _left_out_lines(left_out: list[dict], every: bool) -> list[str]: """What --no-game-data left out, named per category, ten each unless every.""" lines = [] @@ -2340,7 +2371,7 @@ SYSTEM_SHORT_NAMES = { "supergrafx"), "nec-pc-engine"), **dict.fromkeys(("pcecd", "tgcd", "turbografxcd"), "nec-pc-engine-cd"), "pcfx": "nec-pc-fx", - **dict.fromkeys(("mvs", "aes"), "snk-neogeo"), + **dict.fromkeys(("neogeo", "mvs", "aes"), "snk-neogeo"), **dict.fromkeys(("neocd", "neogeocd"), "snk-neogeo-cd"), **dict.fromkeys(("msx", "msx2"), "microsoft-msx"), "c64": "commodore-c64", "c128": "commodore-c128", @@ -2354,17 +2385,44 @@ SYSTEM_SHORT_NAMES = { "cpc": "amstrad-cpc", "bbc": "acorn-bbc-micro", "apple2": "apple-ii", } +# A system some lists file under another id, by the file that stands for it. +# The Neo Geo card offers --system snk-neogeo, and RetroArch's list keeps +# neogeo.zip under arcade: the run suggests arcade, never takes it. +SYSTEM_KEY_FILES = {"snk-neogeo": "neogeo.zip"} + + +def key_file_holders(files: list[dict], omitted: list[dict]) -> dict[str, list[str]]: + """The systems a list files each SYSTEM_KEY_FILES file under, for the + systems the list does not name itself.""" + entries = files + omitted + known = {system for entry in entries for system in entry_systems(entry)} + holders: dict[str, list[str]] = {} + for system, name in SYSTEM_KEY_FILES.items(): + if system in known: + continue + held = sorted({ + holder for entry in entries + if PurePosixPath(str(entry.get("dest", ""))).name.lower() == name + for holder in entry_systems(entry) + }) + if held: + holders[system] = held + return holders + def did_you_mean( typed: list[str], known: list[str], listing: str, aliases: dict[str, str] | None = None, inside_from: int = 4, + meant: dict[str, list[str]] | None = None, ) -> str: """The closest known names to what was typed, and the option listing them all. A typo was answered with every name the platform has, up to 400 on one line, with no hint that playstation is sony-playstation. An alias close to what was typed suggests the name it stands for. A name holding what - was typed, inside_from letters or more, is suggested too. + was typed, inside_from letters or more, is suggested too. meant gives, + by the letters and digits of a typed name, the names it stands for + here, suggested instead of the close ones. """ import difflib # noqa: PLC0415 @@ -2378,6 +2436,9 @@ def did_you_mean( key = _name_key(word) if not key: continue + if meant and key in meant: + close += [name for name in meant[key] if name not in close] + continue matches = difflib.get_close_matches(key, list(by_key), n=3, cutoff=0.6) if len(key) >= inside_from: matches += sorted(k for k in by_key if key in k and k not in matches) @@ -3617,6 +3678,7 @@ def available_choices(files: list[dict], omitted: list[dict]) -> dict[str, list[ def resolve_choices( values: list[str], known: list[str], label: str, aliases: dict[str, str] | None = None, + holders: dict[str, list[str]] | None = None, ) -> list[str]: """Names given on the command line, matched without regard to case. @@ -3626,6 +3688,8 @@ def resolve_choices( carrying on would install something other than what was asked. aliases maps another name of a known one to it, as Batocera's `psx` stands for sony-playstation. A known name always wins over an alias. + holders maps a system the list files under other ids to those ids + (key_file_holders): an unknown name for it is answered with them. """ aliases = {alias: name for alias, name in (aliases or {}).items() if name in known} by_lower = {name.lower(): name for name in known} @@ -3658,14 +3722,28 @@ def resolve_choices( listing = {"system": "--list-systems", "emulator": "--list-emulators"}.get( label, "--list-cores") hints, inside_from = aliases, 4 + held = holders or {} + stands_for: dict[str, str] = {} + notes = "" if label == "system": # gba is the end of nintendo-gba: three letters are enough here. hints = {**{short: name for short, name in SYSTEM_SHORT_NAMES.items() if name in known}, **aliases} inside_from = 3 + stands_for = {_name_key(system): system for system in held} + for short, name in SYSTEM_SHORT_NAMES.items(): + if name in held: + stands_for.setdefault(_name_key(short), name) + named = dict.fromkeys( + stands_for[_name_key(token)] for token in unknown + if _name_key(token) in stands_for) + notes = "".join( + f"This list files {SYSTEM_KEY_FILES[system]} under " + f"{', '.join(held[system])}. " for system in named) + meant = {key: held[system] for key, system in stands_for.items()} raise ValueError( - f"unknown {label}: {', '.join(unknown)}. " - f"{did_you_mean(unknown, known, listing, hints, inside_from)}" + f"unknown {label}: {', '.join(unknown)}. {notes}" + f"{did_you_mean(unknown, known, listing, hints, inside_from, meant)}" ) if values and not chosen: raise ValueError(f"--{label} needs at least one name") @@ -4696,6 +4774,7 @@ class Invocation: ("--every-board", args.every_board), ("--installed-cores", args.installed_cores and not args.cores_dir), ("--no-game-data", args.no_game_data), + ("--declared-only", args.declared_only), ("--standalone-copies", args.standalone_copies), ("--check", args.check), ("--dry-run", args.dry_run), @@ -5972,12 +6051,27 @@ def _same_choice_hint(plat: str, systems: list[str], cores: list[str], regions: def system_aliases(manifest: dict | None) -> dict[str, str]: - """The platform's own names for its systems, each with the system id.""" - return { + """The platform's own names for its systems, then the titles the site + and the installer print (SNK Neo Geo CD), each with the system id. + + A title two systems share names neither, and the platform's own name + wins over a title spelled the same. + """ + aliases = { alias: system for system, names in ((manifest or {}).get("system_names") or {}).items() for alias in names } + taken = {_name_key(alias) for alias in aliases} + titles = (manifest or {}).get("system_titles") or {} + counts: dict[str, int] = {} + for title in titles.values(): + counts[_name_key(title)] = counts.get(_name_key(title), 0) + 1 + for system, title in titles.items(): + key = _name_key(title) + if key and counts[key] == 1 and key not in taken: + aliases[title] = system + return aliases def _system_label(name: str, manifest: dict | None) -> str: @@ -6214,7 +6308,8 @@ def _requested_narrowing( choices = available_choices(files, omitted) try: systems = resolve_choices( - args.system or [], choices["systems"], "system", system_aliases(manifest) + args.system or [], choices["systems"], "system", system_aliases(manifest), + key_file_holders(files, omitted), ) cores = resolve_choices(args.core or [], choices["cores"], "core") regions = resolve_regions(args.region or []) @@ -7815,6 +7910,12 @@ def main() -> None: help="leave out game data (files a game reads, not firmware) and " "install the firmware only", ) + parser.add_argument( + "--declared-only", + action="store_true", + help="keep only the files the platform's own list names, not the extra " + "files cores load: --check then answers for the list the platform checks", + ) parser.add_argument( "--list-systems", action="store_true", @@ -7941,6 +8042,23 @@ def main() -> None: parser.error("--quiet applies to an install, not to --check, --dry-run or a listing") if args.tui and args.plain: parser.error("--tui and --plain ask for two different interfaces") + if args.declared_only: + # Applied or refused, never ignored. + for given, reason in ( + (args.tui, "the full-screen interface has no choice for --declared-only"), + (args.list_platforms or args.list_targets, + "--declared-only narrows a file list, not --list-platforms or --list-targets"), + (args.core or args.list_cores, + "--declared-only leaves out the files cores load, which --core and " + "--list-cores choose among"), + (args.list_emulators, + "--list-emulators names emulators, and --declared-only narrows files"), + (args.standalone_copies, + "--standalone-copies copies into standalone emulators' folders, outside " + "the platform's own list that --declared-only keeps"), + ): + if given: + parser.error(reason) if args.tui: reason = tui_blocker(args) if reason: @@ -7953,7 +8071,7 @@ def main() -> None: alone = ("platform", "dest", "target", "every_board", "check", "list_platforms", "list_targets", "standalone_copies", "emulator", "installed_cores", "cores_dir", "save_dir", "system", "core", "region", "no_game_data", - "list_systems", + "declared_only", "list_systems", "list_cores", "romm_firmware", "list_emulators", "events", "quiet", "yes", "verbose", "from_pack", "manifest", "from") given = [name for name in alone if getattr(args, name)] @@ -7980,7 +8098,9 @@ def main() -> None: if args.no_game_data and (args.list_platforms or args.list_targets): parser.error("--no-game-data narrows a file list, not --list-platforms " "or --list-targets") - if args.tui or not (plain_requested(args) or tui_blocker(args)): + # The screens have no choice for --declared-only: the questions ask the + # rest, and the run keeps the platform's own list. + if args.tui or not (plain_requested(args) or args.declared_only or tui_blocker(args)): sys.exit(run_tui(args)) if args.events: # The events own stdout. The text a person reads goes to stderr, so a @@ -8361,7 +8481,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # which the file lists name: an ES-DE user with DuckStation and # PCSX2 was sent to RetroArch's folder, which nothing here reads. narrowing = (args.system or args.core or args.emulator or args.region - or args.target or args.installed_cores) + or args.target or args.installed_cores or args.declared_only) if not listing and not narrowing and _install_standalone_folders( args, os_type, term, interactive ): @@ -8522,6 +8642,12 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: for line in _left_out_lines(left_out, args.verbose): print(line) + if args.declared_only: + before = len(files) + files, omitted_files = keep_declared(files, omitted_files) + print(f" The platform's own list: {before} -> {len(files)} files " + "(--declared-only)") + if args.list_targets: targets = fetch_targets(plat_name) if targets is None: @@ -8640,13 +8766,15 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: files, omitted_files = installed print("\n Same choice without questions: " f"--platform {plat_name} --installed-cores" - + (" --no-game-data" if args.no_game_data else "")) + + (" --no-game-data" if args.no_game_data else "") + + (" --declared-only" if args.declared_only else "")) _INVOCATION.chosen[plat_name] = [("--installed-cores", None)] selection = None elif selection is not None: print("\n Same choice without questions: " + _same_choice_hint(plat_name, *selection) - + (" --no-game-data" if args.no_game_data else "")) + + (" --no-game-data" if args.no_game_data else "") + + (" --declared-only" if args.declared_only else "")) systems, cores, regions = selection _INVOCATION.chosen[plat_name] = [ *(("--system", name) for name in systems), @@ -8746,7 +8874,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: narrowed = bool( target or args.system or args.core or args.region or args.installed_cores - or len(files) + game_data_cut + len(unplaced) + or args.declared_only or len(files) + game_data_cut + len(unplaced) != len(manifest.get("files", [])) + len(variants if case_kept else []) ) data_dirs = select_data_directories(manifest, files, narrowed) diff --git a/install.sh b/install.sh index cb584208..c35fc21e 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="c90676dcc433061bd97aca94930cd7acd74295624840cb5c85a22897f9c1dd12" +DEFAULT_INSTALL_SHA256="7bc94c44f3696a0b777e5c280754633105a06756932ee52547f0a9c775a895b9" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/schemas/install-events.schema.json b/schemas/install-events.schema.json new file mode 100644 index 00000000..13debfab --- /dev/null +++ b/schemas/install-events.schema.json @@ -0,0 +1,704 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/install-events.schema.json", + "title": "RetroBIOS installer event", + "description": "One line of install.py --events, event schema 1. Written from EVENT_KEYS in install.py by scripts/generate_event_schema.py. The schema gains kinds, never keys on an existing kind: a reader skips a kind it does not know.", + "type": "object", + "required": [ + "event" + ], + "properties": { + "event": { + "enum": [ + "start", + "detect", + "error", + "scanned", + "file", + "system", + "checking", + "check", + "declared", + "found", + "unreadable", + "download", + "progress", + "transfer", + "downloaded", + "summary", + "end" + ] + } + }, + "oneOf": [ + { + "$ref": "#/$defs/start" + }, + { + "$ref": "#/$defs/detect" + }, + { + "$ref": "#/$defs/error" + }, + { + "$ref": "#/$defs/scanned" + }, + { + "$ref": "#/$defs/file" + }, + { + "$ref": "#/$defs/system" + }, + { + "$ref": "#/$defs/checking" + }, + { + "$ref": "#/$defs/check" + }, + { + "$ref": "#/$defs/declared" + }, + { + "$ref": "#/$defs/found" + }, + { + "$ref": "#/$defs/unreadable" + }, + { + "$ref": "#/$defs/download" + }, + { + "$ref": "#/$defs/progress" + }, + { + "$ref": "#/$defs/transfer" + }, + { + "$ref": "#/$defs/downloaded" + }, + { + "$ref": "#/$defs/summary" + }, + { + "$ref": "#/$defs/end" + } + ], + "$defs": { + "start": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "schema", + "installer", + "mode" + ], + "properties": { + "event": { + "const": "start" + }, + "schema": { + "const": 1 + }, + "installer": { + "type": "string" + }, + "mode": { + "enum": [ + "install", + "check", + "dry-run", + "list" + ] + } + } + }, + "detect": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "os", + "os_name", + "platforms" + ], + "properties": { + "event": { + "const": "detect" + }, + "os": { + "type": "string" + }, + "os_name": { + "type": "string" + }, + "platforms": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "error": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "stage", + "reason", + "platform", + "message" + ], + "properties": { + "event": { + "const": "error" + }, + "stage": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "message": { + "type": "string" + } + } + }, + "scanned": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "folder", + "files", + "truncated", + "other_drives" + ], + "properties": { + "event": { + "const": "scanned" + }, + "folder": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "truncated": { + "type": "boolean" + }, + "other_drives": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "file": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "dest", + "systems", + "size", + "state" + ], + "properties": { + "event": { + "const": "file" + }, + "platform": { + "type": "string" + }, + "dest": { + "type": "string" + }, + "systems": { + "type": "array", + "items": { + "type": "string" + } + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "state": { + "enum": [ + "present", + "wrong_hash", + "missing" + ] + } + } + }, + "system": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "system", + "name", + "files", + "present", + "missing", + "wrong_hash", + "bytes_needed" + ], + "properties": { + "event": { + "const": "system" + }, + "platform": { + "type": "string" + }, + "system": { + "type": "string" + }, + "name": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "present": { + "type": "integer", + "minimum": 0 + }, + "missing": { + "type": "integer", + "minimum": 0 + }, + "wrong_hash": { + "type": "integer", + "minimum": 0 + }, + "bytes_needed": { + "type": "integer", + "minimum": 0 + } + } + }, + "checking": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "files_done", + "files_total" + ], + "properties": { + "event": { + "const": "checking" + }, + "files_done": { + "type": "integer", + "minimum": 0 + }, + "files_total": { + "type": "integer", + "minimum": 0 + } + } + }, + "check": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "name", + "bios_dir", + "files", + "present", + "missing", + "wrong_hash", + "bytes_needed", + "data_missing", + "not_installable" + ], + "properties": { + "event": { + "const": "check" + }, + "platform": { + "type": "string" + }, + "name": { + "type": "string" + }, + "bios_dir": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "present": { + "type": "integer", + "minimum": 0 + }, + "missing": { + "type": "integer", + "minimum": 0 + }, + "wrong_hash": { + "type": "integer", + "minimum": 0 + }, + "bytes_needed": { + "type": "integer", + "minimum": 0 + }, + "data_missing": { + "type": "integer", + "minimum": 0 + }, + "not_installable": { + "type": "integer", + "minimum": 0 + } + } + }, + "declared": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "files", + "present", + "missing", + "wrong_hash", + "bytes_needed", + "dests" + ], + "properties": { + "event": { + "const": "declared" + }, + "platform": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "present": { + "type": "integer", + "minimum": 0 + }, + "missing": { + "type": "integer", + "minimum": 0 + }, + "wrong_hash": { + "type": "integer", + "minimum": 0 + }, + "bytes_needed": { + "type": "integer", + "minimum": 0 + }, + "dests": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "found": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "dest", + "folder", + "path", + "member", + "reason" + ], + "properties": { + "event": { + "const": "found" + }, + "platform": { + "type": "string" + }, + "dest": { + "type": "string" + }, + "folder": { + "type": "string" + }, + "path": { + "type": "string" + }, + "member": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "unreadable": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "folder", + "path", + "member", + "reason" + ], + "properties": { + "event": { + "const": "unreadable" + }, + "platform": { + "type": "string" + }, + "folder": { + "type": "string" + }, + "path": { + "type": "string" + }, + "member": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "download": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "files", + "bytes" + ], + "properties": { + "event": { + "const": "download" + }, + "platform": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "bytes": { + "type": "integer", + "minimum": 0 + } + } + }, + "progress": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "files_done", + "files_total", + "bytes_done", + "bytes_total" + ], + "properties": { + "event": { + "const": "progress" + }, + "files_done": { + "type": "integer", + "minimum": 0 + }, + "files_total": { + "type": "integer", + "minimum": 0 + }, + "bytes_done": { + "type": "integer", + "minimum": 0 + }, + "bytes_total": { + "type": "integer", + "minimum": 0 + } + } + }, + "transfer": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "bytes_local", + "bytes_network" + ], + "properties": { + "event": { + "const": "transfer" + }, + "bytes_local": { + "type": "integer", + "minimum": 0 + }, + "bytes_network": { + "type": "integer", + "minimum": 0 + } + } + }, + "downloaded": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "dest", + "ok", + "reason", + "size", + "files_done", + "files_total", + "bytes_done", + "bytes_total" + ], + "properties": { + "event": { + "const": "downloaded" + }, + "dest": { + "type": "string" + }, + "ok": { + "type": "boolean" + }, + "reason": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "files_done": { + "type": "integer", + "minimum": 0 + }, + "files_total": { + "type": "integer", + "minimum": 0 + }, + "bytes_done": { + "type": "integer", + "minimum": 0 + }, + "bytes_total": { + "type": "integer", + "minimum": 0 + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "mode", + "installed", + "up_to_date", + "pending", + "failed", + "skipped", + "left_out", + "kept", + "interrupted" + ], + "properties": { + "event": { + "const": "summary" + }, + "mode": { + "enum": [ + "install", + "check", + "dry-run", + "list" + ] + }, + "installed": { + "type": "integer", + "minimum": 0 + }, + "up_to_date": { + "type": "integer", + "minimum": 0 + }, + "pending": { + "type": "integer", + "minimum": 0 + }, + "failed": { + "type": "array", + "items": { + "type": "object" + } + }, + "skipped": { + "type": "array", + "items": { + "type": "object" + } + }, + "left_out": { + "type": "array", + "items": { + "type": "object" + } + }, + "kept": { + "type": "array", + "items": { + "type": "object" + } + }, + "interrupted": { + "type": "boolean" + } + } + }, + "end": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "code" + ], + "properties": { + "event": { + "const": "end" + }, + "code": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/scripts/generate_event_schema.py b/scripts/generate_event_schema.py new file mode 100644 index 00000000..e3579c4a --- /dev/null +++ b/scripts/generate_event_schema.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Write schemas/install-events.schema.json from install.py's EVENT_KEYS. + + python scripts/generate_event_schema.py rewrite the schema + python scripts/generate_event_schema.py --check exit 1 when it is stale +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +OUTPUT = ROOT / "schemas" / "install-events.schema.json" + +_spec = importlib.util.spec_from_file_location("install", ROOT / "install.py") +_install = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_install) + +EVENT_KEYS: dict[str, tuple[str, ...]] = _install.EVENT_KEYS +EVENT_SCHEMA: int = _install.EVENT_SCHEMA + +_TEXT = {"type": "string"} +_COUNT = {"type": "integer", "minimum": 0} +_FLAG = {"type": "boolean"} +_NAMES = {"type": "array", "items": {"type": "string"}} +_RECORDS = {"type": "array", "items": {"type": "object"}} + +# The JSON type of each key, whatever kind carries it. A key EVENT_KEYS +# gains without a line here stops the generator. +KEY_TYPES: dict[str, dict] = { + **dict.fromkeys(( + "installer", "os", "os_name", "stage", "reason", "platform", "message", + "folder", "dest", "system", "name", "bios_dir", "path", "member", + ), _TEXT), + "mode": {"enum": ["install", "check", "dry-run", "list"]}, + "state": {"enum": ["present", "wrong_hash", "missing"]}, + **dict.fromkeys(( + "files", "files_done", "files_total", "bytes", "bytes_done", "bytes_total", + "bytes_local", "bytes_network", "size", "present", "missing", "wrong_hash", + "bytes_needed", "data_missing", "not_installable", "installed", "up_to_date", + "pending", "code", + ), _COUNT), + **dict.fromkeys(("truncated", "ok", "interrupted"), _FLAG), + **dict.fromkeys(("systems", "other_drives", "dests"), _NAMES), + **dict.fromkeys(("platforms", "failed", "skipped", "left_out", "kept"), _RECORDS), +} + + +def _kind(kind: str, keys: tuple[str, ...]) -> dict: + properties: dict[str, dict] = {"event": {"const": kind}} + for key in keys: + properties[key] = ( + {"const": EVENT_SCHEMA} if (kind, key) == ("start", "schema") else KEY_TYPES[key] + ) + return { + "type": "object", + "additionalProperties": False, + "required": ["event", *keys], + "properties": properties, + } + + +def build_schema() -> dict: + """One line of the stream: an object of one of the kinds, with exactly its keys.""" + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/install-events.schema.json", + "title": "RetroBIOS installer event", + "description": ( + f"One line of install.py --events, event schema {EVENT_SCHEMA}. Written from " + "EVENT_KEYS in install.py by scripts/generate_event_schema.py. The schema " + "gains kinds, never keys on an existing kind: a reader skips a kind it " + "does not know." + ), + "type": "object", + "required": ["event"], + "properties": {"event": {"enum": list(EVENT_KEYS)}}, + "oneOf": [{"$ref": f"#/$defs/{kind}"} for kind in EVENT_KEYS], + "$defs": {kind: _kind(kind, keys) for kind, keys in EVENT_KEYS.items()}, + } + + +def render(schema: dict) -> str: + return json.dumps(schema, indent=2, ensure_ascii=False) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--check", action="store_true", + help="exit 1 when the file differs from EVENT_KEYS, writing nothing") + parser.add_argument("--output", type=Path, default=OUTPUT, help=argparse.SUPPRESS) + args = parser.parse_args(argv) + text = render(build_schema()) + current = args.output.read_text(encoding="utf-8") if args.output.is_file() else None + if current == text: + return 0 + if args.check: + print(f"{args.output} differs from EVENT_KEYS: run " + "python scripts/generate_event_schema.py", file=sys.stderr) + return 1 + args.output.write_text(text, encoding="utf-8") + print(f"Wrote {args.output}") + return 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 5e002a78..57bf5bfa 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -554,6 +554,23 @@ def _semantic_install_checks(path: Path, manifest: dict) -> list[str]: return out +def _event_schema_errors() -> list[str]: + """install-events.schema.json is EVENT_KEYS written out, and a valid schema.""" + import generate_event_schema # noqa: PLC0415 + + path = generate_event_schema.OUTPUT + label = path.relative_to(ROOT).as_posix() + if not path.is_file(): + return [f"{label}: missing, run python scripts/generate_event_schema.py"] + if path.read_text(encoding="utf-8") != generate_event_schema.render( + generate_event_schema.build_schema() + ): + return [f"{label}: differs from EVENT_KEYS in install.py, run " + "python scripts/generate_event_schema.py"] + _validator(path.name) + return [] + + def unchecked_formats() -> list[str]: """Formats the schemas declare that this jsonschema install cannot check. @@ -598,6 +615,7 @@ def main() -> int: errors.extend(_repository_layout_in_profiles()) errors.extend(_save_directory_blocks()) errors.extend(_registry_contract()) + errors.extend(_event_schema_errors()) if not args.source_only: database_path = ROOT / "database.json" diff --git a/tests/test_install.py b/tests/test_install.py index a7877ea2..4da52996 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3864,6 +3864,7 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase): system=None, core=None, emulator=None, region=None, target=None, cores_dir=None, every_board=False, installed_cores=False, standalone_copies=False, check=False, dry_run=False, no_game_data=False, + declared_only=False, ) flags = list(argv) while flags: @@ -3908,6 +3909,13 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase): self.assertIn("| sudo sh -s -- --platform retroarch --dest /media/ro " "--system atari-lynx --region us", advice) + def test_the_platforms_own_list_is_kept_in_the_hint(self): + self._record("--declared-only", "--system", "atari-lynx") + install._INVOCATION.os_type = "linux" + with unittest.mock.patch.dict(os.environ, {"RETROBIOS_BOOTSTRAP": ""}): + command = install._INVOCATION.command("retroarch", None) + self.assertIn("--system atari-lynx --declared-only", command) + def test_windows_has_no_sudo(self): self._record("--system", "atari-lynx") install._INVOCATION.os_type = "windows" @@ -5070,6 +5078,10 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): "bios_dir": str(self.bios), "files": 4, "present": 1, "missing": 2, "wrong_hash": 1, "bytes_needed": 245, "data_missing": 0, "not_installable": 1}, + # Every file of this list is the platform's own (cores: null). + {"event": "declared", "platform": "retroarch", "files": 4, "present": 1, + "missing": 2, "wrong_hash": 1, "bytes_needed": 245, + "dests": ["a.bin", "b.bin", "c.bin", "d.bin"]}, ] SKIPPED = [{"reason": "not_found", "label": "not found", "files": 1, "required": 1}] @@ -5153,6 +5165,70 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): for key in keys: self.assertIn(f"`{key}`", row, f"{kind}.{key}") + def test_a_run_is_valid_against_the_published_schema(self): + try: + from jsonschema import Draft202012Validator # noqa: PLC0415 + except ImportError: + self.skipTest("jsonschema is not installed") + schema = json.loads(EVENT_SCHEMA_PATH.read_text(encoding="utf-8")) + validator = Draft202012Validator(schema) + for argv in (("--check",), ()): + _proc, events = self._run(*argv) + for event in events: + errors = [error.message for error in validator.iter_errors(event)] + self.assertEqual(errors, [], event) + self.assertTrue(list(validator.iter_errors({"event": "file", "platform": "x"}))) + self.assertTrue(list(validator.iter_errors( + {"event": "end", "code": 0, "extra": 1}))) + + +EVENT_SCHEMA_PATH = REPO_ROOT / "schemas" / "install-events.schema.json" + + +class TheEventSchemaFileIsEventKeys(unittest.TestCase): + """The table in the wiki was the only description of --events a reader + outside Python could check a stream against.""" + + def _generator(self): + sys.path.insert(0, str(REPO_ROOT / "scripts")) + import generate_event_schema # noqa: PLC0415 + + return generate_event_schema + + def test_the_committed_file_is_the_generated_one(self): + generator = self._generator() + self.assertEqual(EVENT_SCHEMA_PATH.read_text(encoding="utf-8"), + generator.render(generator.build_schema()), + "run python scripts/generate_event_schema.py") + + def test_each_kind_has_exactly_its_keys(self): + schema = json.loads(EVENT_SCHEMA_PATH.read_text(encoding="utf-8")) + self.assertEqual(schema["properties"]["event"]["enum"], list(install.EVENT_KEYS)) + for kind, keys in install.EVENT_KEYS.items(): + shape = schema["$defs"][kind] + self.assertEqual(shape["required"], ["event", *keys], kind) + self.assertEqual(list(shape["properties"]), ["event", *keys], kind) + self.assertIs(shape["additionalProperties"], False, kind) + start = schema["$defs"]["start"]["properties"]["schema"] + self.assertEqual(start, {"const": install.EVENT_SCHEMA}) + + def test_a_key_without_a_type_stops_the_generator(self): + generator = self._generator() + with unittest.mock.patch.dict(install.EVENT_KEYS, {"new": ("untyped_key",)}), \ + unittest.mock.patch.object(generator, "EVENT_KEYS", install.EVENT_KEYS), \ + self.assertRaises(KeyError): + generator.build_schema() + + def test_the_check_command_fails_on_a_stale_file(self): + generator = self._generator() + scratch = Path(_tempdir(self)) / "events.schema.json" + scratch.write_text("{}\n", encoding="utf-8") + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(generator.main(["--check", "--output", str(scratch)]), 1) + self.assertEqual(generator.main(["--output", str(scratch)]), 0) + self.assertEqual(generator.main(["--check", "--output", str(scratch)]), 0) + class NetworkFailuresInPlainWords(unittest.TestCase): """Offline, the run printed "[Errno -2] Name or service not known". A diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 3577b6da..97b969b3 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -1296,5 +1296,197 @@ class AnUnknownNativeNameIsRefusedBeforeAnyWork(unittest.TestCase): self.assertEqual(json.loads(proc.stdout)["batocera"]["systems"], ["sony-playstation"]) +class ThePlatformsOwnListAlone(unittest.TestCase): + """A setup that checks its own list asked --check about 14,508 files on + Batocera, 353 of them its own, and had no option to ask about those: + --declared-only keeps the entries the platform declares (cores: null).""" + + FILES = [ + {"dest": "a.bin", "size": 1, "cores": None, "systems": ["sys-a"], + "alternatives": [{"sha1": "2" * 40, "size": 2, "cores": ["core-b"]}]}, + {"dest": "b.bin", "size": 1, "cores": None, "systems": ["sys-a"], + "sha1": "1" * 40, + "slot_choices": [{"sha1": "3" * 40, "size": 3, "cores": ["core-b"]}], + "declared_choices": [{"sha1": "1" * 40, "size": 1, "systems": ["sys-a"]}]}, + {"dest": "core/c.bin", "size": 1, "cores": ["core-b"], "systems": ["sys-a"]}, + ] + OMITTED = [ + {"dest": "d.bin", "name": "d.bin", "system": "sys-a", "systems": ["sys-a"], + "required": True, "reason": "not_found", "cores": None}, + {"dest": "core/e.bin", "name": "e.bin", "system": "sys-a", "systems": ["sys-a"], + "required": True, "reason": "not_found", "cores": ["core-b"]}, + ] + + def test_the_cut(self): + files, omitted = install.keep_declared(list(self.FILES), list(self.OMITTED)) + self.assertEqual([f["dest"] for f in files], ["a.bin", "b.bin"]) + self.assertEqual([o["dest"] for o in omitted], ["d.bin"]) + # The platform answers its own slot, and no core copy is left to + # stand in for it when a later narrowing drops the system. + self.assertEqual(files[1]["sha1"], "1" * 40) + for entry in files: + self.assertIsNone(entry["cores"]) + self.assertFalse({"alternatives", "slot_choices", "declared_choices"} & set(entry)) + self.assertEqual(install.available_choices(files, omitted)["cores"], []) + + def test_install_check_and_dry_run_apply_it(self): + with FakeRepository() as repo: + proc = repo.run("--declared-only") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("5 -> 3 files", proc.stdout) + self.assertEqual(repo.installed(), + ["gba_bios.bin", "scph5500.bin", "scph5501.bin"]) + for mode in ("--check", "--dry-run"): + with self.subTest(mode=mode), FakeRepository() as repo: + proc = repo.run("--declared-only", mode) + self.assertIn("5 -> 3 files", proc.stdout, proc.stderr) + self.assertIn("0/3 present", proc.stdout) + self.assertEqual(repo.installed(), []) + + def test_it_composes_with_the_other_narrowings(self): + with FakeRepository() as repo: + proc = repo.run("--declared-only", "--system", "sony-playstation", + "--region", "us") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(repo.installed(), ["scph5501.bin"]) + with FakeRepository() as repo: + listing = repo.run("--declared-only", "--list-systems") + self.assertEqual(listing.returncode, 0, listing.stderr) + row = next(line for line in listing.stdout.splitlines() + if "sony-playstation" in line) + self.assertIn("2 files", row) + + def test_a_local_file_list_applies_it(self): + with FakeRepository() as repo: + proc = repo.run("--declared-only", "--dry-run", "--manifest", + str(repo.root / "install" / "retroarch.json")) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("5 -> 3 files", proc.stdout) + + def test_every_mode_that_cannot_apply_it_refuses_it(self): + """Applied or refused, never ignored: each of these would have run + as if the option was not there.""" + refused = [ + ["--list-platforms"], + ["--list-targets"], + ["--list-cores"], + ["--list-emulators"], + ["--core", "mgba"], + ["--standalone-copies"], + ["--tui"], + ] + with FakeRepository() as repo: + for extra in refused: + with self.subTest(extra=extra): + proc = repo.run("--declared-only", *extra) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("--declared-only", proc.stderr) + self.assertEqual(repo.installed(), []) + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--restore", + str(REPO_ROOT / "tmp"), "--declared-only"], + capture_output=True, text=True, timeout=60, check=False, + stdin=subprocess.DEVNULL) + self.assertEqual(proc.returncode, 2) + self.assertIn("takes no --declared-only", proc.stderr) + + def test_events_count_the_platforms_own_files(self): + with FakeRepository() as repo: + proc = repo.run("--check", "--events") + events = [json.loads(line) for line in proc.stdout.splitlines()] + declared = [e for e in events if e["event"] == "declared"] + self.assertEqual(declared, [{ + "event": "declared", "platform": "retroarch", "files": 3, "present": 0, + "missing": 3, "wrong_hash": 0, "bytes_needed": 22, + "dests": ["scph5500.bin", "scph5501.bin", "gba_bios.bin"], + }]) + kinds = [e["event"] for e in events] + self.assertEqual(kinds.index("declared"), kinds.index("check") + 1) + narrowed = [json.loads(line) for line in + repo.run("--check", "--events", "--declared-only").stdout.splitlines()] + check = next(e for e in narrowed if e["event"] == "check") + self.assertEqual(check["files"], 3) + + +class AnIdAnotherSetupUses(unittest.TestCase): + """The Neo Geo card offers `--system snk-neogeo`, and RetroArch, which + files neogeo.zip under arcade, answered "Did you mean: snk-neogeo-cd?".""" + + KNOWN = ["arcade", "snk-neogeo-cd", "sony-playstation"] + HOLDERS = {"snk-neogeo": ["arcade"]} + + def _refusal(self, typed: str, holders=None) -> str: + with self.assertRaises(ValueError) as ctx: + install.resolve_choices([typed], self.KNOWN, "system", + holders=self.HOLDERS if holders is None else holders) + return str(ctx.exception) + + def test_the_system_holding_its_file_is_suggested_never_taken(self): + for typed in ("snk-neogeo", "SNK-NeoGeo", "neogeo", "mvs", "aes"): + with self.subTest(typed=typed): + text = self._refusal(typed) + self.assertIn("Did you mean: arcade?", text) + self.assertIn("neogeo.zip", text) + self.assertNotIn("snk-neogeo-cd", text) + + def test_a_typo_still_gets_the_closest_names(self): + self.assertIn("Did you mean: snk-neogeo-cd", self._refusal("snk-neogeo-dc", {})) + + def test_holders_are_read_from_the_file_list(self): + files = [ + {"dest": "neogeo.zip", "systems": ["arcade"]}, + {"dest": "fbneo/neogeo.zip", "systems": ["arcade"]}, + {"dest": "neocd/neocd.bin", "systems": ["snk-neogeo-cd"]}, + {"dest": "Other/neogeo.zip", "systems": []}, + ] + self.assertEqual(install.key_file_holders(files, []), {"snk-neogeo": ["arcade"]}) + own = [{"dest": "neogeo.zip", "systems": ["snk-neogeo"]}] + self.assertEqual(install.key_file_holders(own, []), {}) + + def test_retroarch_suggests_arcade(self): + never = REPO_ROOT / "tmp" / "never" + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--manifest", str(REPO_ROOT / "install" / "retroarch.json"), + "--dest", str(never), "--system", "snk-neogeo", "--dry-run", "--no-input"], + capture_output=True, text=True, timeout=120, check=False, + stdin=subprocess.DEVNULL) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertIn("Did you mean: arcade?", proc.stderr) + self.assertNotIn("snk-neogeo-cd", proc.stderr) + self.assertFalse(never.exists()) + + +class TheTitleASetupPrints(unittest.TestCase): + """The installer and the site print "SNK Neo Geo CD": typed back, the + title was refused.""" + + MANIFEST = { + "system_names": {"sony-playstation": ["psx"]}, + "system_titles": {"snk-neogeo-cd": "SNK Neo Geo CD", "arcade": "Arcade games", + "sony-playstation": "Sony PlayStation", + "twin-a": "Twin", "twin-b": "Twin"}, + } + KNOWN = ["arcade", "snk-neogeo-cd", "sony-playstation", "twin-a", "twin-b"] + + def _resolve(self, typed: str) -> list[str]: + return install.resolve_choices([typed], self.KNOWN, "system", + install.system_aliases(self.MANIFEST)) + + def test_a_title_is_taken_for_its_id(self): + self.assertEqual(self._resolve("SNK Neo Geo CD"), ["snk-neogeo-cd"]) + self.assertEqual(self._resolve("arcade games"), ["arcade"]) + self.assertEqual(self._resolve("psx"), ["sony-playstation"]) + + def test_a_title_two_systems_share_is_not_taken(self): + with self.assertRaises(ValueError): + self._resolve("Twin") + + def test_the_platforms_own_name_wins_over_a_title(self): + manifest = {"system_names": {"arcade": ["Sony PlayStation"]}, + "system_titles": {"sony-playstation": "Sony PlayStation"}} + self.assertEqual(install.system_aliases(manifest)["Sony PlayStation"], "arcade") + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index ea072e16..888d8e0c 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -630,11 +630,12 @@ what the options name, everything by default. | `--installed-cores` | Keep only what the cores already installed read (RetroArch, Lakka, RetroPie). See [Installed cores](#installed-cores) | | `--cores-dir PATH` | The folder RetroArch loads its cores from, for `--installed-cores` when it is not found. Implies `--installed-cores` | | `--save-dir PATH` | The frontend's save directory (RetroArch's `savefile_directory`), for the files some cores read there, when the run cannot find it. The core's own folder is still added below it when the frontend sorts saves by core | -| `--system NAMES` | Install only these systems. Comma-separated or repeated. `--list-systems` prints the names, `other` included. A short name the platform does not use (`gba`, `psx`) is refused with the ids it may stand for, never taken in their place | +| `--system NAMES` | Install only these systems. Comma-separated or repeated. `--list-systems` prints the names, `other` included, and the title a system is printed with is taken too. A short name the platform does not use (`gba`, `psx`) is refused with the ids it may stand for, never taken in their place. So is an id the list files under another one: RetroArch keeps `neogeo.zip` under `arcade`, which `--system snk-neogeo` suggests | | `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names | | `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included | | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | | `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | +| `--declared-only` | Keep only the files the platform's own list names (`cores: null` in the manifest), not the extra files cores load. With `--check`, the answer is about the list the platform itself checks. Refused with `--core`, the listings it does not change, `--standalone-copies` and `--tui` | | `--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart | | `--list-cores` | Print the cores whose extra files a platform installs | | `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | @@ -1186,7 +1187,10 @@ the text a person reads goes to standard error. A program driving the installer (the [handheld port](#on-a-handheld-the-retrobios-port) is one) reads the objects and never parses the text. Every object has an `event` key naming its kind, and each kind has exactly the keys below, in this order. The schema is -`EVENT_KEYS` in `install.py`, and a test holds this table to it. +`EVENT_KEYS` in `install.py`, and a test holds this table to it. The same +keys, with their types, are in `schemas/install-events.schema.json`, written +from `EVENT_KEYS` by `scripts/generate_event_schema.py` and published with the +other schemas under `api/v1/schemas/`. | Kind | Keys | When | |------|------|------| @@ -1198,6 +1202,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is | `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` | | `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id, and with `--system` only the systems named. Files with no system count under `other`. `name` is the platform's own name for it | | `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply | +| `declared` | `platform`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `dests` | Right after `check`, the same counts over the files the platform's own list names (`cores: null`), and their `dests` in the order of the `file` events. A setup that checks its own list reads its answer here, with or without `--declared-only` | | `found` | `platform`, `dest`, `folder`, `path`, `member`, `reason` | Each file to install that this machine already holds, after `check` and before anything is written, in `--check` and `--dry-run` too. `folder` is the `--from` folder as given, or the `bios_dir` of the check when the BIOS folder holds the file at another place, and then `reason` says where in words (`nested folder: ...` for a misplaced file, `the same file is at ...` for one the list installs there too). `path` is the file, and `member` the ZIP member holding the bytes when `path` is an archive, an empty string otherwise. Its copy is reported by a `downloaded` event | | `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) | | `download` | `platform`, `files`, `bytes` | Before the downloads start. Counts the files copied from this machine too | @@ -1207,7 +1212,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is | `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `kept`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. `kept` lists `{folder, files, bytes}`, one per backup folder, for the files in place the run moved aside before replacing them (with `--dry-run`, would move), and is empty when none. Not written for a listing | | `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) | -Schema 1 can gain a kind (`found` came with `--from`), never a key of an +Schema 1 can gain a kind (`found` came with `--from`, `declared` with `--declared-only`), never a key of an existing kind: a reader skips a kind it does not know. An option refused before the run starts (`install.py: error:`, code 2) writes no event at all. A program reads a missing `end` as that case. diff --git a/wiki/tools.md b/wiki/tools.md index 5983d766..6f57056b 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -684,6 +684,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6 | `region_audit.py` | Confront every profile `region:` with the regions No-Intro names for the same dump (`--json`, `--db`, `--emulators-dir`). An opinion, never an authority: the code decides, and a disagreement is a line to read in the emulator's source | | `validate_pr.py` | Validate BIOS files in pull requests, post markdown report | | `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs | +| `generate_event_schema.py` | Write `schemas/install-events.schema.json` from `EVENT_KEYS` in `install.py` (`--check` exits 1 when the file is stale, which `validate_schemas.py` also reports) | | `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline). A download is accepted only when every cryptographic hash the entry declares (sha256, sha1, md5) matches. crc32 and size find a file the database already holds, never accept a download. An entry with no cryptographic hash is reported and not searched | | `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones, used by CI) | | `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) | From 7143d20ef8ce23d3821033d17eedfb0ab7ca2cc0 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:23:59 +0200 Subject: [PATCH 2/3] fix: check the event schema under the given root validate_schemas.py read install-events.schema.json from the real tree while a test pointed ROOT at a scratch one, and test_schema_semantic_gate.py called unittest.main() before its last class. --- scripts/validate_schemas.py | 4 ++-- tests/test_schema_semantic_gate.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 57bf5bfa..1a3b888c 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -558,7 +558,7 @@ def _event_schema_errors() -> list[str]: """install-events.schema.json is EVENT_KEYS written out, and a valid schema.""" import generate_event_schema # noqa: PLC0415 - path = generate_event_schema.OUTPUT + path = ROOT / "schemas" / generate_event_schema.OUTPUT.name label = path.relative_to(ROOT).as_posix() if not path.is_file(): return [f"{label}: missing, run python scripts/generate_event_schema.py"] @@ -567,7 +567,7 @@ def _event_schema_errors() -> list[str]: ): return [f"{label}: differs from EVENT_KEYS in install.py, run " "python scripts/generate_event_schema.py"] - _validator(path.name) + Draft202012Validator.check_schema(_load_json(path)) return [] diff --git a/tests/test_schema_semantic_gate.py b/tests/test_schema_semantic_gate.py index 746d2822..2710455a 100644 --- a/tests/test_schema_semantic_gate.py +++ b/tests/test_schema_semantic_gate.py @@ -94,10 +94,6 @@ class SemanticManifestValidation(unittest.TestCase): self.assertIn("total_files does not equal len(files)", errors[0]) -if __name__ == "__main__": - unittest.main() - - try: import jsonschema except ImportError: @@ -122,3 +118,7 @@ class DataRootIsKeyedBySystem(unittest.TestCase): self.assertTrue(self._errors({"amiga": {"path": "x", "source_ref": "y"}})) self.assertTrue(self._errors({"windows": {"path": "%APPDATA%/X"}})) self.assertTrue(self._errors({})) + + +if __name__ == "__main__": + unittest.main() From d6782913cfba06dea6a1e8309842e332c844bf8a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:24:07 +0200 Subject: [PATCH 3/3] feat: add --offline and bound the connect time --offline (or RETROBIOS_INSTALL_OFFLINE=1, which the runs the full-screen interface starts inherit) makes no network request. The file list stops the run at once with an error event of reason offline unless --manifest or --from-pack gives it, a file the run would download fails at once with its own advice, and board detection is skipped. --target and --list-targets refuse it. RETROBIOS_OFFLINE is not reused: CI and the test suite set it for every process, and the installer tests serve their files over loopback. A connection now has 4 s to open, then the read keeps its 30 or 60 s timeout: three tries that never connect end in about 15 s instead of 93, and a slow but open transfer is not cut. A --dry-run prints its totals first. A plan taller than the screen names the ten largest systems, and --verbose lists every system and file. Events are unchanged. --- install.ps1 | 2 +- install.py | 163 ++++++++++++++++++++++++++++----- install.sh | 2 +- tests/test_install_offline.py | 167 ++++++++++++++++++++++++++++++++++ wiki/installer.md | 14 ++- 5 files changed, 322 insertions(+), 26 deletions(-) diff --git a/install.ps1 b/install.ps1 index 211913bd..b09c5836 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "240faa427aaa602deca1a3180f770c6f335095b1cf1d5bb28fc060eb55f54917" + $defaultInstallSha256 = "ea2777500114b796bb4a23ea9e0dd0a15bc4499bb1289e6e426943beece48865" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 9b2029ba..ad61b4cf 100644 --- a/install.py +++ b/install.py @@ -98,6 +98,11 @@ RELEASE_URL = ( "https://github.com/Abdess/retrobios/releases/download/large-files/{asset}" ) MAX_RETRIES = 3 +# Seconds a connection has to open. A dropped network cost three tries of a +# 30-second timeout, 93 s, before the error: a connection that does not open +# now fails in this time, and once open a read waits as long as before, so +# a slow line is not cut. +CONNECT_TIMEOUT = 4.0 MAX_MANIFEST_BYTES = 16 * 1024 * 1024 MAX_TARGETS_BYTES = 4 * 1024 * 1024 MAX_MANIFEST_FILES = 100_000 @@ -3150,10 +3155,13 @@ def network_words( ) -> tuple[str, str]: """The reason a request failed, and the same in words a beginner reads. - The reason is clock, no_ca or network. Offline, the run printed + The reason is clock, no_ca, network, or offline for a request the run + was told not to make. Offline, the run printed "[Errno -2] Name or service not known": the raw text now follows the words only under --verbose. """ + if isinstance(exc, _Offline): + return "offline", str(exc) reason = getattr(exc, "reason", None) inner = reason if isinstance(reason, BaseException) else exc raw = str(reason or exc) @@ -3208,6 +3216,49 @@ def _address_lines(url: str) -> list[str]: return lines +def _open_socket( + address: tuple[str, int], timeout: object, source_address: object = None, +) -> socket.socket: + """A connection given CONNECT_TIMEOUT to open, then the caller's timeout.""" + bounded = isinstance(timeout, (int, float)) + limit = min(CONNECT_TIMEOUT, timeout) if bounded else CONNECT_TIMEOUT + sock = socket.create_connection(address, limit, source_address) + sock.settimeout(timeout if bounded else socket.getdefaulttimeout()) + return sock + + +class _BoundedHTTPConnection(http.client.HTTPConnection): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._create_connection = _open_socket + + +class _BoundedHTTPHandler(urllib.request.HTTPHandler): + def do_open(self, http_class: Any, req: Any, **kwargs: Any) -> Any: + return super().do_open(_BoundedHTTPConnection, req, **kwargs) + + +def install_bounded_opener() -> None: + """Every request of the run opens its connection in CONNECT_TIMEOUT. + + The proxy variables are read as before: build_opener keeps its + ProxyHandler, and only the HTTP and HTTPS handlers are replaced. + """ + handlers: list[type] = [_BoundedHTTPHandler] + if hasattr(http.client, "HTTPSConnection"): + class _BoundedHTTPSConnection(http.client.HTTPSConnection): + def __init__(self, *args: Any, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self._create_connection = _open_socket + + class _BoundedHTTPSHandler(urllib.request.HTTPSHandler): + def do_open(self, http_class: Any, req: Any, **kwargs: Any) -> Any: + return super().do_open(_BoundedHTTPSConnection, req, **kwargs) + + handlers.append(_BoundedHTTPSHandler) + urllib.request.install_opener(urllib.request.build_opener(*handlers)) + + def _fetch_json( url: str, limit: int, label: str, attempts: int = MAX_RETRIES, timeout: float = 30, ) -> object: @@ -3293,6 +3344,11 @@ def fetch_manifest(plat: str) -> dict: EVENTS.emit("error", stage="manifest", reason=code, platform=plat, message=words if code != "network" else f"the file list could not be fetched: {words}") + if code == "offline": + print(f" No file list for {plat}: {OFFLINE_WORDS}. Give it with " + "--manifest FILE (a copy of install/.json) or " + "--from-pack PACK, or run without --offline.", file=sys.stderr) + sys.exit(1) if code != "network": print(f" Could not fetch the file list for {plat}.", file=sys.stderr) print(f" {words}.", file=sys.stderr) @@ -4343,7 +4399,8 @@ def _download_one( if LOCAL.pack is not None: return _extract_from_pack(f, bios_path, dest, progress) - _network_allowed(f["dest"]) + if offline_requested(): + return f["dest"], False, OfflineReason(f"not fetched: {OFFLINE_WORDS}") if f.get("release_asset"): url = RELEASE_URL.format(asset=urllib.parse.quote(f["release_asset"], safe="")) else: @@ -4776,6 +4833,7 @@ class Invocation: ("--no-game-data", args.no_game_data), ("--declared-only", args.declared_only), ("--standalone-copies", args.standalone_copies), + ("--offline", getattr(args, "offline", False)), ("--check", args.check), ("--dry-run", args.dry_run), ): @@ -4892,6 +4950,10 @@ def _plural(count: int, word: str) -> str: return f"{count} {word}" if count == 1 else f"{count} {word}s" +class OfflineReason(str): + """A file not fetched because the run was told to make no request.""" + + class NetworkReason(str): """A failure the connection caused: cut, timed out, refused by a server or a proxy. The summary names the network instead of the folder.""" @@ -4989,11 +5051,15 @@ def _report_outcome( "(or delete them), then run again.") if any(isinstance(reason, NetworkReason) for _dest, reason in failures or []): advice.append(NETWORK_FAILURE_ADVICE) + if any(isinstance(reason, OfflineReason) for _dest, reason in failures or []): + advice.append(" These files come from the network, which --offline does not " + "use: run without --offline, or name a pack with --from-pack.") paths = paths + [folder for folder in elsewhere if folder not in paths] if errors and not downloaded: print(f"\nNothing was installed. {_plural(errors, 'file')} failed.") - if failures and all(isinstance(r, NetworkReason) for _d, r in failures): + if failures and all(isinstance(r, (NetworkReason, OfflineReason)) + for _d, r in failures): # The folder was never the problem: naming it as the place being # written to sent people looking at their disk. print(f"Destination: {_locations(paths)}") @@ -5104,6 +5170,12 @@ def installer_id() -> str: return "unknown" +# A --dry-run plan: file names per system, the systems shown when the +# listing is taller than the screen, and the lines around the listing. +PLAN_FILES_PER_SYSTEM = 8 +PLAN_LARGEST_SYSTEMS = 10 +PLAN_FRAME_LINES = 6 + # Answers that stop: every menu before this one taught q as quit. _REFUSALS = ("n", "q", "quit", "exit", "stop", "abort") @@ -5119,42 +5191,61 @@ def _print_plan( trees: list[tuple[dict, dict[str, dict]]] = (), kept: dict[Path, list[int]] | None = None, copied: list[str] = (), + verbose: bool = False, ) -> None: - """What a real run would fetch, by system, and nothing written. + """What a real run would fetch: the totals first, then by system, and + nothing written. The data trees count in the total and come before the closing line: printed after it, they left 2636 RetroArch files out of the figure a - user plans an SD card from. + user plans an SD card from. A listing taller than the screen pushed the + total out of sight (599 lines for RetroArch): it gives way to the + largest systems, and --verbose lists every file. """ total = sum(f.get("size", 0) for f in to_download) trees = [(entry, needed) for entry, needed in trees if needed] tree_files = sum(len(needed) for _entry, needed in trees) tree_size = sum(meta.get("size", 0) for _entry, needed in trees for meta in needed.values()) - for line in copied: - print(line) - if not to_download and not trees: - if not copied: - print("\n All files up to date. A real run would download nothing.") - return - print("\n Nothing to download.") parts = [] if to_download: parts.append(f"{_plural(len(to_download), 'file')} ({format_size(total)})") if trees: parts.append(f"{_plural(tree_files, 'data file')} ({format_size(tree_size)}, " f"extracted from {_plural(len(trees), 'archive')})") + if not parts and not copied: + print("\n All files up to date. A real run would download nothing.") + return if parts: print(f"\nWould download {' and '.join(parts)} to {bios_path}:") + for line in copied: + print(line) + if not parts: + print("\n Nothing to download.") by_system: dict[str, list[dict]] = {} for entry in to_download: system = ", ".join(entry_systems(entry)) or "other" by_system.setdefault(system, []).append(entry) + shown = None if verbose else PLAN_FILES_PER_SYSTEM + listing: list[str] = [] for system in sorted(by_system): - print(f" {term.bold(system)} {_sized(by_system[system])}") - for entry in sorted(by_system[system], key=lambda e: e["dest"])[:8]: - print(f" {entry['dest']} {term.dim(format_size(entry.get('size', 0)))}") - if len(by_system[system]) > 8: - print(f" ... and {len(by_system[system]) - 8} more") + entries = sorted(by_system[system], key=lambda e: e["dest"]) + listing.append(f" {term.bold(system)} {_sized(entries)}") + listing += [f" {entry['dest']} {term.dim(format_size(entry.get('size', 0)))}" + for entry in entries[:shown]] + if shown is not None and len(entries) > shown: + listing.append(f" ... and {len(entries) - shown} more") + screen = shutil.get_terminal_size((80, 24)).lines + if verbose or len(listing) + len(trees) + PLAN_FRAME_LINES <= screen: + for line in listing: + print(line) + else: + largest = sorted(by_system, key=lambda s: (-_bytes(by_system[s]), s)) + print(f" {_plural(len(largest), 'system')}, the largest first:") + for system in largest[:PLAN_LARGEST_SYSTEMS]: + print(f" {term.bold(system)} {_sized(by_system[system])}") + if len(largest) > PLAN_LARGEST_SYSTEMS: + print(f" ... and {len(largest) - PLAN_LARGEST_SYSTEMS} more systems: " + "--verbose lists every system and its files.") for entry, needed in trees: source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"] print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, " @@ -5573,7 +5664,8 @@ def _install_standalone_folders( pending += len(to_download) continue if args.dry_run: - _print_plan(to_download, folder, term, kept=plan_kept(mismatched, folder)) + _print_plan(to_download, folder, term, kept=plan_kept(mismatched, folder), + verbose=args.verbose) continue if not to_download: print(" All files up to date.") @@ -6596,7 +6688,8 @@ class LocalSources: @property def offline(self) -> bool: - return self.pack is not None + """No network request in this run: a pack is read, or --offline.""" + return self.pack is not None or offline_requested() def reset(self) -> None: if self.pack is not None: @@ -6614,9 +6707,22 @@ class _Offline(OSError): """A network request in a run told to make none.""" +# --offline, or this variable set to anything but 0, which every run the +# full-screen interface starts inherits. RETROBIOS_OFFLINE stays the build +# scripts' own: CI and the test suite set it for every process they start. +OFFLINE_VARIABLE = "RETROBIOS_INSTALL_OFFLINE" +OFFLINE_WORDS = "this run makes no network request (--offline)" + + +def offline_requested() -> bool: + return os.environ.get(OFFLINE_VARIABLE, "") not in ("", "0") + + def _network_allowed(what: str) -> None: - if LOCAL.offline: + if LOCAL.pack is not None: raise _Offline(f"{what}: this run reads a pack and makes no network request") + if offline_requested(): + raise _Offline(f"{what}: {OFFLINE_WORDS}") def read_local_manifest(path: Path, plat: str) -> dict: @@ -7980,6 +8086,13 @@ def main() -> None: "(the others are found beside it) or the folder holding them. " "Makes no network request, and checks every file's hash", ) + parser.add_argument( + "--offline", + action="store_true", + help="make no network request: --check, --from, --from-pack and --manifest " + "work without one, and a file the run would download fails at once " + f"({OFFLINE_VARIABLE}=1 does the same)", + ) parser.add_argument( "--from", action="append", @@ -8088,6 +8201,13 @@ def main() -> None: sys.exit(restore_kept(args.restore, args.dry_run)) if args.every_board and args.target: parser.error("--every-board installs every board; --target names one") + if args.offline: + # The runs the full-screen interface starts read it from here. + os.environ[OFFLINE_VARIABLE] = "1" + if offline_requested() and (args.target or args.list_targets): + parser.error("--target and --list-targets read a core list from the network, " + "which --offline does not use") + install_bounded_opener() if args.from_pack and args.target: parser.error("--target reads a core list from the network, which --from-pack " "does not use") @@ -9050,7 +9170,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: ], bios_path) _print_plan([f for f in to_download if id(f) not in copies], bios_path, term, trees, kept, copy_plan_lines(to_download, copies, bios_index, - bios_path, files, args.verbose, True)) + bios_path, files, args.verbose, True), + verbose=args.verbose) for line in unreadable_lines(unread, args.verbose): print(line) for line in hint: diff --git a/install.sh b/install.sh index 585d8c01..0d16300b 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="240faa427aaa602deca1a3180f770c6f335095b1cf1d5bb28fc060eb55f54917" +DEFAULT_INSTALL_SHA256="ea2777500114b796bb4a23ea9e0dd0a15bc4499bb1289e6e426943beece48865" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_offline.py b/tests/test_install_offline.py index 98267760..efd19431 100644 --- a/tests/test_install_offline.py +++ b/tests/test_install_offline.py @@ -21,6 +21,7 @@ import sys import tempfile import time import unittest +import unittest.mock import zipfile from pathlib import Path @@ -589,5 +590,171 @@ class TheSummaryNamesTheBulk(unittest.TestCase): self.assertLess(text.index("Done."), text.index("arcade pulls")) +class OfflineMakesNoRequest(Scratch): + """A handheld with no Wi-Fi ran --check against a loopback address to + learn its platform, since nothing said "make no request".""" + + def _served(self, *options: str, env: dict | None = None): + import test_install_selection # noqa: PLC0415 + + hits: list[str] = [] + with FakeRepository() as repo, unittest.mock.patch.object( + test_install_selection._QuietHandler, "log_message", + lambda handler, *args: hits.append(handler.path)): + started = time.monotonic() + proc = subprocess.run( + repo.command(*options), env={**repo.env(), **(env or {})}, + capture_output=True, text=True, timeout=60, check=False, + stdin=subprocess.DEVNULL) + elapsed = time.monotonic() - started + installed = repo.installed() + return proc, hits, elapsed, installed + + def test_check_names_the_platform_and_stops_at_the_file_list(self): + for options, env in (((("--offline",)), None), + ((), {"RETROBIOS_INSTALL_OFFLINE": "1"})): + with self.subTest(options=options, env=env): + proc, hits, elapsed, installed = self._served( + *options, "--check", "--events", env=env) + self.assertEqual(hits, []) + self.assertLess(elapsed, 30) + self.assertEqual(proc.returncode, 1, proc.stderr) + events = [json.loads(line) for line in proc.stdout.splitlines()] + self.assertEqual([e["event"] for e in events], + ["start", "detect", "error", "end"]) + self.assertEqual(events[1]["platforms"][0]["platform"], "retroarch") + self.assertEqual((events[2]["stage"], events[2]["reason"]), + ("manifest", "offline")) + self.assertIn("no network request", events[2]["message"]) + self.assertIn("--manifest", proc.stderr) + self.assertNotIn("attempts", proc.stderr) + self.assertEqual(installed, []) + + def test_a_copy_lands_and_the_rest_fails_at_once(self): + old = self.root / "old" + old.mkdir() + (old / "scph5501.bin").write_bytes(US) + proc = self.run_installer("--offline", "--manifest", str(self.list), + "--from", str(old), "--no-input") + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertEqual((self.dest / "scph5501.bin").read_bytes(), US) + self.assertFalse((self.dest / "gba_bios.bin").exists()) + self.assertIn("no network request", proc.stdout + proc.stderr) + self.assertIn("run without --offline", proc.stdout + proc.stderr) + self.assertNotIn("trying again", proc.stderr) + + def test_modes_that_need_the_network_refuse_it(self): + for extra in (["--target", "switch"], ["--list-targets"]): + with self.subTest(extra=extra): + proc = self.run_installer("--offline", *extra) + self.assertEqual(proc.returncode, 2, proc.stderr) + self.assertIn("--offline", proc.stderr) + + def test_a_listing_and_a_pack_need_no_network_anyway(self): + proc = self.run_installer("--offline", "--list-platforms") + self.assertEqual(proc.returncode, 0, proc.stderr) + + +class ANetworkThatDroppedFailsFast(unittest.TestCase): + """A dropped network cost 93 s before the line-mode error: three tries + of a 30-second timeout spent waiting for a connection that never opened.""" + + def test_the_connection_is_bounded_and_the_read_is_not(self): + calls: list[tuple] = [] + + class Sock: + def settimeout(self, value): + calls.append(("read", value)) + + def create(address, timeout, source_address=None): + calls.append(("connect", timeout)) + return Sock() + + with unittest.mock.patch.object(install.socket, "create_connection", create): + install._open_socket(("example.org", 443), 60) + install._open_socket(("example.org", 443), 2) + self.assertEqual(calls, [("connect", install.CONNECT_TIMEOUT), ("read", 60), + ("connect", 2), ("read", 2)]) + + def test_three_tries_that_never_connect_end_within_fifteen_seconds(self): + waited = sum(install._retry_delay(n, None) for n in range(1, install.MAX_RETRIES)) + self.assertLessEqual(install.MAX_RETRIES * install.CONNECT_TIMEOUT + waited, 15) + + def test_a_slow_answer_on_an_open_connection_still_arrives(self): + import http.server # noqa: PLC0415 + import threading # noqa: PLC0415 + + delay = install.CONNECT_TIMEOUT + 1 + + class Slow(http.server.BaseHTTPRequestHandler): + def do_GET(self): # noqa: N802 - stdlib name + time.sleep(delay) + body = b'{"ok": true}' + self.send_response(200) + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + return + + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Slow) + threading.Thread(target=server.serve_forever, daemon=True).start() + self.addCleanup(server.server_close) + self.addCleanup(server.shutdown) + install.install_bounded_opener() + self.addCleanup(install.urllib.request.install_opener, None) + url = f"http://127.0.0.1:{server.server_address[1]}/list.json" + started = time.monotonic() + self.assertEqual(install._fetch_json(url, 1024, "file list", attempts=1), {"ok": True}) + self.assertGreaterEqual(time.monotonic() - started, delay - 0.5) + + +class ADryRunOpensOnItsTotals(Scratch): + """A full --dry-run printed 599 lines, the total scrolled away above + eight file names for each of 66 systems.""" + + def setUp(self): + super().setUp() + files = [ + {"dest": f"sys{n:02d}/file{k}.bin", "sha1": _sha1(b"%d-%d" % (n, k)), + "size": 1000 * (n + 1), "repo_path": f"bios/sys{n:02d}_{k}", "cores": None, + "systems": [f"system-{n:02d}"]} + for n in range(40) for k in range(3) + ] + self.list.write_text(json.dumps({"manifest_version": 2, "platform": "retroarch", + "files": files})) + + def _plan(self, *options: str) -> list[str]: + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", str(self.dest), "--manifest", str(self.list), "--dry-run", + "--no-input", *options], + env={**_offline_env(), "LINES": "24", "COLUMNS": "80"}, + capture_output=True, text=True, timeout=120, check=False, + stdin=subprocess.DEVNULL) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + lines = proc.stdout.splitlines() + return lines[next(i for i, line in enumerate(lines) + if line.startswith("Would download")):] + + def test_the_total_comes_first_and_the_largest_systems_after(self): + plan = self._plan() + self.assertTrue(plan[0].startswith("Would download 120 files"), plan[0]) + self.assertLessEqual(len(plan), 24, "\n".join(plan)) + self.assertIn("system-39", plan[2]) + self.assertTrue(any("30 more systems" in line and "--verbose" in line + for line in plan), "\n".join(plan)) + self.assertFalse(any("file0.bin" in line for line in plan)) + self.assertEqual(plan[-1].strip(), "Nothing was written. Run without --dry-run to install.") + + def test_verbose_lists_every_system_and_its_files(self): + plan = self._plan("--verbose") + self.assertTrue(plan[0].startswith("Would download 120 files"), plan[0]) + for n in range(40): + self.assertTrue(any(line.strip().startswith(f"system-{n:02d}") for line in plan), n) + self.assertTrue(any("sys00/file0.bin" in line for line in plan)) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index 3524f2c0..8ec527fe 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -648,12 +648,13 @@ what the options name, everything by default. | `--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) | | `--no-input` | Never ask a question, even at a terminal | | `--check` | Report and exit without writing | -| `--dry-run` | Show what would be downloaded, by system and with sizes, without writing anything | +| `--dry-run` | Show what would be downloaded, the totals first, then by system and with sizes, without writing anything. A plan taller than the screen names the ten largest systems, and `--verbose` lists every system and file | | `--yes`, `-y` | Do not ask before a download of 1 GB or more | | `--quiet`, `-q` | Print only failures and the final summary. Implies `--no-input` | | `--events` | Write the run as JSON events on standard output, one object per line, and the usual text on standard error. Implies `--no-input`. See [Machine-readable events](#machine-readable-events) | | `--from FOLDER` | Copy the BIOS files a folder holds (an old SD card, a backup, a ZIP) instead of downloading them, each one recognised by its hash and copied, never moved. Repeat for several folders. See [Copy from an old card](#copy-from-an-old-card) | | `--from-pack PATH` | Install from a pack already downloaded: its ZIP, one of its parts (the others are found in the same folder), or the folder holding them. Makes no network request and checks every file's hash. See [Without a network](#without-a-network-offline-install) | +| `--offline` | Make no network request. `--check`, `--from`, `--from-pack` and `--manifest` work without one, the file list stops the run at once when none of them gives it, and a file the run would download fails at once. `RETROBIOS_INSTALL_OFFLINE=1` does the same. Refused with `--target` and `--list-targets`, which read a core list from the network | | `--manifest FILE` | Read the file list from FILE, a copy of `install/.json`, instead of downloading it. See [Without a network](#without-a-network-offline-install) | | `--restore FOLDER` | Put back the files a run replaced and kept in FOLDER, a dated folder inside `retrobios-backup`. Takes no other option but `--dry-run` and `--no-input`. See [Files you already had](#files-you-already-had) | | `--tui` | Open the full-screen interface, or stop with the reason it cannot open here (code 2) instead of falling back to the questions. Wins over `RETROBIOS_PLAIN` and `ACCESSIBLE`. See [The full-screen interface](#the-full-screen-interface) | @@ -1095,6 +1096,10 @@ pack: `--system`, `--core`, `--region` and `--no-game-data` then apply as they do online, and a file of the list the pack does not hold fails as `not in the pack`. The list's platform has to be the one installed for. +`--offline` forbids every request, for a device with no Wi-Fi: `--check` +still names the platform and its folder, then stops at the file list unless +`--manifest` or `--from-pack` gives it. + ## Where the files come from Each manifest entry names either `repo_path`, served from the repository at the @@ -1247,7 +1252,7 @@ other schemas under `api/v1/schemas/`. |------|------|------| | `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` | | `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found | -| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). `message` says it in words | +| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). `message` says it in words | | `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered | | `checking` | `files_done`, `files_total` | While the files already in the folder are read and hashed, at most every half second and once at the end. `files_total` counts the files found in place | | `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` | @@ -1274,6 +1279,7 @@ no event at all. A program reads a missing `end` as that case. |----------|--------| | `RETROBIOS_REF` | Branch or tag the file list and the files committed to the repository are read from. Defaults to `main`. Files the manifest marks `release_asset` always come from the `large-files` release, whatever the ref, and that release replaces an asset in place when it is rebuilt. Each release deletes the previous release tags, so a tag holds a file list until the next release. For contributors: [release process](release-process.md#cutting-a-release) | | `RETROBIOS_BASE_URL` | Base URL for the manifest and the files it declares. HTTPS only, plain HTTP allowed to loopback for end-to-end tests. Whoever controls this base controls the expected hashes, hence the check. A file list that cannot be fetched names the address, and this variable when it is set | +| `RETROBIOS_INSTALL_OFFLINE` | Set to anything but `0`: no network request, as with `--offline`. The runs the full-screen interface starts inherit it. `RETROBIOS_OFFLINE` is the build scripts' own and leaves the installer alone | | `RETROBIOS_OS` | Names the host outright (`linux`, `wsl`, `windows`, `darwin`, `android`) instead of detecting it | | `EXTERNAL_STORAGE` | Android only, the shared storage root RetroArch builds its default directories from. Defaults to `/storage/emulated/0` | | `RETROBIOS_INSTALL_URL` | Where the bootstrap fetches `install.py`. HTTPS only | @@ -1295,7 +1301,9 @@ The common cases, with the short answer, are on The installer fetches its manifest, then downloads only the repository or `large-files` release objects it needs. When a download fails, check that `https://github.com` opens. Behind a proxy, set `HTTPS_PROXY` in the -environment. A file that fails three attempts is counted and reported at the +environment. A connection that does not open within 4 seconds counts as +none, so a dropped network gives its error in about 15 seconds. Once a +connection is open, a slow transfer keeps the full timeout. A file that fails three attempts is counted and reported at the end, and the next run fetches only what is still missing. The reason is said in words: no internet connection or a name that could not