diff --git a/install.ps1 b/install.ps1 index 96a8a487..f7a8d2f5 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "c90676dcc433061bd97aca94930cd7acd74295624840cb5c85a22897f9c1dd12" + $defaultInstallSha256 = "7bc94c44f3696a0b777e5c280754633105a06756932ee52547f0a9c775a895b9" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index d7d51353..36fde413 100644 --- a/install.py +++ b/install.py @@ -526,6 +526,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { "checking": ("files_done", "files_total"), "check": ("platform", "name", "bios_dir", "files", "present", "missing", "wrong_hash", "bytes_needed", "data_missing", "not_installable"), + # Right after check: the same counts over the files the platform's own + # list declares (cores: null), and their dests in file-event order. A + # kind of its own, since schema 1 never adds a key to the file event. + "declared": ("platform", "files", "present", "missing", "wrong_hash", + "bytes_needed", "dests"), # A file to install that this machine already holds: misplaced in the # BIOS folder (folder is its bios_dir, reason says why in words) or in # a folder named with --from (reason empty). member names the ZIP member @@ -616,7 +621,8 @@ def _emit_check( mismatched: list[dict], data_missing: int, not_installable: int, chosen: list[str] | None = None, ) -> None: - """One event per file, per system and per platform after a check. + """One event per file, per system and per platform after a check, then + one for the files of the platform's own list. With systems chosen, only those are described: a file of SkyEmu's carries four systems, and --system nintendo-gba described gb, gbc and @@ -654,6 +660,15 @@ def _emit_check( bytes_needed=sum(entry.get("size", 0) for entry in needed), data_missing=data_missing, not_installable=not_installable, ) + own = [entry for entry in files if entry.get("cores") is None] + own_needed = [entry for entry in own if id(entry) not in present] + own_wrong = sum(1 for entry in own if id(entry) in wrong) + EVENTS.emit( + "declared", platform=plat, files=len(own), present=len(own) - len(own_needed), + missing=len(own_needed) - own_wrong, wrong_hash=own_wrong, + bytes_needed=sum(entry.get("size", 0) for entry in own_needed), + dests=[entry["dest"] for entry in own], + ) def leave_out_game_data( @@ -672,6 +687,22 @@ def leave_out_game_data( ) +def keep_declared(files: list[dict], omitted: list[dict]) -> tuple[list[dict], list[dict]]: + """The files and omissions the platform's own list declares (cores: null). + + A slot a core contradicts gets the platform's copy, and no core copy + stays behind as an alternative: narrowing by cores afterwards would + otherwise put a core's file back where a dropped system's was. + """ + def own(entries: list[dict]) -> list[dict]: + return [ + {k: v for k, v in _settle_slot(entry, set()).items() if k != "alternatives"} + for entry in entries if entry.get("cores") is None + ] + + return own(files), own(omitted) + + def _left_out_lines(left_out: list[dict], every: bool) -> list[str]: """What --no-game-data left out, named per category, ten each unless every.""" lines = [] @@ -2340,7 +2371,7 @@ SYSTEM_SHORT_NAMES = { "supergrafx"), "nec-pc-engine"), **dict.fromkeys(("pcecd", "tgcd", "turbografxcd"), "nec-pc-engine-cd"), "pcfx": "nec-pc-fx", - **dict.fromkeys(("mvs", "aes"), "snk-neogeo"), + **dict.fromkeys(("neogeo", "mvs", "aes"), "snk-neogeo"), **dict.fromkeys(("neocd", "neogeocd"), "snk-neogeo-cd"), **dict.fromkeys(("msx", "msx2"), "microsoft-msx"), "c64": "commodore-c64", "c128": "commodore-c128", @@ -2354,17 +2385,44 @@ SYSTEM_SHORT_NAMES = { "cpc": "amstrad-cpc", "bbc": "acorn-bbc-micro", "apple2": "apple-ii", } +# A system some lists file under another id, by the file that stands for it. +# The Neo Geo card offers --system snk-neogeo, and RetroArch's list keeps +# neogeo.zip under arcade: the run suggests arcade, never takes it. +SYSTEM_KEY_FILES = {"snk-neogeo": "neogeo.zip"} + + +def key_file_holders(files: list[dict], omitted: list[dict]) -> dict[str, list[str]]: + """The systems a list files each SYSTEM_KEY_FILES file under, for the + systems the list does not name itself.""" + entries = files + omitted + known = {system for entry in entries for system in entry_systems(entry)} + holders: dict[str, list[str]] = {} + for system, name in SYSTEM_KEY_FILES.items(): + if system in known: + continue + held = sorted({ + holder for entry in entries + if PurePosixPath(str(entry.get("dest", ""))).name.lower() == name + for holder in entry_systems(entry) + }) + if held: + holders[system] = held + return holders + def did_you_mean( typed: list[str], known: list[str], listing: str, aliases: dict[str, str] | None = None, inside_from: int = 4, + meant: dict[str, list[str]] | None = None, ) -> str: """The closest known names to what was typed, and the option listing them all. A typo was answered with every name the platform has, up to 400 on one line, with no hint that playstation is sony-playstation. An alias close to what was typed suggests the name it stands for. A name holding what - was typed, inside_from letters or more, is suggested too. + was typed, inside_from letters or more, is suggested too. meant gives, + by the letters and digits of a typed name, the names it stands for + here, suggested instead of the close ones. """ import difflib # noqa: PLC0415 @@ -2378,6 +2436,9 @@ def did_you_mean( key = _name_key(word) if not key: continue + if meant and key in meant: + close += [name for name in meant[key] if name not in close] + continue matches = difflib.get_close_matches(key, list(by_key), n=3, cutoff=0.6) if len(key) >= inside_from: matches += sorted(k for k in by_key if key in k and k not in matches) @@ -3617,6 +3678,7 @@ def available_choices(files: list[dict], omitted: list[dict]) -> dict[str, list[ def resolve_choices( values: list[str], known: list[str], label: str, aliases: dict[str, str] | None = None, + holders: dict[str, list[str]] | None = None, ) -> list[str]: """Names given on the command line, matched without regard to case. @@ -3626,6 +3688,8 @@ def resolve_choices( carrying on would install something other than what was asked. aliases maps another name of a known one to it, as Batocera's `psx` stands for sony-playstation. A known name always wins over an alias. + holders maps a system the list files under other ids to those ids + (key_file_holders): an unknown name for it is answered with them. """ aliases = {alias: name for alias, name in (aliases or {}).items() if name in known} by_lower = {name.lower(): name for name in known} @@ -3658,14 +3722,28 @@ def resolve_choices( listing = {"system": "--list-systems", "emulator": "--list-emulators"}.get( label, "--list-cores") hints, inside_from = aliases, 4 + held = holders or {} + stands_for: dict[str, str] = {} + notes = "" if label == "system": # gba is the end of nintendo-gba: three letters are enough here. hints = {**{short: name for short, name in SYSTEM_SHORT_NAMES.items() if name in known}, **aliases} inside_from = 3 + stands_for = {_name_key(system): system for system in held} + for short, name in SYSTEM_SHORT_NAMES.items(): + if name in held: + stands_for.setdefault(_name_key(short), name) + named = dict.fromkeys( + stands_for[_name_key(token)] for token in unknown + if _name_key(token) in stands_for) + notes = "".join( + f"This list files {SYSTEM_KEY_FILES[system]} under " + f"{', '.join(held[system])}. " for system in named) + meant = {key: held[system] for key, system in stands_for.items()} raise ValueError( - f"unknown {label}: {', '.join(unknown)}. " - f"{did_you_mean(unknown, known, listing, hints, inside_from)}" + f"unknown {label}: {', '.join(unknown)}. {notes}" + f"{did_you_mean(unknown, known, listing, hints, inside_from, meant)}" ) if values and not chosen: raise ValueError(f"--{label} needs at least one name") @@ -4696,6 +4774,7 @@ class Invocation: ("--every-board", args.every_board), ("--installed-cores", args.installed_cores and not args.cores_dir), ("--no-game-data", args.no_game_data), + ("--declared-only", args.declared_only), ("--standalone-copies", args.standalone_copies), ("--check", args.check), ("--dry-run", args.dry_run), @@ -5972,12 +6051,27 @@ def _same_choice_hint(plat: str, systems: list[str], cores: list[str], regions: def system_aliases(manifest: dict | None) -> dict[str, str]: - """The platform's own names for its systems, each with the system id.""" - return { + """The platform's own names for its systems, then the titles the site + and the installer print (SNK Neo Geo CD), each with the system id. + + A title two systems share names neither, and the platform's own name + wins over a title spelled the same. + """ + aliases = { alias: system for system, names in ((manifest or {}).get("system_names") or {}).items() for alias in names } + taken = {_name_key(alias) for alias in aliases} + titles = (manifest or {}).get("system_titles") or {} + counts: dict[str, int] = {} + for title in titles.values(): + counts[_name_key(title)] = counts.get(_name_key(title), 0) + 1 + for system, title in titles.items(): + key = _name_key(title) + if key and counts[key] == 1 and key not in taken: + aliases[title] = system + return aliases def _system_label(name: str, manifest: dict | None) -> str: @@ -6214,7 +6308,8 @@ def _requested_narrowing( choices = available_choices(files, omitted) try: systems = resolve_choices( - args.system or [], choices["systems"], "system", system_aliases(manifest) + args.system or [], choices["systems"], "system", system_aliases(manifest), + key_file_holders(files, omitted), ) cores = resolve_choices(args.core or [], choices["cores"], "core") regions = resolve_regions(args.region or []) @@ -7815,6 +7910,12 @@ def main() -> None: help="leave out game data (files a game reads, not firmware) and " "install the firmware only", ) + parser.add_argument( + "--declared-only", + action="store_true", + help="keep only the files the platform's own list names, not the extra " + "files cores load: --check then answers for the list the platform checks", + ) parser.add_argument( "--list-systems", action="store_true", @@ -7941,6 +8042,23 @@ def main() -> None: parser.error("--quiet applies to an install, not to --check, --dry-run or a listing") if args.tui and args.plain: parser.error("--tui and --plain ask for two different interfaces") + if args.declared_only: + # Applied or refused, never ignored. + for given, reason in ( + (args.tui, "the full-screen interface has no choice for --declared-only"), + (args.list_platforms or args.list_targets, + "--declared-only narrows a file list, not --list-platforms or --list-targets"), + (args.core or args.list_cores, + "--declared-only leaves out the files cores load, which --core and " + "--list-cores choose among"), + (args.list_emulators, + "--list-emulators names emulators, and --declared-only narrows files"), + (args.standalone_copies, + "--standalone-copies copies into standalone emulators' folders, outside " + "the platform's own list that --declared-only keeps"), + ): + if given: + parser.error(reason) if args.tui: reason = tui_blocker(args) if reason: @@ -7953,7 +8071,7 @@ def main() -> None: alone = ("platform", "dest", "target", "every_board", "check", "list_platforms", "list_targets", "standalone_copies", "emulator", "installed_cores", "cores_dir", "save_dir", "system", "core", "region", "no_game_data", - "list_systems", + "declared_only", "list_systems", "list_cores", "romm_firmware", "list_emulators", "events", "quiet", "yes", "verbose", "from_pack", "manifest", "from") given = [name for name in alone if getattr(args, name)] @@ -7980,7 +8098,9 @@ def main() -> None: if args.no_game_data and (args.list_platforms or args.list_targets): parser.error("--no-game-data narrows a file list, not --list-platforms " "or --list-targets") - if args.tui or not (plain_requested(args) or tui_blocker(args)): + # The screens have no choice for --declared-only: the questions ask the + # rest, and the run keeps the platform's own list. + if args.tui or not (plain_requested(args) or args.declared_only or tui_blocker(args)): sys.exit(run_tui(args)) if args.events: # The events own stdout. The text a person reads goes to stderr, so a @@ -8361,7 +8481,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # which the file lists name: an ES-DE user with DuckStation and # PCSX2 was sent to RetroArch's folder, which nothing here reads. narrowing = (args.system or args.core or args.emulator or args.region - or args.target or args.installed_cores) + or args.target or args.installed_cores or args.declared_only) if not listing and not narrowing and _install_standalone_folders( args, os_type, term, interactive ): @@ -8522,6 +8642,12 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: for line in _left_out_lines(left_out, args.verbose): print(line) + if args.declared_only: + before = len(files) + files, omitted_files = keep_declared(files, omitted_files) + print(f" The platform's own list: {before} -> {len(files)} files " + "(--declared-only)") + if args.list_targets: targets = fetch_targets(plat_name) if targets is None: @@ -8640,13 +8766,15 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: files, omitted_files = installed print("\n Same choice without questions: " f"--platform {plat_name} --installed-cores" - + (" --no-game-data" if args.no_game_data else "")) + + (" --no-game-data" if args.no_game_data else "") + + (" --declared-only" if args.declared_only else "")) _INVOCATION.chosen[plat_name] = [("--installed-cores", None)] selection = None elif selection is not None: print("\n Same choice without questions: " + _same_choice_hint(plat_name, *selection) - + (" --no-game-data" if args.no_game_data else "")) + + (" --no-game-data" if args.no_game_data else "") + + (" --declared-only" if args.declared_only else "")) systems, cores, regions = selection _INVOCATION.chosen[plat_name] = [ *(("--system", name) for name in systems), @@ -8746,7 +8874,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: narrowed = bool( target or args.system or args.core or args.region or args.installed_cores - or len(files) + game_data_cut + len(unplaced) + or args.declared_only or len(files) + game_data_cut + len(unplaced) != len(manifest.get("files", [])) + len(variants if case_kept else []) ) data_dirs = select_data_directories(manifest, files, narrowed) diff --git a/install.sh b/install.sh index cb584208..c35fc21e 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="c90676dcc433061bd97aca94930cd7acd74295624840cb5c85a22897f9c1dd12" +DEFAULT_INSTALL_SHA256="7bc94c44f3696a0b777e5c280754633105a06756932ee52547f0a9c775a895b9" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/schemas/install-events.schema.json b/schemas/install-events.schema.json new file mode 100644 index 00000000..13debfab --- /dev/null +++ b/schemas/install-events.schema.json @@ -0,0 +1,704 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/install-events.schema.json", + "title": "RetroBIOS installer event", + "description": "One line of install.py --events, event schema 1. Written from EVENT_KEYS in install.py by scripts/generate_event_schema.py. The schema gains kinds, never keys on an existing kind: a reader skips a kind it does not know.", + "type": "object", + "required": [ + "event" + ], + "properties": { + "event": { + "enum": [ + "start", + "detect", + "error", + "scanned", + "file", + "system", + "checking", + "check", + "declared", + "found", + "unreadable", + "download", + "progress", + "transfer", + "downloaded", + "summary", + "end" + ] + } + }, + "oneOf": [ + { + "$ref": "#/$defs/start" + }, + { + "$ref": "#/$defs/detect" + }, + { + "$ref": "#/$defs/error" + }, + { + "$ref": "#/$defs/scanned" + }, + { + "$ref": "#/$defs/file" + }, + { + "$ref": "#/$defs/system" + }, + { + "$ref": "#/$defs/checking" + }, + { + "$ref": "#/$defs/check" + }, + { + "$ref": "#/$defs/declared" + }, + { + "$ref": "#/$defs/found" + }, + { + "$ref": "#/$defs/unreadable" + }, + { + "$ref": "#/$defs/download" + }, + { + "$ref": "#/$defs/progress" + }, + { + "$ref": "#/$defs/transfer" + }, + { + "$ref": "#/$defs/downloaded" + }, + { + "$ref": "#/$defs/summary" + }, + { + "$ref": "#/$defs/end" + } + ], + "$defs": { + "start": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "schema", + "installer", + "mode" + ], + "properties": { + "event": { + "const": "start" + }, + "schema": { + "const": 1 + }, + "installer": { + "type": "string" + }, + "mode": { + "enum": [ + "install", + "check", + "dry-run", + "list" + ] + } + } + }, + "detect": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "os", + "os_name", + "platforms" + ], + "properties": { + "event": { + "const": "detect" + }, + "os": { + "type": "string" + }, + "os_name": { + "type": "string" + }, + "platforms": { + "type": "array", + "items": { + "type": "object" + } + } + } + }, + "error": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "stage", + "reason", + "platform", + "message" + ], + "properties": { + "event": { + "const": "error" + }, + "stage": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "platform": { + "type": "string" + }, + "message": { + "type": "string" + } + } + }, + "scanned": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "folder", + "files", + "truncated", + "other_drives" + ], + "properties": { + "event": { + "const": "scanned" + }, + "folder": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "truncated": { + "type": "boolean" + }, + "other_drives": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "file": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "dest", + "systems", + "size", + "state" + ], + "properties": { + "event": { + "const": "file" + }, + "platform": { + "type": "string" + }, + "dest": { + "type": "string" + }, + "systems": { + "type": "array", + "items": { + "type": "string" + } + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "state": { + "enum": [ + "present", + "wrong_hash", + "missing" + ] + } + } + }, + "system": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "system", + "name", + "files", + "present", + "missing", + "wrong_hash", + "bytes_needed" + ], + "properties": { + "event": { + "const": "system" + }, + "platform": { + "type": "string" + }, + "system": { + "type": "string" + }, + "name": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "present": { + "type": "integer", + "minimum": 0 + }, + "missing": { + "type": "integer", + "minimum": 0 + }, + "wrong_hash": { + "type": "integer", + "minimum": 0 + }, + "bytes_needed": { + "type": "integer", + "minimum": 0 + } + } + }, + "checking": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "files_done", + "files_total" + ], + "properties": { + "event": { + "const": "checking" + }, + "files_done": { + "type": "integer", + "minimum": 0 + }, + "files_total": { + "type": "integer", + "minimum": 0 + } + } + }, + "check": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "name", + "bios_dir", + "files", + "present", + "missing", + "wrong_hash", + "bytes_needed", + "data_missing", + "not_installable" + ], + "properties": { + "event": { + "const": "check" + }, + "platform": { + "type": "string" + }, + "name": { + "type": "string" + }, + "bios_dir": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "present": { + "type": "integer", + "minimum": 0 + }, + "missing": { + "type": "integer", + "minimum": 0 + }, + "wrong_hash": { + "type": "integer", + "minimum": 0 + }, + "bytes_needed": { + "type": "integer", + "minimum": 0 + }, + "data_missing": { + "type": "integer", + "minimum": 0 + }, + "not_installable": { + "type": "integer", + "minimum": 0 + } + } + }, + "declared": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "files", + "present", + "missing", + "wrong_hash", + "bytes_needed", + "dests" + ], + "properties": { + "event": { + "const": "declared" + }, + "platform": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "present": { + "type": "integer", + "minimum": 0 + }, + "missing": { + "type": "integer", + "minimum": 0 + }, + "wrong_hash": { + "type": "integer", + "minimum": 0 + }, + "bytes_needed": { + "type": "integer", + "minimum": 0 + }, + "dests": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "found": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "dest", + "folder", + "path", + "member", + "reason" + ], + "properties": { + "event": { + "const": "found" + }, + "platform": { + "type": "string" + }, + "dest": { + "type": "string" + }, + "folder": { + "type": "string" + }, + "path": { + "type": "string" + }, + "member": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "unreadable": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "folder", + "path", + "member", + "reason" + ], + "properties": { + "event": { + "const": "unreadable" + }, + "platform": { + "type": "string" + }, + "folder": { + "type": "string" + }, + "path": { + "type": "string" + }, + "member": { + "type": "string" + }, + "reason": { + "type": "string" + } + } + }, + "download": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "platform", + "files", + "bytes" + ], + "properties": { + "event": { + "const": "download" + }, + "platform": { + "type": "string" + }, + "files": { + "type": "integer", + "minimum": 0 + }, + "bytes": { + "type": "integer", + "minimum": 0 + } + } + }, + "progress": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "files_done", + "files_total", + "bytes_done", + "bytes_total" + ], + "properties": { + "event": { + "const": "progress" + }, + "files_done": { + "type": "integer", + "minimum": 0 + }, + "files_total": { + "type": "integer", + "minimum": 0 + }, + "bytes_done": { + "type": "integer", + "minimum": 0 + }, + "bytes_total": { + "type": "integer", + "minimum": 0 + } + } + }, + "transfer": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "bytes_local", + "bytes_network" + ], + "properties": { + "event": { + "const": "transfer" + }, + "bytes_local": { + "type": "integer", + "minimum": 0 + }, + "bytes_network": { + "type": "integer", + "minimum": 0 + } + } + }, + "downloaded": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "dest", + "ok", + "reason", + "size", + "files_done", + "files_total", + "bytes_done", + "bytes_total" + ], + "properties": { + "event": { + "const": "downloaded" + }, + "dest": { + "type": "string" + }, + "ok": { + "type": "boolean" + }, + "reason": { + "type": "string" + }, + "size": { + "type": "integer", + "minimum": 0 + }, + "files_done": { + "type": "integer", + "minimum": 0 + }, + "files_total": { + "type": "integer", + "minimum": 0 + }, + "bytes_done": { + "type": "integer", + "minimum": 0 + }, + "bytes_total": { + "type": "integer", + "minimum": 0 + } + } + }, + "summary": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "mode", + "installed", + "up_to_date", + "pending", + "failed", + "skipped", + "left_out", + "kept", + "interrupted" + ], + "properties": { + "event": { + "const": "summary" + }, + "mode": { + "enum": [ + "install", + "check", + "dry-run", + "list" + ] + }, + "installed": { + "type": "integer", + "minimum": 0 + }, + "up_to_date": { + "type": "integer", + "minimum": 0 + }, + "pending": { + "type": "integer", + "minimum": 0 + }, + "failed": { + "type": "array", + "items": { + "type": "object" + } + }, + "skipped": { + "type": "array", + "items": { + "type": "object" + } + }, + "left_out": { + "type": "array", + "items": { + "type": "object" + } + }, + "kept": { + "type": "array", + "items": { + "type": "object" + } + }, + "interrupted": { + "type": "boolean" + } + } + }, + "end": { + "type": "object", + "additionalProperties": false, + "required": [ + "event", + "code" + ], + "properties": { + "event": { + "const": "end" + }, + "code": { + "type": "integer", + "minimum": 0 + } + } + } + } +} diff --git a/scripts/generate_event_schema.py b/scripts/generate_event_schema.py new file mode 100644 index 00000000..e3579c4a --- /dev/null +++ b/scripts/generate_event_schema.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Write schemas/install-events.schema.json from install.py's EVENT_KEYS. + + python scripts/generate_event_schema.py rewrite the schema + python scripts/generate_event_schema.py --check exit 1 when it is stale +""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +OUTPUT = ROOT / "schemas" / "install-events.schema.json" + +_spec = importlib.util.spec_from_file_location("install", ROOT / "install.py") +_install = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_install) + +EVENT_KEYS: dict[str, tuple[str, ...]] = _install.EVENT_KEYS +EVENT_SCHEMA: int = _install.EVENT_SCHEMA + +_TEXT = {"type": "string"} +_COUNT = {"type": "integer", "minimum": 0} +_FLAG = {"type": "boolean"} +_NAMES = {"type": "array", "items": {"type": "string"}} +_RECORDS = {"type": "array", "items": {"type": "object"}} + +# The JSON type of each key, whatever kind carries it. A key EVENT_KEYS +# gains without a line here stops the generator. +KEY_TYPES: dict[str, dict] = { + **dict.fromkeys(( + "installer", "os", "os_name", "stage", "reason", "platform", "message", + "folder", "dest", "system", "name", "bios_dir", "path", "member", + ), _TEXT), + "mode": {"enum": ["install", "check", "dry-run", "list"]}, + "state": {"enum": ["present", "wrong_hash", "missing"]}, + **dict.fromkeys(( + "files", "files_done", "files_total", "bytes", "bytes_done", "bytes_total", + "bytes_local", "bytes_network", "size", "present", "missing", "wrong_hash", + "bytes_needed", "data_missing", "not_installable", "installed", "up_to_date", + "pending", "code", + ), _COUNT), + **dict.fromkeys(("truncated", "ok", "interrupted"), _FLAG), + **dict.fromkeys(("systems", "other_drives", "dests"), _NAMES), + **dict.fromkeys(("platforms", "failed", "skipped", "left_out", "kept"), _RECORDS), +} + + +def _kind(kind: str, keys: tuple[str, ...]) -> dict: + properties: dict[str, dict] = {"event": {"const": kind}} + for key in keys: + properties[key] = ( + {"const": EVENT_SCHEMA} if (kind, key) == ("start", "schema") else KEY_TYPES[key] + ) + return { + "type": "object", + "additionalProperties": False, + "required": ["event", *keys], + "properties": properties, + } + + +def build_schema() -> dict: + """One line of the stream: an object of one of the kinds, with exactly its keys.""" + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://abdess.github.io/retrobios/api/v1/schemas/install-events.schema.json", + "title": "RetroBIOS installer event", + "description": ( + f"One line of install.py --events, event schema {EVENT_SCHEMA}. Written from " + "EVENT_KEYS in install.py by scripts/generate_event_schema.py. The schema " + "gains kinds, never keys on an existing kind: a reader skips a kind it " + "does not know." + ), + "type": "object", + "required": ["event"], + "properties": {"event": {"enum": list(EVENT_KEYS)}}, + "oneOf": [{"$ref": f"#/$defs/{kind}"} for kind in EVENT_KEYS], + "$defs": {kind: _kind(kind, keys) for kind, keys in EVENT_KEYS.items()}, + } + + +def render(schema: dict) -> str: + return json.dumps(schema, indent=2, ensure_ascii=False) + "\n" + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(allow_abbrev=False, description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--check", action="store_true", + help="exit 1 when the file differs from EVENT_KEYS, writing nothing") + parser.add_argument("--output", type=Path, default=OUTPUT, help=argparse.SUPPRESS) + args = parser.parse_args(argv) + text = render(build_schema()) + current = args.output.read_text(encoding="utf-8") if args.output.is_file() else None + if current == text: + return 0 + if args.check: + print(f"{args.output} differs from EVENT_KEYS: run " + "python scripts/generate_event_schema.py", file=sys.stderr) + return 1 + args.output.write_text(text, encoding="utf-8") + print(f"Wrote {args.output}") + return 0 + + +if __name__ == "__main__": + from common import run_cli + + run_cli(main) diff --git a/scripts/validate_schemas.py b/scripts/validate_schemas.py index 5e002a78..57bf5bfa 100644 --- a/scripts/validate_schemas.py +++ b/scripts/validate_schemas.py @@ -554,6 +554,23 @@ def _semantic_install_checks(path: Path, manifest: dict) -> list[str]: return out +def _event_schema_errors() -> list[str]: + """install-events.schema.json is EVENT_KEYS written out, and a valid schema.""" + import generate_event_schema # noqa: PLC0415 + + path = generate_event_schema.OUTPUT + label = path.relative_to(ROOT).as_posix() + if not path.is_file(): + return [f"{label}: missing, run python scripts/generate_event_schema.py"] + if path.read_text(encoding="utf-8") != generate_event_schema.render( + generate_event_schema.build_schema() + ): + return [f"{label}: differs from EVENT_KEYS in install.py, run " + "python scripts/generate_event_schema.py"] + _validator(path.name) + return [] + + def unchecked_formats() -> list[str]: """Formats the schemas declare that this jsonschema install cannot check. @@ -598,6 +615,7 @@ def main() -> int: errors.extend(_repository_layout_in_profiles()) errors.extend(_save_directory_blocks()) errors.extend(_registry_contract()) + errors.extend(_event_schema_errors()) if not args.source_only: database_path = ROOT / "database.json" diff --git a/tests/test_install.py b/tests/test_install.py index a7877ea2..4da52996 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3864,6 +3864,7 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase): system=None, core=None, emulator=None, region=None, target=None, cores_dir=None, every_board=False, installed_cores=False, standalone_copies=False, check=False, dry_run=False, no_game_data=False, + declared_only=False, ) flags = list(argv) while flags: @@ -3908,6 +3909,13 @@ class HintsGiveTheUsersOwnCommandBack(unittest.TestCase): self.assertIn("| sudo sh -s -- --platform retroarch --dest /media/ro " "--system atari-lynx --region us", advice) + def test_the_platforms_own_list_is_kept_in_the_hint(self): + self._record("--declared-only", "--system", "atari-lynx") + install._INVOCATION.os_type = "linux" + with unittest.mock.patch.dict(os.environ, {"RETROBIOS_BOOTSTRAP": ""}): + command = install._INVOCATION.command("retroarch", None) + self.assertIn("--system atari-lynx --declared-only", command) + def test_windows_has_no_sudo(self): self._record("--system", "atari-lynx") install._INVOCATION.os_type = "windows" @@ -5070,6 +5078,10 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): "bios_dir": str(self.bios), "files": 4, "present": 1, "missing": 2, "wrong_hash": 1, "bytes_needed": 245, "data_missing": 0, "not_installable": 1}, + # Every file of this list is the platform's own (cores: null). + {"event": "declared", "platform": "retroarch", "files": 4, "present": 1, + "missing": 2, "wrong_hash": 1, "bytes_needed": 245, + "dests": ["a.bin", "b.bin", "c.bin", "d.bin"]}, ] SKIPPED = [{"reason": "not_found", "label": "not found", "files": 1, "required": 1}] @@ -5153,6 +5165,70 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): for key in keys: self.assertIn(f"`{key}`", row, f"{kind}.{key}") + def test_a_run_is_valid_against_the_published_schema(self): + try: + from jsonschema import Draft202012Validator # noqa: PLC0415 + except ImportError: + self.skipTest("jsonschema is not installed") + schema = json.loads(EVENT_SCHEMA_PATH.read_text(encoding="utf-8")) + validator = Draft202012Validator(schema) + for argv in (("--check",), ()): + _proc, events = self._run(*argv) + for event in events: + errors = [error.message for error in validator.iter_errors(event)] + self.assertEqual(errors, [], event) + self.assertTrue(list(validator.iter_errors({"event": "file", "platform": "x"}))) + self.assertTrue(list(validator.iter_errors( + {"event": "end", "code": 0, "extra": 1}))) + + +EVENT_SCHEMA_PATH = REPO_ROOT / "schemas" / "install-events.schema.json" + + +class TheEventSchemaFileIsEventKeys(unittest.TestCase): + """The table in the wiki was the only description of --events a reader + outside Python could check a stream against.""" + + def _generator(self): + sys.path.insert(0, str(REPO_ROOT / "scripts")) + import generate_event_schema # noqa: PLC0415 + + return generate_event_schema + + def test_the_committed_file_is_the_generated_one(self): + generator = self._generator() + self.assertEqual(EVENT_SCHEMA_PATH.read_text(encoding="utf-8"), + generator.render(generator.build_schema()), + "run python scripts/generate_event_schema.py") + + def test_each_kind_has_exactly_its_keys(self): + schema = json.loads(EVENT_SCHEMA_PATH.read_text(encoding="utf-8")) + self.assertEqual(schema["properties"]["event"]["enum"], list(install.EVENT_KEYS)) + for kind, keys in install.EVENT_KEYS.items(): + shape = schema["$defs"][kind] + self.assertEqual(shape["required"], ["event", *keys], kind) + self.assertEqual(list(shape["properties"]), ["event", *keys], kind) + self.assertIs(shape["additionalProperties"], False, kind) + start = schema["$defs"]["start"]["properties"]["schema"] + self.assertEqual(start, {"const": install.EVENT_SCHEMA}) + + def test_a_key_without_a_type_stops_the_generator(self): + generator = self._generator() + with unittest.mock.patch.dict(install.EVENT_KEYS, {"new": ("untyped_key",)}), \ + unittest.mock.patch.object(generator, "EVENT_KEYS", install.EVENT_KEYS), \ + self.assertRaises(KeyError): + generator.build_schema() + + def test_the_check_command_fails_on_a_stale_file(self): + generator = self._generator() + scratch = Path(_tempdir(self)) / "events.schema.json" + scratch.write_text("{}\n", encoding="utf-8") + with contextlib.redirect_stdout(io.StringIO()), \ + contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(generator.main(["--check", "--output", str(scratch)]), 1) + self.assertEqual(generator.main(["--output", str(scratch)]), 0) + self.assertEqual(generator.main(["--check", "--output", str(scratch)]), 0) + class NetworkFailuresInPlainWords(unittest.TestCase): """Offline, the run printed "[Errno -2] Name or service not known". A diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 3577b6da..97b969b3 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -1296,5 +1296,197 @@ class AnUnknownNativeNameIsRefusedBeforeAnyWork(unittest.TestCase): self.assertEqual(json.loads(proc.stdout)["batocera"]["systems"], ["sony-playstation"]) +class ThePlatformsOwnListAlone(unittest.TestCase): + """A setup that checks its own list asked --check about 14,508 files on + Batocera, 353 of them its own, and had no option to ask about those: + --declared-only keeps the entries the platform declares (cores: null).""" + + FILES = [ + {"dest": "a.bin", "size": 1, "cores": None, "systems": ["sys-a"], + "alternatives": [{"sha1": "2" * 40, "size": 2, "cores": ["core-b"]}]}, + {"dest": "b.bin", "size": 1, "cores": None, "systems": ["sys-a"], + "sha1": "1" * 40, + "slot_choices": [{"sha1": "3" * 40, "size": 3, "cores": ["core-b"]}], + "declared_choices": [{"sha1": "1" * 40, "size": 1, "systems": ["sys-a"]}]}, + {"dest": "core/c.bin", "size": 1, "cores": ["core-b"], "systems": ["sys-a"]}, + ] + OMITTED = [ + {"dest": "d.bin", "name": "d.bin", "system": "sys-a", "systems": ["sys-a"], + "required": True, "reason": "not_found", "cores": None}, + {"dest": "core/e.bin", "name": "e.bin", "system": "sys-a", "systems": ["sys-a"], + "required": True, "reason": "not_found", "cores": ["core-b"]}, + ] + + def test_the_cut(self): + files, omitted = install.keep_declared(list(self.FILES), list(self.OMITTED)) + self.assertEqual([f["dest"] for f in files], ["a.bin", "b.bin"]) + self.assertEqual([o["dest"] for o in omitted], ["d.bin"]) + # The platform answers its own slot, and no core copy is left to + # stand in for it when a later narrowing drops the system. + self.assertEqual(files[1]["sha1"], "1" * 40) + for entry in files: + self.assertIsNone(entry["cores"]) + self.assertFalse({"alternatives", "slot_choices", "declared_choices"} & set(entry)) + self.assertEqual(install.available_choices(files, omitted)["cores"], []) + + def test_install_check_and_dry_run_apply_it(self): + with FakeRepository() as repo: + proc = repo.run("--declared-only") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("5 -> 3 files", proc.stdout) + self.assertEqual(repo.installed(), + ["gba_bios.bin", "scph5500.bin", "scph5501.bin"]) + for mode in ("--check", "--dry-run"): + with self.subTest(mode=mode), FakeRepository() as repo: + proc = repo.run("--declared-only", mode) + self.assertIn("5 -> 3 files", proc.stdout, proc.stderr) + self.assertIn("0/3 present", proc.stdout) + self.assertEqual(repo.installed(), []) + + def test_it_composes_with_the_other_narrowings(self): + with FakeRepository() as repo: + proc = repo.run("--declared-only", "--system", "sony-playstation", + "--region", "us") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(repo.installed(), ["scph5501.bin"]) + with FakeRepository() as repo: + listing = repo.run("--declared-only", "--list-systems") + self.assertEqual(listing.returncode, 0, listing.stderr) + row = next(line for line in listing.stdout.splitlines() + if "sony-playstation" in line) + self.assertIn("2 files", row) + + def test_a_local_file_list_applies_it(self): + with FakeRepository() as repo: + proc = repo.run("--declared-only", "--dry-run", "--manifest", + str(repo.root / "install" / "retroarch.json")) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("5 -> 3 files", proc.stdout) + + def test_every_mode_that_cannot_apply_it_refuses_it(self): + """Applied or refused, never ignored: each of these would have run + as if the option was not there.""" + refused = [ + ["--list-platforms"], + ["--list-targets"], + ["--list-cores"], + ["--list-emulators"], + ["--core", "mgba"], + ["--standalone-copies"], + ["--tui"], + ] + with FakeRepository() as repo: + for extra in refused: + with self.subTest(extra=extra): + proc = repo.run("--declared-only", *extra) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn("--declared-only", proc.stderr) + self.assertEqual(repo.installed(), []) + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--restore", + str(REPO_ROOT / "tmp"), "--declared-only"], + capture_output=True, text=True, timeout=60, check=False, + stdin=subprocess.DEVNULL) + self.assertEqual(proc.returncode, 2) + self.assertIn("takes no --declared-only", proc.stderr) + + def test_events_count_the_platforms_own_files(self): + with FakeRepository() as repo: + proc = repo.run("--check", "--events") + events = [json.loads(line) for line in proc.stdout.splitlines()] + declared = [e for e in events if e["event"] == "declared"] + self.assertEqual(declared, [{ + "event": "declared", "platform": "retroarch", "files": 3, "present": 0, + "missing": 3, "wrong_hash": 0, "bytes_needed": 22, + "dests": ["scph5500.bin", "scph5501.bin", "gba_bios.bin"], + }]) + kinds = [e["event"] for e in events] + self.assertEqual(kinds.index("declared"), kinds.index("check") + 1) + narrowed = [json.loads(line) for line in + repo.run("--check", "--events", "--declared-only").stdout.splitlines()] + check = next(e for e in narrowed if e["event"] == "check") + self.assertEqual(check["files"], 3) + + +class AnIdAnotherSetupUses(unittest.TestCase): + """The Neo Geo card offers `--system snk-neogeo`, and RetroArch, which + files neogeo.zip under arcade, answered "Did you mean: snk-neogeo-cd?".""" + + KNOWN = ["arcade", "snk-neogeo-cd", "sony-playstation"] + HOLDERS = {"snk-neogeo": ["arcade"]} + + def _refusal(self, typed: str, holders=None) -> str: + with self.assertRaises(ValueError) as ctx: + install.resolve_choices([typed], self.KNOWN, "system", + holders=self.HOLDERS if holders is None else holders) + return str(ctx.exception) + + def test_the_system_holding_its_file_is_suggested_never_taken(self): + for typed in ("snk-neogeo", "SNK-NeoGeo", "neogeo", "mvs", "aes"): + with self.subTest(typed=typed): + text = self._refusal(typed) + self.assertIn("Did you mean: arcade?", text) + self.assertIn("neogeo.zip", text) + self.assertNotIn("snk-neogeo-cd", text) + + def test_a_typo_still_gets_the_closest_names(self): + self.assertIn("Did you mean: snk-neogeo-cd", self._refusal("snk-neogeo-dc", {})) + + def test_holders_are_read_from_the_file_list(self): + files = [ + {"dest": "neogeo.zip", "systems": ["arcade"]}, + {"dest": "fbneo/neogeo.zip", "systems": ["arcade"]}, + {"dest": "neocd/neocd.bin", "systems": ["snk-neogeo-cd"]}, + {"dest": "Other/neogeo.zip", "systems": []}, + ] + self.assertEqual(install.key_file_holders(files, []), {"snk-neogeo": ["arcade"]}) + own = [{"dest": "neogeo.zip", "systems": ["snk-neogeo"]}] + self.assertEqual(install.key_file_holders(own, []), {}) + + def test_retroarch_suggests_arcade(self): + never = REPO_ROOT / "tmp" / "never" + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--manifest", str(REPO_ROOT / "install" / "retroarch.json"), + "--dest", str(never), "--system", "snk-neogeo", "--dry-run", "--no-input"], + capture_output=True, text=True, timeout=120, check=False, + stdin=subprocess.DEVNULL) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertIn("Did you mean: arcade?", proc.stderr) + self.assertNotIn("snk-neogeo-cd", proc.stderr) + self.assertFalse(never.exists()) + + +class TheTitleASetupPrints(unittest.TestCase): + """The installer and the site print "SNK Neo Geo CD": typed back, the + title was refused.""" + + MANIFEST = { + "system_names": {"sony-playstation": ["psx"]}, + "system_titles": {"snk-neogeo-cd": "SNK Neo Geo CD", "arcade": "Arcade games", + "sony-playstation": "Sony PlayStation", + "twin-a": "Twin", "twin-b": "Twin"}, + } + KNOWN = ["arcade", "snk-neogeo-cd", "sony-playstation", "twin-a", "twin-b"] + + def _resolve(self, typed: str) -> list[str]: + return install.resolve_choices([typed], self.KNOWN, "system", + install.system_aliases(self.MANIFEST)) + + def test_a_title_is_taken_for_its_id(self): + self.assertEqual(self._resolve("SNK Neo Geo CD"), ["snk-neogeo-cd"]) + self.assertEqual(self._resolve("arcade games"), ["arcade"]) + self.assertEqual(self._resolve("psx"), ["sony-playstation"]) + + def test_a_title_two_systems_share_is_not_taken(self): + with self.assertRaises(ValueError): + self._resolve("Twin") + + def test_the_platforms_own_name_wins_over_a_title(self): + manifest = {"system_names": {"arcade": ["Sony PlayStation"]}, + "system_titles": {"sony-playstation": "Sony PlayStation"}} + self.assertEqual(install.system_aliases(manifest)["Sony PlayStation"], "arcade") + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index ea072e16..888d8e0c 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -630,11 +630,12 @@ what the options name, everything by default. | `--installed-cores` | Keep only what the cores already installed read (RetroArch, Lakka, RetroPie). See [Installed cores](#installed-cores) | | `--cores-dir PATH` | The folder RetroArch loads its cores from, for `--installed-cores` when it is not found. Implies `--installed-cores` | | `--save-dir PATH` | The frontend's save directory (RetroArch's `savefile_directory`), for the files some cores read there, when the run cannot find it. The core's own folder is still added below it when the frontend sorts saves by core | -| `--system NAMES` | Install only these systems. Comma-separated or repeated. `--list-systems` prints the names, `other` included. A short name the platform does not use (`gba`, `psx`) is refused with the ids it may stand for, never taken in their place | +| `--system NAMES` | Install only these systems. Comma-separated or repeated. `--list-systems` prints the names, `other` included, and the title a system is printed with is taken too. A short name the platform does not use (`gba`, `psx`) is refused with the ids it may stand for, never taken in their place. So is an id the list files under another one: RetroArch keeps `neogeo.zip` under `arcade`, which `--system snk-neogeo` suggests | | `--core NAMES` | Keep the platform's own list and only the extra files these cores load. `--list-cores` prints the names | | `--emulator NAMES` | Install what these emulators read on the detected platform, and nothing else: their own files and the platform's files of the systems they run. Each emulator page on the site gives this command. `--list-emulators` prints the names, buildbot names such as `mednafen_psx` included | | `--region LIST` | Preferred regions, best first (`us,eu,jp`): keep one BIOS per region where a system has several. Accepts the names and aliases of `generate_pack.py --region` | | `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | +| `--declared-only` | Keep only the files the platform's own list names (`cores: null` in the manifest), not the extra files cores load. With `--check`, the answer is about the list the platform itself checks. Refused with `--core`, the listings it does not change, `--standalone-copies` and `--tui` | | `--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart | | `--list-cores` | Print the cores whose extra files a platform installs | | `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | @@ -1186,7 +1187,10 @@ the text a person reads goes to standard error. A program driving the installer (the [handheld port](#on-a-handheld-the-retrobios-port) is one) reads the objects and never parses the text. Every object has an `event` key naming its kind, and each kind has exactly the keys below, in this order. The schema is -`EVENT_KEYS` in `install.py`, and a test holds this table to it. +`EVENT_KEYS` in `install.py`, and a test holds this table to it. The same +keys, with their types, are in `schemas/install-events.schema.json`, written +from `EVENT_KEYS` by `scripts/generate_event_schema.py` and published with the +other schemas under `api/v1/schemas/`. | Kind | Keys | When | |------|------|------| @@ -1198,6 +1202,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is | `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` | | `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id, and with `--system` only the systems named. Files with no system count under `other`. `name` is the platform's own name for it | | `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply | +| `declared` | `platform`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `dests` | Right after `check`, the same counts over the files the platform's own list names (`cores: null`), and their `dests` in the order of the `file` events. A setup that checks its own list reads its answer here, with or without `--declared-only` | | `found` | `platform`, `dest`, `folder`, `path`, `member`, `reason` | Each file to install that this machine already holds, after `check` and before anything is written, in `--check` and `--dry-run` too. `folder` is the `--from` folder as given, or the `bios_dir` of the check when the BIOS folder holds the file at another place, and then `reason` says where in words (`nested folder: ...` for a misplaced file, `the same file is at ...` for one the list installs there too). `path` is the file, and `member` the ZIP member holding the bytes when `path` is an archive, an empty string otherwise. Its copy is reported by a `downloaded` event | | `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) | | `download` | `platform`, `files`, `bytes` | Before the downloads start. Counts the files copied from this machine too | @@ -1207,7 +1212,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is | `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `kept`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. `kept` lists `{folder, files, bytes}`, one per backup folder, for the files in place the run moved aside before replacing them (with `--dry-run`, would move), and is empty when none. Not written for a listing | | `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) | -Schema 1 can gain a kind (`found` came with `--from`), never a key of an +Schema 1 can gain a kind (`found` came with `--from`, `declared` with `--declared-only`), never a key of an existing kind: a reader skips a kind it does not know. An option refused before the run starts (`install.py: error:`, code 2) writes no event at all. A program reads a missing `end` as that case. diff --git a/wiki/tools.md b/wiki/tools.md index 5983d766..6f57056b 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -684,6 +684,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6 | `region_audit.py` | Confront every profile `region:` with the regions No-Intro names for the same dump (`--json`, `--db`, `--emulators-dir`). An opinion, never an authority: the code decides, and a disagreement is a line to read in the emulator's source | | `validate_pr.py` | Validate BIOS files in pull requests, post markdown report | | `validate_schemas.py` | Validate the data contracts: schemas and semantic invariants. `--source-only` checks `emulators/` and `platforms/` alone, which is what PR validation runs | +| `generate_event_schema.py` | Write `schemas/install-events.schema.json` from `EVENT_KEYS` in `install.py` (`--check` exits 1 when the file is stale, which `validate_schemas.py` also reports) | | `auto_fetch.py` | Fetch missing BIOS files from known sources (4-step pipeline). A download is accepted only when every cryptographic hash the entry declares (sha256, sha1, md5) matches. crc32 and size find a file the database already holds, never accept a download. An entry with no cryptographic hash is reported and not searched | | `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones, used by CI) | | `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) |