From e534f11713ed04c61f3522c9206c4dcc04cac256 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:31:33 +0200 Subject: [PATCH 1/5] fix: second audit of the full-screen interface - A handheld card mounted on a computer is found by the files its firmware writes there: muOS, Onion, spruceOS, CrossMix, Allium and Koriki from their device rules read below the card root, KNULLI and ROCKNIX from the share their first boot seeds. It is a setup of its own ("muOS SD card (E:\)") with the folder that system reads, and --dest writes there too. --platform muos with a folder that is no muOS card is refused with the folder to name. - Space (X on PortMaster, Y on muxterm) opens the confirm screen from Install and never installs, closes or quits by itself. - Short of space, Choose another folder comes first, and the result says the card is full in words, with no log lines and no command naming the interface's own copy of the list. - The plan keeps the BIOS folder second, cut in its middle, a --from folder by its name, and the config that named the folder. - The result says how many files came from the old card, how many were downloaded and how many are still missing, as the line mode. - The "other" row is named and explained, files shared by several systems are counted apart, and a file that cannot be supplied is named with its system, only where it matters. - A work folder left by SIGKILL is removed by the next run, a failed run's log after a week. A full card keeps no log. - Smaller: the sudo advice names the folder, removing a --from folder says so, the preview counts its files only, a proxy is named, a list that is no JSON and the clock error read as plain sentences. --- install.ps1 | 2 +- install.py | 532 +++++++++++++++++--- install.sh | 2 +- tests/fixtures/tui/confirm-120x40.txt | 2 +- tests/fixtures/tui/confirm-40x12.txt | 4 +- tests/fixtures/tui/confirm-40x15.txt | 4 +- tests/fixtures/tui/confirm-80x24.txt | 2 +- tests/fixtures/tui/options-120x40.txt | 2 +- tests/fixtures/tui/options-40x12.txt | 2 +- tests/fixtures/tui/options-40x15.txt | 2 +- tests/fixtures/tui/options-80x24.txt | 2 +- tests/fixtures/tui/plan-120x40.txt | 4 +- tests/fixtures/tui/plan-40x12.txt | 4 +- tests/fixtures/tui/plan-40x15.txt | 4 +- tests/fixtures/tui/plan-80x24.txt | 4 +- tests/fixtures/tui/plan-in-place-120x40.txt | 2 +- tests/fixtures/tui/plan-in-place-40x12.txt | 4 +- tests/fixtures/tui/plan-in-place-40x15.txt | 4 +- tests/fixtures/tui/plan-in-place-80x24.txt | 2 +- tests/fixtures/tui/result-120x40.txt | 2 +- tests/fixtures/tui/result-40x12.txt | 2 +- tests/fixtures/tui/result-40x15.txt | 2 +- tests/fixtures/tui/result-80x24.txt | 4 +- tests/test_install_from.py | 3 +- tests/test_install_tui.py | 481 +++++++++++++++++- wiki/installer.md | 32 +- 26 files changed, 999 insertions(+), 111 deletions(-) diff --git a/install.ps1 b/install.ps1 index 0a3b78fc..379d2f1f 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "bcbde4c289b32c02ada78001b75274a87b125c1efd3d8fb1064d7f5482ef3997" + $defaultInstallSha256 = "7ac3e6030b12bacd262964fea5373f27fb09e62296255bc3c22255a1aaad5fe8" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 7f816c98..14c9e699 100644 --- a/install.py +++ b/install.py @@ -7594,6 +7594,11 @@ def main() -> None: if args.cores_dir: args.installed_cores = True _INVOCATION.record(args) + if os.environ.get(TUI_LIST_ENV): + # The full screen's own copy of the list, deleted when it closes: a + # command naming it fails once the screen is gone. + _INVOCATION.options = [option for option in _INVOCATION.options + if option[0] != "--manifest"] if not 1 <= args.jobs <= 32: parser.error("--jobs must be between 1 and 32") if args.region: @@ -7818,6 +7823,106 @@ def card_platform(root: Path) -> str | None: return None +# A system that runs from its SD card names the card by where the device +# mounts it: on a computer the card's root stands in for that mount point, +# and its device rule then reads the same below the root. +_CARD_MOUNTS = ("/mnt/SDCARD/", "/mnt/mmc/") + +# Systems whose card a computer sees as their data partition, by the file +# their own build writes there, paths below that partition. KNULLI's is +# /userdata: S12populateshare:15,105 copies datainit there, and every board's +# datainit/system/batocera.conf sets system.hostname=KNULLI (knulli-cfw +# distribution ab3a6809, board/batocera/*/fsoverlay/usr/share/batocera/). +# ROCKNIX's is /storage: userconfig-setup:12 copies /usr/config into +# /storage/.config, and its system.cfg sets root.password=rocknix (ROCKNIX +# distribution 8b70f693, projects/ROCKNIX/packages/rocknix/config/system/ +# configs/system.cfg:152). +_SHARE_RULES: tuple[_OsRule, ...] = ( + _OsRule(_STANDIN_MANIFESTS["KNULLI"], "KNULLI", "file_matches", + ("system/batocera.conf", r"^system\.hostname=KNULLI\s*$"), "bios"), + _OsRule("rocknix", "ROCKNIX", "file_matches", + (".config/system/configs/system.cfg", r"^root\.password=rocknix\s*$"), + "roms/bios"), +) + + +def _card_rules() -> list[_OsRule]: + """The device rules that name a card, their paths below its root.""" + rules = list(_SHARE_RULES) + for rule in _EMBEDDED_RULES: + if rule.method != "file_exists" or not rule.manifest or not rule.bios: + continue + mount = next((m for m in _CARD_MOUNTS if rule.bios.startswith(m) + and all(value.startswith(m) for value in rule.values)), "") + if mount: + rules.append(rule._replace( + values=tuple(value[len(mount):] for value in rule.values), + bios=rule.bios[len(mount):], configs=())) + return rules + + +def _card_rule_matches(rule: _OsRule, root: Path) -> bool: + if rule.method == "file_matches": + path, pattern = rule.values + try: + text = (root / path).read_text(encoding="utf-8", errors="replace") + except OSError: + return False + return re.search(pattern, text, re.MULTILINE) is not None + return all((root / value).exists() for value in rule.values) + + +def card_system(root: Path) -> tuple[str, Path, str] | None: + """(file list, BIOS folder, system name) of the card mounted at root. + + None when nothing on it names a system, or when two systems that would + write to different folders both match: the card is then the user's to + name, never a guess. + """ + plat = card_platform(root) + if plat: + return plat, root, display_name(plat) + matched = [rule for rule in _card_rules() if _card_rule_matches(rule, root)] + bases = {rule.base for rule in matched if rule.base} + matched = [rule for rule in matched if rule.name not in bases] + targets = {(rule.manifest, rule.bios) for rule in matched} + if len(targets) != 1: + return None + rule = matched[0] + return rule.manifest, root / rule.bios, rule.name + + +def card_dest(named: str, plat: str, dest: Path) -> tuple[str, Path]: + """The file list and folder for a --dest that names a handheld's card. + + The card's root is where the user points, its BIOS folder is below it: + `--platform muos --dest /media/me/SD1` wrote to the root, which muOS + never reads. A system named on the command line whose card this is not + is refused with the folder it reads. + """ + found = card_system(dest) + if found and (not plat or found[0] == plat) and found[1] != dest: + print(f" {found[2]} card: {display_name(found[0])} files, in {found[1]}") + return found[0], found[1] + # Only a system named by its own name: --platform retroarch or knulli + # says which list, not which card. + key = _name_key(named) + rule = next((rule for rule in _card_rules() if key and key == _name_key(rule.name) + and key != _name_key(display_name(rule.manifest))), None) + if rule is None or found: + return plat, dest + # Its folder on the card, or one its device reads (/run/muos/storage/bios). + own = {device.bios for device in _EMBEDDED_RULES if device.name == rule.name} + if dest.as_posix() in own or dest.as_posix().casefold().endswith(rule.bios.casefold()): + return plat, dest + marker = rule.values[0].split("/")[0] + print(f"Error: {dest} is not a {rule.name} card: it holds no {marker}. " + f"{rule.name} reads its BIOS files from {rule.bios} on its card: name the card's " + f"root (the folder holding {marker}) or that folder: --dest /{rule.bios}", + file=sys.stderr) + sys.exit(1) + + def _check_card_root(plat: str, dest: Path) -> None: """Refuse a folder inside the card where its root is meant. @@ -7915,8 +8020,12 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # Platform detection or override. A destination taken from the default # table, the platform not being found here, is only a guess. assumed_dests: set[Path] = set() + named = args.platform or "" if args.platform: args.platform = normalize_platform(args.platform) + if args.dest: + plat, args.dest = card_dest(named, args.platform or "", args.dest) + args.platform = plat or args.platform if args.platform and args.dest: if args.platform in CARD_PLATFORMS: _check_card_root(args.platform, args.dest) @@ -8638,6 +8747,8 @@ TUI_MAX_WIDTH = 100 # past this wait an Esc with nothing after it is the key. TUI_ESC_WAIT = 0.04 TUI_FRAME = 0.1 +# Set for a child run handed the interface's own copy of the file list. +TUI_LIST_ENV = "RETROBIOS_TUI_LIST" _PASTE_START = "\x1b[200~" _PASTE_END = "\x1b[201~" # Turned on by the interface, so a pasted path arrives as one piece and its @@ -8971,8 +9082,9 @@ class TuiState: self.found: list = [] # What the browser is choosing: the BIOS folder, or one to copy from. self.browse_for: str = "dest" - # Files an install copies from this machine, whose bytes are no download. - self.local_dests: set = set() + # Files an install copies from this machine, whose bytes are no + # download: dest to (folder, reason), reason set for a misplaced one. + self.local_dests: dict = {} self.local_done: int = 0 # The child run in progress: check, confirm, preview or install. self.run: str = "" @@ -8987,6 +9099,9 @@ class TuiState: self.dry: dict | None = None self.dry_files: list = [] self.dry_summary: dict | None = None + # The confirm screen waits for A even when the plan is unchanged: + # it was opened by Space, which never installs by itself. + self.must_confirm: bool = False # Choices, each one an option of the command line. self.ticked: set = set() self.cli_systems: list = [] @@ -9161,6 +9276,8 @@ def _tui_system_label(row: dict, titles: dict | None = None) -> str: title = (titles or {}).get(row["system"]) if isinstance(title, str) and title.strip(): return title.strip() + if row["system"] == OTHER_SYSTEMS: + return OTHER_TITLE name = row.get("name") or row["system"] if name == row["system"]: return name @@ -9169,6 +9286,12 @@ def _tui_system_label(row: dict, titles: dict | None = None) -> str: return f"{row['system']} ({name})" +# What the "other" row of the systems holds, in words (OTHER_SYSTEMS). +OTHER_TITLE = "Other systems and game engines" +OTHER_HELP = ("Other systems and game engines: what cores load for systems this setup " + "does not list, such as game engines and extra computers.") + + def _matches(text: str, needle: str) -> bool: return not needle or _folded(needle) in _folded(text) @@ -9279,8 +9402,9 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: size = check.get("bytes_needed", 0) label = f"Install {format_size(size)}" if size >= CONFIRM_FROM_BYTES else "Install" rows = [TuiRow("install", label)] - if state.facts.get("writable") is False: - # Every file would fail alike: the way out is another folder. + if state.facts.get("writable") is False or _short_of_space(state, check): + # Every file would fail alike, or the run would stop before the + # first one: the way out is another folder. rows = [TuiRow("folder", "Choose another folder"), TuiRow("install", "Install anyway"), *rows[1:]] if records_selection((state.manifest or {}).get("files", [])) and len(state.systems) > 1: @@ -9291,23 +9415,51 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: return rows + [TuiRow("options", "Options"), TuiRow("quit", "Quit")] +def _short_of_space(state: TuiState, check: dict) -> bool: + free = state.facts.get("free") + return free is not None and check.get("bytes_needed", 0) > free + + def _system_rows(state: TuiState) -> list[TuiRow]: needle = state.filters.get("systems", "") titles = _system_titles(state) rows = [TuiRow("all", "All"), TuiRow("none", "None"), _filter_row(state)] + shared = _shared_needed(state) for row in _sorted_systems(state): label = _tui_system_label(row, titles) if not _matches(f"{label} {row['system']}", needle): continue box = "[x]" if row["system"] in state.ticked else "[ ]" need = _needed(row) - right = (f"{_plural(need, 'file')}, {format_size(row['bytes_needed'])}" - if need else "in place") + count, size = shared.get(row["system"], (0, 0)) + own, own_size = need - count, row["bytes_needed"] - size + if not need: + right = "in place" + elif not count: + right = f"{_plural(need, 'file')}, {format_size(row['bytes_needed'])}" + elif not own: + right = f"{_plural(need, 'file')}, {format_size(size)}, all shared" + else: + right = f"{_plural(own, 'file')}, {format_size(own_size)} + {count} shared" rows.append(TuiRow("system", f"{box} {label}", right, row["system"], "" if need else "dim")) return rows +def _shared_needed(state: TuiState) -> dict[str, tuple[int, int]]: + """Per system, the files to download that other systems carry too, and + their size: `3DO 1704 files, 349.0 MB` read as the 3DO BIOS, when 1689 + are the MAME files six systems share. The line mode counts them apart + the same way (_system_sized).""" + shared: dict[str, tuple[int, int]] = {} + for event in state.files: + if len(event["systems"]) > 1: + for system in event["systems"]: + count, size = shared.get(system, (0, 0)) + shared[system] = (count + 1, size + event["size"]) + return shared + + def _cores_dir_given(state: TuiState) -> bool: return any(flag == "--cores-dir" for flag, _value in state.passthrough) @@ -9325,7 +9477,9 @@ def _option_rows(state: TuiState) -> list[TuiRow]: + (", ".join(map(region_title, state.regions)) or "any"), "--region")) if manifest.get("categorized"): - rows.append(TuiRow("game_data", f"Leave out game data: {_yes(state.no_game_data)}", + # "Leave out game data: no" was a double negative. + rows.append(TuiRow("game_data", "Game data: " + + ("left out, firmware only" if state.no_game_data else "kept"), "--no-game-data")) found = len(state.facts.get("cores", [])) if _cores_dir_given(state): @@ -9343,7 +9497,7 @@ def _option_rows(state: TuiState) -> list[TuiRow]: TuiRow("jobs", f"Parallel downloads: {state.jobs}", "--jobs"), TuiRow("folder", f"BIOS folder: {_short_path(state.dest, state.home)}", "--dest", style="path"), - TuiRow("from", f"Copy from: {copied}", "--from"), + TuiRow("from", f"Copy from: {copied}", "--from", style="path"), TuiRow("preview", "Preview the file list", "--dry-run"), TuiRow("command", "Show the command line"), ] @@ -9420,7 +9574,7 @@ def _start_run(state: TuiState, run: str) -> list[tuple]: state.dry, state.dry_files, state.dry_summary = None, [], None else: state.download = state.progress = state.reading = None - state.samples, state.local_dests, state.local_done = [], set(), 0 + state.samples, state.local_dests, state.local_done = [], {}, 0 state.finished, state.summary, state.code, state.stopping = [], None, None, False return [("run", run)] @@ -9442,6 +9596,10 @@ def _start_check(state: TuiState) -> list[tuple]: def _choose(state: TuiState, plat: str, dest: Path) -> list[tuple]: + card = card_system(dest) + if card and card[0] == plat and card[1] != dest: + # A muOS or KNULLI card's root: that system reads a folder below it. + dest = card[1] state.platform, state.dest = plat, dest if plat in CARD_PLATFORMS and card_platform(dest) is None: # The engine refuses a folder inside the card, the list writing @@ -9466,6 +9624,11 @@ def _open_folder_screen(state: TuiState, default: Path | None) -> list[tuple]: return _push(state, "folder") +def _mount_name(mount: Path) -> str: + """How a person names a mounted card: E:\\ on Windows, its label elsewhere.""" + return mount.name or str(mount) + + def _mounted_cards() -> list[Path]: """Mount points of removable media, where a card or a share appears. @@ -9507,8 +9670,8 @@ def _browse_rows(folder: Path | None, zips: bool = False) -> tuple[list[TuiRow], rows = [TuiRow("use", "Use this folder"), TuiRow("up", "..", value=folder.parent)] for card in _mounted_cards(): if card != folder: - plat = card_platform(card) - what = f"{display_name(plat)} card" if plat else "card or drive" + system = card_system(card) + what = f"{system[2]} card" if system else "card or drive" rows.append(TuiRow("dir", f"{card} ({what})", value=card)) try: entries = list(os.scandir(folder)) @@ -9698,9 +9861,10 @@ def _option(state: TuiState, action: str, step: int) -> list[tuple]: elif action == "folder": return _open_folder_screen(state, state.dest) elif action == "from" and step < 0: + # Said, and the screen stays: Back to the plan counts again. if state.from_folders: - state.from_folders.pop() - return _start_check(state) + removed = state.from_folders.pop() + state.status = f"No longer copying from {_last_name(str(removed))}." elif action == "from": return _open_from_browser(state) elif action == "preview": @@ -9740,6 +9904,7 @@ def _begin_install(state: TuiState) -> list[tuple]: if state.systems and not state.ticked: state.status = "No system is ticked: tick one, or choose All." return [] + state.must_confirm = False if tui_flags(state) != state.checked_flags: _push(state, "confirm") return _start_run(state, "confirm") @@ -9965,12 +10130,34 @@ def tui_update(state: TuiState, key: TuiKey) -> list[tuple]: return _option(state, row.action, 1 if name == "right" else -1) \ if row.action in ("game_data", "installed", "jobs") \ or (row.action == "from" and name == "left") else [] - elif name == "enter" or char == " ": + elif char == " ": + return _space(state, row) + elif name == "enter": return _activate(state, row) state.focus[screen] = focus return [] +# Rows Space leaves alone: it is X (tick) on PortMaster and Y on muxterm, a +# key pressed for something else, and these end the run or write. +_SPACE_INERT = ("install", "close", "quit") + + +def _space(state: TuiState, row: TuiRow) -> list[tuple]: + """Space opens what a row leads to, and never installs by itself: on + the plan's Install it opens the confirm screen, where A installs.""" + if row.action == "install" and state.screen == "plan": + if state.systems and not state.ticked: + state.status = "No system is ticked: tick one, or choose All." + return [] + state.must_confirm = True + _push(state, "confirm") + return _start_run(state, "confirm") + if row.action in _SPACE_INERT: + return [] + return _activate(state, row) + + def _take_event(state: TuiState, event: dict) -> None: kind = event.get("event") if kind == "checking" and state.run in ("check", "install"): @@ -9978,7 +10165,7 @@ def _take_event(state: TuiState, event: dict) -> None: elif kind == "found" and state.run == "check": state.found.append(event) elif kind == "found" and state.run == "install": - state.local_dests.add(event["dest"]) + state.local_dests[event["dest"]] = (event["folder"], event["reason"]) elif state.run == "check": if kind == "check": state.check = event @@ -10049,7 +10236,7 @@ def tui_receive(state: TuiState, message: tuple) -> list[tuple]: elif state.run == "confirm" and state.screen == "confirm": if state.dry is None: return _fail(state, tail, code) - if _same_plan(state.check, state.dry): + if _same_plan(state.check, state.dry) and not state.must_confirm: state.stack.pop() return _install_now(state) elif state.run == "install": @@ -10110,6 +10297,10 @@ def _text_lines(items: list, width: int) -> list[TuiLine]: lines: list[TuiLine] = [] for item in items: style = getattr(item, "style", "") + if style == "path": + # One line, cut in its middle: the end of a path tells it apart. + lines.append(_styled(middle(str(item), width))) + continue for piece in _wrap(str(item), width): lines.append(_styled(fit(piece, width), style)) return lines @@ -10161,7 +10352,9 @@ def _compose( return body + footer_lines -def _pager(state: TuiState, width: int, height: int, items: list) -> list[TuiLine]: +def _pager( + state: TuiState, width: int, height: int, items: list, lead: int = 0, +) -> list[TuiLine]: """The items in view, from the first shown on, each wrapped alone: the 8,000 lines of a full preview were wrapped again at every key.""" first = max(0, min(state.focus.get(state.screen, 0), _pager_last(state, items))) @@ -10177,7 +10370,9 @@ def _pager(state: TuiState, width: int, height: int, items: list) -> list[TuiLin index += 1 whole = index if len(lines) <= room else index - 1 lines = lines[:room] - return lines + [_styled(f"{first + 1}-{max(first + 1, whole)} of {len(items)}", "dim")] + start = max(1, first + 1 - lead) + return lines + [_styled(f"{start}-{max(start, whole - lead)} of {len(items) - lead}", + "dim")] def _tui_misplaced(state: TuiState) -> list[dict]: @@ -10194,6 +10389,11 @@ def _tui_found_elsewhere(state: TuiState) -> dict[str, list[dict]]: return found +def _last_name(folder: str) -> str: + """A folder or ZIP by its last name, a drive by itself (E:\\).""" + return Path(folder).name or folder + + def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: need = _needed(check) data = check.get("data_missing", 0) @@ -10206,7 +10406,9 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: fetch_bytes = max(0, check["bytes_needed"] - sum(sizes[dest_] for dest_ in copied)) elsewhere = _tui_found_elsewhere(state) if check is state.check else {} if elsewhere: - where = ", ".join(f"{len(events)} found in {folder}" + # The folder by its last name: the whole path pushed the BIOS + # folder off a 40x15 screen. The Options row spells it out. + where = ", ".join(f"{len(events)} found in {_last_name(folder)}" for folder, events in elsewhere.items()) headline = (f"{where}, {fetch} to download ({format_size(fetch_bytes)})" if fetch else f"{where}, nothing to download") @@ -10221,35 +10423,46 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: headline = (f"{_plural(data, 'data file')} to fetch" if data else "Everything is in place") wrong = check["wrong_hash"] + # The folder comes second: on a 40x12 screen it is what is left once + # the rows are drawn, and nothing is written before it is seen. lines: list = [ _styled(headline, "title"), - f"{check['present']} in place, {check['missing']} missing, " - f"{wrong} wrong {'copy' if wrong == 1 else 'copies'}", - f"BIOS folder: {_short_path(dest, state.home)}", + _styled(f"BIOS folder: {_short_path(dest, state.home)}", "path"), ] + if state.facts.get("writable") is False: + lines.append("You cannot write to this folder: choose another one.") + lines.append(f"{check['present']} in place, {check['missing']} missing, " + f"{wrong} wrong {'copy' if wrong == 1 else 'copies'}") + found_in = _RETROARCH_CONFIGS.get(dest) if dest is not None else None + if found_in: + lines.append(_styled(f"Named by {_short_path(Path(found_in[0]), state.home)}", "path")) free = state.facts.get("free") if free is not None: lines.append(f"Free space: {format_size(free)}") if check["bytes_needed"] > free: - lines.append("Not enough free space for this download: free some, or " - "choose another folder in Options.") + lines.append(f"Not enough free space: {format_size(check['bytes_needed'])} " + "needed. Free some, or choose another folder.") if wrong: lines.append("They are replaced, and the old copies are kept in a backup folder.") misplaced = _tui_misplaced(state) if check is state.check else [] if misplaced: - lines.insert(1, f"{_plural(len(misplaced), 'file')} " + lines.insert(3, f"{_plural(len(misplaced), 'file')} " f"{'is' if len(misplaced) == 1 else 'are'} in the wrong place, " f"{'it' if len(misplaced) == 1 else 'they'} will be copied where the " "emulators read " f"{'it' if len(misplaced) == 1 else 'them'}.") if state.facts.get("writable") is False: - lines.insert(1, "You cannot write to this folder: choose another one, or run " - + ("the installer as an administrator." if state.os_type == "windows" - else "the installer with sudo.")) + if state.os_type == "windows": + lines.append("Or run the installer as an administrator.") + else: + # sudo changes the home folder, and with it what is detected: + # the folder is named so root writes here. + hint = Invocation() + hint.os_type = state.os_type + lines.append("Or run it as root, this folder named: " + + hint.command(state.platform, dest, elevated=True)) if check.get("not_installable"): - lines.append(f"{_plural(check['not_installable'], 'file')} of the list " - f"{'is' if check['not_installable'] == 1 else 'are'} not in " - "the collection.") + lines += _unavailable_lines(state) return lines @@ -10367,14 +10580,17 @@ def _choice_line(state: TuiState) -> list[str]: def _view_plan(state: TuiState, width: int, height: int) -> tuple: check = state.check or {} lines = _plan_lines(check, state.dest, state) - lines[1:1] = _choice_line(state) + lines[2:2] = _choice_line(state) if state.status: lines.append(state.status) rows = tui_rows(state) if width < 54: rows = [row._replace(label="Copy from your old card or a folder") if row.action == "fromcard" else row for row in rows] - return (display_name(state.platform, state.manifest), + # A muOS card gets RetroArch's files: the title says both. + system = detected_os(state.platform, state.dest) if state.dest is not None else "" + name = display_name(state.platform, state.manifest) + return (f"{system} ({name} files)" if system and system != name else name, _compose(state, width, height, lines, rows), ("A/Enter Install", "B/Esc Back", "Y/o Options", "? Help")) @@ -10398,7 +10614,7 @@ def _view_systems(state: TuiState, width: int, height: int) -> tuple: if width < 60: # The name first: the size alone stays beside it, the count is in # the total below. - rows = [row._replace(right=row.right.split(", ")[-1]) for row in rows] + rows = [row._replace(right=re.sub(r"^\d+ files?, ", "", row.right)) for row in rows] elif width >= 78: # The id --system takes, beside a name that does not show it. # Only where it fits whole: a cut id ("maker26-co") said nothing. @@ -10411,6 +10627,13 @@ def _view_systems(state: TuiState, width: int, height: int) -> tuple: f"{_plural(len(chosen), 'file')}, " f"{format_size(sum(e['size'] for e in chosen))}" + (" before your options" if _counts_predate_options(state) else "")] + focused = rows[min(state.focus.get("systems", 0), len(rows) - 1)] + if focused.value == OTHER_SYSTEMS: + footer.insert(0, OTHER_HELP) + elif "shared" in focused.right: + count = _shared_needed(state)[focused.value][0] + footer.insert(0, f"{count} of these files are shared with other systems, " + "counted once in the total.") needle = state.filters.get("systems", "") after = [f"No system matches '{needle}'."] if len(rows) == 3 and needle else [] if state.status: @@ -10433,12 +10656,19 @@ def _view_options(state: TuiState, width: int, height: int) -> tuple: rows = [row._replace(right="") for row in rows] # A folder is cut in its middle, its last levels being what tells it apart. room = width - 4 - (0 if width < 60 else 2 + max(cell_width(r.right) for r in rows)) - rows = [row._replace(label="BIOS folder: " + middle(row.label[13:], room - 13), style="") - if row.style == "path" else row for row in rows] + rows = [_middle_label(row, room) if row.style == "path" else row for row in rows] + if state.status: + after.append(state.status) return ("Options", _compose(state, width, height, before, rows, after), ("A/Enter Change", "B/Esc Back", "Left/Right Set", "? Help")) +def _middle_label(row: TuiRow, room: int) -> TuiRow: + """A row naming a folder, the folder cut in its middle to fit.""" + name, _sep, value = row.label.partition(": ") + return row._replace(label=f"{name}: " + middle(value, room - len(name) - 2), style="") + + def _view_regions(state: TuiState, width: int, height: int) -> tuple: before = ["Tick the regions to keep, best first. Where a system has one BIOS " "per region, the best one found is installed."] @@ -10480,7 +10710,9 @@ def _pager_items(state: TuiState) -> list: def _view_preview(state: TuiState, width: int, height: int) -> tuple: - return ("Preview", _pager(state, width, height, _pager_items(state)), + # The count and the blank line above the files are not among them. + lead = 2 if state.dry is not None and not state.running else 0 + return ("Preview", _pager(state, width, height, _pager_items(state), lead), ("A/Enter Back", "Up/Down Scroll")) @@ -10548,6 +10780,83 @@ def _view_installing(state: TuiState, width: int, height: int) -> tuple: ("B/Esc Stop", "? Help")) +_SPACE_NEEDED = re.compile(r"Needed: (\S+ \S+)\. Free: (\S+ \S+)\.") + + +def _space_stop(state: TuiState) -> re.Match | None: + """Needed and free, when the run stopped for lack of space.""" + return next(filter(None, (_SPACE_NEEDED.search(line) for line in state.tail)), None) + + +def _space_words(state: TuiState) -> str: + """The run's stop for lack of space, in words, "" for another stop.""" + match = _space_stop(state) + if match is None: + return "" + dest = (state.dest or Path(".")).absolute() + card = any((mount == dest or mount in dest.parents) and card_system(mount) + for mount in _mounted_cards()) + return f"The {'card' if card else 'drive'} is full: {match[1]} needed, {match[2]} free." + + +def _tui_sources(state: TuiState, installed: int, failed: int) -> str: + """Where the installed files came from, as the line mode says it: + "6 files copied from /media/me/OLD, 42 downloaded, 2 still missing." """ + ok = {dest for dest, good, _why in state.finished if good} + folders: dict[str, int] = {} + fixed = 0 + for dest, (folder, reason) in state.local_dests.items(): + if dest not in ok: + continue + if reason: + fixed += 1 + else: + # By its name, as the plan said "6 found in old card". + name = _last_name(folder) + folders[name] = folders.get(name, 0) + 1 + copied = fixed + sum(folders.values()) + return sources_line(list(folders.items()), fixed, max(0, installed - copied), failed) + + +def _unavailable(state: TuiState) -> tuple[list[dict], int]: + """The list's files RetroBIOS cannot supply for the systems chosen: the + ones an emulator needs to start, and how many others.""" + listed = {row["system"] for row in state.systems} + chosen = state.ticked if state.systems else None + needed, others = [], 0 + for entry in (state.manifest or {}).get("omitted_files") or []: + system = entry.get("system") if entry.get("system") in listed else OTHER_SYSTEMS + if chosen is not None and system not in chosen: + continue + if state.no_game_data and entry.get("category") == "game_data": + continue + if entry.get("required"): + needed.append(entry) + else: + others += 1 + return needed, others + + +def _unavailable_lines(state: TuiState) -> list[str]: + """The files that cannot be supplied and stop an emulator, by name and + system: "1 file ... an emulator needs it" named neither.""" + needed, _others = _unavailable(state) + if not needed: + return [] + titles = _system_titles(state) + rows = {row["system"]: row for row in state.systems} + + def system(name: str) -> str: + return _tui_system_label(rows.get(name) or {"system": name}, titles) + + names = [f"{entry['dest']} ({system(entry['system'])})" if entry.get("system") + else entry["dest"] for entry in needed[:3]] + more = f" and {len(needed) - 3} more" if len(needed) > 3 else "" + return [f"Not available here, and needed to start " + f"{'its emulator' if len(needed) == 1 else 'their emulators'}: " + f"{', '.join(names)}{more}. Your own copy goes in the BIOS folder."] + + def tui_outcome(state: TuiState) -> tuple[str, list[str]]: """The result in words, first what happened, then where and what next.""" summary = state.summary or {} @@ -10557,6 +10866,7 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: failed = summary.get("failed") or [] up_to_date = summary.get("up_to_date", 0) total = (state.download or {}).get("files", 0) + space = "" if summary else _space_words(state) if summary.get("interrupted") or state.code == 130: title = "Stopped" head = (f"Stopped: {installed} of {total} installed, run again to finish." @@ -10567,6 +10877,9 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: elif failed: title = "Nothing installed" head = f"Nothing was installed: {_plural(len(failed), 'file')} failed." + elif not summary and space: + title = "Not enough space" + head = space elif not summary: title = "Stopped" head = f"The installer stopped with code {state.code}." @@ -10578,7 +10891,13 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: else: title = "Done" head = "Everything is in place." - lines = [head, f"BIOS folder: {state.dest}"] + lines = [head] + sources = _tui_sources(state, installed, len(failed)) + if sources: + lines.append(sources) + lines.append(f"BIOS folder: {state.dest}") + if space: + lines.append("Free some space, or run again and choose another folder.") causes: dict[str, list[str]] = {} for failure in failed: causes.setdefault(failure["reason"], []).append(failure["dest"]) @@ -10586,8 +10905,8 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: for reason, dests in causes.items(): if len(dests) > 3: # A read-only folder fails every file alike: said once. - lines.append(f" {len(dests)} files failed ({reason or 'no reason given'}), " - f"the first {dests[0]}") + lines.append(f" {len(dests)} files failed, all for one reason: " + f"{reason or 'none given'}.") continue for dest in dests: if shown < 20: @@ -10595,7 +10914,7 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: shown += 1 if shown > 20: lines.append(f" ... and {shown - 20} more") - if not summary: + if not summary and not space: lines += [f" {line}" for line in state.tail[-8:]] run = Invocation() run.os_type = state.os_type @@ -10603,15 +10922,8 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: lines.append(f"The files it replaced are kept in {kept['folder']}") lines.append(f"To put them back: {run.launcher()} --restore " f"{run.word(str(kept['folder']))}") - skipped = summary.get("skipped") or [] - if skipped: - count = sum(group["files"] for group in skipped) - needed = sum(group["required"] for group in skipped) - lines.append(f"{_plural(count, 'file')} of the list " - f"{'is' if count == 1 else 'are'} not in the collection" - + ("." if not needed - else ", and an emulator needs it to start." if count == 1 - else f", {needed} of them needed to start an emulator.")) + if summary and not summary.get("interrupted") and installed + up_to_date: + lines += _unavailable_lines(state) for group in summary.get("left_out") or []: lines.append(f"Left out as {group['label']}: {_plural(group['files'], 'file')}.") lines += ["Same choice without questions:", f" {tui_command(state)}"] @@ -10625,7 +10937,9 @@ def tui_keeps_log(state: TuiState) -> bool: 57 folders of logs from stopped and finished runs were left in /tmp.""" if state.screen == "crashed": return True - return bool(state.log) and state.code not in (0, 130, None) + # A full card is said in words on the screen: its log adds nothing. + return (bool(state.log) and state.code not in (0, 130, None) + and _space_stop(state) is None) def _view_result(state: TuiState, width: int, height: int) -> tuple: @@ -10750,15 +11064,21 @@ def _install_kind(path: Path) -> str: return word if (path.parent / "retroarch.cfg").is_file(): return "portable" - return "/".join(path.parts[-2:]) + return "" def tell_apart(setups: list[tuple]) -> list[tuple]: """Two setups that read alike get the word that separates them: the - Flatpak and the native RetroArch were both "RetroArch" at 40 columns.""" + Flatpak and the native RetroArch were both "RetroArch" at 40 columns. + The one no package names is "installed", and two of those keep the + end of their folder, the only thing that tells them apart.""" labels = [label for _plat, _path, label in setups] - return [(plat, path, f"{label} ({_install_kind(path)})" if labels.count(label) > 1 - else label) for plat, path, label in setups] + kinds = [_install_kind(path) or "installed" for _plat, path, _label in setups] + named = list(zip(labels, kinds)) + return [(plat, path, label if labels.count(label) == 1 + else f"{label} ({kind})" if named.count((label, kind)) == 1 + else f"{label} ({'/'.join(path.parts[-2:])})") + for (plat, path, label), kind in zip(setups, kinds)] def _tui_passthrough(args: argparse.Namespace) -> list[tuple[str, str | None]]: @@ -10805,10 +11125,16 @@ def tui_start( with contextlib.redirect_stdout(captured), contextlib.redirect_stderr(captured): found = detect_platforms(os_type) frontends = detect_frontends(os_type) - cards = [(card_platform(mount), mount) for mount in _mounted_cards()] + cards = [(card_system(mount), mount) for mount in _mounted_cards()] state.setups = [(name, path, _setup_label(name, path)) for name, path in found] - state.setups += [(card, mount, f"{display_name(card)} SD card") - for card, mount in cards if card and (card, mount) not in found] + # On the device itself its own card is the system found, by its + # own path (muOS reads /run/muos/storage/bios, a bind of MUOS/bios). + here = {detected_os(name, path) for name, path in found} + for card, mount in cards: + if card and (card[0], card[1]) not in found and card[2] not in here: + _DETECTED_OS.setdefault((card[0], card[1]), card[2]) + state.setups.append((card[0], card[1], + f"{card[2]} SD card ({_mount_name(mount)})")) state.setups = tell_apart(state.setups) if "esde" in frontends: state.notes.append("ES-DE found: its BIOS files go to each emulator, " @@ -10859,11 +11185,16 @@ def _log_tail(path: Path, count: int = 12) -> list[str]: class _EngineRun: """This installer run as a child with --events, read line by line.""" - def __init__(self, argv: list[str], log: Path, inbox: queue.Queue, run_id: int) -> None: + def __init__( + self, argv: list[str], log: Path, inbox: queue.Queue, run_id: int, + env: dict | None = None, + ) -> None: self.log = log # Its own session: a signal typed at the terminal reaches the # interface alone, which forwards one Ctrl-C and waits for the summary. options: dict = {} if os.name == "nt" else {"start_new_session": True} + if env is not None: + options["env"] = env with open(log, "w", encoding="utf-8", errors="replace") as handle: self.proc = subprocess.Popen( argv, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=handle, @@ -10938,6 +11269,10 @@ def _tui_fetch(plat: str, inbox: queue.Queue, log: io.StringIO) -> None: return except ValueError as exc: title, lines = "The file list is not valid", [f"What arrived is not a file list: {exc}."] + if isinstance(exc.__cause__, (json.JSONDecodeError, UnicodeDecodeError)): + # The parser's "Expecting value: line 2 column 1" told no one anything. + lines = ["What arrived is not a file list: a Wi-Fi sign-in page or a cut " + "download looks like this.", "Check the connection, then choose Try again."] except urllib.error.HTTPError as exc: title = "No file list" lines = [f"There is no file list for {display_name(plat)} at this address." @@ -10946,10 +11281,18 @@ def _tui_fetch(plat: str, inbox: queue.Queue, log: io.StringIO) -> None: code, words = network_words(exc) title = {"clock": "The clock is wrong", "no_ca": "No trusted certificates"}.get(code, "No connection") - lines = [words[0].upper() + words[1:] + "."] + # The screen's first row is Try again: "then run again" was a second way. + lines = [words[0].upper() + words[1:].replace(", then run again", + ", then choose Try again") + "."] if code == "network": lines = [f"The list of BIOS files could not be downloaded: {words}.", "Check the Wi-Fi or the cable, then choose Try again."] + proxy = next((name for name in ("HTTPS_PROXY", "https_proxy", "ALL_PROXY", + "all_proxy") if os.environ.get(name)), "") + if proxy: + # Named, never shown: the value can hold a password. + lines.append(f"{proxy} is set, so the connection goes through a proxy: " + "check that it answers.") if os.environ.get("RETROBIOS_BASE_URL"): lines.append(f"RETROBIOS_BASE_URL chose the address: {url}") inbox.put(("manifest_failed", plat, title, lines, 1)) @@ -11078,6 +11421,66 @@ class _TuiTerminal: time.sleep(0.01) +TUI_WORK_PREFIX = "retrobios-tui-" +# The interface's process id, in its work folder: a folder whose process is +# gone was left by a kill no handler sees (SIGKILL, a power cut). +TUI_OWNER = "owner" +# Written when a failed run's log is kept: it stays a week, then goes. +TUI_KEPT = "kept" +TUI_KEEP_SECONDS = 7 * 24 * 3600 + + +def _owner_alive(folder: Path) -> bool | None: + """Whether the run that made folder is still running, None when unknown.""" + try: + pid = int((folder / TUI_OWNER).read_text(encoding="utf-8").strip()) + except (OSError, ValueError): + return None + if os.name == "nt": + # os.kill(pid, 0) ends a process on Windows instead of asking. + return None + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + except OSError: + return None + return True + + +def _mark_kept(work: Path) -> None: + try: + (work / TUI_KEPT).write_text("", encoding="utf-8") + except OSError as exc: + # The log is still there, only its week is not counted. + print(f"The log folder {work} could not be marked: {exc}", file=sys.stderr) + + +def _prune_work_folders(root: Path, now: float) -> list[Path]: + """Remove what earlier runs of the interface left in root: the folder of + a run killed outright, and a failed run's log after a week. A folder + whose run is still going, or unknown and recent, stays.""" + removed = [] + try: + folders = [path for path in root.glob(TUI_WORK_PREFIX + "*") if path.is_dir()] + except OSError: + return removed + for folder in folders: + try: + age = now - folder.stat().st_mtime + kept = (folder / TUI_KEPT).exists() + except OSError: + continue + alive = _owner_alive(folder) + if alive or ((alive is None or kept) and age < TUI_KEEP_SECONDS): + continue + shutil.rmtree(folder, ignore_errors=True) + removed.append(folder) + return removed + + def tui_farewell(state: TuiState) -> list[str]: """What stays in the terminal once the full screen is gone.""" return [screen_text(line) for line in _farewell_lines(state)] @@ -11109,7 +11512,9 @@ def run_tui(args: argparse.Namespace) -> int: _enter_wsl() _open_local_sources(args) state, effects = tui_start(args, os_type) - work = Path(tempfile.mkdtemp(prefix="retrobios-tui-")) + work = Path(tempfile.mkdtemp(prefix=TUI_WORK_PREFIX)) + (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") + _prune_work_folders(work.parent, time.time()) script = str(Path(__file__).resolve()) inbox: queue.Queue = queue.Queue() current: list[_EngineRun] = [] @@ -11144,10 +11549,11 @@ def run_tui(args: argparse.Namespace) -> int: log = work / f"{effect[1]}.log" if effect[1] == "install": state.log = str(log) + own_list = state.manifest_path in written.values() try: current.append(_EngineRun( tui_argv(state, effect[1], sys.executable, script), log, inbox, - state.run_id)) + state.run_id, {**os.environ, TUI_LIST_ENV: "1"} if own_list else None)) except OSError as exc: inbox.put(("ended", state.run_id, 1, [f"The installer could not be started: {exc}"])) @@ -11254,6 +11660,8 @@ def run_tui(args: argparse.Namespace) -> int: path.unlink() if not tui_keeps_log(state): shutil.rmtree(work, ignore_errors=True) + else: + _mark_kept(work) stream = sys.stderr if state.screen in ("error", "crashed") else sys.stdout lines = tui_farewell(state) if problems: diff --git a/install.sh b/install.sh index 98cf7f82..d99d7ebe 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="bcbde4c289b32c02ada78001b75274a87b125c1efd3d8fb1064d7f5482ef3997" +DEFAULT_INSTALL_SHA256="7ac3e6030b12bacd262964fea5373f27fb09e62296255bc3c22255a1aaad5fe8" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/fixtures/tui/confirm-120x40.txt b/tests/fixtures/tui/confirm-120x40.txt index 66f8307e..89ee7e0e 100644 --- a/tests/fixtures/tui/confirm-120x40.txt +++ b/tests/fixtures/tui/confirm-120x40.txt @@ -1,8 +1,8 @@ RetroBIOS: Confirm 1 file to download, 16.0 KB - 0 in place, 1 missing, 0 wrong copies BIOS folder: ~/.config/retroarch/system + 0 in place, 1 missing, 0 wrong copies Free space: 50.0 GB > Install diff --git a/tests/fixtures/tui/confirm-40x12.txt b/tests/fixtures/tui/confirm-40x12.txt index 69373976..e6b28e26 100644 --- a/tests/fixtures/tui/confirm-40x12.txt +++ b/tests/fixtures/tui/confirm-40x12.txt @@ -1,12 +1,12 @@ RetroBIOS: Confirm 1 file to download, 16.0 KB + BIOS folder: ~/.co.../retroarch/system 0 in place, 1 missing, 0 wrong copies - BIOS folder: - ~/.config/retroarch/system Free space: 50.0 GB > Install Back + A/Enter Install B/Esc Back ? Help diff --git a/tests/fixtures/tui/confirm-40x15.txt b/tests/fixtures/tui/confirm-40x15.txt index 7bf203fd..d44fb000 100644 --- a/tests/fixtures/tui/confirm-40x15.txt +++ b/tests/fixtures/tui/confirm-40x15.txt @@ -1,9 +1,8 @@ RetroBIOS: Confirm 1 file to download, 16.0 KB + BIOS folder: ~/.co.../retroarch/system 0 in place, 1 missing, 0 wrong copies - BIOS folder: - ~/.config/retroarch/system Free space: 50.0 GB > Install @@ -12,4 +11,5 @@ + A/Enter Install B/Esc Back ? Help diff --git a/tests/fixtures/tui/confirm-80x24.txt b/tests/fixtures/tui/confirm-80x24.txt index 9e63bd66..6e210148 100644 --- a/tests/fixtures/tui/confirm-80x24.txt +++ b/tests/fixtures/tui/confirm-80x24.txt @@ -1,8 +1,8 @@ RetroBIOS: Confirm 1 file to download, 16.0 KB - 0 in place, 1 missing, 0 wrong copies BIOS folder: ~/.config/retroarch/system + 0 in place, 1 missing, 0 wrong copies Free space: 50.0 GB > Install diff --git a/tests/fixtures/tui/options-120x40.txt b/tests/fixtures/tui/options-120x40.txt index b3b47fd2..21aae272 100644 --- a/tests/fixtures/tui/options-120x40.txt +++ b/tests/fixtures/tui/options-120x40.txt @@ -3,7 +3,7 @@ Each row is an option of the command line, named on the right. > Preferred regions: North America, Japan --region - Leave out game data: no --no-game-data + Game data: kept --no-game-data Installed cores only: no (1 found) --installed-cores Cores: all --core Parallel downloads: 8 --jobs diff --git a/tests/fixtures/tui/options-40x12.txt b/tests/fixtures/tui/options-40x12.txt index 81190984..0aad95e4 100644 --- a/tests/fixtures/tui/options-40x12.txt +++ b/tests/fixtures/tui/options-40x12.txt @@ -1,7 +1,7 @@ RetroBIOS: Options > Preferred regions: North America, Ja - Leave out game data: no + Game data: kept Installed cores only: no (1 found) Cores: all Parallel downloads: 8 diff --git a/tests/fixtures/tui/options-40x15.txt b/tests/fixtures/tui/options-40x15.txt index 1508976a..0d94cbea 100644 --- a/tests/fixtures/tui/options-40x15.txt +++ b/tests/fixtures/tui/options-40x15.txt @@ -1,7 +1,7 @@ RetroBIOS: Options > Preferred regions: North America, Ja - Leave out game data: no + Game data: kept Installed cores only: no (1 found) Cores: all Parallel downloads: 8 diff --git a/tests/fixtures/tui/options-80x24.txt b/tests/fixtures/tui/options-80x24.txt index 041866a9..b4e6e1da 100644 --- a/tests/fixtures/tui/options-80x24.txt +++ b/tests/fixtures/tui/options-80x24.txt @@ -3,7 +3,7 @@ Each row is an option of the command line, named on the right. > Preferred regions: North America, Japan --region - Leave out game data: no --no-game-data + Game data: kept --no-game-data Installed cores only: no (1 found) --installed-cores Cores: all --core Parallel downloads: 8 --jobs diff --git a/tests/fixtures/tui/plan-120x40.txt b/tests/fixtures/tui/plan-120x40.txt index b4fabf6b..0c0ad949 100644 --- a/tests/fixtures/tui/plan-120x40.txt +++ b/tests/fixtures/tui/plan-120x40.txt @@ -1,11 +1,10 @@ RetroBIOS: RetroArch 4 files to download, 1.0 MB - 1 in place, 3 missing, 1 wrong copy BIOS folder: ~/.config/retroarch/system + 1 in place, 3 missing, 1 wrong copy Free space: 50.0 GB They are replaced, and the old copies are kept in a backup folder. - 2 files of the list are not in the collection. > Install Choose systems @@ -35,6 +34,7 @@ + A/Enter Install B/Esc Back Y/o Options ? Help diff --git a/tests/fixtures/tui/plan-40x12.txt b/tests/fixtures/tui/plan-40x12.txt index b0fbb0d5..80132bf2 100644 --- a/tests/fixtures/tui/plan-40x12.txt +++ b/tests/fixtures/tui/plan-40x12.txt @@ -1,9 +1,9 @@ RetroBIOS: RetroArch 4 files to download, 1.0 MB + BIOS folder: ~/.co.../retroarch/system 1 in place, 3 missing, 1 wrong copy - BIOS folder: - ~/.config/retroarch/system + Free space: 50.0 GB > Install Choose systems Copy from your old card or a folder diff --git a/tests/fixtures/tui/plan-40x15.txt b/tests/fixtures/tui/plan-40x15.txt index 94738418..709f289f 100644 --- a/tests/fixtures/tui/plan-40x15.txt +++ b/tests/fixtures/tui/plan-40x15.txt @@ -1,12 +1,12 @@ RetroBIOS: RetroArch 4 files to download, 1.0 MB + BIOS folder: ~/.co.../retroarch/system 1 in place, 3 missing, 1 wrong copy - BIOS folder: - ~/.config/retroarch/system Free space: 50.0 GB They are replaced, and the old copies are kept in a backup folder. + > Install Choose systems Copy from your old card or a folder diff --git a/tests/fixtures/tui/plan-80x24.txt b/tests/fixtures/tui/plan-80x24.txt index 0e320fc2..ee2684f4 100644 --- a/tests/fixtures/tui/plan-80x24.txt +++ b/tests/fixtures/tui/plan-80x24.txt @@ -1,11 +1,10 @@ RetroBIOS: RetroArch 4 files to download, 1.0 MB - 1 in place, 3 missing, 1 wrong copy BIOS folder: ~/.config/retroarch/system + 1 in place, 3 missing, 1 wrong copy Free space: 50.0 GB They are replaced, and the old copies are kept in a backup folder. - 2 files of the list are not in the collection. > Install Choose systems @@ -21,4 +20,5 @@ + A/Enter Install B/Esc Back Y/o Options ? Help diff --git a/tests/fixtures/tui/plan-in-place-120x40.txt b/tests/fixtures/tui/plan-in-place-120x40.txt index 97d74f80..8f604834 100644 --- a/tests/fixtures/tui/plan-in-place-120x40.txt +++ b/tests/fixtures/tui/plan-in-place-120x40.txt @@ -1,8 +1,8 @@ RetroBIOS: RetroArch Everything is in place - 5 in place, 0 missing, 0 wrong copies BIOS folder: ~/.config/retroarch/system + 5 in place, 0 missing, 0 wrong copies Free space: 50.0 GB > Close diff --git a/tests/fixtures/tui/plan-in-place-40x12.txt b/tests/fixtures/tui/plan-in-place-40x12.txt index 8755e793..0bd14655 100644 --- a/tests/fixtures/tui/plan-in-place-40x12.txt +++ b/tests/fixtures/tui/plan-in-place-40x12.txt @@ -1,12 +1,12 @@ RetroBIOS: RetroArch Everything is in place + BIOS folder: ~/.co.../retroarch/system 5 in place, 0 missing, 0 wrong copies - BIOS folder: - ~/.config/retroarch/system Free space: 50.0 GB > Close Options + A Close B Back Y Options ? Help diff --git a/tests/fixtures/tui/plan-in-place-40x15.txt b/tests/fixtures/tui/plan-in-place-40x15.txt index 0aa6b200..f4907958 100644 --- a/tests/fixtures/tui/plan-in-place-40x15.txt +++ b/tests/fixtures/tui/plan-in-place-40x15.txt @@ -1,9 +1,8 @@ RetroBIOS: RetroArch Everything is in place + BIOS folder: ~/.co.../retroarch/system 5 in place, 0 missing, 0 wrong copies - BIOS folder: - ~/.config/retroarch/system Free space: 50.0 GB > Close @@ -12,4 +11,5 @@ + A Close B Back Y Options ? Help diff --git a/tests/fixtures/tui/plan-in-place-80x24.txt b/tests/fixtures/tui/plan-in-place-80x24.txt index a73a9173..8cf344f6 100644 --- a/tests/fixtures/tui/plan-in-place-80x24.txt +++ b/tests/fixtures/tui/plan-in-place-80x24.txt @@ -1,8 +1,8 @@ RetroBIOS: RetroArch Everything is in place - 5 in place, 0 missing, 0 wrong copies BIOS folder: ~/.config/retroarch/system + 5 in place, 0 missing, 0 wrong copies Free space: 50.0 GB > Close diff --git a/tests/fixtures/tui/result-120x40.txt b/tests/fixtures/tui/result-120x40.txt index 4f0bc6ec..16b4dffc 100644 --- a/tests/fixtures/tui/result-120x40.txt +++ b/tests/fixtures/tui/result-120x40.txt @@ -6,7 +6,6 @@ The files it replaced are kept in /home/user/.config/retroarch/retrobios-backup/2026-10-23_1200 To put them back: python3 install.py --restore /home/user/.config/retroarch/retrobios-backup/2026-10-23_1200 - 2 files of the list are not in the collection, 1 of them needed to start an emulator. Same choice without questions: python3 install.py --platform retroarch --dest /home/user/.config/retroarch/system Details: /tmp/retrobios-x/install.log @@ -35,6 +34,7 @@ + A/Enter Exit Up/Down Scroll diff --git a/tests/fixtures/tui/result-40x12.txt b/tests/fixtures/tui/result-40x12.txt index 85613a75..e1ef9b13 100644 --- a/tests/fixtures/tui/result-40x12.txt +++ b/tests/fixtures/tui/result-40x12.txt @@ -8,5 +8,5 @@ /home/user/.config/retroarch/ retrobios-backup/2026-10-23_1200 To put them back: python3 install.py - 1-4 of 9 + 1-4 of 8 A/Enter Exit Up/Down Scroll diff --git a/tests/fixtures/tui/result-40x15.txt b/tests/fixtures/tui/result-40x15.txt index 86cb62b4..c551897a 100644 --- a/tests/fixtures/tui/result-40x15.txt +++ b/tests/fixtures/tui/result-40x15.txt @@ -11,5 +11,5 @@ --restore /home/user/.config/retroarch/ retrobios-backup/2026-10-23_1200 - 1-5 of 9 + 1-5 of 8 A/Enter Exit Up/Down Scroll diff --git a/tests/fixtures/tui/result-80x24.txt b/tests/fixtures/tui/result-80x24.txt index 5e8e6cb9..ee5cd165 100644 --- a/tests/fixtures/tui/result-80x24.txt +++ b/tests/fixtures/tui/result-80x24.txt @@ -7,8 +7,6 @@ /home/user/.config/retroarch/retrobios-backup/2026-10-23_1200 To put them back: python3 install.py --restore /home/user/.config/retroarch/retrobios-backup/2026-10-23_1200 - 2 files of the list are not in the collection, 1 of them needed to start an - emulator. Same choice without questions: python3 install.py --platform retroarch --dest /home/user/.config/retroarch/system @@ -21,4 +19,6 @@ + + A/Enter Exit Up/Down Scroll diff --git a/tests/test_install_from.py b/tests/test_install_from.py index 056c3c8f..7cb60f7f 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -349,7 +349,8 @@ class TheInterfaceReadsAFoundEvent(unittest.TestCase): state, _before = self.receive("check") self.assertEqual(state.found, [self.FOUND]) state, _before = self.receive("install") - self.assertEqual(state.local_dests, {"scph5501.bin"}) + self.assertEqual(list(state.local_dests), ["scph5501.bin"]) + self.assertEqual(state.local_dests["scph5501.bin"][0], self.FOUND["folder"]) def test_a_dry_run_ignores_it(self): for run in ("confirm", "preview"): diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 2bebdce9..7feade1c 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -28,7 +28,7 @@ import threading import time import unittest import warnings -from pathlib import Path +from pathlib import Path, PureWindowsPath from unittest import mock REPO_ROOT = Path(__file__).resolve().parent.parent @@ -656,9 +656,10 @@ class Flow(unittest.TestCase): mock.patch.object(install, "detect_frontends", return_value=[]): state, effects = install.tui_start(_args(), "linux") self.assertEqual((state.screen, effects), ("setup", [])) - self.assertEqual(state.setups, [("minui", card, "MinUI SD card")]) + label = f"MinUI SD card ({card.name})" + self.assertEqual(state.setups, [("minui", card, label)]) self.assertEqual([row.label for row in install.tui_rows(state)], - ["MinUI SD card", "Another setup or folder"]) + [label, "Another setup or folder"]) def test_an_error_offers_another_try(self): state = fresh_state() @@ -717,9 +718,11 @@ class PlanWarnings(unittest.TestCase): def test_a_folder_that_cannot_take_the_files_says_so_first(self): state = checked_state({"free": 1024, "writable": False, "cores": []}) text = " ".join(install.tui_view(state, 120, 40)) - self.assertIn("Not enough free space for this download", text) - self.assertIn("You cannot write to this folder: choose another one, or run the " - "installer with sudo.", text) + self.assertIn("Not enough free space: 1.0 MB needed.", text) + self.assertIn("You cannot write to this folder: choose another one.", text) + # sudo changes the home folder: the advice names this one. + self.assertIn("Or run it as root, this folder named: sudo python3 install.py " + f"--platform retroarch --dest {SYSTEM_DIR}", " ".join(text.split())) rows = install.tui_rows(state) self.assertEqual([row.action for row in rows[:2]], ["folder", "install"]) self.assertEqual(state.focus.get("plan", 0), 0) @@ -1248,8 +1251,9 @@ class CleanupSurvivesABrokenTerminal(unittest.TestCase): code = install.run_tui(_args()) self.assertIs(sys.stdout, tty) self.assertEqual(code, 1) - # The crash keeps its log, and nothing else of the run. - self.assertEqual(sorted(path.name for path in work.iterdir()), ["error.log"]) + # The crash keeps its log, marked kept, and nothing else of the run. + self.assertEqual(sorted(path.name for path in work.iterdir()), + ["error.log", "kept", "owner"]) self.assertEqual(termios.tcgetattr(slave), before) for number, handler in handlers.items(): self.assertEqual(signal.getsignal(number), handler) @@ -1384,7 +1388,7 @@ class AuditRoundTwo(unittest.TestCase): _title, lines = install.tui_outcome(state) self.assertEqual([line for line in lines if "failed" in line], ["Nothing was installed: 50 files failed.", - " 50 files failed (folder not writable), the first f0.bin"]) + " 50 files failed, all for one reason: folder not writable."]) def test_p2_6_a_folder_inside_a_card_offers_its_root(self): card = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) @@ -1444,7 +1448,12 @@ class AuditRoundTwo(unittest.TestCase): ("retroarch", SYSTEM_DIR, "RetroArch"), ("emudeck", HOME / "Emulation/bios", "EmuDeck")]) self.assertEqual([label for _p, _f, label in setups], - ["RetroArch (Flatpak)", "RetroArch (retroarch/system)", "EmuDeck"]) + ["RetroArch (Flatpak)", "RetroArch (installed)", "EmuDeck"]) + # Two that no package names keep the end of their folder. + twice = install.tell_apart([("retroarch", SYSTEM_DIR, "RetroArch"), + ("retroarch", HOME / "ra2/system", "RetroArch")]) + self.assertEqual([label for _p, _f, label in twice], + ["RetroArch (retroarch/system)", "RetroArch (ra2/system)"]) state = fresh_state() state.setups = setups self.assertIn("RetroArch (Flatpak)", "\n".join(install.tui_view(state, 40, 15))) @@ -1634,7 +1643,8 @@ class CopyFromAnOldCard(unittest.TestCase): self.recheck(state, [found("scph5500.bin", str(old)), found("gba_bios.bin", str(old))]) self.assertEqual(state.screen, "plan") text = "".join("".join(install.tui_view(state, 120, 40)).split()) - self.assertIn("".join(f"2 found in {old}, 2 to download (512.0 KB)".split()), text) + # The folder by its name: the whole path pushed the BIOS folder off. + self.assertIn("".join(f"2 found in {old.name}, 2 to download (512.0 KB)".split()), text) # A second folder from the options: --from twice. press(state, "o") state.focus["options"] = next(i for i, r in enumerate(install.tui_rows(state)) @@ -1655,11 +1665,15 @@ class CopyFromAnOldCard(unittest.TestCase): def test_left_takes_the_last_folder_back(self): state = checked_state() state.from_folders = [Path("/media/old")] + state.checked_flags = install.tui_flags(state) state.stack.append("options") state.focus["options"] = next(i for i, r in enumerate(install.tui_rows(state)) if r.action == "from") - self.assertEqual(press(state, "left"), [("run", "check")]) - self.assertEqual(state.from_folders, []) + # Said, and the screen stays: Back to the plan counts again. + self.assertEqual(press(state, "left"), []) + self.assertEqual((state.screen, state.from_folders), ("options", [])) + self.assertIn("No longer copying from old.", "\n".join(install.tui_view(state, 80, 24))) + self.assertEqual(press(state, "esc"), [("run", "check")]) def test_the_command_line_folders_are_kept(self): with mock.patch.object(install, "detect_platforms", return_value=[]), \ @@ -1710,6 +1724,438 @@ class CopyFromAnOldCard(unittest.TestCase): self.assertIn("1.0 MB/s, 1:38 left", text) +class AuditRoundThree(unittest.TestCase): + """tui_audit_2.md: its P1 and P2 findings, each held by a test.""" + + CARDS = { + "muOS": ({"MUOS/": ""}, "retroarch", "MUOS/bios"), + "Onion": ({".tmp_update/onionVersion/version.txt": "4.3.1\n"}, "retroarch", "BIOS"), + "spruceOS": ({"spruce/": ""}, "retroarch", "BIOS"), + "KNULLI": ({"system/batocera.conf": "a=1\nsystem.hostname=KNULLI\n"}, "knulli", + "bios"), + "ROCKNIX": ({".config/system/configs/system.cfg": "root.password=rocknix\n"}, + "rocknix", "roms/bios"), + } + + def card(self, files: dict) -> Path: + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + for name, text in files.items(): + path = root / name + if name.endswith("/"): + path.mkdir(parents=True, exist_ok=True) + else: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + return root + + # P1-1: a handheld's card mounted on a computer. + + def test_p1_1_each_card_by_its_own_files(self): + for name, (files, plat, folder) in self.CARDS.items(): + with self.subTest(card=name): + root = self.card(files) + self.assertEqual(install.card_system(root), (plat, root / folder, name)) + + def test_p1_1_a_card_nothing_names_is_left_alone(self): + self.assertIsNone(install.card_system(self.card({"roms/": "", "BIOS/": ""}))) + # Batocera's share and a KNULLI board seeded with its name read alike. + self.assertIsNone(install.card_system(self.card({ + "system/batocera.conf": "system.hostname=BATOCERA\n"}))) + + def test_p1_1_the_card_rules_are_the_device_rules(self): + # The folder and the marker come from the rule the device reads, so + # they are never written twice. + rules = {rule.name: rule for rule in install._card_rules()} + self.assertLessEqual({"muOS", "Onion", "spruceOS", "CrossMix-OS", "Allium", "Koriki", + "KNULLI", "ROCKNIX"}, set(rules)) + device = {rule.name: rule for rule in install._EMBEDDED_RULES + if rule.values and rule.values[0].startswith(install._CARD_MOUNTS)} + for name, rule in device.items(): + if rule.bios and rule.manifest: + with self.subTest(system=name): + self.assertTrue(rule.bios.endswith("/" + rules[name].bios)) + self.assertEqual(install.DEFAULT_DESTS["knulli"], Path("/userdata") / "bios") + self.assertEqual(install.DEFAULT_DESTS["rocknix"], Path("/storage") / "roms/bios") + + def test_p1_1_the_first_screen_offers_the_card_folder(self): + root = self.card(self.CARDS["muOS"][0]) + with mock.patch.object(install, "_mounted_cards", return_value=[root]), \ + mock.patch.object(install, "detect_platforms", return_value=[]), \ + mock.patch.object(install, "detect_frontends", return_value=[]): + state, effects = install.tui_start(_args(), "linux") + label = f"muOS SD card ({root.name})" + self.assertEqual(state.setups, [("retroarch", root / "MUOS" / "bios", label)]) + text = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn(f"> {label}", text) + self.assertIn(f"BIOS folder: {root / 'MUOS' / 'bios'}", text) + (root / "MUOS" / "bios").mkdir() + self.assertEqual(press(state, "enter"), [("fetch", "retroarch")]) + self.assertEqual(state.dest, root / "MUOS" / "bios") + + def test_p1_1_a_drive_letter_names_the_card(self): + self.assertEqual(install._mount_name(PureWindowsPath("E:\\")), "E:\\") + self.assertEqual(install._mount_name(Path("/media/me/MUOS")), "MUOS") + + def test_p1_1_on_the_device_its_card_is_not_offered_twice(self): + root = self.card(self.CARDS["muOS"][0]) + bound = Path("/run/muos/storage/bios") + with mock.patch.object(install, "_mounted_cards", return_value=[root]), \ + mock.patch.object(install, "detect_platforms", return_value=[("retroarch", + bound)]), \ + mock.patch.object(install, "detect_frontends", return_value=[]), \ + mock.patch.dict(install._DETECTED_OS, {("retroarch", bound): "muOS"}): + state, _effects = install.tui_start(_args(), "linux") + self.assertEqual([setup[1] for setup in state.setups], [bound]) + + def test_p1_1_browsing_to_the_card_root_uses_its_folder(self): + root = self.card(self.CARDS["KNULLI"][0]) + with mock.patch.object(install, "_mounted_cards", return_value=[root]): + rows, _error = install._browse_rows(REPO_ROOT / "tests") + self.assertEqual(rows[2].label, f"{root} (KNULLI card)") + state = fresh_state() + install._choose(state, "knulli", root) + self.assertEqual(state.dest, root / "bios") + # Another platform's list keeps the folder it was given. + install._choose(state, "batocera", root) + self.assertEqual(state.dest, root) + + def test_p1_1_the_command_line_writes_where_muos_reads(self): + root = self.card(self.CARDS["muOS"][0]) + with FakeRepository() as repo: + proc = install_run("--platform", "muos", "--dest", str(root), "--yes", + env=repo.env()) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn(f"muOS card: RetroArch files, in {root / 'MUOS' / 'bios'}", proc.stdout) + self.assertEqual(sorted(path.name for path in root.iterdir()), ["MUOS"]) + self.assertTrue((root / "MUOS" / "bios" / "scph5501.bin").is_file()) + + def test_p1_1_a_folder_that_is_no_muos_card_is_refused(self): + root = self.card({"roms/": ""}) + proc = install_run("--platform", "muos", "--dest", str(root), "--yes") + self.assertEqual(proc.returncode, 1) + self.assertIn(f"{root} is not a muOS card: it holds no MUOS.", proc.stderr) + self.assertIn("--dest /MUOS/bios", proc.stderr) + self.assertEqual(list(root.iterdir()), [root / "roms"]) + # Its BIOS folder named outright is taken as given. + self.assertEqual(install.card_dest("muos", "retroarch", root / "MUOS" / "bios"), + ("retroarch", root / "MUOS" / "bios")) + # On the device, the folder muOS binds it to. + bound = Path("/run/muos/storage/bios") + self.assertEqual(install.card_dest("muos", "retroarch", bound), ("retroarch", bound)) + # A list named by its own name says which list, not which card. + self.assertEqual(install.card_dest("retroarch", "retroarch", root), + ("retroarch", root)) + + # P1-2: Space is X on PortMaster and Y on muxterm. + + def test_p1_2_space_on_install_opens_the_confirm_screen(self): + state = checked_state() + self.assertEqual(install.tui_rows(state)[0].action, "install") + self.assertEqual(press(state, " "), [("run", "confirm")]) + self.assertEqual(state.screen, "confirm") + # The same plan comes back: the screen still waits for A. + install.tui_receive(state, ("event", state.run_id, dict(CHECK))) + self.assertEqual(install.tui_receive(state, ("ended", state.run_id, 0, [])), []) + self.assertEqual(state.screen, "confirm") + self.assertEqual(install.tui_rows(state)[0].action, "install") + self.assertEqual(press(state, " "), []) + self.assertEqual(state.screen, "confirm") + self.assertEqual(press(state, "enter"), [("run", "install")]) + + def test_p1_2_space_never_closes_or_quits(self): + for action in ("quit", "close"): + with self.subTest(row=action): + state = checked_state() + if action == "close": + state.check = {**CHECK, "missing": 0, "wrong_hash": 0, "bytes_needed": 0} + rows = install.tui_rows(state) + state.focus["plan"] = next(i for i, r in enumerate(rows) if r.action == action) + self.assertEqual(press(state, " "), []) + self.assertIsNone(state.exit_code) + state = checked_state() + state.focus["plan"] = next(i for i, r in enumerate(install.tui_rows(state)) + if r.action == "options") + press(state, " ") + self.assertEqual(state.screen, "options") + + def test_p1_2_enter_still_installs_in_three_presses(self): + state = checked_state() + self.assertEqual(press(state, "enter"), [("run", "install")]) + self.assertEqual(state.screen, "installing") + + # P2-1: a card too small for the download. + + def test_p2_1_another_folder_comes_first_when_space_is_short(self): + state = checked_state({"free": 1024, "writable": True, "cores": []}) + self.assertEqual([row.label for row in install.tui_rows(state)[:2]], + ["Choose another folder", "Install anyway"]) + self.assertEqual(press(state, "enter"), []) + self.assertEqual(state.screen, "folder") + + FULL_TAIL = [ + " Checking existing files...", + " 0/50 present (0 verified, 0 wrong hash)", + " Downloading 50 files (845.7 KB)...", + "Not enough space on the drive holding /media/me/MUOS/MUOS/bios.", + " Needed: 1.2 GB. Free: 300.0 MB.", + " Free up 920.0 MB, or install to another drive:", + " python3 install.py --platform retroarch --dest ''", + ] + + def test_p2_1_a_full_card_is_said_in_words(self): + root = self.card(self.CARDS["muOS"][0]) + state = checked_state() + state.dest, state.log = root / "MUOS" / "bios", "/tmp/retrobios-tui-x/install.log" + state.stack.append("result") + state.code, state.tail = 2, list(self.FULL_TAIL) + with mock.patch.object(install, "_mounted_cards", return_value=[root]): + title, lines = install.tui_outcome(state) + self.assertEqual(title, "Not enough space") + self.assertEqual(lines[0], "The card is full: 1.2 GB needed, 300.0 MB free.") + self.assertIn("Free some space, or run again and choose another folder.", lines) + text = "\n".join(lines) + for raw in ("Checking existing files", "Downloading", "stopped with code", "Details:"): + self.assertNotIn(raw, text) + self.assertFalse(install.tui_keeps_log(state)) + + def test_p2_1_advice_never_names_the_interface_list(self): + if hasattr(os, "geteuid") and os.geteuid() == 0: + self.skipTest("root writes to a read-only folder") + folder = self.card({}) + folder.chmod(0o555) + self.addCleanup(folder.chmod, 0o755) + with FakeRepository() as repo: + listed = repo.root / "install" / "retroarch.json" + proc = install_run("--platform", "retroarch", "--dest", str(folder), + f"--manifest={listed}", "--yes", + env={**repo.env(), install.TUI_LIST_ENV: "1"}) + self.assertIn("sudo", proc.stdout + proc.stderr) + self.assertNotIn("--manifest", proc.stdout + proc.stderr) + + # P2-2: the BIOS folder line on a handheld screen. + + def test_p2_2_the_plan_keeps_the_folder_at_handheld_sizes(self): + zipped = "/home/me/a/long/path/to/the/Downloads/PSX BIOS.zip" + for cols, rows in ((40, 12), (40, 15)): + with self.subTest(size=f"{cols}x{rows}"): + state = checked_state() + state.dest = Path("/home/someone/with/a/long/path/.config/retroarch/system") + state.found = [found("scph5500.bin", zipped)] + state.cols, state.rows = cols, rows + lines = [line.strip() for line in install.tui_view(state, cols, rows)] + folder = next(line for line in lines if line.startswith("BIOS folder: ")) + self.assertTrue(folder.endswith("retroarch/system"), folder) + self.assertIn("...", folder) + self.assertIn("1 found in PSX BIOS.zip,", " ".join(lines)) + self.assertNotIn("/home/me/a/long", " ".join(lines)) + + # P2-3: where the files came from. + + def test_p2_3_the_result_says_what_came_from_the_card(self): + state = checked_state() + state.stack.append("result") + state.code, state.download = 0, {"files": 5, "bytes": 10} + state.local_dests = {"scph5500.bin": ("/media/me/OLD", ""), + "scph5501.bin": ("/media/me/OLD", ""), + "gba_bios.bin": (str(SYSTEM_DIR), "one folder too deep")} + state.finished = [("scph5500.bin", True, ""), ("scph5501.bin", True, ""), + ("gba_bios.bin", True, ""), ("pcsx/extra.bin", True, ""), + ("fc/disksys.rom", False, "HTTP 404")] + state.summary = {"installed": 4, "up_to_date": 0, "failed": [ + {"dest": "fc/disksys.rom", "reason": "HTTP 404"}]} + _title, lines = install.tui_outcome(state) + self.assertEqual(lines[1], "2 files copied from OLD, 1 fixed from copies " + "already in the BIOS folder, 1 downloaded, 1 still missing.") + # The line mode's own sentence, from the same counts. + self.assertEqual(lines[1], install.sources_line([("OLD", 2)], 1, 1, 1)) + + def test_p2_3_a_run_that_copied_nothing_adds_no_line(self): + state = install.TuiState() + state.summary, state.finished = {"installed": 2, "failed": []}, [("a", True, "")] + self.assertEqual(install._tui_sources(state, 2, 0), "") + + # P2-4 and P2-5: the systems list. + + def systems_state(self) -> install.TuiState: + state = checked_state() + state.systems = [*state.systems, + {"system": "other", "name": "other", "files": 2, "present": 0, + "missing": 2, "wrong_hash": 0, "bytes_needed": 300}, + {"system": "3do", "name": "3do", "files": 3, "present": 0, + "missing": 3, "wrong_hash": 0, "bytes_needed": 600}] + state.files = [*state.files, + {"event": "file", "dest": "panafz1.bin", "systems": ["3do"], + "size": 100, "state": "missing"}, + {"event": "file", "dest": "mame/hash/a.xml", "systems": ["3do", "mac"], + "size": 200, "state": "missing"}, + {"event": "file", "dest": "mame/hash/b.xml", "systems": ["3do", "mac"], + "size": 300, "state": "missing"}] + state.ticked |= {"other", "3do"} + state.stack.append("systems") + return state + + def test_p2_4_the_other_row_is_named_and_explained(self): + state = self.systems_state() + rows = install.tui_rows(state) + other = next(i for i, row in enumerate(rows) if row.value == "other") + self.assertEqual(rows[other].label, "[x] Other systems and game engines") + state.focus["systems"] = other + text = " ".join(" ".join(install.tui_view(state, 80, 24)).split()) + self.assertIn("Other systems and game engines: what cores load for systems this " + "setup does not list", text) + + def test_p2_5_shared_files_are_counted_apart(self): + state = self.systems_state() + rows = install.tui_rows(state) + three = next(i for i, row in enumerate(rows) if row.value == "3do") + self.assertEqual(rows[three].right, "1 file, 100 B + 2 shared") + state.focus["systems"] = three + text = " ".join(" ".join(install.tui_view(state, 80, 24)).split()) + self.assertIn("2 of these files are shared with other systems, counted once", text) + # The line mode counts the same files apart. + self.assertIn("and 2 files, 500 B shared with other systems", install._system_sized([ + {"dest": "panafz1.bin", "size": 100, "systems": ["3do"]}, + {"dest": "a.xml", "size": 200, "systems": ["3do", "mac"]}, + {"dest": "b.xml", "size": 300, "systems": ["3do", "mac"]}])) + narrow = install.tui_view(state, 40, 15) + self.assertTrue(any("100 B + 2 shared" in line for line in narrow), narrow) + + # P2-6: a file that cannot be supplied, by name, where it matters. + + def unavailable_state(self) -> install.TuiState: + state = checked_state() + state.manifest = {**MANIFEST, "system_titles": {"nintendo-gba": "Game Boy Advance"}, + "omitted_files": [ + {"dest": "gba_extra.bin", "system": "nintendo-gba", + "required": True, "reason": "not_found"}, + {"dest": "optional.bin", "system": "nintendo-gba", + "required": False, "reason": "not_found"}]} + state.stack.append("result") + state.code, state.download = 0, {"files": 1, "bytes": 1} + state.summary = {"installed": 1, "up_to_date": 0, "failed": []} + return state + + def test_p2_6_the_missing_file_is_named_with_its_system(self): + state = self.unavailable_state() + _title, lines = install.tui_outcome(state) + self.assertIn("Not available here, and needed to start its emulator: gba_extra.bin " + "(Game Boy Advance). Your own copy goes in the BIOS folder.", lines) + self.assertNotIn("collection", " ".join(lines)) + + def test_p2_6_it_is_left_out_when_it_does_not_concern_the_run(self): + state = self.unavailable_state() + state.ticked = {"sony-playstation"} + self.assertEqual(install._unavailable_lines(state), []) + state = self.unavailable_state() + state.summary = {"installed": 1, "failed": [], "interrupted": True} + self.assertFalse(any("Not available" in line for line in install.tui_outcome(state)[1])) + + # P3: small fixes the audit listed. + + def fetch_failure(self, error: BaseException, env: dict | None = None) -> tuple: + inbox = __import__("queue").Queue() + with mock.patch.object(install, "_fetch_json", side_effect=error), \ + mock.patch.dict(os.environ, env or {}): + for name in ("HTTPS_PROXY", "https_proxy", "ALL_PROXY", "all_proxy"): + if name not in (env or {}): + os.environ.pop(name, None) + install._tui_fetch("retroarch", inbox, io.StringIO()) + _kind, _plat, title, lines, _code = inbox.get_nowait() + return title, lines + + def test_p3_the_clock_names_the_row_to_choose(self): + import ssl + + error = urllib_error(ssl.SSLCertVerificationError("certificate is not yet valid")) + words = install.network_words(error, today=install.datetime.date(1980, 1, 1)) + with mock.patch.object(install, "network_words", return_value=words): + _title, lines = self.fetch_failure(error) + self.assertTrue(lines[0].endswith("then choose Try again."), lines[0]) + + def test_p3_a_proxy_in_the_way_is_named_not_shown(self): + error = urllib_error(ConnectionRefusedError(111, "Connection refused")) + _title, lines = self.fetch_failure( + error, {"HTTPS_PROXY": "http://me:secret@10.0.0.1:3128"}) + self.assertIn("HTTPS_PROXY is set, so the connection goes through a proxy: check " + "that it answers.", lines) + self.assertNotIn("secret", " ".join(lines)) + _title, lines = self.fetch_failure(error) + self.assertFalse(any("PROXY" in line for line in lines)) + + def test_p3_a_list_that_is_no_json_is_said_in_words(self): + try: + install.json.loads("") + except install.json.JSONDecodeError as cause: + error = ValueError(f"invalid file list: {cause}") + error.__cause__ = cause + title, lines = self.fetch_failure(error) + self.assertEqual(title, "The file list is not valid") + self.assertNotIn("Expecting value", " ".join(lines)) + self.assertIn("a Wi-Fi sign-in page or a cut download looks like this.", lines[0]) + + def test_p3_the_preview_counts_its_files_only(self): + state = checked_state() + state.stack += ["options", "preview"] + state.dry = dict(CHECK) + state.dry_files = [{**FILES[0], "dest": f"f{n}.bin"} for n in range(40)] + state.rows = 24 + lines = [line.strip() for line in install.tui_view(state, 80, 24)] + self.assertIn("1-18 of 40", lines) + + def test_p3_game_data_reads_without_a_double_negative(self): + state = checked_state() + label = next(row.label for row in install._option_rows(state) + if row.action == "game_data") + self.assertEqual(label, "Game data: kept") + state.no_game_data = True + label = next(row.label for row in install._option_rows(state) + if row.action == "game_data") + self.assertEqual(label, "Game data: left out, firmware only") + + def test_p3_the_plan_says_which_config_named_the_folder(self): + config = HOME / ".config/retroarch/retroarch.cfg" + with mock.patch.dict(install._RETROARCH_CONFIGS, {SYSTEM_DIR: (config, config.parent)}): + text = "\n".join(install.tui_view(checked_state(), 80, 24)) + self.assertIn("Named by ~/.config/retroarch/retroarch.cfg", text) + + # Work folders: only a kill no handler sees leaves one, and the next + # run takes it away. + + def test_work_folders_left_behind_are_pruned(self): + import subprocess + + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + gone = subprocess.Popen([sys.executable, "-c", "pass"]) + gone.wait() + now = time.time() + week = install.TUI_KEEP_SECONDS + + def folder(name: str, pid: int | None, kept: bool, age: float) -> Path: + path = root / f"{install.TUI_WORK_PREFIX}{name}" + path.mkdir() + if pid is not None: + (path / install.TUI_OWNER).write_text(str(pid)) + if kept: + (path / install.TUI_KEPT).write_text("") + os.utime(path, (now - age, now - age)) + return path + + killed = folder("killed", gone.pid, False, 60) + running = folder("running", os.getpid(), False, 2 * week) + failed = folder("failed", gone.pid, True, 60) + failed_old = folder("failed-old", gone.pid, True, 2 * week) + unknown = folder("unknown", None, False, 60) + unknown_old = folder("unknown-old", None, False, 2 * week) + other = root / "something-else" + other.mkdir() + removed = install._prune_work_folders(root, now) + self.assertEqual(sorted(removed), sorted([killed, failed_old, unknown_old])) + self.assertEqual(sorted(path.name for path in root.iterdir()), + sorted(path.name for path in (running, failed, unknown, other))) + + class Crash(unittest.TestCase): """An unexpected error ends on one sentence and a log, never a traceback.""" @@ -2030,11 +2476,14 @@ class EndToEnd(unittest.TestCase): term.send("\r") screen = term.wait("found in") # The folder wraps at a slash: compared without the spaces. - self.assertIn("".join(f"3 found in {old}, 2 to download (8 B)".split()), - "".join(screen.split())) + self.assertIn("3 found in oldcard, 2 to download (8 B)", screen) # Checked again, the plan opens on Install. term.send("\r") - term.wait("files installed.") + screen = term.wait("files installed.") + # Where each came from, as the line mode says it. + self.assertIn("5 files installed.", screen) + self.assertIn("3 files copied from", screen) + self.assertIn("2 downloaded.", screen) term.send("\r") self.assertEqual(term.close(), 0) self.assertEqual(len(repo.installed()), 5) diff --git a/wiki/installer.md b/wiki/installer.md index 40098811..9c8e07c1 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -67,7 +67,10 @@ curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh \ ``` `--dest` is the BIOS folder itself: on a card, `retroarch/cores/system` for -RetroArch on Switch, `RetroArch/system` on Android. +RetroArch on Switch, `RetroArch/system` on Android. The root of a handheld's +card works too when the installer knows that card (muOS, Onion, spruceOS, +KNULLI, ROCKNIX and the MinUI family, see +[Platform detection](#platform-detection)): the files go where its system reads. PowerShell needs another form. `iex` takes the script text as its own argument, so anything appended to `irm ... | iex` is read as an argument to `iex` and @@ -454,6 +457,9 @@ python install.py --from ~/Downloads --from /media/me/backup python install.py --check --from /media/me/STOCK ``` +The first line names the root of the muOS card, the one holding `MUOS`: the +files go to `MUOS/bios` on it, where muOS reads them, and the run says so. + Every file is recognised by its contents, never by its name: `PSX BIOS (USA).bin` in a subfolder lands as `scph5501.bin` when it holds those bytes, and a file of the right name with other bytes (an empty file left by a @@ -742,6 +748,26 @@ family writes the release name as the first line of `.system/version.txt` at the card root (`MinUI-`, `NextUI-`, `MyMinUI-`), and that line names the file list. +A card that holds a system's own files is read the same way, without +`--platform` or with the one that names its list, by the rules the table above +gives for the device, below the card's root instead of its mount point: + +| On the card | System | Files go to | +|-------------|--------|-------------| +| `MUOS/` | muOS | `MUOS/bios`, RetroArch's list | +| `.tmp_update/onionVersion/version.txt` | Onion | `BIOS`, RetroArch's list | +| `spruce/` | spruceOS | `BIOS`, RetroArch's list | +| `System/usr/trimui/crossmix-version.txt` | CrossMix-OS | `BIOS`, RetroArch's list | +| `.allium/` | Allium | `BIOS`, RetroArch's list | +| `Koriki/version.txt` | Koriki | `BIOS`, RetroArch's list | +| `system/batocera.conf` with `system.hostname=KNULLI`, as KNULLI's first boot writes its share | KNULLI | `bios`, KNULLI's list | +| `.config/system/configs/system.cfg` with `root.password=rocknix`, as ROCKNIX's first boot writes its storage | ROCKNIX | `roms/bios`, ROCKNIX's list | + +The full screen lists such a card on its first screen ("muOS SD card (E:\)"). +Named by its system (`--platform muos`, `--platform onion`), a `--dest` that +is neither such a card nor its BIOS folder is refused, with the folder that +system reads. + Desktop installs are read from their own configuration, and several can be found on one machine. @@ -1157,6 +1183,10 @@ fields, the installer refuses to narrow rather than install everything. | `130` | stopped with Ctrl+C, or with `q`, Ctrl+C or Ctrl+D at a question. Run the same command again to carry on | | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) | +The full screen ended by SIGTERM (Start+Select on a handheld) before any +install exits `143`. During an install it stops the install first, and exits +`130`. + `install.ps1` passes the installer's code on unchanged: run as a file it exits with it, run through `irm \| iex` it sets `$LASTEXITCODE` (an `exit` there would close the PowerShell window). It declares no parameter of its own, so From bbbf551140d9679b6ab24f019030b14b763887ee Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:31:33 +0200 Subject: [PATCH 2/5] docs: first install for the full screen The one-liner opens the full screen, and the page walked through the line questions. It now shows the first screen, the keys and the controller, Install, the copy from an old card and the result titles. Troubleshooting no longer asks to move a nested bios folder by hand. --- tests/test_doc_pages.py | 29 +++++++++-------- tests/test_site_r2_wiki.py | 14 ++++++--- tests/test_site_r3_wiki.py | 20 +++++++++--- wiki/first-install.md | 64 ++++++++++++++++++-------------------- wiki/troubleshooting.md | 3 +- 5 files changed, 73 insertions(+), 57 deletions(-) diff --git a/tests/test_doc_pages.py b/tests/test_doc_pages.py index 025cde83..73d75bfd 100644 --- a/tests/test_doc_pages.py +++ b/tests/test_doc_pages.py @@ -699,13 +699,14 @@ class FirstInstallWalkthrough(unittest.TestCase): step3 = _section(self.run, "The file index and the download") step4 = _section(self.run, "The end of the run") step5 = _section(self.run, "A later check") - # First install names the three endings a run prints, and no count: - # a figure from one setup read as the size of every install. + # First install names the three endings the full screen shows as + # its title, and no count: a figure from one setup read as the size + # of every install. source = (ROOT / "install.py").read_text(encoding="utf-8") - for ending in ("Done.", "Finished with errors.", "Nothing was installed."): + for ending in ("Done", "Finished with errors", "Nothing installed"): with self.subTest(ending=ending): - self.assertIn(f"`{ending}`", _section(self.text, "5. Read the last line")) - self.assertIn(f'"\\n{ending} ', source) + self.assertIn(f"**{ending}**", _section(self.text, "5. Read the result")) + self.assertIn(f'title = "{ending}"', source) self.assertNotRegex(self.text, r"Done\. \d+ files") count, size = re.search(r"\n (\d+) files \(([\d.]+ [KMG]B)\)", step3).groups() self.assertIn(f"0/{count} present", step3) @@ -726,15 +727,17 @@ class FirstInstallWalkthrough(unittest.TestCase): self.assertIn(f"all {required} belong", step4) self.assertIn(f"other {total - required} are optional", step4) - def test_no_platform_detected_reads_like_the_installer(self): - source = (ROOT / "install.py").read_text(encoding="utf-8") - branch = source.split('print(" No supported platform detected.")', 1)[1][:1600] - self.assertIn("if not interactive or listing:", branch) - self.assertIn("_prompt_manual_platform(os_type)", branch) + def test_nothing_found_reads_like_the_installer(self): + # The one-liner hands the terminal to the installer, which opens the + # full screen: its first screen says the heading and offers the row. + import install + self.assertIn('"$INSTALLER" "$@" ", section) self.assertNotIn("Batocera", section) def test_the_diagram_follows_the_steps(self): text = _wiki("first-install.md") - self.assertLess(text.index("## 5. Read the last line"), text.index("")) + self.assertLess(text.index("## 5. Read the result"), text.index("")) class OneNamePerPage(unittest.TestCase): diff --git a/tests/test_site_r3_wiki.py b/tests/test_site_r3_wiki.py index 78970435..3fadac8b 100644 --- a/tests/test_site_r3_wiki.py +++ b/tests/test_site_r3_wiki.py @@ -251,11 +251,21 @@ class SmallCopyFixes(unittest.TestCase): self.assertIn("`retroarch/cores/system/` on the SD card", guides) self.assertIn("--dest E:\\retroarch\\cores\\system", section) - def test_the_end_line_is_the_printed_one(self): - section = _section(_wiki("first-install.md"), "5. Read the last line") - self.assertIn("`N files installed, M already up to date.`", section) - source = (REPO_ROOT / "install.py").read_text(encoding="utf-8") - self.assertIn("installed, \"\n f\"{_plural(up_to_date, 'file')} already up to date.", source) + def test_the_end_is_the_shown_one(self): + # The one-liner opens the full screen: the page names the titles + # its result screen shows, as tui_outcome writes them. + import install + + section = _section(_wiki("first-install.md"), "5. Read the result") + state = install.TuiState() + state.stack, state.code = ["result"], 0 + for summary in ({"installed": 3, "up_to_date": 0, "failed": []}, + {"installed": 2, "failed": [{"dest": "a.bin", "reason": "HTTP 404"}]}, + {"installed": 0, "failed": [{"dest": "a.bin", "reason": "HTTP 404"}]}): + state.summary = summary + title, _lines = install.tui_outcome(state) + with self.subTest(title=title): + self.assertIn(f"**{title}**", section) class CommunityAnswers(unittest.TestCase): diff --git a/wiki/first-install.md b/wiki/first-install.md index 6b461a83..33a97475 100644 --- a/wiki/first-install.md +++ b/wiki/first-install.md @@ -67,44 +67,44 @@ curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | On Batocera, Recalbox and similar setups, see [Get your files](../which-pack.md). -## 2. Check the folder it found +## 2. Check the setup it found -It shows your setup and its BIOS folder: +It opens a menu: ``` - Found at + RetroBIOS: Where do you play? + Found on this machine: + > + Another setup or folder + BIOS folder: ``` -- **Right folder?** Go on. -- **Wrong folder?** Press Ctrl+C. Run it again with your folder: +- **Right folder?** Press Enter. +- **Wrong folder?** Choose **Another setup or folder**, then browse to yours. +- **A handheld's SD card in a reader?** It is listed, as **muOS SD card**. - -```sh -curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --dest ~/path/to/bios -``` +Arrows move, Enter chooses, Esc goes back. A controller's A and B do the same. -## 3. Choose what to install +## 3. Install -``` - Enter) install everything - i) only what the installed cores read - c) choose systems, cores and regions - q) quit -``` +The next screen counts the missing files and the free space. +Press Enter on **Install**. **Choose systems** keeps the consoles you +play. -**Enter** installs everything. **c** lets you pick consoles. +**Old card?** In a card reader, choose **Copy the BIOS files from your old +card or a folder**, then the card. It stays unchanged. ## 4. Wait -Ctrl+C stops it. Run the command again to carry on. +B or Ctrl+C stops it. Run it again to carry on. -## 5. Read the last line +## 5. Read the result -- **`Done.`** `N files installed, M already up to date.` All files are in place. -- **`Finished with errors.`** Some files failed. Run the same command again. - It retries only those. -- **`Nothing was installed.`** The lines above it say why. See - [If it stops](#if-it-stops). +- **Done**: all files are in place. +- **Finished with errors**: run the command again. It retries only those. +- **Nothing installed**: the screen says why. See [If it stops](#if-it-stops). + +For questions line by line (screen readers), add `--plain`. ## What the command does @@ -127,18 +127,14 @@ All good: ## Undo -Every file it replaces goes to a `retrobios-backup` folder next to your BIOS -folder. The last lines say how to put them back. +Files it replaces go to a `retrobios-backup` folder beside your BIOS folder. +The result says how to restore them. ## If it stops -### No supported platform detected. +### Nothing was found here. -At a terminal, a menu follows. Pick your setup by its number, then type its -BIOS folder, or press Enter for the usual one. - -With `--no-input`, or without a terminal, it stops and prints the command to -use. +Choose **Another setup or folder**, then your setup and its folder. ### Error: Python 3.8 or newer is required. @@ -157,9 +153,9 @@ The download was cut short. Nothing was changed. Run the command again. You cannot write there. Do what the message says. -### Not enough space on the drive holding ... +### The card is full -Nothing was written. Free some space, or pick another drive with `--dest`. +Nothing was written. Free some space, or choose another folder. ### The game still does not start diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md index af29ce2a..deec29fe 100644 --- a/wiki/troubleshooting.md +++ b/wiki/troubleshooting.md @@ -38,7 +38,8 @@ Run it without `--check`. It adds missing files and replaces wrong ones. - **Right folder?** [Get your files](../which-pack.md) names it. - **Exact name?** `scph5501.bin` and `SCPH-5501.BIN` differ. On Linux, capitals count too. -- **A `bios` folder inside your BIOS folder?** Move its files up one level. +- **A `bios` folder inside your BIOS folder?** Run the installer: it copies + those files up one level. - **Unzipped by hand on Linux?** The command adds names that differ by capitals. - **A ZIP in parts, like `Pack.part1of2.zip`?** Unzip every part into one From 4be68b1eeebfe0679a669d4e604f48985887c4e2 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:43:11 +0200 Subject: [PATCH 3/5] fix: keep the example profile out of core lists 00_example is the libretro-core-info .info template, not a core the buildbot publishes. As type libretro, all_libretro resolved it for RetroArch, Lakka, EmuDeck, RetroDECK and ROCKNIX. type test keeps it out, as the site already did for that type. --- emulators/00_example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/emulators/00_example.yml b/emulators/00_example.yml index d87e75be..b7e3a710 100644 --- a/emulators/00_example.yml +++ b/emulators/00_example.yml @@ -1,5 +1,5 @@ emulator: "00_example" -type: libretro +type: test core_classification: pure_libretro source: "https://github.com/libretro/libretro-core-info" profiled_date: "2026-03-22" From cffd9eeb029a17536b9972374d2251ce3d8be093 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:43:18 +0200 Subject: [PATCH 4/5] feat: read the engine's transfer and scan events The speed is bytes_network over time, the interface's own guess at local bytes is gone. The plan says when a --from folder was read only in part or holds another drive, and how many files on the old card could not be read, with a list of each reason. The --from folders are only checked before the screen opens, never indexed there. --- install.ps1 | 2 +- install.py | 85 ++++++++++++++++++++++++++++++--------- install.sh | 2 +- tests/test_install_tui.py | 71 +++++++++++++++++++++++++++++++- 4 files changed, 138 insertions(+), 22 deletions(-) diff --git a/install.ps1 b/install.ps1 index 1dcc2992..9f56ce58 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "e0a65cc08918a2f53032f63d08974995542a0beabb7528ac3de35d0fdb698e36" + $defaultInstallSha256 = "1a26e1066a699413ab3d19f50ee552185ee2e9cbf74ee44548d3c9ea66f1b512" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 09f75b88..f2daef16 100644 --- a/install.py +++ b/install.py @@ -9468,7 +9468,10 @@ class TuiState: # Files an install copies from this machine, whose bytes are no # download: dest to (folder, reason), reason set for a misplaced one. self.local_dests: dict = {} - self.local_done: int = 0 + # Per --from folder, its scanned event, and the files on those + # folders that could not be read (unreadable events). + self.scanned: list = [] + self.unreadable: list = [] # The child run in progress: check, confirm, preview or install. self.run: str = "" self.running: bool = False @@ -9498,9 +9501,11 @@ class TuiState: # The install. self.download: dict | None = None self.progress: dict | None = None - # (time, bytes done) of the last progress events: a partial copy - # resumed counts in the first one and never in a speed. + # (time, bytes from the network) of the last transfer events: what + # came from this machine (a copy, the pack, a partial file resumed) + # is never in a speed. self.samples: list = [] + self.network: int = 0 self.finished: list = [] self.summary: dict | None = None self.code: int | None = None @@ -9644,7 +9649,7 @@ class TuiRow(NamedTuple): # Screens that scroll text instead of listing rows. -_PAGERS = ("preview", "command", "result", "misplaced") +_PAGERS = ("preview", "command", "result", "misplaced", "unreadable") _CHECKLISTS = ("systems", "cores") @@ -9795,6 +9800,8 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: rows.append(TuiRow("fromcard", "Copy the BIOS files from your old card or a folder")) if _tui_misplaced(state): rows.append(TuiRow("misplaced", "Show the files in the wrong place")) + if state.unreadable: + rows.append(TuiRow("unreadable", "Show the files that could not be read")) return rows + [TuiRow("options", "Options"), TuiRow("quit", "Quit")] @@ -9951,13 +9958,13 @@ def _start_run(state: TuiState, run: str) -> list[tuple]: # The systems of an earlier check go first: ticks read against them # dropped the --system the command line gave. state.check, state.files, state.systems, state.reading = None, [], [], None - state.found = [] + state.found, state.scanned, state.unreadable = [], [], [] state.checked_flags = tui_flags(state) elif run in ("confirm", "preview"): state.dry, state.dry_files, state.dry_summary = None, [], None else: state.download = state.progress = state.reading = None - state.samples, state.local_dests, state.local_done = [], {}, 0 + state.samples, state.local_dests, state.network = [], {}, 0 state.finished, state.summary, state.code, state.stopping = [], None, None, False return [("run", run)] @@ -10174,8 +10181,8 @@ def _activate(state: TuiState, row: TuiRow) -> list[tuple]: return _open_folder_screen(state, state.dest) if action == "fromcard": return _open_from_browser(state) - if action == "misplaced": - return _push(state, "misplaced") + if action in ("misplaced", "unreadable"): + return _push(state, action) # On the first system, so A continues and the D-pad ticks. return _push(state, action, 3 if action == "systems" else 0) if screen == "systems": @@ -10547,6 +10554,10 @@ def _take_event(state: TuiState, event: dict) -> None: state.reading = (event["files_done"], event["files_total"]) elif kind == "found" and state.run == "check": state.found.append(event) + elif kind == "scanned" and state.run == "check": + state.scanned.append(event) + elif kind == "unreadable" and state.run == "check": + state.unreadable.append(event) elif kind == "found" and state.run == "install": state.local_dests[event["dest"]] = (event["folder"], event["reason"]) elif state.run == "check": @@ -10566,15 +10577,12 @@ def _take_event(state: TuiState, event: dict) -> None: elif state.run == "install": if kind == "download": state.download = event + elif kind == "transfer": + state.network = event["bytes_network"] + state.samples = [sample for sample in state.samples + if sample[0] >= state.now - 10] + [(state.now, state.network)] elif kind in ("progress", "downloaded"): state.progress = event - if kind == "downloaded" and event["dest"] in state.local_dests: - # A copy from an old card or a folder: its bytes come from - # this machine, and a speed counting them was not the network's. - state.local_done += event["size"] - state.samples = [sample for sample in state.samples - if sample[0] >= state.now - 10] + [ - (state.now, event["bytes_done"] - state.local_done)] if kind == "downloaded": state.finished.append((event["dest"], event["ok"], event["reason"])) elif kind == "summary": @@ -10667,6 +10675,7 @@ _ROW_VERBS = { "preview": "Open", "folder": "Choose", "game_data": "Change", "installed": "Change", "jobs": "Change", "retry": "Try again", "back": "Back", "stay": "Stay", "fromcard": "Choose", "from": "Add a folder", "misplaced": "Show", "zip": "Copy from it", + "unreadable": "Show", } @@ -10772,6 +10781,20 @@ def _tui_found_elsewhere(state: TuiState) -> dict[str, list[dict]]: return found +def _scan_lines(state: TuiState) -> list[str]: + """What the reading of the --from folders left out, one line each.""" + lines = [] + for scan in state.scanned: + if scan["truncated"]: + lines.append(f"Read the first {MAX_SCANNED_FILES:,} files of {scan['folder']}.") + lines += [f"{drive} is another drive: add it to copy from it too." + for drive in scan["other_drives"]] + if state.unreadable: + count = len(state.unreadable) + lines.append(f"{_plural(count, 'file')} on your old card could not be read.") + return lines + + def _last_name(folder: str) -> str: """A folder or ZIP by its last name, a drive by itself (E:\\).""" return Path(folder).name or folder @@ -10844,6 +10867,8 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: hint.os_type = state.os_type lines.append("Or run it as root, this folder named: " + hint.command(state.platform, dest, elevated=True)) + if check is state.check: + lines += _scan_lines(state) if check.get("not_installable"): lines += _unavailable_lines(state) return lines @@ -11073,6 +11098,9 @@ def _pager_items(state: TuiState) -> list: """The text of a screen that scrolls, the same for its view and its keys.""" if state.screen == "misplaced": return [f"{event['dest']}: {event['reason']}" for event in _tui_misplaced(state)] + if state.screen == "unreadable": + return [f"{event['path']}" + (f", {event['member']}" if event["member"] else "") + + f": {event['reason']}" for event in state.unreadable] if state.screen == "result": return tui_outcome(state)[1] if state.screen == "command": @@ -11099,6 +11127,12 @@ def _view_preview(state: TuiState, width: int, height: int) -> tuple: ("A/Enter Back", "Up/Down Scroll")) +def _view_unreadable(state: TuiState, width: int, height: int) -> tuple: + return ("Files that could not be read", + _pager(state, width, height, _pager_items(state)), + ("A/Enter Back", "Up/Down Scroll")) + + def _view_misplaced(state: TuiState, width: int, height: int) -> tuple: return ("Files in the wrong place", _pager(state, width, height, _pager_items(state)), ("A/Enter Back", "Up/Down Scroll")) @@ -11136,10 +11170,9 @@ def _view_installing(state: TuiState, width: int, height: int) -> tuple: lines.append(_bar(min(done / total, 1.0) if total else 1.0, width)) lines.append(f"{format_size(done)} of {format_size(total)}") window = [sample for sample in state.samples if sample[0] >= state.now - 5] - fetched = done - state.local_done - start, base = window[0] if window else (state.now, fetched) + start, base = window[0] if window else (state.now, state.network) seconds = state.now - start - speed = (fetched - base) / seconds if seconds >= 1 else 0 + speed = (state.network - base) / seconds if seconds >= 1 else 0 if speed > 0: lines.append(f"{format_size(int(speed))}/s, " f"{format_duration((total - done) / speed)} left") @@ -11351,6 +11384,7 @@ _TUI_VIEWS: dict[str, Callable[[TuiState, int, int], tuple]] = { "preview": _view_preview, "command": _view_command, "confirm": _view_confirm, "installing": _view_installing, "result": _view_result, "error": _view_error, "quit": _view_quit, "cardroot": _view_cardroot, "misplaced": _view_misplaced, + "unreadable": _view_unreadable, } @@ -11893,7 +11927,20 @@ def run_tui(args: argparse.Namespace) -> int: os_type = _INVOCATION.os_type or detect_os() if os_type == "wsl": _enter_wsl() - _open_local_sources(args) + # The --from folders are read by each check, not here: indexing a large + # card held the first frame back. Only whether each can be read is asked. + folders = getattr(args, "from") or [] + setattr(args, "from", None) + try: + _open_local_sources(args) + finally: + setattr(args, "from", folders) + for folder in folders: + problem = from_folder_problem(str(folder)) + if problem is not None: + print(f"Error: {folder} {problem[1]}: --from names the folder to copy BIOS " + "files from.", file=sys.stderr) + return 1 state, effects = tui_start(args, os_type) work = Path(tempfile.mkdtemp(prefix=TUI_WORK_PREFIX)) (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") diff --git a/install.sh b/install.sh index d2ad8edf..5a10ae72 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="e0a65cc08918a2f53032f63d08974995542a0beabb7528ac3de35d0fdb698e36" +DEFAULT_INSTALL_SHA256="1a26e1066a699413ab3d19f50ee552185ee2e9cbf74ee44548d3c9ea66f1b512" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 7feade1c..6a907996 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -883,6 +883,7 @@ def golden_states() -> dict: installing.run, installing.running = "install", True installing.download = {"event": "download", "files": 4, "bytes": 1064964} installing.samples, installing.now = [(100.0, 12672), (104.0, 540672)], 104.0 + installing.network = 540672 installing.progress = {"files_done": 2, "files_total": 4, "bytes_done": 540672, "bytes_total": 1064964} installing.finished = [("gba_bios.bin", True, ""), ("scph5500.bin", False, "HTTP 404")] @@ -1103,6 +1104,10 @@ class SmallFixes(unittest.TestCase): install.tui_receive(state, ("event", state.run_id, { "event": "progress", "files_done": 0, "files_total": 2, "bytes_done": done, "bytes_total": 1000 * mega})) + # The partial copy resumed is counted local by the engine. + install.tui_receive(state, ("event", state.run_id, { + "event": "transfer", "bytes_local": 500 * mega, + "bytes_network": done - 500 * mega})) text = "\n".join(install.tui_view(state, 80, 24)) self.assertIn("1.0 MB/s, 8:18 left", text) @@ -1713,10 +1718,17 @@ class CopyFromAnOldCard(unittest.TestCase): (10.0, {"event": "downloaded", "dest": "big.bin", "ok": True, "reason": "", "size": 200 * mega, "files_done": 1, "files_total": 2, "bytes_done": 200 * mega, "bytes_total": 300 * mega}), + # The engine says which bytes came from this machine. + (10.0, {"event": "transfer", "bytes_local": 200 * mega, + "bytes_network": 0}), (11.0, {"event": "progress", "files_done": 1, "files_total": 2, "bytes_done": 201 * mega, "bytes_total": 300 * mega}), + (11.0, {"event": "transfer", "bytes_local": 200 * mega, + "bytes_network": mega}), (12.0, {"event": "progress", "files_done": 1, "files_total": 2, - "bytes_done": 202 * mega, "bytes_total": 300 * mega})] + "bytes_done": 202 * mega, "bytes_total": 300 * mega}), + (12.0, {"event": "transfer", "bytes_local": 200 * mega, + "bytes_network": 2 * mega})] for when, event in events: state.now = when install.tui_receive(state, ("event", state.run_id, event)) @@ -2119,6 +2131,63 @@ class AuditRoundThree(unittest.TestCase): text = "\n".join(install.tui_view(checked_state(), 80, 24)) self.assertIn("Named by ~/.config/retroarch/retroarch.cfg", text) + # The engine's own events (main, after the second audit). + + def test_the_speed_is_the_network_bytes(self): + state = checked_state() + state.stack.append("installing") + state.run, state.running = "install", True + install.tui_receive(state, ("event", state.run_id, + {"event": "download", "files": 4, "bytes": 4 << 20})) + for now, done, network in ((100.0, 1 << 20, 0), (102.0, 3 << 20, 1 << 20)): + state.now = now + install.tui_receive(state, ("event", state.run_id, { + "event": "progress", "files_done": 1, "files_total": 4, "bytes_done": done, + "bytes_total": 4 << 20})) + install.tui_receive(state, ("event", state.run_id, { + "event": "transfer", "bytes_local": done - network, "bytes_network": network})) + text = "\n".join(install.tui_view(state, 80, 24)) + # 1 MB from the network in 2 s, whatever came from the old card. + self.assertIn("512.0 KB/s, 0:02 left", text) + + def test_the_plan_says_what_the_scan_left_out(self): + state = checked_state() + state.from_folders = [Path("E:\\")] + install._start_check(state) + events = [{"event": "scanned", "folder": "E:\\", "files": 200000, "truncated": True, + "other_drives": ["E:\\D"]}, + {"event": "unreadable", "platform": "retroarch", "folder": "E:\\", + "path": "E:\\bios\\scph5501.bin", "member": "", "reason": "Permission denied"}, + {"event": "unreadable", "platform": "retroarch", "folder": "E:\\", + "path": "E:\\backup.zip", "member": "gba_bios.bin", + "reason": "it is encrypted, a password is needed"}] + for event in [*FILES, *SYSTEMS, *events, CHECK]: + install.tui_receive(state, ("event", state.run_id, event)) + with mock.patch.object(install, "_tui_folder_facts", return_value={ + "free": 50 * 1024 ** 3, "writable": True, "cores": []}): + install.tui_receive(state, ("ended", state.run_id, 1, [])) + text = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn("Read the first 200,000 files of E:\\.", text) + self.assertIn("E:\\D is another drive: add it to copy from it too.", text) + self.assertIn("2 files on your old card could not be read.", text) + rows = install.tui_rows(state) + state.focus["plan"] = next(i for i, row in enumerate(rows) if row.action == "unreadable") + press(state, "enter") + self.assertEqual(state.screen, "unreadable") + self.assertEqual(install._pager_items(state), [ + "E:\\bios\\scph5501.bin: Permission denied", + "E:\\backup.zip, gba_bios.bin: it is encrypted, a password is needed"]) + + def test_the_interface_does_not_index_the_folders_itself(self): + missing = REPO_ROOT / "tmp" / "no-such-old-card" + err = io.StringIO() + with mock.patch.object(install, "FolderIndex", side_effect=AssertionError("indexed")), \ + mock.patch.object(install, "detect_os", return_value="linux"), \ + mock.patch.object(install, "tui_start", side_effect=AssertionError("opened")), \ + mock.patch.object(sys, "stderr", err): + self.assertEqual(install.run_tui(_args(**{"from": [missing]})), 1) + self.assertIn(f"Error: {missing} does not exist: --from names the folder", err.getvalue()) + # Work folders: only a kill no handler sees leaves one, and the next # run takes it away. From 50b42823caba064be89274610a94aa1d34e8ae5a Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:44:25 +0200 Subject: [PATCH 5/5] feat: add a per-os data_root to profiles Apps such as AetherSX2 named their user data folder only in prose, so the site could neither print it nor show only the OS tabs that apply. data_root keys android, ios, windows, macos and linux, each with the path the user finds and the source_ref that builds it. --- schemas/emulator.schema.json | 16 ++++++++++++++++ tests/test_schema_semantic_gate.py | 26 ++++++++++++++++++++++++++ wiki/profiling.md | 1 + 3 files changed, 43 insertions(+) diff --git a/schemas/emulator.schema.json b/schemas/emulator.schema.json index 23c3356c..3b6062aa 100644 --- a/schemas/emulator.schema.json +++ b/schemas/emulator.schema.json @@ -912,6 +912,22 @@ "type": "string", "description": "Why source_commit is held on purpose rather than followed. The profile is then judged against its own revision, like one pinned to a superseded tag: comparing it to HEAD says nothing and recaling would repoint its refs at code the build it documents never had." }, + "data_root": { + "type": "object", + "minProperties": 1, + "propertyNames": {"enum": ["android", "ios", "windows", "macos", "linux"]}, + "additionalProperties": { + "type": "object", + "additionalProperties": false, + "required": ["path", "source_ref"], + "properties": { + "path": {"type": "string", "minLength": 1, "description": "The folder as the user finds it on that OS, with the environment variable or home prefix the code expands (%APPDATA%, ~, Android/data/)."}, + "source_ref": {"type": "string", "minLength": 1}, + "note": {"type": "string"} + } + }, + "description": "Per operating system, the folder the app reads its user data from, under which standalone_path destinations sit. Only the systems the app runs on are keyed." + }, "upstream_frozen": { "type": "string", "description": "Why upstream_commit is held on purpose rather than followed: the port vendors that upstream tree, synced to that release, or merged that commit. The upstream view is judged against its own revision and a bump leaves it, the way pin_frozen holds source_commit." diff --git a/tests/test_schema_semantic_gate.py b/tests/test_schema_semantic_gate.py index 6c06b3d0..746d2822 100644 --- a/tests/test_schema_semantic_gate.py +++ b/tests/test_schema_semantic_gate.py @@ -96,3 +96,29 @@ class SemanticManifestValidation(unittest.TestCase): if __name__ == "__main__": unittest.main() + + +try: + import jsonschema +except ImportError: + jsonschema = None + + +@unittest.skipIf(jsonschema is None, "jsonschema unavailable") +class DataRootIsKeyedBySystem(unittest.TestCase): + """AetherSX2 gave Android/data/xyz.aethersx2.android only in its notes: + the site could not print the folder, nor which OS tabs apply.""" + + def _errors(self, value): + schema = json.loads((REPO_ROOT / "schemas/emulator.schema.json").read_text(encoding="utf-8")) + validator = jsonschema.Draft202012Validator(schema["properties"]["data_root"]) + return list(validator.iter_errors(value)) + + def test_a_system_with_its_path_and_reference(self): + self.assertEqual(self._errors({"android": { + "path": "Android/data/xyz.aethersx2.android/files", "source_ref": "a.kt:1"}}), []) + + def test_an_unknown_system_or_a_path_without_reference(self): + self.assertTrue(self._errors({"amiga": {"path": "x", "source_ref": "y"}})) + self.assertTrue(self._errors({"windows": {"path": "%APPDATA%/X"}})) + self.assertTrue(self._errors({})) diff --git a/wiki/profiling.md b/wiki/profiling.md index 420c6336..0e106927 100644 --- a/wiki/profiling.md +++ b/wiki/profiling.md @@ -591,6 +591,7 @@ does not look. | `known_hash_adler32` | expected Adler-32 hash (Dolphin's DSP ROMs). Pair with `adler32_byteswap` when the code hashes 16-bit byte-swapped data | | `adler32_pairs_with`, `adler32_pairs` | the file hashed together with this one and the combinations the code accepts, this file's value first, in table order: Dolphin warns on a DSP ROM pair outside its `known_roms` even when each file passes alone | | `pin_frozen` | why `source_commit` is held on purpose rather than followed. The profile is then judged against its own revision, like one pinned to a superseded tag | +| `data_root` | per operating system (`android`, `ios`, `windows`, `macos`, `linux`), the folder the app reads its user data from, under which `standalone_path` destinations sit: `path` as the user finds it (`Android/data//files`, `%APPDATA%/...`), with the `source_ref` that builds it. Only the systems the app runs on are keyed | | `upstream_frozen` | why `upstream_commit` is held: the port vendors that upstream tree, synced to that release, or merged that commit. The upstream view is read at its own pin and a bump leaves it. Without it, a port that is a GitHub fork of its upstream follows the merge base of the two | | `prose_written_at` | the pin a reader confirmed the prose citations against. When the profile's history shows a text living under several pins, profile_sync takes this one as the revision the text describes, so a bump is no longer refused for a citation already reread. It never overrides a history with a single candidate | | `version_source` | for a project published outside any forge, `{url, pattern}`: the page that announces releases and a regular expression whose first group is the latest version. `profile_sync --check-version` compares it with `core_version`, and `check_freshness --profiles` calls the profile stale when they differ. `no_genuine_copy` names an announced version found only rebuilt or re-signed: the profile stays on the newest unmodified build and is not stale until the page announces a later one |