From cf3b61a17baf60ae90ccf8add5b88fdcc06ba755 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:22:42 +0200 Subject: [PATCH 01/19] fix: exit on the window's signal when it closes A closed window under a changing screen failed the draw (EIO, or a termios.error from the SIGCONT that comes with SIGHUP) before the signal was read, and the run ended on 1 with a crash log kept. --- install.ps1 | 2 +- install.py | 41 ++++++++++++++++++++++++++++++++------- install.sh | 2 +- tests/test_install_tui.py | 34 ++++++++++++++++++++++++++++---- 4 files changed, 66 insertions(+), 13 deletions(-) diff --git a/install.ps1 b/install.ps1 index 70f6b4e50..9a1ae9148 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "e2cc5f073a8226fe26b0513ebd25f00de6f3f613d8c9851473c2d30b53356f48" + $defaultInstallSha256 = "c2a66e6b26004e96ef5f59a7fb5a2ed65bcaa9afc7602bd3e0d71f50cc561056" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index b0eca90ef..ee80f7923 100644 --- a/install.py +++ b/install.py @@ -12500,13 +12500,14 @@ def run_tui(args: argparse.Namespace) -> int: if terminating["signal"] and state.screen == "result": # The result is printed in the terminal on the way out. state.exit_code = state.code if state.code is not None else 1 - except Exception: # noqa: BLE001 - # A traceback is where every reporter of #71, #73 and #74 stopped: - # it goes to a file, and the terminal says one sentence. - crash = work / "error.log" - crash.write_text(traceback.format_exc(), encoding="utf-8") - state.log, state.code, state.exit_code = str(crash), 1, 1 - state.stack = ["crashed"] + except Exception as exc: # noqa: BLE001 + if flags["term"] or _closed_terminal(exc): + # The window closed under a screen that was changing, or the + # SIGCONT that comes with its SIGHUP put the screen back: the + # write failed before the signal was read. That is its exit. + state.exit_code = 128 + (flags["term"] or getattr(signal, "SIGHUP", 1)) + else: + _tui_crashed(state, work) finally: # Each step on its own: a write the closed window refuses, or a # second signal while the install writes its summary, must not @@ -12560,6 +12561,32 @@ def _tui_startup(args: argparse.Namespace) -> tuple[TuiState, list[tuple]] | Non return tui_start(args, os_type) +# What a write to a closed window or a closed pipe answers. +_CLOSED_TERMINAL = frozenset({errno.EIO, errno.EPIPE, errno.ENXIO}) + + +def _closed_terminal(exc: BaseException) -> bool: + """Whether exc is the terminal gone: an OSError, or a termios.error, + whose first argument is the errno, answering EIO, EPIPE or ENXIO.""" + if isinstance(exc, OSError): + return exc.errno in _CLOSED_TERMINAL + if type(exc).__name__ == "error" and type(exc).__module__ == "termios": + return bool(exc.args) and exc.args[0] in _CLOSED_TERMINAL + return False + + +def _tui_crashed(state: TuiState, work: Path) -> None: + """A traceback is where every reporter of #71, #73 and #74 stopped: it + goes to a file, and the terminal says one sentence.""" + crash = work / "error.log" + try: + crash.write_text(traceback.format_exc(), encoding="utf-8") + except OSError as exc: + print(f"The error could not be written to {crash}: {exc}", file=sys.stderr) + state.log, state.code, state.exit_code = str(crash), 1, 1 + state.stack = ["crashed"] + + def _restore_handlers(handlers: dict) -> None: for number, previous in handlers.items(): signal.signal(number, previous) diff --git a/install.sh b/install.sh index 539267b40..6c8d516c4 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="e2cc5f073a8226fe26b0513ebd25f00de6f3f613d8c9851473c2d30b53356f48" +DEFAULT_INSTALL_SHA256="c2a66e6b26004e96ef5f59a7fb5a2ed65bcaa9afc7602bd3e0d71f50cc561056" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 392ff1b97..8fe485e15 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -1444,13 +1444,14 @@ class CleanupSurvivesABrokenTerminal(unittest.TestCase): mock.patch.object(sys, "stderr", out): code = install.run_tui(_args()) self.assertIs(sys.stdout, out) - self.assertEqual(code, 1) + # A window that answers EIO is a closed window: its signal's code, + # not a crash. + self.assertEqual(code, 129) # The farewell found no terminal: what still flushes at exit goes to # the null device, and the exit code stays the one returned. self.assertEqual(os.fstat(screen_fd).st_rdev, os.stat(os.devnull).st_rdev) - # The crash keeps its log, marked kept, and nothing else of the run. - self.assertEqual(sorted(path.name for path in work.iterdir()), - ["error.log", "kept", "owner"]) + # No crash log is kept, and nothing else of the run. + self.assertFalse(work.exists()) self.assertEqual(termios.tcgetattr(slave), before) for number, handler in handlers.items(): self.assertEqual(signal.getsignal(number), handler) @@ -2859,6 +2860,31 @@ class EndToEnd(unittest.TestCase): self.assertTrue(os.WIFEXITED(status)) self.assertEqual(term.status, 129) + def test_closing_the_window_while_it_installs(self): + """The changing screen fails to draw on the closed window: that is + the window's signal, not a crash with a log kept.""" + repo = self.repo(GameDataRepository, slow=True) + scratch = repo.root / "scratch" + scratch.mkdir() + self.env["TMPDIR"] = str(scratch) + term = self.start("--platform", "retroarch", "--dest", str(repo.dest), "--jobs", "1") + term.wait("> Install") + term.send("\r") + term.wait("1 of 6 files") + os.close(term.fd) + end = time.monotonic() + 20 + while time.monotonic() < end: + pid, status = os.waitpid(term.pid, os.WNOHANG) + if pid: + break + time.sleep(0.05) + else: + os.kill(term.pid, 9) + self.fail("still running once its window closed") + term.status = os.WEXITSTATUS(status) + self.assertEqual(term.status, 129) + self.assertEqual(list(scratch.glob("retrobios-tui-*/error.log")), []) + def test_a_desktop_names_its_keys_and_a_pad_its_buttons(self): self.repo() for env, bar in (({"DISPLAY": ":0"}, "Enter Continue Esc Quit ? Help"), From 5869da409bc56ba8cd37b3cf2750de06308cc504 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:24:40 +0200 Subject: [PATCH 02/19] fix: open the interface whatever owner files hold An owner file holding a number past a pid raised OverflowError before the screen opened, and a full temp disk stopped the run on the owner write. 0 and negative numbers are unknown owners, not process groups. --- install.ps1 | 2 +- install.py | 30 ++++++++++++++++++++++++------ install.sh | 2 +- tests/test_install_tui.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/install.ps1 b/install.ps1 index 9a1ae9148..f6295a5b6 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "c2a66e6b26004e96ef5f59a7fb5a2ed65bcaa9afc7602bd3e0d71f50cc561056" + $defaultInstallSha256 = "b23c6da4ee131f898c25540bd53677ea4112964fd00b209e4fe85dfdba8427d1" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index ee80f7923..0bcd62db4 100644 --- a/install.py +++ b/install.py @@ -12279,8 +12279,9 @@ def _owner_alive(folder: Path) -> bool | None: pid = int((folder / TUI_OWNER).read_text(encoding="utf-8").strip()) except (OSError, ValueError): return None - if os.name == "nt": - # os.kill(pid, 0) ends a process on Windows instead of asking. + if os.name == "nt" or pid <= 0: + # os.kill(pid, 0) ends a process on Windows instead of asking, and + # 0 or a negative number names a process group, not the run. return None try: os.kill(pid, 0) @@ -12288,7 +12289,8 @@ def _owner_alive(folder: Path) -> bool | None: return False except PermissionError: return True - except OSError: + except (OSError, OverflowError): + # A number past what a pid holds is no process this run can ask. return None return True @@ -12375,9 +12377,20 @@ def run_tui(args: argparse.Namespace) -> int: state, effects = started if flags["term"]: state.exit_code = 128 + flags["term"] - work = Path(tempfile.mkdtemp(prefix=TUI_WORK_PREFIX)) - (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") - _prune_work_folders(work.parent, time.time()) + try: + work = Path(tempfile.mkdtemp(prefix=TUI_WORK_PREFIX)) + except OSError as exc: + _restore_handlers(handlers) + print(f"Error: no work folder could be made in {tempfile.gettempdir()}: {exc}. " + "Free some space there, or point TMPDIR (TEMP on Windows) at another folder.", file=sys.stderr) + return 1 + try: + (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") + except OSError as exc: + # A full temp disk: the folder has no owner, and a later run leaves + # it a week like any unknown one. The logs fail as they come. + print(f"The work folder {work} could not be marked as this run's: {exc}", + file=sys.stderr) script = str(Path(__file__).resolve()) inbox: queue.Queue = queue.Queue() current: list[_EngineRun] = [] @@ -12445,6 +12458,11 @@ def run_tui(args: argparse.Namespace) -> int: if number is not None: handlers[number] = signal.signal(number, handler) sys.stdout = sys.stderr = buffer + try: + _prune_work_folders(work.parent, time.time()) + except OSError as exc: + # What earlier runs left stays until a later run: this one opens. + print(f"Earlier work folders could not be removed: {exc}", file=sys.stderr) if state.exit_code is None: # Stopped while it started: the screen never opens. terminal.enter() diff --git a/install.sh b/install.sh index 6c8d516c4..e2217f75e 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="c2a66e6b26004e96ef5f59a7fb5a2ed65bcaa9afc7602bd3e0d71f50cc561056" +DEFAULT_INSTALL_SHA256="b23c6da4ee131f898c25540bd53677ea4112964fd00b209e4fe85dfdba8427d1" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 8fe485e15..ef7c2f674 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -10,6 +10,7 @@ installs it). from __future__ import annotations import argparse +import errno import fcntl import functools import hashlib @@ -1801,6 +1802,31 @@ class WorkFolders(unittest.TestCase): work, out = self.run_to(130) self.assertFalse(work.exists()) + def test_a_full_temp_disk_or_a_failing_pruner_still_opens(self): + real_write = Path.write_text + + def full(path, *args, **kwargs): + if path.name == install.TUI_OWNER: + raise OSError(errno.ENOSPC, "No space left on device") + return real_write(path, *args, **kwargs) + + with mock.patch.object(Path, "write_text", full): + work, out = self.run_to(0) + self.assertFalse(work.exists()) + with mock.patch.object(install, "_prune_work_folders", + side_effect=OSError(errno.EACCES, "denied")): + work, out = self.run_to(0) + self.assertFalse(work.exists()) + err = io.StringIO() + with mock.patch.object(install.tempfile, "mkdtemp", + side_effect=OSError(errno.ENOSPC, "No space left")), \ + mock.patch.object(install, "tui_start", return_value=(fresh_state(), [])), \ + mock.patch.object(install, "_open_local_sources"), \ + mock.patch.object(install, "detect_os", return_value="linux"), \ + mock.patch.object(sys, "stderr", err): + self.assertEqual(install.run_tui(_args()), 1) + self.assertIn("no work folder could be made", err.getvalue()) + def test_a_stop_while_it_starts_ends_on_the_farewell(self): """Start+Select during the detection: the signal is the interface's, never the default action that ended Python without a word.""" @@ -2456,8 +2482,17 @@ class AuditRoundThree(unittest.TestCase): unknown_old = folder("unknown-old", None, False, 2 * week) other = root / "something-else" other.mkdir() + # A stray or hostile owner file is an unknown owner: a number past + # what a pid holds raised OverflowError, and 0 or -1 name a group. + hostile = [folder(f"hostile{index}", None, False, 60) + for index in range(3)] + for path, text in zip(hostile, ("9" * 40, "0", "-1")): + (path / install.TUI_OWNER).write_text(text) + self.assertIsNone(install._owner_alive(path)) removed = install._prune_work_folders(root, now) self.assertEqual(sorted(removed), sorted([killed, failed_old, unknown_old])) + for path in hostile: + shutil.rmtree(path) self.assertEqual(sorted(path.name for path in root.iterdir()), sorted(path.name for path in (running, failed, unknown, other))) From ea48a54329c92e8e4151bb93efbd9aca0a349137 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:36:53 +0200 Subject: [PATCH 03/19] fix: judge a declared slot by the platform's copy --declared-only settled each slot on its first declaration before the cores narrowed the list, and the declared event judged the core copy served in its place. Both now read the declaration whose systems remain, as the platform checks it. --- install.ps1 | 2 +- install.py | 95 ++++++++++++++++++++++++++++++------- install.sh | 2 +- tests/test_install_cores.py | 50 +++++++++++++++++++ 4 files changed, 129 insertions(+), 20 deletions(-) diff --git a/install.ps1 b/install.ps1 index f6295a5b6..ab608c8f6 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "b23c6da4ee131f898c25540bd53677ea4112964fd00b209e4fe85dfdba8427d1" + $defaultInstallSha256 = "5fedfbf308ba04a174ce0682380a28150b6fe5eab514a605e3b318193ce37c7a" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 0bcd62db4..8f2464c7b 100644 --- a/install.py +++ b/install.py @@ -665,17 +665,45 @@ def _emit_check( bytes_needed=sum(entry.get("size", 0) for entry in needed), data_missing=data_missing, not_installable=not_installable, ) - own = [entry for entry in files if entry.get("cores") is None] - own_needed = [entry for entry in own if id(entry) not in present] - own_wrong = sum(1 for entry in own if id(entry) in wrong) + # The copy the platform itself checks at each of its slots, which is + # what --declared-only checks: a core's copy served in its place is + # judged against the platform's. + own_states = [] + for entry in files: + if entry.get("cores") is not None: + continue + copy = declared_copy(entry) + if copy is entry or copy.get("sha1") == entry.get("sha1"): + state = ("present" if id(entry) in present + else "wrong_hash" if id(entry) in wrong else "missing") + else: + state = _declared_state(copy, entry, bios_path) + own_states.append((entry["dest"], copy.get("size", 0), state)) EVENTS.emit( - "declared", platform=plat, files=len(own), present=len(own) - len(own_needed), - missing=len(own_needed) - own_wrong, wrong_hash=own_wrong, - bytes_needed=sum(entry.get("size", 0) for entry in own_needed), - dests=[entry["dest"] for entry in own], + "declared", platform=plat, files=len(own_states), + present=sum(1 for _d, _s, state in own_states if state == "present"), + missing=sum(1 for _d, _s, state in own_states if state == "missing"), + wrong_hash=sum(1 for _d, _s, state in own_states if state == "wrong_hash"), + bytes_needed=sum(size for _d, size, state in own_states if state != "present"), + dests=[dest for dest, _s, _state in own_states], ) +def _declared_state(copy: dict, entry: dict, bios_path: Path) -> str: + """present, missing or wrong_hash: the file at entry's place against copy.""" + try: + path = _entry_path(bios_path, entry) + except ValueError: + return "missing" + if not path.is_file(): + return "missing" if not path.exists() else "wrong_hash" + try: + verdict = _classify_local(copy, path) + except OSError: + return "wrong_hash" + return "present" if verdict == "up_to_date" else "wrong_hash" + + def leave_out_game_data( files: list[dict], omitted: list[dict] ) -> tuple[list[dict], list[dict], list[dict]]: @@ -697,17 +725,42 @@ def keep_declared(files: list[dict], omitted: list[dict]) -> tuple[list[dict], l A slot a core contradicts gets the platform's copy, and no core copy stays behind as an alternative: narrowing by cores afterwards would - otherwise put a core's file back where a dropped system's was. + otherwise put a core's file back where a dropped system's was. Which of + the platform's declarations answers is left to that narrowing, as + without --declared-only: the first whose systems remain. """ def own(entries: list[dict]) -> list[dict]: - return [ - {k: v for k, v in _settle_slot(entry, set()).items() if k != "alternatives"} - for entry in entries if entry.get("cores") is None - ] + result = [] + for entry in entries: + if entry.get("cores") is not None: + continue + kept = {k: v for k, v in _settle_slot(entry, set()).items() + if k not in ("alternatives", "declared_copy")} + declared = entry.get("declared_choices") or [] + if len(declared) > 1: + kept["declared_choices"] = declared + result.append(kept) + return result return own(files), own(omitted) +def declared_copy(entry: dict) -> dict: + """The copy the platform's own list checks at a platform file's slot. + + A narrowing that serves a core's copy there records the platform's + answer beside it, and an entry not yet settled answers by its first + declaration. Otherwise the entry is the platform's copy itself. + """ + recorded = entry.get("declared_copy") + if recorded is not None: + return recorded + declared = entry.get("declared_choices") + if declared: + return declared[0] + return entry + + def _left_out_lines(left_out: list[dict], every: bool) -> list[str]: """What --no-game-data left out, named per category, ten each unless every.""" lines = [] @@ -3428,22 +3481,28 @@ def _settle_slot( declared = entry.get("declared_choices") if declared is None: return entry + platform_copy = next( + (c for c in declared + if kept_system is None or any(kept_system(s) for s in c["systems"])), + declared[0], + ) copy = next( (c for c in entry.get("slot_choices", []) if chosen.intersection(c["cores"])), None, ) if copy is None or "sha1" not in copy: - copy = next( - (c for c in declared - if kept_system is None or any(kept_system(s) for s in c["systems"])), - declared[0], - ) + copy = platform_copy kept = { k: v for k, v in entry.items() - if k not in _COPY_FIELDS and k not in ("slot_choices", "declared_choices") + if k not in _COPY_FIELDS + and k not in ("slot_choices", "declared_choices", "declared_copy") } kept.update((k, copy[k]) for k in _COPY_FIELDS if k in copy and k != "cores") kept["cores"] = None + if copy is not platform_copy and copy.get("sha1") != platform_copy.get("sha1"): + # What the platform checks here, for the `declared` event. + kept["declared_copy"] = {k: platform_copy[k] for k in _COPY_FIELDS + if k in platform_copy and k != "cores"} return kept diff --git a/install.sh b/install.sh index e2217f75e..47b0c4dab 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="b23c6da4ee131f898c25540bd53677ea4112964fd00b209e4fe85dfdba8427d1" +DEFAULT_INSTALL_SHA256="5fedfbf308ba04a174ce0682380a28150b6fe5eab514a605e3b318193ce37c7a" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_cores.py b/tests/test_install_cores.py index 3d104654e..7a855642e 100644 --- a/tests/test_install_cores.py +++ b/tests/test_install_cores.py @@ -288,6 +288,56 @@ class APlatformFileSettledByCoresAndSystems(unittest.TestCase): self.assertEqual(self.served(["dsi"]), "c") self.assertEqual(self.served(["nds"]), "b") + def test_declared_only_keeps_the_declaration_whose_system_remains(self): + """--declared-only settled the slot on the first declaration before + the cores narrowed it: on a target without melonds it checked the + nintendo-dsi copy, the platform the nintendo-ds one.""" + manifest = self.manifest() + for names, letter in ((["nds"], "b"), (["dsi"], "c"), (["melonds"], "c"), + (["desmume"], "b")): + with self.subTest(names=names): + files, omitted = install.keep_declared(manifest["files"], []) + files, _ = install.narrow_to_cores(manifest, names, files, omitted) + self.assertEqual(files[0]["sha1"][0], letter) + + def test_the_declared_event_judges_the_copy_the_platform_checks(self): + """melonds' copy is served at the slot, the platform checks its own: + the event and a --declared-only check give the same answer.""" + import hashlib + + tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, tmp, True) + payloads = {letter: letter.encode() * 3 for letter in "abc"} + manifest = self.manifest() + entry = manifest["files"][0] + + def real(copy: dict, letter: str) -> None: + copy.update(sha1=hashlib.sha1(payloads[letter]).hexdigest(), + sha256=hashlib.sha256(payloads[letter]).hexdigest(), size=3) + + real(entry, "a") + real(entry["slot_choices"][1], "a") + real(entry["declared_choices"][0], "c") + real(entry["declared_choices"][1], "b") + (tmp / "dsi_bios9.bin").write_bytes(payloads["c"]) + + def declared_event(files: list[dict]) -> dict: + _todo, fine, wrong = install.check_local(files, tmp) + with mock.patch.object(install.EVENTS, "emit") as emit: + install._emit_check("retroarch", manifest, tmp, files, fine, wrong, 0, 0) + return next(call.kwargs for call in emit.call_args_list + if call.args[0] == "declared") + + for names in (None, ["melonds"], ["desmume", "melonds"]): + with self.subTest(names=names): + served = manifest["files"] + own, _ = install.keep_declared(manifest["files"], []) + if names is not None: + served, _ = install.narrow_to_cores(manifest, names) + own, _ = install.narrow_to_cores(manifest, names, own, []) + self.assertEqual(declared_event(served), declared_event(own)) + self.assertEqual(declared_event(own)["present"], 1) + def test_the_boundary_checks_every_copy(self): install._validate_manifest(self.manifest(), "retroarch") bad_choices = ( From 4b8e2f9de2e8a5734f71f8670c3125e9c70fa2a5 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:37:03 +0200 Subject: [PATCH 04/19] fix: check system_titles at the boundary A list or a number among the titles passed validation and raised later, in --system matching or the interface. --- install.ps1 | 2 +- install.py | 9 +++++++++ install.sh | 2 +- tests/test_install_selection.py | 13 +++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index ab608c8f6..1ddca47ec 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "5fedfbf308ba04a174ce0682380a28150b6fe5eab514a605e3b318193ce37c7a" + $defaultInstallSha256 = "6d8af0dbca6697ba71692e5e482c8a16d9e8b9896652fec722d9c84f23bba047" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 8f2464c7b..cb583f7f1 100644 --- a/install.py +++ b/install.py @@ -3069,6 +3069,15 @@ def _validate_manifest(data: object, plat: str) -> dict: _check_tags(mapping, key, field) if mapping[key] is None: raise ValueError(f"invalid {field} for {key}") + titles = data.get("system_titles") + if titles is not None: + if not isinstance(titles, dict) or len(titles) > MAX_MANIFEST_FILES: + raise ValueError("invalid system_titles") + for key, title in titles.items(): + if not isinstance(key, str) or not 0 < len(key) <= 256: + raise ValueError("invalid system_titles key") + if not isinstance(title, str) or not 0 < len(title) <= 256: + raise ValueError(f"invalid system_titles for {key}") copies = data.get("standalone_copies", []) if not isinstance(copies, list) or len(copies) > 10_000: diff --git a/install.sh b/install.sh index 47b0c4dab..e85ff322c 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="5fedfbf308ba04a174ce0682380a28150b6fe5eab514a605e3b318193ce37c7a" +DEFAULT_INSTALL_SHA256="6d8af0dbca6697ba71692e5e482c8a16d9e8b9896652fec722d9c84f23bba047" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 97b969b33..5cf8f4819 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -493,6 +493,19 @@ class TheBoundaryChecksTheNewFields(unittest.TestCase): with self.subTest(field=field, value=bad), self.assertRaises(ValueError): install._validate_manifest(manifest, "retroarch") + def test_the_system_titles(self): + """A title is read as text by --system and the interface: a list or + a number there raised mid-run instead of at the boundary.""" + good = self._manifest() + good["system_titles"] = {"sony-playstation": "Sony PlayStation"} + install._validate_manifest(good, "retroarch") + for bad in (["x"], {"s": ["Sony"]}, {"s": None}, {"s": 5}, {"s": ""}, + {"": "Sony"}, {"s": "x" * 257}, {"s" * 257: "Sony"}): + manifest = self._manifest() + manifest["system_titles"] = bad + with self.subTest(value=bad), self.assertRaises(ValueError): + install._validate_manifest(manifest, "retroarch") + class _QuietHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format, *args): # noqa: A002 - stdlib signature From 3203c766fc09faa6a6d102d4d5d697ee532b7e35 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:37:14 +0200 Subject: [PATCH 05/19] fix: name a folder or unreadable file in place A folder or a file the user cannot read at a destination ended --check and the install on a traceback from the hashing pool. --check now gives the reason per file, and the install fails that file alone, before any download for a folder. --- install.ps1 | 2 +- install.py | 21 ++++++++++++++++++--- install.sh | 2 +- tests/test_install_selection.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/install.ps1 b/install.ps1 index 1ddca47ec..c0e93dca3 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "6d8af0dbca6697ba71692e5e482c8a16d9e8b9896652fec722d9c84f23bba047" + $defaultInstallSha256 = "c361b511d9550b2115d918a0233115cd9f42920bce881f41395c3d7614be534d" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index cb583f7f1..4568f8e54 100644 --- a/install.py +++ b/install.py @@ -4046,9 +4046,18 @@ def _classify_local(entry: dict, dest: Path) -> str: for name in ("sha256", "sha1") if entry.get(name) } + if not dest.is_file(): + # A folder at the file's place: --check says so, an install fails + # on that file alone. + return "mismatched" if not expected: return "up_to_date" - actual = _digest_file(dest, tuple(expected)) + try: + actual = _digest_file(dest, tuple(expected)) + except OSError: + # Unreadable (no permission, a read error on a failing card): its + # contents are not known to be the right ones. + return "mismatched" if all(actual[name] == value for name, value in expected.items()): return "up_to_date" return "mismatched" @@ -4444,6 +4453,9 @@ def _download_one( if _write_refused(exc): return f["dest"], False, NOT_WRITABLE return f["dest"], False, exc.strerror or str(exc) + if dest.is_dir(): + # Fetched, it failed only at the rename, on an errno and two paths. + return f["dest"], False, FOLDER_IN_PLACE if copy is not None: result = _copy_local(f, bios_path, dest, copy, progress) @@ -4834,6 +4846,7 @@ READ_ONLY = "file in place is read-only" # A file in place whose content differs, which could not be moved aside: it # is left as it is rather than lost. NOT_KEPT = "file in place could not be kept aside" +FOLDER_IN_PLACE = "a folder sits where this file goes: move it away, then run again" def _clear_read_only(dest: Path) -> None: @@ -7539,12 +7552,14 @@ def _wrong_contents( owners: dict[str, list[dict]] | None = None, ) -> str: """Why the file at an entry's place is not the one the list installs.""" + if path.is_dir(): + return FOLDER_IN_PLACE try: size = path.stat().st_size with open(path, "rb") as handle: head = handle.read(4) - except OSError: - return "" + except OSError as exc: + return f"could not be read ({exc.strerror or exc}): its contents are unknown" sha1 = index.sha1(path) owner = _content_owner(sha1, entry, files, owners) if owner: diff --git a/install.sh b/install.sh index e85ff322c..34c27f9d4 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="6d8af0dbca6697ba71692e5e482c8a16d9e8b9896652fec722d9c84f23bba047" +DEFAULT_INSTALL_SHA256="c361b511d9550b2115d918a0233115cd9f42920bce881f41395c3d7614be534d" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 5cf8f4819..3b4e23531 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -1501,5 +1501,38 @@ class TheTitleASetupPrints(unittest.TestCase): self.assertEqual(install.system_aliases(manifest)["Sony PlayStation"], "arcade") +class SomethingElseAtADestination(unittest.TestCase): + """A folder, or a file this user cannot read, at a destination ended + --check and the install on an IsADirectoryError traceback.""" + + def prepare(self, repo: FakeRepository) -> None: + (repo.dest / "scph5500.bin").mkdir(parents=True) + unreadable = repo.dest / "scph5501.bin" + unreadable.write_bytes(b"x") + unreadable.chmod(0) + + @unittest.skipIf(os.name == "nt" or (hasattr(os, "geteuid") and os.geteuid() == 0), + "a mode of 0 stops no read on Windows or as root") + def test_each_says_why_in_the_check_and_the_install(self): + with FakeRepository() as repo: + self.prepare(repo) + proc = repo.run("--check") + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertNotIn("Traceback", proc.stderr) + self.assertIn(" wrong hash: scph5500.bin\n a folder sits where this " + "file goes: move it away, then run again", proc.stdout) + self.assertIn(" wrong hash: scph5501.bin\n could not be read " + "(Permission denied)", proc.stdout) + proc = repo.run("--yes") + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertNotIn("Traceback", proc.stderr) + self.assertIn("failed: scph5500.bin (a folder sits where this file goes", + proc.stdout) + self.assertIn("failed: scph5501.bin (file in place could not be kept aside", + proc.stdout) + self.assertTrue((repo.dest / "scph5500.bin").is_dir()) + self.assertIn("gba_bios.bin", repo.installed()) + + if __name__ == "__main__": unittest.main() From 3523e46595506e31ca193744fc6f8efea09ab86b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:40:57 +0200 Subject: [PATCH 06/19] fix: open each archive of a folder once Each ZIP member read reopened its archive and parsed its directory again: 4000 members of one size made 4000 parses. A folder index keeps the archives it reads open, 16 at most, read under a lock from the copy threads, and every index is closed at the end of the run. Archives the list installs are still opened and closed at once. --- install.ps1 | 2 +- install.py | 80 +++++++++++++++++++++++++++++++++++--- install.sh | 2 +- tests/test_install_from.py | 49 +++++++++++++++++++++++ 4 files changed, 126 insertions(+), 7 deletions(-) diff --git a/install.ps1 b/install.ps1 index c0e93dca3..561ddaedf 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "c361b511d9550b2115d918a0233115cd9f42920bce881f41395c3d7614be534d" + $defaultInstallSha256 = "5a7dc2fe2aa2abed511a94b3711f1846461a306ea185cd34407efac6c526cab0" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 4568f8e54..43d1dca02 100644 --- a/install.py +++ b/install.py @@ -45,6 +45,7 @@ import unicodedata import urllib.error import urllib.parse import urllib.request +import weakref import xml.etree.ElementTree as ET import zipfile import zlib @@ -6766,6 +6767,8 @@ class LocalSources: self.pack: PackSource | None = None # The folders named with --from, read once for every platform. self.folders: list[FolderIndex] = [] + # Every folder index of the run still in use, closed at its end. + self.indexes: weakref.WeakSet[FolderIndex] = weakref.WeakSet() @property def offline(self) -> bool: @@ -6775,10 +6778,16 @@ class LocalSources: def reset(self) -> None: if self.pack is not None: self.pack.close() + self.close_indexes() self.manifest = None self.pack = None self.folders = [] + def close_indexes(self) -> None: + for index in list(self.indexes): + index.close() + self.indexes = weakref.WeakSet() + LOCAL = LocalSources() NOT_IN_PACK = "not in the pack" @@ -6860,7 +6869,7 @@ def _open_copy(copy: LocalCopy) -> Iterator[BinaryIO]: with open(copy.path, "rb") as handle: yield handle return - with zipfile.ZipFile(copy.path) as archive, archive.open(copy.member) as handle: + with copy.index.open_member(copy.path, copy.member) as handle: yield handle @@ -7014,6 +7023,10 @@ class LocalCopy(NamedTuple): return f"{self.member} inside {self.path}" if self.member else str(self.path) +# The archives a FolderIndex keeps open at once. +OPEN_ARCHIVES = 16 + + class FolderIndex: """The files under a folder, by name and by size, hashed on demand. @@ -7032,6 +7045,7 @@ class FolderIndex: self.archives: list[Path] = [] # Name a ._ file stands beside (casefolded) -> the ._ files. self.apple_double: dict[str, list[Path]] = {} + LOCAL.indexes.add(self) # The list's destinations, casefolded (an archive among them is not # unpacked for its members) and as the list writes them. self.sealed: set[str] = set() @@ -7050,6 +7064,10 @@ class FolderIndex: self._member_named: dict[int, dict[str, list[tuple[Path, str]]]] = {} self._scans: dict[tuple[str, int, str], _HashScan] = {} self._member_size: dict[tuple[Path, str], int] = {} + # The archives read so far, open, the last read at the end: a ZIP + # was opened again, its directory parsed again, for each member. + self._zips: dict[Path, zipfile.ZipFile] = {} + self._zip_lock = threading.Lock() # (path, member or "") -> (its size or None when unknown, why) of each # file, member or folder below root that could not be read. self.unreadable: dict[tuple[Path, str], tuple[int | None, str]] = {} @@ -7121,6 +7139,50 @@ class FolderIndex: return False return True + def _archive(self, path: Path) -> zipfile.ZipFile: + """The archive at path, opened once. Called with _zip_lock held. + + A few stay open, the least recently read closed first: a card of + MAME sets would otherwise hold a descriptor per archive. Closing + one leaves a member being read open until it is closed itself. + Only archives the list does not install are read this way. + """ + archive = self._zips.pop(path, None) + if archive is None: + archive = zipfile.ZipFile(path) + while len(self._zips) >= OPEN_ARCHIVES: + self._zips.pop(next(iter(self._zips))).close() + self._zips[path] = archive + return archive + + def infolist(self, path: Path) -> list[zipfile.ZipInfo]: + with self._zip_lock: + return self._archive(path).infolist() + + @contextlib.contextmanager + def open_member(self, path: Path, member: str) -> Iterator[BinaryIO]: + """A member of an archive under the folder, read from any thread.""" + with self._zip_lock: + handle = self._archive(path).open(member) + try: + yield handle + finally: + # The archive counts its open members: the count moves under + # the lock, as when the member was opened. + with self._zip_lock: + handle.close() + + def open_archives(self) -> int: + with self._zip_lock: + return len(self._zips) + + def close(self) -> None: + """Close the archives the index holds open.""" + with self._zip_lock: + for archive in self._zips.values(): + archive.close() + self._zips.clear() + def seed(self, path: Path, sha1: str) -> None: """A SHA-1 already known, so the file is not read again.""" self._sha1[path] = sha1 @@ -7147,7 +7209,7 @@ class FolderIndex: if key not in self._digests: try: if member: - with zipfile.ZipFile(path) as archive, archive.open(member) as handle: + with self.open_member(path, member) as handle: value = _digest_handle(handle, (algorithm,))[algorithm] else: value = _digest_file(path, (algorithm,))[algorithm] @@ -7192,6 +7254,8 @@ class FolderIndex: def zip_members(self, path: Path) -> list[str]: if path not in self._members: try: + # Read once and closed: the archive may be a file of the + # list, which Windows will not replace while it is open. with zipfile.ZipFile(path) as archive: self._members[path] = [n.rsplit("/", 1)[-1].casefold() for n in archive.namelist()] @@ -7209,8 +7273,7 @@ class FolderIndex: or self.relative(path).casefold() in self.sealed): continue try: - with zipfile.ZipFile(path) as archive: - infos = archive.infolist() + infos = self.infolist(path) except OSError as exc: # A ZIP that will not open may hold anything: named. A # damaged one is a download cut short, not a copy. @@ -8332,7 +8395,14 @@ def main() -> None: def _run(args: argparse.Namespace) -> None: - """The run, its text quiet or not.""" + """The run, its text quiet or not, the archives it read closed after.""" + try: + _run_text(args) + finally: + LOCAL.close_indexes() + + +def _run_text(args: argparse.Namespace) -> None: if not args.quiet: _install(args, sys.stdout) return diff --git a/install.sh b/install.sh index 34c27f9d4..a5588df75 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="c361b511d9550b2115d918a0233115cd9f42920bce881f41395c3d7614be534d" +DEFAULT_INSTALL_SHA256="5a7dc2fe2aa2abed511a94b3711f1846461a306ea185cd34407efac6c526cab0" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_from.py b/tests/test_install_from.py index fafc353e1..ccc91de6b 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -18,6 +18,7 @@ import io import json import os import subprocess +import threading import sys import time import unittest @@ -843,6 +844,54 @@ class ManySameSizedFiles(FromFolder): self.assertEqual(len(copies), 3) self.assertLess(elapsed, 30, f"{elapsed:.1f} s") + def test_one_archive_of_thousands_of_members_is_opened_once(self): + """Each member read reopened its ZIP and parsed its directory again: + 4000 members of one size made 4000 parses of 4000 entries.""" + count = 4000 + _zip(self.old / "romset.zip", {f"m{n:04}.bin": b"m%015d" % n for n in range(count)}) + wanted = [_entry(f"z{n}.bin", b"m%015d" % n, ["x"]) for n in (count - 1, 3, 1234)] + index = install.FolderIndex(self.old) + opened = [] + real = zipfile.ZipFile + + def counting(*args, **kwargs): + opened.append(args[0]) + return real(*args, **kwargs) + + start = time.monotonic() + with mock.patch.object(install.zipfile, "ZipFile", side_effect=counting): + copies = install.plan_local_copies(wanted, self.dest, None, [index]) + self.assertEqual(len(copies), 3) + # The copy step reads members of the archive the plan opened, + # from threads. + results = [] + threads = [threading.Thread(target=lambda e=e: results.append( + install._copy_local(e, self.dest, self.dest / e["dest"], copies[id(e)], None))) + for e in wanted] + self.dest.mkdir(parents=True, exist_ok=True) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + elapsed = time.monotonic() - start + self.assertEqual(sorted(ok for _d, ok, _r in results), [True] * 3, results) + self.assertEqual(len(opened), 1) + self.assertLess(elapsed, 20, f"{elapsed:.1f} s") + index.close() + self.assertEqual(index.open_archives(), 0) + + def test_a_card_of_many_archives_holds_few_open(self): + count = install.OPEN_ARCHIVES * 3 + for n in range(count): + _zip(self.old / f"sets/s{n:03}.zip", {f"m{n}.bin": b"n%015d" % n}) + wanted = [_entry(f"z{n}.bin", b"n%015d" % n, ["x"]) for n in range(count)] + index = install.FolderIndex(self.old) + copies = install.plan_local_copies(wanted, self.dest, None, [index]) + self.assertEqual(len(copies), count) + self.assertLessEqual(index.open_archives(), install.OPEN_ARCHIVES) + install.LOCAL.close_indexes() + self.assertEqual(index.open_archives(), 0) + def test_a_file_of_the_same_name_comes_first(self): self.keep("a/other.bin", US) self.keep("b/scph5501.bin", US) From 246b6c6efc77612b87111f3b232876d944ffd12c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:42:55 +0200 Subject: [PATCH 07/19] fix: print found paths as the terminal shows them The line output printed the --from folder, the files found there and the BIOS folder paths as they are: a name holding ESC drove the terminal. screen_text moves out of the interface section and cleans every line of the check, the copy plan and the unreadable list. --- install.ps1 | 2 +- install.py | 62 +++++++++++++++++++++----------------- install.sh | 2 +- tests/test_install_from.py | 23 ++++++++++++++ 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/install.ps1 b/install.ps1 index 561ddaedf..cf6f068af 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "5a7dc2fe2aa2abed511a94b3711f1846461a306ea185cd34407efac6c526cab0" + $defaultInstallSha256 = "1097dfe33e56238089af275bfea662d29613ff592f68ecdb427a86b5bcbc3e81" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 43d1dca02..4ce73ef17 100644 --- a/install.py +++ b/install.py @@ -7003,6 +7003,24 @@ _MEMBER_ERRORS = (OSError, EOFError, RuntimeError, NotImplementedError, zipfile.BadZipFile, zlib.error) +# C0 and C1 controls, DEL and lone surrogates: a file, folder or core name +# holding ESC would drive the terminal (clear it, or set the clipboard with +# OSC 52) instead of being shown. +_CONTROLS = re.compile("[\x00-\x1f\x7f-\x9f\ud800-\udfff]") + + +def screen_text(text: str) -> str: + """Text as a terminal shows it, every control character a "?".""" + return _CONTROLS.sub("?", text) + + +def shown_lines(lines: list[str]) -> list[str]: + """Lines for the terminal, a blank line opening one kept as it is: the + paths in them come from folders anyone may have filled.""" + return [("\n" + screen_text(line[1:])) if line.startswith("\n") else screen_text(line) + for line in lines] + + class LocalCopy(NamedTuple): """A file found on this machine with the contents an entry wants.""" index: FolderIndex @@ -7016,11 +7034,12 @@ class LocalCopy(NamedTuple): def where(self) -> str: """The copy as the folder it was found in names it.""" rel = self.index.relative(self.path) - return f"{rel}, inside as {self.member}" if self.member else rel + return screen_text(f"{rel}, inside as {self.member}" if self.member else rel) def named(self) -> str: """The copy by its full path, the member first when it is one.""" - return f"{self.member} inside {self.path}" if self.member else str(self.path) + return screen_text(f"{self.member} inside {self.path}" if self.member + else str(self.path)) # The archives a FolderIndex keeps open at once. @@ -7727,7 +7746,7 @@ def check_lines( if len(shown) < len(plain): lines.append(f" ... and {len(plain) - len(shown)} more missing " "(--verbose lists them all)") - return lines + return shown_lines(lines) def bios_folder_index( @@ -7829,7 +7848,7 @@ def _check_copy_lines( them = "it" if count == 1 else "them" lines.append(f" {label} holds {count} of these files: a run without --check " f"copies {them}.") - return lines + return shown_lines(lines) def _counts_by_folder( @@ -7885,7 +7904,7 @@ def copy_plan_lines( lines.append(f" {entry['dest']}: {why}" if why else f" {entry['dest']}") if len(chosen) > 10 and not every: lines.append(f" ... and {len(chosen) - 10} more (--verbose lists them all)") - return lines + return shown_lines(lines) def emit_copies( @@ -7914,7 +7933,7 @@ class Unreadable(NamedTuple): def line(self) -> str: where = f"{self.member} inside {self.path}" if self.member else str(self.path) - return f"{where} could not be read: {self.why}" + return screen_text(f"{where} could not be read: {self.why}") def unreadable_copies( @@ -7936,7 +7955,7 @@ def unreadable_lines(found: list[Unreadable], every: bool) -> list[str]: if len(found) > len(lines): lines.append(f" ... and {len(found) - len(lines)} more could not be read " "(--verbose lists them all)") - return lines + return shown_lines(lines) def emit_unreadable(plat: str, found: list[Unreadable]) -> None: @@ -7951,7 +7970,7 @@ def sources_line( ) -> str: """Where the files of a run that copied some came from, in words: "38 files copied from E:\\, 4 downloaded, 2 still missing." """ - parts = [(count, f"copied from {label}") for label, count in folders] + parts = [(count, f"copied from {screen_text(label)}") for label, count in folders] parts.append((fixed, "fixed from copies already in the BIOS folder")) parts.append((shared, "copied from another place of the BIOS folder")) if not any(count for count, _what in parts): @@ -8515,17 +8534,17 @@ def _open_from_folders(folders: list[Path]) -> None: reason, what = problem message = f"{label} {what}: --from names the folder to copy BIOS files from" EVENTS.emit("error", stage="from", reason=reason, platform="", message=message) - print(f"Error: {message}.", file=sys.stderr) + print(f"Error: {screen_text(message)}.", file=sys.stderr) sys.exit(1) index = FolderIndex(Path(os.path.expanduser(label)), label) LOCAL.folders.append(index) - print(f" Copying BIOS files from: {label}") + print(f" Copying BIOS files from: {screen_text(label)}") if index.truncated: - print(f" Read the first {MAX_SCANNED_FILES:,} files of {label} only. Name " - "the folder that holds the BIOS files to search it all.") + print(f" Read the first {MAX_SCANNED_FILES:,} files of {screen_text(label)} " + "only. Name the folder that holds the BIOS files to search it all.") for drive in index.other_drives: - print(f" Did not go into {drive}: it is another drive. Name it with --from " - "to read it too.") + print(f" Did not go into {screen_text(str(drive))}: it is another drive. Name " + "it with --from to read it too.") EVENTS.emit("scanned", folder=label, files=index.files, truncated=index.truncated, other_drives=[str(drive) for drive in index.other_drives]) @@ -9701,17 +9720,6 @@ def cell_width(text: str) -> int: return sum(_char_width(char) for char in text) -# C0 and C1 controls, DEL and lone surrogates: a file, folder or core name -# holding ESC would drive the terminal (clear it, or set the clipboard with -# OSC 52) instead of being shown. -_CONTROLS = re.compile("[\x00-\x1f\x7f-\x9f\ud800-\udfff]") - - -def screen_text(text: str) -> str: - """Text as a terminal shows it, every control character a "?".""" - return _CONTROLS.sub("?", text) - - def ascii_text(text: str) -> str: """The text in ASCII, accents dropped (Edition) and any other character as one ? per cell it took, so the columns stay where they were.""" @@ -12726,8 +12734,8 @@ def _tui_startup(args: argparse.Namespace) -> tuple[TuiState, list[tuple]] | Non for folder in folders: problem = from_folder_problem(str(folder)) if problem is not None: - print(f"Error: {folder} {problem[1]}: --from names the folder to copy BIOS " - "files from.", file=sys.stderr) + print(f"Error: {screen_text(str(folder))} {problem[1]}: --from names the folder " + "to copy BIOS files from.", file=sys.stderr) return None return tui_start(args, os_type) diff --git a/install.sh b/install.sh index a5588df75..2d1e27a99 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="5a7dc2fe2aa2abed511a94b3711f1846461a306ea185cd34407efac6c526cab0" +DEFAULT_INSTALL_SHA256="1097dfe33e56238089af275bfea662d29613ff592f68ecdb427a86b5bcbc3e81" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_from.py b/tests/test_install_from.py index ccc91de6b..b4aef6ca6 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -17,6 +17,7 @@ import errno import io import json import os +import re import subprocess import threading import sys @@ -597,6 +598,28 @@ class UnreadableCopies(FromFolder): self.assertIn(line, proc.stdout) self.assertNotIn("notes.txt", proc.stdout) + def test_names_on_the_card_cannot_drive_the_terminal(self): + """The plain output printed the paths it found as they are: a name + holding ESC cleared the screen or set the clipboard (OSC 52).""" + hostile = "x\x1b]52;c;eA==\x07\x1b[2J" + self.locked(f"{hostile}/scph5501.bin", US) + self.keep(f"{hostile}/gba_bios.bin", GBA) + (self.dest / f"pcsx{hostile}").mkdir(parents=True) + (self.dest / f"pcsx{hostile}" / "extra.bin").write_bytes(EXTRA) + for options in (("--check",), ("--dry-run",), ("--check", "--verbose"), ()): + with self.subTest(options=options): + proc = self.install(*options) + self.assertNotIn("Traceback", proc.stderr) + for stream in (proc.stdout, proc.stderr): + self.assertIsNone(re.search(r"[\x00-\x09\x0b-\x1f\x7f-\x9f]", stream), + stream) + self.assertIn("x?]52;c;eA==??[2J", proc.stdout) + proc = self.run_installer("--manifest", str(self.list), "--no-input", + "--from", str(self.root / hostile)) + self.assertEqual(proc.returncode, 1) + self.assertNotIn("\x1b", proc.stderr) + self.assertIn("x?]52;c;eA==??[2J", proc.stderr) + def test_an_unreadable_copy_is_an_event(self): path = self.locked("psx/scph5501.bin", US).resolve() proc = self.install("--check", "--events") From ccc8af7a47ad71f8e50139a6ec9a36b656ff797d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:43:38 +0200 Subject: [PATCH 08/19] fix: compare a card's BIOS folder by folders --dest NOTMUOS/bios ended with MUOS/bios as text and passed as a muOS card's folder. A card two systems claim now says so in one line, and the system named among them decides the folder. --- install.ps1 | 2 +- install.py | 34 ++++++++++++++++++++++++++++------ install.sh | 2 +- tests/test_install_tui.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/install.ps1 b/install.ps1 index cf6f068af..11bffe1d5 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "1097dfe33e56238089af275bfea662d29613ff592f68ecdb427a86b5bcbc3e81" + $defaultInstallSha256 = "2103f380bac5e2dc6ab91183283d268cccce81971da89f2963bab752d1518ef2" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 4ce73ef17..8ddbcfc1b 100644 --- a/install.py +++ b/install.py @@ -8667,16 +8667,29 @@ def card_system(root: Path) -> tuple[str, Path, str] | None: plat = card_platform(root) if plat: return plat, root, display_name(plat) - matched = [rule for rule in _card_rules() if _card_rule_matches(rule, root)] - bases = {rule.base for rule in matched if rule.base} - matched = [rule for rule in matched if rule.name not in bases] - targets = {(rule.manifest, rule.bios) for rule in matched} - if len(targets) != 1: + matched = _card_matches(root) + if len({(rule.manifest, rule.bios) for rule in matched}) != 1: return None rule = matched[0] return rule.manifest, root / rule.bios, rule.name +def _card_matches(root: Path) -> list[_OsRule]: + """The card rules root matches, a system another one is built on left + out for it.""" + matched = [rule for rule in _card_rules() if _card_rule_matches(rule, root)] + bases = {rule.base for rule in matched if rule.base} + return [rule for rule in matched if rule.name not in bases] + + +def _ends_with(path: Path, relative: str) -> bool: + """Whether path's last folders are relative's, compared folder by folder + and without case: NOTMUOS/bios does not end with MUOS/bios.""" + tail = [part.casefold() for part in PurePosixPath(relative).parts] + parts = [part.casefold() for part in path.parts] + return len(parts) >= len(tail) and parts[len(parts) - len(tail):] == tail + + def card_dest(named: str, plat: str, dest: Path) -> tuple[str, Path]: """The file list and folder for a --dest that names a handheld's card. @@ -8694,12 +8707,21 @@ def card_dest(named: str, plat: str, dest: Path) -> tuple[str, Path]: key = _name_key(named) rule = next((rule for rule in _card_rules() if key and key == _name_key(rule.name) and key != _name_key(display_name(rule.manifest))), None) + matched = [] if found else _card_matches(dest) if rule is None or found: + if len({(match.manifest, match.bios) for match in matched}) > 1: + names = " and ".join(sorted({match.name for match in matched})) + print(f" {screen_text(str(dest))} holds the files of {names}: installing into " + "it as named. Name the card's BIOS folder with --dest to write there.") return plat, dest # Its folder on the card, or one its device reads (/run/muos/storage/bios). own = {device.bios for device in _EMBEDDED_RULES if device.name == rule.name} - if dest.as_posix() in own or dest.as_posix().casefold().endswith(rule.bios.casefold()): + if dest.as_posix() in own or _ends_with(dest, rule.bios): return plat, dest + if any(match.name == rule.name for match in matched): + # A card two systems claim, the one named among them. + print(f" {rule.name} card: {display_name(plat)} files, in {dest / rule.bios}") + return plat, dest / rule.bios marker = rule.values[0].split("/")[0] print(f"Error: {dest} is not a {rule.name} card: it holds no {marker}. " f"{rule.name} reads its BIOS files from {rule.bios} on its card: name the card's " diff --git a/install.sh b/install.sh index 2d1e27a99..b0c2495d6 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="1097dfe33e56238089af275bfea662d29613ff592f68ecdb427a86b5bcbc3e81" +DEFAULT_INSTALL_SHA256="2103f380bac5e2dc6ab91183283d268cccce81971da89f2963bab752d1518ef2" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index ef7c2f674..73eee5ea3 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -10,6 +10,7 @@ installs it). from __future__ import annotations import argparse +import contextlib import errno import fcntl import functools @@ -2122,6 +2123,33 @@ class AuditRoundThree(unittest.TestCase): self.assertEqual(install.card_dest("retroarch", "retroarch", root), ("retroarch", root)) + def test_p1_1_the_folder_named_is_compared_by_folders(self): + """NOTMUOS/bios ended with MUOS/bios as text and was taken for a + muOS card's folder.""" + root = self.card({"roms/": ""}) + stray = root / "NOTMUOS" / "bios" + with self.assertRaises(SystemExit), \ + contextlib.redirect_stderr(io.StringIO()) as err: + install.card_dest("muos", "retroarch", stray) + self.assertIn("is not a muOS card", err.getvalue()) + self.assertEqual(install.card_dest("muos", "retroarch", root / "muos" / "BIOS"), + ("retroarch", root / "muos" / "BIOS")) + + def test_p1_1_a_card_two_systems_claim_says_so(self): + root = self.card({"MUOS/": "", "spruce/": ""}) + self.assertIsNone(install.card_system(root)) + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(install.card_dest("retroarch", "retroarch", root), + ("retroarch", root)) + lines = out.getvalue().splitlines() + self.assertEqual(len(lines), 1, lines) + self.assertIn("holds the files of muOS and spruceOS", lines[0]) + # The system named among them decides. + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(install.card_dest("muos", "retroarch", root), + ("retroarch", root / "MUOS/bios")) + self.assertIn("muOS card", out.getvalue()) + # P1-2: Space is X on PortMaster and Y on muxterm. def test_p1_2_space_on_install_opens_the_confirm_screen(self): From abba99d352f2925444ceb37c67697d8f3e49f02c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:47:36 +0200 Subject: [PATCH 09/19] fix: find the save directory for a named folder A run given --dest detected nothing, so retroarch.cfg was never read: the full screen, which names every folder, sent Azahar's keys.txt to a guessed RetroArch/saves or dropped it without a word, where line mode wrote it to savefile_directory. The configs are now read once when a save directory is asked for, and the plan and the result say what was left out. --- install.ps1 | 2 +- install.py | 43 +++++++++++++++++- install.sh | 2 +- tests/test_install_tui.py | 96 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/install.ps1 b/install.ps1 index 11bffe1d5..2bdb4e8d2 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "2103f380bac5e2dc6ab91183283d268cccce81971da89f2963bab752d1518ef2" + $defaultInstallSha256 = "7065a9d52aa9016066c273ec8a5907b09fcce7eaad5a423583c79515f09edb99" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 8ddbcfc1b..339551942 100644 --- a/install.py +++ b/install.py @@ -3949,6 +3949,27 @@ def _cfg_true(value: str | None) -> bool: return (value or "").strip().lower() == "true" +_CONFIGS_READ: list[bool] = [] + + +def _retroarch_config_for(bios_path: Path) -> tuple[Path, Path] | None: + """The retroarch.cfg whose system_directory is bios_path, and the + folder ':' expands to there. + + A run given --dest detects nothing, so the configs are read here, once + and silently, the first time a save directory is asked for: the + interface names every folder with --dest, and its runs sent Azahar's + keys.txt to a guessed folder where line mode read savefile_directory. + """ + found = _RETROARCH_CONFIGS.get(bios_path) + if found is not None or _CONFIGS_READ: + return found + _CONFIGS_READ.append(True) + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + detect_platforms(_INVOCATION.os_type or detect_os()) + return _RETROARCH_CONFIGS.get(bios_path) + + def save_directory_for( bios_path: Path, manifest: dict, override: Path | None = None, ) -> SavePlace | str: @@ -3975,7 +3996,7 @@ def save_directory_for( folder = layout["folder"] listed = bool(layout["by_core"]) by_core = listed - found = None if layout.get("by_system") else _RETROARCH_CONFIGS.get(bios_path) + found = None if layout.get("by_system") else _retroarch_config_for(bios_path) if found is not None: cfg_path, app_dir = found for key in ("savefiles_in_content_dir", "sort_savefiles_by_content_enable"): @@ -9920,6 +9941,8 @@ class TuiState: self.dry: dict | None = None self.dry_files: list = [] self.dry_summary: dict | None = None + # The check run's summary: what it left out and why. + self.check_summary: dict | None = None # The confirm screen waits for A even when the plan is unchanged: # it was opened by Space, which never installs by itself. self.must_confirm: bool = False @@ -10427,6 +10450,7 @@ def _start_run(state: TuiState, run: str) -> list[tuple]: # dropped the --system the command line gave. state.check, state.files, state.systems, state.reading = None, [], [], None state.found, state.scanned, state.unreadable = [], [], [] + state.check_summary = None state.checked_flags = tui_flags(state) elif run in ("confirm", "preview"): state.dry, state.dry_files, state.dry_summary = None, [], None @@ -11060,6 +11084,8 @@ def _take_event(state: TuiState, event: dict) -> None: state.files.append(event) elif kind == "system": state.systems.append(event) + elif kind == "summary": + state.check_summary = event elif state.run in ("confirm", "preview"): if kind == "check": state.dry = event @@ -11413,9 +11439,23 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: lines += _scan_lines(state) if check.get("not_installable"): lines += _unavailable_lines(state) + if check is state.check: + lines += _unplaced_lines(state.check_summary, state) return lines +def _unplaced_lines(summary: dict | None, state: TuiState) -> list[str]: + """The files read from the frontend's save directory that the run left + out, that directory being unknown: said, never silent.""" + count = sum(group.get("files", 0) for group in (summary or {}).get("skipped") or [] + if group.get("reason") == "no_save_directory") + if not count: + return [] + name = display_name(state.platform) if state.platform else "the frontend" + return [f"{_plural(count, 'file')} read from {name}'s save directory left out: " + "its place is not known. Name it with --save-dir."] + + def _view_setup(state: TuiState, width: int, height: int) -> tuple: rows = tui_rows(state) focus = rows[min(state.focus.get("setup", 0), len(rows) - 1)] @@ -11895,6 +11935,7 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: lines += _unavailable_lines(state) for group in summary.get("left_out") or []: lines.append(f"Left out as {group['label']}: {_plural(group['files'], 'file')}.") + lines += _unplaced_lines(summary, state) lines += ["Same choice without questions:", f" {tui_command(state)}"] if tui_keeps_log(state): lines.append(f"Details: {state.log}") diff --git a/install.sh b/install.sh index b0c2495d6..559077c83 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="2103f380bac5e2dc6ab91183283d268cccce81971da89f2963bab752d1518ef2" +DEFAULT_INSTALL_SHA256="7065a9d52aa9016066c273ec8a5907b09fcce7eaad5a423583c79515f09edb99" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 73eee5ea3..f9bd298d3 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -2525,6 +2525,36 @@ class AuditRoundThree(unittest.TestCase): sorted(path.name for path in (running, failed, unknown, other))) +class AuditRoundFour(unittest.TestCase): + """tui_audit_3.md, each finding held where it was found.""" + + UNPLACED = {"reason": "no_save_directory", "label": "no save directory", "files": 2, + "required": 0} + + def test_p1_1_files_left_out_of_the_save_directory_are_said(self): + state = fresh_state() + state.stack = ["setup"] + with mock.patch.object(install.Path, "exists", return_value=True): + install.tui_update(state, Key("enter")) + install.tui_receive(state, ("manifest", "retroarch", MANIFEST)) + summary = {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 0, + "pending": 5, "failed": [], "skipped": [self.UNPLACED], "left_out": [], + "kept": [], "interrupted": False} + for event in [*FILES, *SYSTEMS, CHECK, summary]: + install.tui_receive(state, ("event", state.run_id, event)) + with mock.patch.object(install, "_tui_folder_facts", return_value={ + "free": 50 * 1024 ** 3, "writable": True, "cores": []}): + install.tui_receive(state, ("ended", state.run_id, 1, [])) + line = ("2 files read from RetroArch's save directory left out: its place is " + "not known. Name it with --save-dir.") + plan = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn(line, plan) + state.stack.append("result") + state.code = 0 + state.summary = {**summary, "mode": "install", "installed": 5} + self.assertIn(line, install.tui_outcome(state)[1]) + + class Crash(unittest.TestCase): """An unexpected error ends on one sentence and a log, never a traceback.""" @@ -2684,6 +2714,29 @@ class GameDataRepository(FakeRepository): return self +class SaveDirectoryRepository(FakeRepository): + """The fake repository with one file read from RetroArch's save + directory, as Azahar's keys.txt and 105 others of the real list.""" + + KEYS = b"azahar keys" + + def __enter__(self) -> SaveDirectoryRepository: + super().__enter__() + path = self.root / "install" / "retroarch.json" + manifest = json.loads(path.read_text()) + (self.root / "bios" / "keys.txt").write_bytes(self.KEYS) + manifest["files"].append({ + "dest": "Azahar/Azahar/sysdata/keys.txt", "root": "save", + "core_folder": "Azahar", "sha1": hashlib.sha1(self.KEYS).hexdigest(), + "sha256": hashlib.sha256(self.KEYS).hexdigest(), "size": len(self.KEYS), + "repo_path": "bios/keys.txt", "cores": ["azahar"], + "systems": ["nintendo-3ds"]}) + manifest["save_directory"] = {"folder": "saves", "by_core": True} + manifest["base_destination"] = "system" + path.write_text(json.dumps(manifest)) + return self + + @unittest.skipUnless(pyte and pty and hasattr(pty, "fork"), "screen tests need pyte (pip install pyte==0.8.2) and a pty") class EndToEnd(unittest.TestCase): @@ -2735,6 +2788,49 @@ class EndToEnd(unittest.TestCase): # The cursor and the main screen come back before the result is printed. self.assertIn(b"\x1b[?25h\x1b[?1049l", term.raw) + def test_the_save_directory_is_the_one_line_mode_finds(self): + """The interface named the BIOS folder with --dest, so its runs never + read retroarch.cfg: keys.txt went to a guessed RetroArch/saves, or + was dropped without a word, where line mode writes it to the + savefile_directory RetroArch reads.""" + import subprocess + + def written(repo, saves: Path) -> list[str]: + return sorted([*repo.installed(), *( + "saves/" + p.relative_to(saves).as_posix() + for p in saves.rglob("*") if p.is_file())]) + + results = [] + for mode in ("line", "interface"): + with self.subTest(mode=mode): + repo = self.repo(SaveDirectoryRepository) + home = Path(self.env["HOME"]) + saves = home / "MySaves" + saves.mkdir() + cfg = home / ".config" / "retroarch" / "retroarch.cfg" + cfg.write_text(f'system_directory = "{repo.dest}"\n' + f'savefile_directory = "{saves}"\n') + if mode == "line": + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", + "retroarch", "--yes"], env=self.env, capture_output=True, + text=True, timeout=60, check=False, stdin=subprocess.DEVNULL) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("6 files installed", proc.stdout) + else: + term = self.start() + term.wait("Where do you play?") + term.send("\r") + screen = term.wait("files to download") + self.assertIn("6 files to download", screen) + term.send("\r") + term.wait("6 files installed.") + term.send("\r") + self.assertEqual(term.close(), 0) + results.append(written(repo, saves)) + self.assertIn("saves/Azahar/Azahar/sysdata/keys.txt", results[-1]) + self.assertEqual(results[0], results[1]) + def test_one_system_from_the_checklist(self): repo = self.repo() term = self.start("--platform", "retroarch", "--dest", str(repo.dest), cols=40, rows=15) From a296bbfdffde903bb04ccbd505e84086b6fa205f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:49:23 +0200 Subject: [PATCH 10/19] fix: keep why a data tree or a copy failed A damaged member of a data tree's archive (zlib.error, EOFError) ended the run on a traceback, and a failed write left its .part. --offline refusing a data tree, or the download after a --from copy failed, lost its kind, so the advice named the network instead of --offline. --- install.ps1 | 2 +- install.py | 84 ++++++++++++++++++++++++-------------- install.sh | 2 +- tests/test_install.py | 35 ++++++++++++++++ tests/test_install_from.py | 12 ++++++ 5 files changed, 103 insertions(+), 32 deletions(-) diff --git a/install.ps1 b/install.ps1 index 2bdb4e8d2..3224f28a3 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "7065a9d52aa9016066c273ec8a5907b09fcce7eaad5a423583c79515f09edb99" + $defaultInstallSha256 = "079fcba3b83003932af8f5ed513e556675832ab20e55bbbbc356581b2daf1a14" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 339551942..de751a655 100644 --- a/install.py +++ b/install.py @@ -4497,7 +4497,9 @@ def _download_one( or why.startswith(NOT_KEPT)): return fetched both = f"{result[2]}, and {why}" - return fetched[0], False, NetworkReason(both) if isinstance(why, NetworkReason) else both + # The fetch's kind of failure stays, so the advice is the right one. + kind = type(why) if isinstance(why, (NetworkReason, OfflineReason)) else str + return fetched[0], False, kind(both) if LOCAL.pack is not None: return _extract_from_pack(f, bios_path, dest, progress) @@ -5599,40 +5601,62 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict: continue target = _destination_path(root, rel) target.parent.mkdir(parents=True, exist_ok=True) - digest = hashlib.sha1() - size = 0 - with opener() as src, tempfile.NamedTemporaryFile( - dir=target.parent, prefix=f".{target.name}.", suffix=".part", - delete=False, - ) as out: - while size <= meta["size"]: - chunk = src.read(min(1024 * 1024, meta["size"] + 1 - size)) - if not chunk: - break - digest.update(chunk) - size += len(chunk) - out.write(chunk) - if digest.hexdigest() == meta["sha1"] and size == meta["size"]: - slot = f"{entry['dest']}/{rel}" if entry.get("dest") else rel - try: - KEPT.keep(*_kept_slot(bios_path, slot), target, - {"sha1": meta["sha1"]}) - except _NotKept as exc: - Path(out.name).unlink(missing_ok=True) - counts["not_kept"].append((rel, str(exc))) - continue - os.replace(out.name, target) - counts["installed"] += 1 - else: - Path(out.name).unlink(missing_ok=True) - counts["changed"] += 1 - except (urllib.error.URLError, OSError, ValueError, zipfile.BadZipFile, - tarfile.TarError) as exc: + outcome = _install_data_member( + opener, meta, target, bios_path, + f"{entry['dest']}/{rel}" if entry.get("dest") else rel, + rel, counts["not_kept"]) + if outcome: + counts[outcome] += 1 + except _Offline as exc: + # Kept apart, so the advice is the offline one, not the network's. + counts["error"] = OfflineReason(str(exc)) + except (urllib.error.URLError, OSError, ValueError, tarfile.TarError) as exc: counts["error"] = str(getattr(exc, "reason", None) or exc) + except _MEMBER_ERRORS as exc: + # A damaged, encrypted or unsupported member, in words. + counts["error"] = unreadable_words(exc) counts["absent"] = len(needed) return counts +def _install_data_member( + opener: Callable[[], ContextManager[BinaryIO]], meta: dict, target: Path, + bios_path: Path, slot: str, rel: str, not_kept: list[tuple[str, str]], +) -> str: + """Write one member of a data tree's archive at target once its bytes + are the recorded ones: "installed", "changed" when they are not, or "" + when the file in place could not be moved aside (named in not_kept). + A read or write error is raised, and leaves no .part behind.""" + scratch: Path | None = None + try: + digest = hashlib.sha1() + size = 0 + with opener() as src, tempfile.NamedTemporaryFile( + dir=target.parent, prefix=f".{target.name}.", suffix=".part", delete=False, + ) as out: + scratch = Path(out.name) + while size <= meta["size"]: + chunk = src.read(min(1024 * 1024, meta["size"] + 1 - size)) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + out.write(chunk) + if digest.hexdigest() != meta["sha1"] or size != meta["size"]: + return "changed" + try: + KEPT.keep(*_kept_slot(bios_path, slot), target, {"sha1": meta["sha1"]}) + except _NotKept as exc: + not_kept.append((rel, str(exc))) + return "" + os.replace(scratch, target) + scratch = None + return "installed" + finally: + if scratch is not None: + scratch.unlink(missing_ok=True) + + def select_data_directories(manifest: dict, files: list[dict], narrowed: bool) -> list[dict]: """The data trees this install keeps: all, or those of a kept system.""" entries = manifest.get("data_directories") or [] diff --git a/install.sh b/install.sh index 559077c83..1cc244ceb 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="7065a9d52aa9016066c273ec8a5907b09fcce7eaad5a423583c79515f09edb99" +DEFAULT_INSTALL_SHA256="079fcba3b83003932af8f5ed513e556675832ab20e55bbbbc356581b2daf1a14" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 4da52996f..b7cb32854 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3248,6 +3248,41 @@ class DataDirectoriesAreInstalled(unittest.TestCase): self.assertEqual(counts["changed"], 1) self.assertLess(Endless.served, 64 * 1024) + def test_a_damaged_member_is_a_reason_and_leaves_no_part(self): + """zlib.error and EOFError from a damaged member ended the run on a + traceback, and a failed write left its .part beside the tree.""" + import zlib + + work = Path(_tempdir(self)) + entry = self._entry() + entry["files"] = {"Sys/GC/font.bin": {"sha1": "1" * 40, "size": 4}} + for error in (zlib.error("invalid stored block lengths"), EOFError(), + OSError(28, "No space left on device")): + with self.subTest(error=type(error).__name__): + class Broken(io.RawIOBase): + def readable(self): + return True + + def readinto(self, buffer, error=error): + raise error + + def members(_archive, _entry): + yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Broken()) + + with unittest.mock.patch.object(install, "_fetch_archive", + lambda url, d: Path(d) / "a"), \ + unittest.mock.patch.object(install, "_archive_members", members): + counts = install.install_data_directory(entry, work) + self.assertTrue(counts["error"]) + self.assertEqual(counts["installed"], 0) + self.assertEqual([p.name for p in work.rglob("*.part")], []) + + def test_offline_keeps_its_reason(self): + work = Path(_tempdir(self)) + with unittest.mock.patch.dict(os.environ, {install.OFFLINE_VARIABLE: "1"}): + counts = install.install_data_directory(self._entry(), work) + self.assertIsInstance(counts["error"], install.OfflineReason) + def test_a_tarball_maps_through_source_path(self): import io as _io import tarfile diff --git a/tests/test_install_from.py b/tests/test_install_from.py index b4aef6ca6..2f5f2880c 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -520,6 +520,18 @@ class ACopyThatFails(FromFolder): self.assertNotIn(install.NOT_WRITABLE, reason) self.assertIsInstance(reason, install.NetworkReason) + def test_offline_the_fallback_keeps_the_offline_reason(self): + """The copy failed, the download was refused by --offline: the + joined reason lost its kind, and the advice named neither.""" + _unprivileged(self) + self.source.chmod(0) + self.addCleanup(self.source.chmod, 0o644) + with mock.patch.dict(os.environ, {install.OFFLINE_VARIABLE: "1"}): + _dest, ok, reason = install._download_one(self.entry, self.dest, copy=self.copy) + self.assertFalse(ok) + self.assertIsInstance(reason, install.OfflineReason) + self.assertIn("could not be read: Permission denied, and not fetched", reason) + def test_a_read_error_is_the_source_and_not_the_folder(self): class Failing(io.BytesIO): def read(self, *_args): From a23e9a09b87d07b40baa3c92a722cb36d1eaed89 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:50:24 +0200 Subject: [PATCH 11/19] feat: close the first screen when no button comes RETROBIOS_IDLE_EXIT=SECONDS: with no key on the first screen for that long, before any choice, the interface exits 0 with one line. A mapper that never opens the pad no longer freezes a handheld there. --- install.ps1 | 2 +- install.py | 29 +++++++++++++++++++++++++++++ install.sh | 2 +- tests/test_install_tui.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index 3224f28a3..d9cc53e23 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "079fcba3b83003932af8f5ed513e556675832ab20e55bbbbc356581b2daf1a14" + $defaultInstallSha256 = "4d0a09cb5df35b5ccac1addbcd31feaf1ae00153694c11c7a1894e73bc063833" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index de751a655..a02b083f2 100644 --- a/install.py +++ b/install.py @@ -9965,6 +9965,8 @@ class TuiState: self.dry: dict | None = None self.dry_files: list = [] self.dry_summary: dict | None = None + # Closed by RETROBIOS_IDLE_EXIT, no key ever pressed. + self.idle_closed = False # The check run's summary: what it left out and why. self.check_summary: dict | None = None # The confirm screen waits for A even when the plan is unchanged: @@ -10021,6 +10023,24 @@ TUI_PAD_ROUTES = ("gptokeyb", "muxterm", "evmapy", "miyoo") _STEAM_MARKERS = ("SteamGamepadUI", "SteamDeck", "SteamGameId", "GAMESCOPE_WAYLAND_DISPLAY") +# Seconds the first screen waits for a first key before it closes, set by a +# launcher whose mapper may never open the pad: the screen would otherwise +# hold a handheld that has no other way out. +TUI_IDLE_ENV = "RETROBIOS_IDLE_EXIT" +TUI_IDLE_WORDS = "No button pressed, closing." + + +def tui_idle_exit(env: Mapping[str, str] | None = None) -> float | None: + """The seconds RETROBIOS_IDLE_EXIT names, None when unset or not a + positive number.""" + env = os.environ if env is None else env + try: + seconds = float(env.get(TUI_IDLE_ENV, "").strip()) + except ValueError: + return None + return seconds if 0 < seconds < float("inf") else None + + def tui_pad_route(os_type: str, env: Mapping[str, str] | None = None) -> str: """The keys the hints name: a launcher's pad route, "keys" at a desktop, "both" (button and key) where nothing says which one is held.""" @@ -12580,6 +12600,8 @@ def tui_farewell(state: TuiState) -> list[str]: def _farewell_lines(state: TuiState) -> list[str]: + if state.idle_closed: + return [f"RetroBIOS: {TUI_IDLE_WORDS}"] if state.screen != "result" and state.download is not None: # An install ran: what it wrote is said, whatever ended it. title, lines = tui_outcome(state) @@ -12705,6 +12727,8 @@ def run_tui(args: argparse.Namespace) -> int: number = getattr(signal, name, None) if number is not None: handlers[number] = signal.signal(number, handler) + idle = tui_idle_exit() + first_screen, opened, pressed = state.screen, time.monotonic(), False sys.stdout = sys.stderr = buffer try: _prune_work_folders(work.parent, time.time()) @@ -12735,10 +12759,15 @@ def run_tui(args: argparse.Namespace) -> int: state.now = time.monotonic() terminal.draw(tui_view(state, cols, rows), cols) for key in terminal.keys(TUI_FRAME): + pressed = True state.now = time.monotonic() perform(tui_update(state, key)) if state.exit_code is not None: break + if (idle is not None and not pressed and state.exit_code is None + and state.screen == first_screen + and time.monotonic() - opened >= idle): + state.idle_closed, state.exit_code = True, 0 if flags["interrupt"]: flags["interrupt"] = False if time.monotonic() >= ignore_interrupt_until[0]: diff --git a/install.sh b/install.sh index 1cc244ceb..bb28d346f 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="079fcba3b83003932af8f5ed513e556675832ab20e55bbbbc356581b2daf1a14" +DEFAULT_INSTALL_SHA256="4d0a09cb5df35b5ccac1addbcd31feaf1ae00153694c11c7a1894e73bc063833" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index f9bd298d3..64c062766 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -2525,6 +2525,22 @@ class AuditRoundThree(unittest.TestCase): sorted(path.name for path in (running, failed, unknown, other))) +class IdleExit(unittest.TestCase): + def test_the_variable(self): + for value, seconds in (("30", 30.0), (" 2.5 ", 2.5), ("", None), ("0", None), + ("-4", None), ("soon", None), ("inf", None), ("nan", None)): + with self.subTest(value=value): + self.assertEqual(install.tui_idle_exit({install.TUI_IDLE_ENV: value}), + seconds) + self.assertIsNone(install.tui_idle_exit({})) + + def test_the_farewell(self): + state = fresh_state() + state.idle_closed, state.exit_code = True, 0 + self.assertEqual(install.tui_farewell(state), + ["RetroBIOS: No button pressed, closing."]) + + class AuditRoundFour(unittest.TestCase): """tui_audit_3.md, each finding held where it was found.""" @@ -3044,6 +3060,26 @@ class EndToEnd(unittest.TestCase): self.assertEqual(term.status, 129) self.assertEqual(list(scratch.glob("retrobios-tui-*/error.log")), []) + def test_no_key_on_the_first_screen_closes_it(self): + """A mapper that never opens the pad left the first screen up for + good: a handheld then has no way out but the power button.""" + self.repo() + self.env[install.TUI_IDLE_ENV] = "1" + term = self.start() + term.wait("Where do you play?") + self.assertEqual(term.close(), 0) + self.assertIn("No button pressed, closing.", term.after()) + # A key pressed, the screen stays: the person is there. + term = self.start() + term.wait("Where do you play?") + term.send(DOWN) + end = time.monotonic() + 2.5 + while time.monotonic() < end: + term._pump(0.05) + self.assertEqual(os.waitpid(term.pid, os.WNOHANG), (0, 0)) + term.send("q") + self.assertEqual(term.close(), 130) + def test_a_desktop_names_its_keys_and_a_pad_its_buttons(self): self.repo() for env, bar in (({"DISPLAY": ":0"}, "Enter Continue Esc Quit ? Help"), From 36f4b338e165e5ea3b5a52c7174cd534bfd372a1 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:50:59 +0200 Subject: [PATCH 12/19] fix: offer no paging keys on muxterm muxterm keeps PgUp and PgDn for its scrollback and its L1 and R1 send nothing: the help named a page key that never reaches the interface. --- install.ps1 | 2 +- install.py | 7 +++++-- install.sh | 2 +- tests/test_install_tui.py | 7 +++++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/install.ps1 b/install.ps1 index d9cc53e23..7bb0e43c2 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "4d0a09cb5df35b5ccac1addbcd31feaf1ae00153694c11c7a1894e73bc063833" + $defaultInstallSha256 = "3d36f9aaf0bfd17cc2eeff61d0f99d35ff60d7cf967572ec146b166681334e5d" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index a02b083f2..c91ddc7ad 100644 --- a/install.py +++ b/install.py @@ -12044,7 +12044,9 @@ _HELP_KEYS: dict[str, dict[str, str | None]] = { "tick": "X, Space, Right/Left", "options": "Y, Tab, o", "back": "B, Select, Esc", "stop": "B, Esc, Ctrl-C", "kill": None, "enter": "A, Enter", "esc": "B, Esc"}, - "muxterm": {"move": "D-pad, Up/Down", "page": "PgUp/PgDn", "ok": "Start, Enter", + # muxterm keeps PgUp and PgDn for its own scrollback, and its L1 and R1 + # send nothing: no key pages a list there. + "muxterm": {"move": "D-pad, Up/Down", "page": None, "ok": "Start, Enter", "tick": "Y, Space, Right/Left", "options": "o, Tab", "back": "B, Esc, Backspace", "stop": "B, Esc, Ctrl-C", "kill": None, "enter": "Start, Enter", "esc": "Esc"}, @@ -12080,7 +12082,8 @@ def _help_lines(state: TuiState) -> list[tuple[str, str]]: *_route_extras(state)] keys = [(names["move"], "Move")] if len(tui_rows(state)) > 9 or screen in _PAGERS: - keys += [(names["page"], "Page"), ("Home/End, g/G", "First, last")] + keys += [*([(names["page"], "Page")] if names["page"] else []), + ("Home/End, g/G", "First, last")] primary = _primary(state) or _TUI_VIEWS[screen](state, 80, 24)[2][0] ok = names["ok"] if state.pad == "miyoo" and screen in _CHECKLISTS: diff --git a/install.sh b/install.sh index bb28d346f..0a7fce9a8 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="4d0a09cb5df35b5ccac1addbcd31feaf1ae00153694c11c7a1894e73bc063833" +DEFAULT_INSTALL_SHA256="3d36f9aaf0bfd17cc2eeff61d0f99d35ff60d7cf967572ec146b166681334e5d" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 64c062766..1f46b96d9 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -1097,6 +1097,13 @@ class PadRoutes(unittest.TestCase): "Close the terminal"): self.assertIn(words, muxterm) self.assertNotIn("Start+Select", muxterm) + # muxterm keeps PgUp and PgDn for its scrollback, and L1 and R1 + # send nothing: no paging is offered there, on a list or a pager. + for screen in ("systems", "result", "plan"): + with self.subTest(screen=screen): + text = help_text("muxterm", screen) + for words in ("L1", "R1", "PgUp", "PgDn", "Page"): + self.assertNotIn(words, text) miyoo = help_text("miyoo", "systems") for words in ("Start, Enter", "A, Space, Right/Left", "R2, Esc, Backspace", "L2, Tab, o", "Sends no key here"): From ba4db2ab57eacc797ff230f6c4d2ea8bacd54415 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:53:07 +0200 Subject: [PATCH 13/19] fix: keep system names whole at 40 columns The size column took half the line and cut every name to eleven letters: Game Boy, Game Boy Advance and Game Boy Color read alike. A system row now carries shorter size forms, and the longest one that leaves the name whole is shown, none when that is what keeps it. --- install.ps1 | 2 +- install.py | 26 +++++++++++++++++++++++--- install.sh | 2 +- tests/test_install_tui.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/install.ps1 b/install.ps1 index 7bb0e43c2..38b7ee9ec 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "3d36f9aaf0bfd17cc2eeff61d0f99d35ff60d7cf967572ec146b166681334e5d" + $defaultInstallSha256 = "eaa912af47dc6f29e4436ec200a1b431b8ddd8006ca08e57c8c73d3a05717818" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index c91ddc7ad..d7987b3a1 100644 --- a/install.py +++ b/install.py @@ -10178,6 +10178,9 @@ class TuiRow(NamedTuple): right: str = "" value: object = None style: str = "" + # Shorter forms of right, longest first, for a narrow screen: the label + # keeps its room before the right-hand text does. + narrow: tuple = () # Screens that scroll text instead of listing rows. @@ -10358,16 +10361,21 @@ def _system_rows(state: TuiState) -> list[TuiRow]: need = _needed(row) count, size = shared.get(row["system"], (0, 0)) own, own_size = need - count, row["bytes_needed"] - size + narrow: tuple = () if not need: right = "in place" elif not count: right = f"{_plural(need, 'file')}, {format_size(row['bytes_needed'])}" + narrow = (format_size(row["bytes_needed"]),) elif not own: right = f"{_plural(need, 'file')}, {format_size(size)}, all shared" + narrow = (f"{format_size(size)}, shared", format_size(size)) else: right = f"{_plural(own, 'file')}, {format_size(own_size)} + {count} shared" + narrow = (f"{format_size(own_size)} + {count} shared", + f"{format_size(own_size)} + {count} sh.", format_size(own_size)) rows.append(TuiRow("system", f"{box} {label}", right, row["system"], - "" if need else "dim")) + "" if need else "dim", narrow)) return rows @@ -11292,8 +11300,20 @@ _ROW_VERBS = { def _row_line(row: TuiRow, focused: bool, width: int) -> TuiLine: - right = middle(row.right, max(12, width // 2)) if row.right else "" - text = _columns(("> " if focused else " ") + row.label, right, width) + left = ("> " if focused else " ") + row.label + if row.narrow: + # The name first: the longest right-hand form that leaves it whole, + # else none at all when that is what keeps it whole. Nintendo Game + # Boy, Game Boy Advance and Game Boy Color read alike once cut. + whole = width - cell_width(left) - 2 + right = next((form for form in (row.right, *row.narrow) + if cell_width(form) <= whole), "") + if not right and cell_width(left) > width: + right = row.narrow[-1] + text = _columns(left, right, width) + else: + right = middle(row.right, max(12, width // 2)) if row.right else "" + text = _columns(left, right, width) return _styled(text, "focus" if focused else row.style) diff --git a/install.sh b/install.sh index 0a7fce9a8..f64306ffc 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="3d36f9aaf0bfd17cc2eeff61d0f99d35ff60d7cf967572ec146b166681334e5d" +DEFAULT_INSTALL_SHA256="eaa912af47dc6f29e4436ec200a1b431b8ddd8006ca08e57c8c73d3a05717818" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 1f46b96d9..f8ba0aa89 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -2578,6 +2578,39 @@ class AuditRoundFour(unittest.TestCase): self.assertIn(line, install.tui_outcome(state)[1]) +class NarrowSystemNames(unittest.TestCase): + """At 40 columns the size column took half the line: Game Boy, Game Boy + Advance and Game Boy Color all read "Nintendo Gam".""" + + def test_the_name_comes_before_the_size(self): + state = checked_state() + names = {"nintendo-gb": "Nintendo Game Boy", "nintendo-gba": "Nintendo Game Boy Advance", + "nintendo-gbc": "Nintendo Game Boy Color", + "videoton-tvc": "Videoton TV Computer"} + state.manifest = {**state.manifest, "system_titles": names} + state.systems = [ + {"event": "system", "platform": "retroarch", "system": system, "name": system, + "files": 30, "present": 0, "missing": 30, "wrong_hash": 0, + "bytes_needed": 30 * 1024 * 1024} + for system in names] + state.files = [{"event": "file", "platform": "retroarch", "dest": f"s{n}.bin", + "systems": list(names), "size": 1024, "state": "missing"} + for n in range(27)] + state.ticked = set(names) + state.stack.append("systems") + for cols, rows in ((40, 15), (40, 12), (52, 15)): + with self.subTest(size=f"{cols}x{rows}"): + text = "\n".join(install.tui_view(state, cols, rows)) + for name in names.values(): + self.assertIn(name, text) + self.assertNotIn("...", text) + for line in text.splitlines(): + self.assertLessEqual(install.cell_width(line), cols) + # Wide, the full form stays. + self.assertIn("3 files, 30.0 MB + 27 shared", + "\n".join(install.tui_view(state, 120, 40))) + + class Crash(unittest.TestCase): """An unexpected error ends on one sentence and a log, never a traceback.""" From 812d5d157e6e28d7967a6b6ff8557907ccd61e8d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:56:37 +0200 Subject: [PATCH 14/19] fix: tell offline from a lost connection on screen With --offline and no list, the full screen said No connection and offered a Try again that could never work. It now names offline mode and the two ways a run gets its list without the network. With a list, the plan counted downloads the run forbids: it says what this machine can supply and what needs the internet, and the printed command keeps --offline. --- install.ps1 | 2 +- install.py | 46 ++++++++++++++++++++++++++++++++++--- install.sh | 2 +- tests/test_install_tui.py | 48 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/install.ps1 b/install.ps1 index 38b7ee9ec..b00b2a730 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "eaa912af47dc6f29e4436ec200a1b431b8ddd8006ca08e57c8c73d3a05717818" + $defaultInstallSha256 = "1f6b90760303d04cb6beef7fd294371b2e1d4ba733578c59a95e1b0395c2f376" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index d7987b3a1..874efc26e 100644 --- a/install.py +++ b/install.py @@ -10140,6 +10140,8 @@ def tui_flags(state: TuiState) -> list[tuple[str, str | None]]: if state.jobs != 8: flags.append(("--jobs", str(state.jobs))) flags += [("--from", str(folder)) for folder in state.from_folders] + if offline_requested() and not any(flag == "--offline" for flag, _v in flags): + flags.append(("--offline", None)) return flags @@ -10312,7 +10314,8 @@ def tui_rows(state: TuiState) -> list[TuiRow]: return [TuiRow("install", "Install"), TuiRow("back", "Back")] if screen == "error": rows = ([TuiRow("retry", "Try again")] - if state.run == "check" and state.error_title else []) + if state.run == "check" and state.error_title + and state.error_title != TUI_OFFLINE_TITLE else []) return rows + [TuiRow("back", "Choose another setup or folder" if state.run == "check" else "Back"), TuiRow("quit", "Quit")] @@ -10328,6 +10331,10 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: size = check.get("bytes_needed", 0) label = f"Install {format_size(size)}" if size >= CONFIRM_FROM_BYTES else "Install" rows = [TuiRow("install", label)] + if _offline_only() and not _plan_copies(state, check): + # Nothing here to copy, and the network refused: Install would + # only fail every file. + rows = [] if state.facts.get("writable") is False or _short_of_space(state, check): # Every file would fail alike, or the run would stop before the # first one: the way out is another folder. @@ -10343,6 +10350,18 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: return rows + [TuiRow("options", "Options"), TuiRow("quit", "Quit")] +def _offline_only() -> bool: + """--offline without a pack: what is not on this machine cannot come.""" + return offline_requested() and LOCAL.pack is None + + +def _plan_copies(state: TuiState, check: dict) -> list[str]: + """The files to install that this machine holds, by destination.""" + local = {event["dest"] for event in state.found} if check is state.check else set() + sizes = {event["dest"] for event in state.files} + return [dest for dest in local if dest in sizes] + + def _short_of_space(state: TuiState, check: dict) -> bool: free = state.facts.get("free") return free is not None and check.get("bytes_needed", 0) > free @@ -11443,13 +11462,18 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: fetch = max(0, need - len(copied)) fetch_bytes = max(0, check["bytes_needed"] - sum(sizes[dest_] for dest_ in copied)) elsewhere = _tui_found_elsewhere(state) if check is state.check else {} + offline = _offline_only() if elsewhere: # The folder by its last name: the whole path pushed the BIOS # folder off a 40x15 screen. The Options row spells it out. where = ", ".join(f"{len(events)} found in {_last_name(folder)}" for folder, events in elsewhere.items()) - headline = (f"{where}, {fetch} to download ({format_size(fetch_bytes)})" + headline = (f"{where}, {fetch} need the internet (--offline)" if fetch and offline + else f"{where}, {fetch} to download ({format_size(fetch_bytes)})" if fetch else f"{where}, nothing to download") + elif need and fetch and offline: + headline = (f"{_plural(fetch, 'file')} {'needs' if fetch == 1 else 'need'} the " + "internet, which --offline does not use") elif need and fetch: headline = f"{_plural(fetch, 'file')} to download, {format_size(fetch_bytes)}" elif need: @@ -11461,6 +11485,9 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: headline = (f"{_plural(data, 'data file')} to fetch" if data else "Everything is in place") wrong = check["wrong_hash"] + offline_lines = (["Copy them from your old card or a folder, or run the installer " + "again with a pack (--from-pack) or without --offline."] + if offline and fetch else []) # The folder comes second: on a 40x12 screen it is what is left once # the rows are drawn, and nothing is written before it is seen. lines: list = [ @@ -11505,7 +11532,7 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: lines += _unavailable_lines(state) if check is state.check: lines += _unplaced_lines(state.check_summary, state) - return lines + return lines[:2] + offline_lines + lines[2:] def _unplaced_lines(summary: dict | None, state: TuiState) -> list[str]: @@ -12379,6 +12406,9 @@ class _EngineRun: TUI_FETCH_TIMEOUT = 15 +TUI_OFFLINE_TITLE = "No internet: offline mode" + + def _tui_fetch(plat: str, inbox: queue.Queue, log: io.StringIO) -> None: """The file list, fetched once in this process, or what went wrong.""" if LOCAL.manifest is not None or LOCAL.pack is not None: @@ -12407,6 +12437,16 @@ def _tui_fetch(plat: str, inbox: queue.Queue, log: io.StringIO) -> None: if exc.code == 404 else f"The server answered HTTP {exc.code}."] except (urllib.error.URLError, OSError) as exc: code, words = network_words(exc) + if code == "offline": + # Asked for, not a failure: no Try again, and the two ways a + # run without the network gets its list. + inbox.put(("manifest_failed", plat, TUI_OFFLINE_TITLE, [ + "This run makes no network request (--offline), and the list of " + "BIOS files comes from the network.", + "Run the installer again with the list saved as a file " + "(--manifest FILE, a copy of install/.json) or with a pack " + "(--from-pack PACK), or without --offline."], 1)) + return title = {"clock": "The clock is wrong", "no_ca": "No trusted certificates"}.get(code, "No connection") # The screen's first row is Try again: "then run again" was a second way. diff --git a/install.sh b/install.sh index f64306ffc..07943fb3c 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="eaa912af47dc6f29e4436ec200a1b431b8ddd8006ca08e57c8c73d3a05717818" +DEFAULT_INSTALL_SHA256="1f6b90760303d04cb6beef7fd294371b2e1d4ba733578c59a95e1b0395c2f376" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index f8ba0aa89..d9a9fac86 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -19,6 +19,7 @@ import http.server import io import json import os +import queue import re import select import shutil @@ -2578,6 +2579,53 @@ class AuditRoundFour(unittest.TestCase): self.assertIn(line, install.tui_outcome(state)[1]) +class OfflineInTheInterface(unittest.TestCase): + """--offline in the full screen: a "No connection" error offering a Try + again that can never work, and a plan counting downloads the run + forbids.""" + + def setUp(self): + patcher = mock.patch.dict(os.environ, {install.OFFLINE_VARIABLE: "1"}) + patcher.start() + self.addCleanup(patcher.stop) + + def test_no_list_names_the_ways_that_work(self): + inbox = queue.Queue() + install._tui_fetch("retroarch", inbox, io.StringIO()) + kind, plat, title, lines, code = inbox.get_nowait() + self.assertEqual((kind, title, code), ("manifest_failed", + "No internet: offline mode", 1)) + text = " ".join(lines) + for words in ("--manifest", "--from-pack", "without --offline"): + self.assertIn(words, text) + state = fresh_state() + state.stack = ["setup"] + with mock.patch.object(install.Path, "exists", return_value=True): + install.tui_update(state, Key("enter")) + install.tui_receive(state, (kind, plat, title, lines, code)) + self.assertEqual(state.screen, "error") + self.assertNotIn("retry", [row.action for row in install.tui_rows(state)]) + view = "\n".join(install.tui_view(state, 80, 24)) + self.assertIn("No internet: offline mode", view) + self.assertNotIn("Try again", view) + + def test_the_plan_offers_no_download(self): + state = checked_state() + text = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertNotIn("to download", text) + self.assertIn("need the internet, which --offline does not use", text) + self.assertNotIn("install", [row.action for row in install.tui_rows(state)]) + # A file found on this machine can still be copied. + state.found = [{"event": "found", "platform": "retroarch", "dest": FILES[0]["dest"], + "folder": "/media/old", "path": "/media/old/scph5500.bin", + "member": "", "reason": ""}] + text = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn("1 found in old,", text) + self.assertIn("need the internet (--offline)", text) + self.assertEqual(install.tui_rows(state)[0].action, "install") + self.assertIn("--offline", install.tui_command(state)) + + class NarrowSystemNames(unittest.TestCase): """At 40 columns the size column took half the line: Game Boy, Game Boy Advance and Game Boy Color all read "Nintendo Gam".""" From d297cc3ff6e32350bf610d8eac13c971576a1da4 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:58:19 +0200 Subject: [PATCH 15/19] fix: bound the wait for a silent list server A server that took the connection and never answered held line mode 93 s, three waits of 30 s for a first byte. A file list now waits 15 s per answer and asks a silent server twice. Downloads keep their own timeout between chunks. --- install.ps1 | 2 +- install.py | 30 ++++++++++++++++++++++++++---- install.sh | 2 +- tests/test_install.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/install.ps1 b/install.ps1 index b00b2a730..fdba58381 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "1f6b90760303d04cb6beef7fd294371b2e1d4ba733578c59a95e1b0395c2f376" + $defaultInstallSha256 = "aef82904ca3356ded55f7b43bc03fe80dc30da178a28bc598caef8367b3f9766" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 874efc26e..76b83f59a 100644 --- a/install.py +++ b/install.py @@ -3322,16 +3322,34 @@ def install_bounded_opener() -> None: urllib.request.install_opener(urllib.request.build_opener(*handlers)) +# The wait for each answer of a file list, its first byte included: a +# server that takes the connection and never answers (a captive portal, a +# stuck proxy) held the run 93 s on three waits of 30 s. A list is a few +# MB, never a slow stream. +LIST_TIMEOUT = 15.0 +# Attempts after a server that took the connection went silent: once more, +# a stall is rarely a blip. +STALLED_ATTEMPTS = 2 + + +def _stalled(exc: BaseException) -> bool: + """Whether exc is a wait that ran out, the connection made.""" + reason = getattr(exc, "reason", None) + return isinstance(exc, socket.timeout) or isinstance(reason, socket.timeout) + + def _fetch_json( - url: str, limit: int, label: str, attempts: int = MAX_RETRIES, timeout: float = 30, + url: str, limit: int, label: str, attempts: int = MAX_RETRIES, + timeout: float | None = None, ) -> object: """A JSON document, tried MAX_RETRIES times like every file download. A file list fetched once ended the run on one timeout over flaky Wi-Fi. A 4xx answer or an invalid document is final: asking again changes - nothing. + nothing, and a server gone silent is asked once more only. """ _network_allowed(label) + timeout = LIST_TIMEOUT if timeout is None else timeout for attempt in range(1, attempts + 1): try: with urllib.request.urlopen(url, timeout=timeout) as resp: @@ -3344,9 +3362,12 @@ def _fetch_json( except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: code, problem = network_words(exc) # A wrong clock or no certificate list fails every attempt alike. - if attempt == attempts or code != "network": + if (attempt == attempts or code != "network" + or (_stalled(exc) and attempt >= STALLED_ATTEMPTS)): if isinstance(exc, http.client.HTTPException): raise ConnectionError(f"connection cut: {exc!r}") from exc + # How many were made, for the line that says so. + exc.attempts = attempt # type: ignore[union-attr] raise delay = _retry_delay(attempt, None) # Said, or three 30-second attempts read as a frozen program. @@ -3416,7 +3437,8 @@ def fetch_manifest(plat: str) -> dict: print(f" Could not fetch the file list for {plat}.", file=sys.stderr) print(f" {words}.", file=sys.stderr) sys.exit(1) - print(f" Could not fetch the file list for {plat} after {MAX_RETRIES} " + tries = getattr(exc, "attempts", MAX_RETRIES) + print(f" Could not fetch the file list for {plat} after {tries} " f"attempts: {words}", file=sys.stderr) for line in _address_lines(url): print(line, file=sys.stderr) diff --git a/install.sh b/install.sh index 07943fb3c..e1e8a74af 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="1f6b90760303d04cb6beef7fd294371b2e1d4ba733578c59a95e1b0395c2f376" +DEFAULT_INSTALL_SHA256="aef82904ca3356ded55f7b43bc03fe80dc30da178a28bc598caef8367b3f9766" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index b7cb32854..1748ab29c 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -17,6 +17,7 @@ import subprocess import sys import tempfile import threading +import time import unittest from types import SimpleNamespace import unittest.mock @@ -2369,6 +2370,48 @@ class FileListsAreRetried(unittest.TestCase): self.assertIn(f"after {install.MAX_RETRIES} attempts", err.getvalue()) +class ASilentServerIsBounded(unittest.TestCase): + """A server that takes the connection and never answers held line mode + 93 s: three waits of 30 s for a first byte that never came.""" + + def test_two_short_waits_then_the_reason(self): + import socket + import threading + + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(8) + self.addCleanup(listener.close) + held = [] + + def accept(): + while True: + try: + conn, _addr = listener.accept() + except OSError: + return + held.append(conn) + + threading.Thread(target=accept, daemon=True).start() + self.addCleanup(lambda: [conn.close() for conn in held]) + port = listener.getsockname()[1] + err = io.StringIO() + start = time.monotonic() + with unittest.mock.patch.object(install, "LIST_TIMEOUT", 0.5), \ + unittest.mock.patch.object(install, "_retry_delay", return_value=0), \ + unittest.mock.patch.object(install, "MANIFEST_URL", + f"http://127.0.0.1:{port}/{{platform}}.json"), \ + contextlib.redirect_stderr(err): + with self.assertRaises(SystemExit): + install.fetch_manifest("retroarch") + elapsed = time.monotonic() - start + self.assertLess(elapsed, 5) + self.assertEqual(len(held), install.STALLED_ATTEMPTS) + self.assertIn(f"after {install.STALLED_ATTEMPTS} attempts", err.getvalue()) + # A download keeps its own, longer wait between chunks. + self.assertGreater(install.LIST_TIMEOUT, 1) + + class TargetListFailureIsNotAnAnswer(unittest.TestCase): """A timeout is not 'no targets': it sent users to the full pack.""" From 67a6ad3e32a52f5fde44bce7bfecda8b130aac25 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:11:45 +0200 Subject: [PATCH 16/19] chore: wrap the work folder error line --- install.ps1 | 2 +- install.py | 3 ++- install.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/install.ps1 b/install.ps1 index fdba58381..821c2dc57 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "aef82904ca3356ded55f7b43bc03fe80dc30da178a28bc598caef8367b3f9766" + $defaultInstallSha256 = "563ed14fea5ea54f5d6cb98311932293e714e9581014ea60c195325c20a587eb" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 76b83f59a..5d7449676 100644 --- a/install.py +++ b/install.py @@ -12737,7 +12737,8 @@ def run_tui(args: argparse.Namespace) -> int: except OSError as exc: _restore_handlers(handlers) print(f"Error: no work folder could be made in {tempfile.gettempdir()}: {exc}. " - "Free some space there, or point TMPDIR (TEMP on Windows) at another folder.", file=sys.stderr) + "Free some space there, or point TMPDIR (TEMP on Windows) at another " + "folder.", file=sys.stderr) return 1 try: (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") diff --git a/install.sh b/install.sh index e1e8a74af..8959bba12 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="aef82904ca3356ded55f7b43bc03fe80dc30da178a28bc598caef8367b3f9766" +DEFAULT_INSTALL_SHA256="563ed14fea5ea54f5d6cb98311932293e714e9581014ea60c195325c20a587eb" MAX_INSTALLER_BYTES=2097152 cleanup() { From 49b2a69567639ab2d983f8e5192a4ebb553ed024 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:11:45 +0200 Subject: [PATCH 17/19] docs: exit 129, offline refusals and pad variables The exit status names 129 for a closed window and the --offline refusals of --target and --list-targets, and the events say a crash leaves no end line. RETROBIOS_PAD and RETROBIOS_IDLE_EXIT join the environment table, muxterm's missing paging and the save directory found under --dest are said, and the options at a glance list --declared-only and --offline. The first install page no longer promises a controller on a computer, and gives the RetroPie command for Windows too. --- wiki/first-install.md | 10 +++++++--- wiki/installer.md | 37 +++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/wiki/first-install.md b/wiki/first-install.md index 33a974756..1537570cf 100644 --- a/wiki/first-install.md +++ b/wiki/first-install.md @@ -83,7 +83,7 @@ It opens a menu: - **Wrong folder?** Choose **Another setup or folder**, then browse to yours. - **A handheld's SD card in a reader?** It is listed, as **muOS SD card**. -Arrows move, Enter chooses, Esc goes back. A controller's A and B do the same. +Arrows move, Enter chooses, Esc goes back. A handheld's A and B do the same. ## 3. Install @@ -142,8 +142,12 @@ The message shows the command that installs Python. Run it, then run the installer again. A RetroPie 4.8 image has Python 3.7.3, and its update command finds nothing -newer. Run the command on a computer, with -`--platform retropie --dest \\RETROPIE\bios\`. +newer. Run it on a computer, the `bios` share open: + + +```sh +curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --platform retropie --dest +``` ### Error: the downloaded installer does not match its expected fingerprint. diff --git a/wiki/installer.md b/wiki/installer.md index 826692fcc..becbca570 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -26,9 +26,9 @@ How the command protects your machine is under Each links to its row in the [Options](#options) table. - **Where and for what**: [`--platform`](#option-platform), [`--dest`](#option-dest), [`--target`](#option-target), [`--every-board`](#option-every-board), [`--save-dir`](#option-save-dir), [`--romm-firmware`](#option-romm-firmware), [`--standalone-copies`](#option-standalone-copies) -- **Fewer files**: [`--system`](#option-system), [`--core`](#option-core), [`--emulator`](#option-emulator), [`--region`](#option-region), [`--no-game-data`](#option-no-game-data), [`--installed-cores`](#option-installed-cores), [`--cores-dir`](#option-cores-dir) +- **Fewer files**: [`--system`](#option-system), [`--core`](#option-core), [`--emulator`](#option-emulator), [`--region`](#option-region), [`--no-game-data`](#option-no-game-data), [`--installed-cores`](#option-installed-cores), [`--cores-dir`](#option-cores-dir), [`--declared-only`](#option-declared-only) - **Look first**: [`--check`](#option-check), [`--dry-run`](#option-dry-run), [`--list-platforms`](#option-list-platforms), [`--list-targets`](#option-list-targets), [`--list-systems`](#option-list-systems), [`--list-cores`](#option-list-cores), [`--list-emulators`](#option-list-emulators) -- **From files you have**: [`--from`](#option-from), [`--from-pack`](#option-from-pack), [`--manifest`](#option-manifest), [`--restore`](#option-restore) +- **From files you have**: [`--from`](#option-from), [`--from-pack`](#option-from-pack), [`--manifest`](#option-manifest), [`--offline`](#option-offline), [`--restore`](#option-restore) - **How the run talks**: [`--tui`](#option-tui), [`--plain`](#option-plain), [`--no-input`](#option-no-input), [`--yes`](#option-yes), [`--quiet`](#option-quiet), [`--verbose`](#option-verbose), [`--events`](#option-events), [`--jobs`](#option-jobs), [`--version`](#option-version), [`--help`](#option-help) ## One line @@ -232,7 +232,8 @@ below the BIOS folder: Azahar's keys, shared font and NAND files, Citra's `sysdata`, Dolphin's `User` tree. The file list marks them `root: save`, and the run writes them below the save directory, which it names before checking files (`5 files read from RetroArch's save directory go to ...`). For a -RetroArch found by its `retroarch.cfg`, that is `savefile_directory` as +RetroArch found by its `retroarch.cfg` (with `--dest` too, when the folder is +the config's `system_directory`, as in the full screen), that is `savefile_directory` as RetroArch expands it (`:` for its own folder, `~` for the home), `saves/` beside the config when the value is `default`, empty, or names no folder, and the core's own folder below it when `sort_savefiles_enable` is on, as it is by @@ -242,7 +243,7 @@ the save directory beside it (`saves`, `/storage/savefiles` on Lakka). applies, or when `savefiles_in_content_dir` or `sort_savefiles_by_content_enable` keeps saves beside each game, those files are left out and the run says why, like the other entries the list cannot -install. +install. The full screen says so on its plan and its result. Batocera, KNULLI, Recalbox, RetroBat and RetroDECK write the save directory at each launch, one folder per system (`by_system` in the file list). Their lists @@ -1066,7 +1067,9 @@ muxterm, as the muOS app does. Start pressed first also chooses the line in focus: hold Select first. A running install is then stopped the way B stops it, and the last screen says what it wrote. On muOS without PortMaster, muxterm reads the pad itself: Y -ticks a system, Select shows its keyboard, and Menu quits. +ticks a system, Select shows its keyboard, and Menu quits. L1 and R1 do not +page there: muxterm keeps PgUp and PgDn for its own scrollback, so a list +moves with the D-pad. Each launcher tells the interface which buttons reach it, in `RETROBIOS_PAD`, so the hints can name them: @@ -1078,7 +1081,12 @@ Each launcher tells the interface which buttons reach it, in | `evmapy` | `RetroBIOS.sh.keys`, the same buttons as the table | the Batocera and Knulli port | | `miyoo` | buttons that are already keys: A as Space, Start as Enter, L1 and R1 as E and T, L2 as Tab, R2 as Backspace | no launcher yet | -A launcher with no mapper at all (a keyboard) leaves it unset. On a Miyoo, B, +A launcher with no mapper at all (a keyboard) leaves it unset. +A launcher can also set `RETROBIOS_IDLE_EXIT` to a number of seconds: when +no button reaches the first screen for that long, before any choice, the +interface closes with `No button pressed, closing.` and exit status 0. A +mapper that never opened the pad would otherwise hold the device on that +screen. On a Miyoo, B, X, Y and Select are Left Ctrl, Shift, Alt and Right Ctrl: pressed alone, a modifier sends nothing to a terminal, so the interface goes back with R2 instead, and the bottom line says so. @@ -1235,13 +1243,15 @@ fields, the installer refuses to narrow rather than install everything. |------|---------| | `0` | everything asked for is installed or already up to date | | `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, or `--restore` could not put a file back | -| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | +| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`, `--target` or `--list-targets` with `--offline` (or `RETROBIOS_INSTALL_OFFLINE` set), which read a core list from the network. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | | `130` | stopped with Ctrl+C, or with `q`, Ctrl+C or Ctrl+D at a question. Run the same command again to carry on | | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) | The full screen ended by SIGTERM (Start+Select on a handheld) before any install exits `143`. During an install it stops the install first, and exits -`130`. +`130`. Its window or terminal closed (SIGHUP) exits `129`, during an install +too: the install is stopped as Ctrl+C stops it, and the last lines go +nowhere, no one being left to read them. `install.ps1` passes the installer's code on unchanged: run as a file it exits with it, run through `irm | iex` it sets `$LASTEXITCODE` (an `exit` there @@ -1293,7 +1303,9 @@ other schemas under `api/v1/schemas/`. Schema 1 can gain a kind (`found` came with `--from`, `declared` with `--declared-only`), never a key of an existing kind: a reader skips a kind it does not know. An option refused before the run starts (`install.py: error:`, code 2) writes -no event at all. A program reads a missing `end` as that case. +no event at all. A program reads a missing `end` as that case, or as a +crash: an error the installer did not expect ends the process on a Python +traceback, on standard error, without an `end` line. Five lines of a run of `--check --events --system sony-playstation` on a RetroArch folder that holds only `scph5501.bin`, in the order written: @@ -1322,6 +1334,8 @@ RetroArch folder that holds only `scph5501.bin`, in the order written: | `FORCE_COLOR`, `CLICOLOR_FORCE` | Colour even when the output is a pipe or a file, set to any value but `0` or an empty one. `NO_COLOR` wins over both, and the progress line stays one line per file | | `SSL_CERT_FILE`, `SSL_CERT_DIR` | The trusted certificates to check servers with, for a Python that finds none of its own | | `RETROBIOS_PLAIN`, `ACCESSIBLE` | Set to anything but `0`: the questions come line by line, as with `--plain`, instead of the full-screen interface | +| `RETROBIOS_PAD` | The buttons a handheld launcher's mapper sends to the full screen (`gptokeyb`, `muxterm`, `evmapy`, `miyoo`), so its hints name them. See [On a handheld](#on-a-handheld-the-retrobios-port) | +| `RETROBIOS_IDLE_EXIT` | Seconds the full screen's first screen waits for a first button. With none pressed, it closes with exit status 0 | | `HTTPS_PROXY`, `NO_PROXY` | Honoured for every download through the standard library's default proxy handling: one names the proxy, the other the hosts to reach directly | ## When something goes wrong @@ -1336,7 +1350,10 @@ The installer fetches its manifest, then downloads only the repository or `https://github.com` opens. Behind a proxy, set `HTTPS_PROXY` in the environment. A connection that does not open within 4 seconds counts as none, so a dropped network gives its error in about 15 seconds. Once a -connection is open, a slow transfer keeps the full timeout. A file that fails three attempts is counted and reported at the +connection is open, a slow transfer keeps the full timeout. The file list +is the exception: a server that takes the connection and then sends nothing +(a Wi-Fi sign-in page, a stuck proxy) gets 15 seconds per wait and two +attempts, about 30 seconds in all. A file that fails three attempts is counted and reported at the end, and the next run fetches only what is still missing. The reason is said in words: no internet connection or a name that could not From e7d27a264d5c6474e71bdcfff9d1d28791b4ef47 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:29:48 +0200 Subject: [PATCH 18/19] fix: hide the pad from muxterm on every muos build muxterm was allowed vendor and product 0001:0001, the id gpio-keys gives the H700 pad on muOS before 2026-08-28: muxterm and gptokeyb2 both read it and every button acted twice. The allow-list is now 0000:0001, which SDL never reads back for a device (a vendor of 0 makes a name GUID, read back as 0:0), so a pad plugged in later stays hidden. - muxterm keeps PgUp and PgDn for its scrollback: in muxterm L1 and R1 send nothing, and the README and wiki stop promising paging there. - The offline detection runs with RETROBIOS_INSTALL_OFFLINE=1 instead of a closed loopback URL. - Every launcher sets RETROBIOS_IDLE_EXIT=300 for the interface. - ROCKNIX: control.ini's -sudokill goes through a sudo stub that runs nothing, so the mapper starts with -1. - The sentinel ends with its launcher, a closed terminal app writes its status, a mapper lost mid-run is said and named (gptokeyb2 included), and the unwritable-card message stays in view. - Tests: an SDL GUID model, a gptokeyb2 that outlives its kill, an interface that ignores SIGTERM, a zombie, a SIGHUP to the inner run, arrow bytes at the question, ROCKNIX's sudo. --- ports/RetroBIOS/RetroBIOS.sh | 7 +- ports/muos/README.md | 6 +- ports/muos/mux_launch.sh | 14 +- ports/shared/tui.txt | 85 +++++- ports/tools/RetroBIOS.sh | 12 +- tests/test_handheld_launchers.py | 494 +++++++++++++++++++++++++++---- wiki/installer.md | 7 +- 7 files changed, 537 insertions(+), 88 deletions(-) diff --git a/ports/RetroBIOS/RetroBIOS.sh b/ports/RetroBIOS/RetroBIOS.sh index 3c8195734..24610f480 100755 --- a/ports/RetroBIOS/RetroBIOS.sh +++ b/ports/RetroBIOS/RetroBIOS.sh @@ -69,13 +69,10 @@ rb_open_log keep # muOS: PortMaster's mod_muOS.txt points CUR_TTY at /dev/null, so there is no # console to draw on. muxterm, the terminal muOS's Terminal core runs, shows -# the interface instead. It is told to leave the pad alone (an SDL hint) so -# that gptokeyb sends every button once, with this port's mapping: muxterm -# sends Y as Space, where the interface reads Y as Options. The hint allows -# one vendor and product id, 0001:0001, which no pad carries. +# the interface instead, told to leave the pad to gptokeyb (rb_muxterm). RB_MUXTERM="${RB_MUXTERM:-/opt/muos/frontend/muxterm}" if [ -x "$RB_MUXTERM" ] && [ -n "$RB_MAPPER" ]; then - RB_TERMAPP=(env SDL_GAMECONTROLLER_IGNORE_DEVICES_EXCEPT=0x0001/0x0001 "$RB_MUXTERM" --gl --) + rb_muxterm "$RB_MUXTERM" fi export SDL_GAMECONTROLLERCONFIG="$sdl_controllerconfig" diff --git a/ports/muos/README.md b/ports/muos/README.md index b9200a765..8b2facf76 100644 --- a/ports/muos/README.md +++ b/ports/muos/README.md @@ -10,9 +10,13 @@ connection, or from a pack copied to the root of the card without one. It runs in muxterm, muOS's terminal. With PortMaster installed, gptokeyb2 turns every button into a key and muxterm is told to leave the pad alone, so each button is sent once: D-pad to move, A or Start to choose, B or Select to -go back, X to tick a system, Y for the options, L1 and R1 to page. Hold +go back, X to tick a system, Y for the options. L1 and R1 do nothing in +muxterm, which keeps Page Up and Page Down for its own scrollback. Hold Select, then press Start to quit. Without PortMaster, muxterm reads the pad itself (Y then ticks a system) and Menu closes it. +With no button pressed on its first screen for 5 minutes, it closes by +itself: a pad the system cannot read then never leaves it stuck. + Not tried on a device yet: the icon, and whether muxterm honours SDL's controller allow-list as SDL 2.30 does. diff --git a/ports/muos/mux_launch.sh b/ports/muos/mux_launch.sh index 1cdcb98aa..201db1701 100755 --- a/ports/muos/mux_launch.sh +++ b/ports/muos/mux_launch.sh @@ -46,17 +46,9 @@ if [ -x "$PM_DIR/gptokeyb2" ]; then fi if [ -x "$MUXTERM" ]; then RB_ROUTES="app" - if [ -n "${RB_MAPPER:-}" ]; then - # muxterm reads the pad itself and sends Y as Space (MustardOS/terminal - # input.c), where the interface reads Y as Options. The SDL hint allows - # only vendor and product 0001:0001, which no pad carries, so muxterm - # leaves the pad to gptokeyb2 and every button is sent once. - RB_TERMAPP=(env SDL_GAMECONTROLLER_IGNORE_DEVICES_EXCEPT=0x0001/0x0001 "$MUXTERM" --gl --) - else - # No gptokeyb2: muxterm's own buttons are all there is. - RB_TERMAPP=("$MUXTERM" --gl --) - RB_PAD="muxterm" - fi + # With gptokeyb2, muxterm leaves the pad to it. Without, muxterm's own + # buttons are all there is. + rb_muxterm "$MUXTERM" else RB_ROUTES="vt" RB_CONSOLE="${RB_CONSOLE:-/dev/tty0}" diff --git a/ports/shared/tui.txt b/ports/shared/tui.txt index bccb30c63..0b087674e 100644 --- a/ports/shared/tui.txt +++ b/ports/shared/tui.txt @@ -284,17 +284,17 @@ rb_online() { } # The system and BIOS folder install.py finds here, read without a network: -# detection is local, and the run stops where the file list would download. -# That download is sent to a loopback port nothing listens on, so it fails -# at once instead of retrying three times 30 s on a network already found -# missing. +# detection is local, and RETROBIOS_INSTALL_OFFLINE stops the run at once +# where the file list would download (reason offline), instead of three +# tries of 30 s on a network already found missing. Set for this run only: +# the interface's own run reads a pack, which makes no request either. rb_detect() { RB_PLATFORM="" RB_PLATFORM_NAME="" RB_BIOS_DIR="" mkdir -p "$RB_STATE" rb_pycmd - RETROBIOS_BASE_URL="http://127.0.0.1:9" rb_bounded "$RB_DETECT_LIMIT" \ + RETROBIOS_INSTALL_OFFLINE=1 rb_bounded "$RB_DETECT_LIMIT" \ "${RB_PYCMD[@]}" "$RB_INSTALL" --check --events > "$RB_STATE/detect.jsonl" 2>> "$RB_LOG" [ $? = 124 ] && rb_say "the detection run took over $RB_DETECT_LIMIT s and was stopped" local kind a b c @@ -415,6 +415,36 @@ rb_terminal_app() { fi } +# The only pad SDL may show muxterm: vendor 0, product 1, which no device +# can carry. SDL builds a GUID from the ids only when the vendor is not 0, +# else from the name, and reads a name GUID back as vendor 0 and product 0 +# (SDL_CreateJoystickGUID and SDL_GetJoystickGUIDInfo, SDL_joystick.c), so +# a pad plugged in later is hidden too. The first list, 0001:0001, is the +# id gpio-keys gives the H700 pad on muOS before 2026-08-28: every button +# acted twice there. +RB_SDL_NO_PAD="0x0000/0x0001" + +# muxterm, muOS's terminal, as the terminal app. It reads the pad itself +# (Y as Space, where the interface reads Y as Options), so with a mapper it +# is told to leave the pad alone and each button is sent once. muxterm also +# keeps PgUp and PgDn for its scrollback and never writes them to the +# interface (MustardOS/terminal input.c): L1 and R1 send nothing there, +# rather than scroll the interface out of sight. Run after rb_open_log. +rb_muxterm() { + if [ -z "${RB_MAPPER:-}" ]; then + RB_TERMAPP=("$1" --gl --) + RB_PAD="muxterm" + return 0 + fi + RB_TERMAPP=(env "SDL_GAMECONTROLLER_IGNORE_DEVICES_EXCEPT=$RB_SDL_NO_PAD" "$1" --gl --) + if awk '$1 == "l1" || $1 == "r1" { print $1 " = \\\""; next } { print }' \ + "$RB_GPTK" > "$RB_STATE/muxterm.gptk" 2>> "$RB_LOG"; then + RB_GPTK="$RB_STATE/muxterm.gptk" + else + rb_say "$RB_STATE/muxterm.gptk cannot be written: L1 and R1 scroll muxterm" + fi +} + # A console a terminal can be read from. PortMaster's mod_muOS.txt sets # CUR_TTY to /dev/null, a character device that is no console at all. A # console this user cannot open yet (rb_vt_enter opens it up) is judged by @@ -581,10 +611,13 @@ rb_vt_leave() { # wrote. So the target is a sentinel, a process this launcher owns whose # command line alone carries the token: when it dies, the launcher asks the # interface to stop, and the interface gives a running install its 2 s. +# It also ends once the launcher is gone, which a SIGKILL leaves no trap +# to say. rb_stop_start() { RB_STOP_TOKEN="retrobios-stop-${$}x" - bash -c 'exec {f}<> <(:); while :; do read -r -t 3600 -u "$f" _; done' \ - "$RB_STOP_TOKEN" > /dev/null 2>&1 & + # shellcheck disable=SC2016 # expanded by the inner bash + bash -c 'exec {f}<> <(:); while [ -d "/proc/$1" ]; do read -r -t 5 -u "$f" _; done' \ + "$RB_STOP_TOKEN" "$$" > /dev/null 2>&1 & RB_STOP_PID=$! } @@ -628,8 +661,26 @@ rb_mapper_stop() { rb_stop_end } +# The mapper's own name, gptokeyb or gptokeyb2, past sudo and its options. +rb_mapper_name() { + local word + # shellcheck disable=SC2086 # RB_MAPPER is a command and its flags + for word in $RB_MAPPER; do + case "$word" in + sudo|-*|*=*) ;; + *) printf '%s\n' "${word##*/}"; return 0 ;; + esac + done + printf 'gptokeyb\n' +} + +# "run" when the mapper stopped during the run: the interface just went. rb_mapper_text() { - printf '%s\n' "The controller cannot be read: gptokeyb, which turns the buttons into keys, stopped as soon as it started. What it said is in $RB_LOG. $RB_PC_ROUTE" + if [ "${1:-}" = "run" ]; then + printf '%s\n' "The controller stopped responding: $(rb_mapper_name), which turns the buttons into keys, stopped during the run, so RetroBIOS closed. What it said is in $RB_LOG. Start RetroBIOS again." + return 0 + fi + printf '%s\n' "The controller cannot be read: $(rb_mapper_name), which turns the buttons into keys, stopped as soon as it started. What it said is in $RB_LOG. $RB_PC_ROUTE" } # --- Messages ------------------------------------------------------------- @@ -816,6 +867,10 @@ rb_here() { if [ "${RB_FALLBACK:-0}" != "1" ]; then rb_message "$(tail -n 1 "$RB_STATE/tui.err") The messages are in $RB_LOG." fi + elif [ "$RB_STOPPED" = "mapper" ]; then + # The interface disappeared: why, for RB_DEAD_WAIT seconds, since no + # button is left to close it. + rb_message "$(rb_mapper_text run)" elif [ -n "$RB_STOPPED" ] || [ "$status" -ge 128 ]; then # Ended by Start + Select or a signal: the player asked to leave, and no # button may be left to close a message. @@ -891,6 +946,9 @@ rb_launch() { printf '%s\n' "$RB_STATE_PROBLEM" >&2 if rb_is_console "$RB_CONSOLE"; then printf '\r\n%s\r\n' "$RB_STATE_PROBLEM" > "$RB_CONSOLE" 2>/dev/null + # No mapper runs yet to close it with a button: left in view a while + # before the menu draws over it. + rb_nap "$RB_DEAD_WAIT" fi return 1 fi @@ -898,6 +956,10 @@ rb_launch() { export RB_DIR RB_INSTALL RB_EVENTS RB_STATE RB_LOG RB_PACK_PATH RB_FALLBACK export RB_OTHER_CARDS RB_BIOS_DEST RB_STOP_GRACE RB_DEAD_WAIT RB_DETECT_LIMIT export RB_PYTHON RB_PYTHONHOME RB_PYTHON_PROBLEM RB_MESSAGE_WAIT RB_ROUTE_TABLE + # A mapper that runs but never opens the pad (one SDL has no mapping for) + # leaves no button to move the interface or end the run: it closes by + # itself after this many seconds on its first screen with nothing pressed. + export RETROBIOS_IDLE_EXIT="${RETROBIOS_IDLE_EXIT:-300}" # The interface names the buttons of this route in its hints. if [ -z "$RB_PAD" ] && [ -n "${RB_MAPPER:-}" ]; then RB_PAD="gptokeyb" @@ -931,9 +993,10 @@ if [ "${BASH_SOURCE[0]}" = "$0" ]; then case "${1:-}" in here) shift - # The terminal app closed or the launcher signalled: the interface - # gets its time to stop. - trap 'rb_ui_stop; exit 143' TERM HUP + # The terminal app closed (muxterm's Menu) or the launcher signalled: + # the interface gets its time to stop, and the status says the run + # ended, so the launcher opens no other route after it. + trap 'rb_ui_stop; printf "143\n" > "$RB_STATE/status"; exit 143' TERM HUP rb_here "$@" exit $? ;; diff --git a/ports/tools/RetroBIOS.sh b/ports/tools/RetroBIOS.sh index ef197e945..b3bc34854 100755 --- a/ports/tools/RetroBIOS.sh +++ b/ports/tools/RetroBIOS.sh @@ -82,8 +82,16 @@ else # shellcheck source=/dev/null source "$control" declare -F get_controls >/dev/null && get_controls - # It carries its own kill flag (-sudokill). - [ -n "${GPTOKEYB:-}" ] && mapper="$GPTOKEYB" kill_flag="" + # Its kill flag is -sudokill (control-gen.cpp:31), which kills through + # sudo, and ROCKNIX's sudo is a stub that runs nothing (busybox + # scripts/sudo): Start + Select would end the run only 3 s later, + # through the mapper's exit. -1 kills as the user it runs as, root. + if [ -n "${GPTOKEYB:-}" ]; then + mapper="$GPTOKEYB" + case "$mapper" in + *" -sudokill"|*" -1"|*" -k") mapper="${mapper% *}" ;; + esac + fi ESUDO="$saved_esudo" controlfolder="$saved_folder" fi fi diff --git a/tests/test_handheld_launchers.py b/tests/test_handheld_launchers.py index 4a6246d73..a9dc8756f 100644 --- a/tests/test_handheld_launchers.py +++ b/tests/test_handheld_launchers.py @@ -177,13 +177,100 @@ class ControllerMappings(unittest.TestCase): targets = {a["trigger"]: a["target"] for a in keys["actions_player1"]} self.assertEqual((targets["x"], targets["y"]), (["KEY_SPACE"], ["KEY_TAB"])) # muxterm sends Y as Space on its own (MustardOS/terminal input.c): - # where gptokeyb runs beside it, muxterm is told to leave the pad. + # where gptokeyb runs beside it, muxterm is told to leave the pad + # (assert_muxterm_hides_every_pad). for launcher in (PORTS / "muos" / "mux_launch.sh", PORTS / "RetroBIOS" / "RetroBIOS.sh"): - text = launcher.read_text() - self.assertIn("SDL_GAMECONTROLLER_IGNORE_DEVICES_EXCEPT=0x0001/0x0001", text) + self.assertIn("rb_muxterm ", launcher.read_text()) self.assertFalse((PORTS / "muos" / "muxterm.gptk").exists()) +def sdl_vidpid(bus: int, vendor: int, product: int, version: int, name: str) -> tuple[int, int]: + """The vendor and product SDL 2 reads back from the GUID it builds for + a device: SDL_CreateJoystickGUID, then SDL_GetJoystickGUIDInfo + (libsdl-org/SDL release-2.30.x src/joystick/SDL_joystick.c). A vendor of + 0 makes the GUID carry the name instead of the ids.""" + data = bytearray(16) + struct.pack_into(" list[int]: + """SDL_LoadVIDPIDListFromHint: every pair of 0x numbers.""" + entries = [] + for vendor, product in re.findall(r"0x([0-9a-fA-F]+)[^x]*?0x([0-9a-fA-F]+)", hint): + entries.append((int(vendor, 16) & 0xFFFF) << 16 | (int(product, 16) & 0xFFFF)) + return entries + + +# Input devices a muOS device lists in /proc/bus/input/devices: the H700 +# pad before 2026-08-28 (gpio-keys, 0001:0001, MustardOS internal +# f6a14e3121 device/rg35xx-plus/control/gamecontrollerdb/modern.txt), the +# muinput pad after it (0x756d), a USB and a Bluetooth pad, gptokeyb2's +# uinput keyboard, and pads whose vendor is 0. +PADS = [ + (0x19, 0x0001, 0x0001, 0x0100, "Deeplay-keys"), + (0x19, 0x0001, 0x0001, 0x0100, "muOS-Keys"), + (0x06, 0x756d, 0x6b6d, 0x0001, "muOS-Keys"), + (0x03, 0x045e, 0x028e, 0x0114, "Microsoft X-Box 360 pad"), + (0x05, 0x057e, 0x2009, 0x8111, "Pro Controller"), + (0x06, 0x0000, 0x0000, 0x0000, "Fake Keyboard"), +] +PADS += [(bus, 0, product, 1, "pad"[:length] + "x" * max(0, length - 3)) + for bus in (0x03, 0x05, 0x06, 0x19, 0xFF) for product in (0, 1, 0x2009) + for length in range(13)] +PADS += [(bus, vendor, product, 1, "pad") + for bus in (0x03, 0x05, 0x19) for vendor in (1, 0x0079, 0xFFFF) + for product in (0, 1, 0xFFFF)] + + +def assert_muxterm_hides_every_pad(test: unittest.TestCase, hint: str) -> None: + """muxterm reads the pad itself (T/input.c:303-317) while gptokeyb2 + sends its keys, so every button would act twice. The launchers set + SDL_GAMECONTROLLER_IGNORE_DEVICES_EXCEPT, which hides from SDL every + device it does not name (SDL_gamecontroller.c:2145-2150, applied to the + joystick layer by SDL_ShouldIgnoreJoystick, SDL_joystick.c:3006).""" + allowed = sdl_vidpid_list(hint) + # An empty list allows every device. + test.assertTrue(allowed, hint) + for pad in PADS: + vendor, product = sdl_vidpid(*pad) + with test.subTest(pad=pad): + test.assertNotIn(vendor << 16 | product, allowed) + + +def assert_muxterm_mapping(test: unittest.TestCase, args: list[str], shared: Path) -> None: + """muxterm keeps PgUp and PgDn for its scrollback and never writes them + to the terminal (T/input.c:352-360): in muxterm, L1 and R1 send nothing, + so they cannot scroll the interface away. Every other button is + retrobios.gptk's.""" + test.assertEqual(args[-2], "-c") + mapping = read_gptk(Path(args[-1])) + expected = read_gptk(shared) + expected["l1"] = expected["r1"] = ['\\"'] + test.assertEqual(mapping, expected) + + +class TheSdlModel(unittest.TestCase): + def test_it_reads_the_ids_as_sdl_does(self): + self.assertEqual(sdl_vidpid(0x19, 1, 1, 0x100, "Deeplay-keys"), (1, 1)) + self.assertEqual(sdl_vidpid(0x05, 0, 0x2009, 1, "pad"), (0, 0)) + self.assertEqual(sdl_vidpid(0x05, 0, 0x2009, 1, "ab"), (0x6261, 0)) + self.assertEqual(sdl_vidpid_list("0x0001/0x0001, 0x045e/0x028e"), + [0x00010001, 0x045e028e]) + + def test_the_first_hint_allowed_the_h700_pad(self): + with self.assertRaises(AssertionError): + assert_muxterm_hides_every_pad(unittest.TestCase(), "0x0001/0x0001") + + class TheArchives(unittest.TestCase): LAYOUT = { build_port.PORT_ZIP: ("RetroBIOS.sh", "retrobios"), @@ -300,6 +387,8 @@ if "--engine" in sys.argv: signal.signal(signal.SIGINT, summary) signal.signal(signal.SIGTERM, other) + with open(os.path.join(state, "engine.pid"), "w") as fh: + fh.write(str(os.getpid())) note("engine.ready") time.sleep(60) sys.exit(0) @@ -309,17 +398,32 @@ with open(calls, "a") as fh: "stdout": tty(1), "term": os.environ.get("TERM", ""), "pythonhome": os.environ.get("PYTHONHOME", ""), "base_url": os.environ.get("RETROBIOS_BASE_URL", ""), + "offline": os.environ.get("RETROBIOS_INSTALL_OFFLINE", ""), + "idle_exit": os.environ.get("RETROBIOS_IDLE_EXIT"), "pad": os.environ.get("RETROBIOS_PAD")}) + "\n") if "--check" in sys.argv: + # install.py --check --events: detection is local, then the file list. + # With RETROBIOS_INSTALL_OFFLINE the file list stops the run at once + # with reason offline. Otherwise it is downloaded: a closed loopback + # port refuses at once, any other address is three tries of 30 s on a + # device without a network (FAKE_CURL, the probe that found it so). + offline = os.environ.get("RETROBIOS_INSTALL_OFFLINE", "") not in ("", "0") + base_url = os.environ.get("RETROBIOS_BASE_URL", "") + no_network = os.environ.get("FAKE_CURL", "0") in ("5", "6", "7", "28") if os.environ.get("FAKE_DETECT_HANG"): time.sleep(60) + print(json.dumps({"event": "start", "mode": "check"}), flush=True) print(json.dumps({"event": "detect", "os": "linux", "os_name": "Fake OS", "platforms": [{"platform": os.environ.get("FAKE_PLATFORM", "retroarch"), "name": os.environ.get("FAKE_PLATFORM_NAME", "RetroArch"), - "bios_dir": "/roms/bios"}]})) - print(json.dumps({"event": "error", "stage": "manifest", "reason": "network", + "bios_dir": "/roms/bios"}]}), flush=True) + if no_network and not offline and not base_url.startswith("http://127.0.0.1"): + time.sleep(90) + print(json.dumps({"event": "error", "stage": "manifest", + "reason": "offline" if offline else "network", "platform": os.environ.get("FAKE_PLATFORM", "retroarch"), - "message": "unreachable"})) + "message": "this run makes no network request (--offline)" if offline + else "unreachable"})) print(json.dumps({"event": "end", "code": 1})) sys.exit(1) if os.environ.get("FAKE_REFUSE"): @@ -329,6 +433,21 @@ if os.environ.get("FAKE_REFUSE"): if os.environ.get("FAKE_RAW") and os.isatty(0): import tty as _tty _tty.setraw(0) +if os.environ.get("FAKE_DRAIN") and os.isatty(0): + # What the console still holds when the interface starts: bytes a + # question left behind are read as the interface's first keys. + # TCSANOW, as install.py sets its mode: TCSAFLUSH would discard them. + import select as _select + import termios as _termios + import tty as _tty + _tty.setraw(0, _termios.TCSANOW) + left = b"" + while _select.select([0], [], [], 0.3)[0]: + chunk = os.read(0, 64) + if not chunk: + break + left += chunk + note("ui.input", left.hex()) if os.environ.get("FAKE_HOLD"): # The interface up and waiting for a key until a signal ends it. With # FAKE_ENGINE an install runs, and SIGTERM gives it 2 s to say what it @@ -354,9 +473,14 @@ if os.environ.get("FAKE_HOLD"): print("Error: " + os.environ["FAKE_TERM_ERROR"], file=sys.stderr) sys.exit(128 + signum) - if not os.environ.get("FAKE_NO_HANDLER"): + if os.environ.get("FAKE_IGNORE_TERM"): + # An interface stuck where its signal handler never runs. + signal.signal(signal.SIGTERM, signal.SIG_IGN) + elif not os.environ.get("FAKE_NO_HANDLER"): signal.signal(signal.SIGTERM, stop) signal.signal(signal.SIGHUP, stop) + with open(os.path.join(state, "ui.pid"), "w") as fh: + fh.write(str(os.getpid())) note("ui.ready") time.sleep(60) sys.exit(0) @@ -371,12 +495,13 @@ PGREP = shutil.which("pgrep", path="/usr/bin:/bin") or "pgrep" PKILL = shutil.which("pkill", path="/usr/bin:/bin") or "pkill" # gptokeyb as PortsMaster/gptokeyb runs it: kill mode takes the argument -# after the flag (`-1`, `-k` or `-sudokill`) as its target. Start + Select (FAKE_PRESS, once the -# interface is up) is doKillMode in src/util.cpp: pkill -f on the target, -# which also matches gptokeyb's own command line (SDL turns that SIGTERM -# into an event it does not read there), 3 s, then pkill -9 -f and exit. -# Stopped otherwise by a SIGTERM, which it records. FAKE_MAPPER_DIES is a -# mapper that cannot start (no /dev/uinput). +# after the flag (`-1`, `-k` or `-sudokill`) as its target. Start + Select +# (FAKE_PRESS, once FAKE_PRESS_AT exists, by default once the interface is +# up) is doKillMode in src/util.cpp: pkill -f on the target, which also +# matches gptokeyb's own command line (SDL turns that SIGTERM into an event +# it does not read there), 3 s, then pkill -9 -f and exit. `-sudokill` runs +# both through sudo. Stopped otherwise by a SIGTERM, which it records. +# FAKE_MAPPER_DIES is a mapper that cannot start (no /dev/uinput). FAKE_MAPPER = f'''#!/bin/bash printf '%s\\n' "$@" > "$FAKE_STATE/mapper.args" echo $$ > "$FAKE_STATE/mapper.pid" @@ -385,32 +510,39 @@ if [ -n "${{FAKE_MAPPER_DIES:-}}" ]; then echo "gptokeyb: cannot open /dev/uinput" >&2 exit 1 fi -name="" +name="" sudo="" while [ $# -gt 0 ]; do case "$1" in - -1|-k|-sudokill) name="$2"; shift 2 ;; + -1|-k) name="$2"; shift 2 ;; + -sudokill) name="$2" sudo="sudo"; shift 2 ;; -c) shift 2 ;; *) shift ;; esac done killing="" trap '[ -n "$killing" ] || {{ echo stopped > "$FAKE_STATE/mapper.stopped"; exit 0; }}' TERM -until [ -n "${{FAKE_PRESS:-}}" ] && [ -e "$FAKE_STATE/ui.ready" ]; do sleep 0.1; done +until [ -n "${{FAKE_PRESS:-}}" ] && [ -e "$FAKE_STATE/${{FAKE_PRESS_AT:-ui.ready}}" ]; do sleep 0.1; done killing=1 echo "kill $name" >> "$FAKE_STATE/mapper.log" -{PKILL} -f "$name" +$sudo {PKILL} -f "$name" sleep 3 -if {PGREP} -f "$name" > /dev/null; then {PKILL} -9 -f "$name"; fi +if {PGREP} -f "$name" > /dev/null; then $sudo {PKILL} -9 -f "$name"; fi exit 0 ''' # gptokeyb2 as PortsMaster/gptokeyb2 src/util.c kills: SIGKILL only, by # process name (pkill -9 NAME) unless -X, then pgrep -f and kill -9 on every # match but its own ancestry and children. It quits after a kill, and keeps -# running when it has no process name. +# running when it has no process name. FAKE_MAPPER_STAYS is gptokeyb2 +# before 2026-10-05, which keeps running after a -X kill (an inner +# `int status` shadows the result, 23581b62ec src/util.c:486-519). FAKE_GPTOKEYB2 = f'''#!/bin/bash printf '%s\\n' "$@" > "$FAKE_STATE/mapper.args" echo $$ > "$FAKE_STATE/mapper.pid" +if [ -n "${{FAKE_MAPPER_DIES:-}}" ]; then + echo "gptokeyb2: cannot open /dev/uinput" >&2 + exit 1 +fi want_kill="" name="" while [ $# -gt 0 ]; do case "$1" in @@ -421,7 +553,7 @@ while [ $# -gt 0 ]; do esac done trap 'echo stopped > "$FAKE_STATE/mapper.stopped"; exit 0' TERM -until [ -n "${{FAKE_PRESS:-}}" ] && [ -e "$FAKE_STATE/ui.ready" ]; do sleep 0.1; done +until [ -n "${{FAKE_PRESS:-}}" ] && [ -e "$FAKE_STATE/${{FAKE_PRESS_AT:-ui.ready}}" ]; do sleep 0.1; done echo "kill $name" >> "$FAKE_STATE/mapper.log" if [ -z "$name" ]; then while :; do sleep 0.1; done @@ -442,9 +574,20 @@ if [ -n "$want_kill" ]; then else {PKILL} -9 "$name" fi +if [ -n "${{FAKE_MAPPER_STAYS:-}}" ]; then + while :; do sleep 0.1; done +fi exit 0 ''' +# ROCKNIX's /usr/bin/sudo: a stub that runs nothing (ROCKNIX distribution +# packages/sysutils/busybox/scripts/sudo). +ROCKNIX_SUDO = """#!/bin/sh +printf '%s\\n' "sudo $*" >> "$FAKE_STATE/tools.log" +echo " You are 'root' so 'sudo' is not required or supported" +exit 1 +""" + FAKE_RECORDER = """#!/bin/sh printf '%s\\n' "$(basename "$0") $*" >> "$FAKE_STATE/tools.log" """ @@ -472,6 +615,8 @@ done FAKE_TERMAPP = """#!/bin/sh printf '%s\\n' "$(basename "$0") $*" >> "$FAKE_STATE/tools.log" printf '%s\\n' "${SDL_GAMECONTROLLER_IGNORE_DEVICES_EXCEPT:-}" > "$FAKE_STATE/termapp.sdl" +printf '%s\\n' "${SDL_GAMECONTROLLER_IGNORE_DEVICES:-}" > "$FAKE_STATE/termapp.sdl_ignore" +echo $$ > "$FAKE_STATE/termapp.pid" if [ -n "$FAKE_TERM_FAILS" ]; then echo "$(basename "$0"): Can't open display" >&2 exit 1 @@ -540,7 +685,8 @@ class Device: (self.root / "fb").write_text("640,480\n") environ = {k: v for k, v in os.environ.items() if k not in ("DISPLAY", "WAYLAND_DISPLAY", "RETROBIOS_BASE_URL", - "XAUTHORITY", "XDG_RUNTIME_DIR")} + "RETROBIOS_INSTALL_OFFLINE", "RETROBIOS_IDLE_EXIT", + "RETROBIOS_PAD", "XAUTHORITY", "XDG_RUNTIME_DIR")} self.env = { **environ, "PATH": f"{self.bin}{os.pathsep}{os.environ['PATH']}", @@ -621,7 +767,7 @@ class Device: def _reap(self) -> None: """A fake the launcher failed to stop must not outlive the test.""" - for name in ("mapper.pid",): + for name in ("mapper.pid", "ui.pid", "engine.pid"): path = self.state / name if path.exists(): try: @@ -671,6 +817,29 @@ class Device: self.test.assertEqual(termios.tcgetattr(self.slave), self.before) +def sentinels(token: str) -> list[int]: + """The Start + Select sentinels of a run: `bash -c SCRIPT TOKEN`, where + the mapper carries the token after its own path.""" + found = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + argv = (entry / "cmdline").read_bytes().split(b"\0") + except OSError: + continue + if len(argv) > 3 and argv[1] == b"-c" and argv[3] == token.encode(): + found.append(int(entry.name)) + return found + + +def wait_until(test: unittest.TestCase, check, timeout: float, what: str) -> None: + end = time.monotonic() + timeout + while not check(): + test.assertLess(time.monotonic(), end, what) + time.sleep(0.1) + + CONTROL_TXT = ''' directory="{roms}" ESUDO="" @@ -727,6 +896,7 @@ class ThePortMasterPort(unittest.TestCase): installer = str(self.gamedir / "install.py") self.assertEqual(call["argv0"], installer) self.assertEqual(call["pad"], "gptokeyb") + self.assertEqual(call["idle_exit"], "300") # Kill mode on the Start + Select sentinel, never on the installer's # path, which the engine's command line carries too. args = device.mapper_args() @@ -734,6 +904,7 @@ class ThePortMasterPort(unittest.TestCase): self.assertRegex(args[1], r"^retrobios-stop-\d+x$") self.assertEqual(args[2:], ["-c", str(self.gamedir / "retrobios.gptk")]) device.assert_mapper_stopped() + self.assertEqual(sentinels(args[1]), []) device.assert_console_restored() # Reset when the console is taken and again when it is handed back. self.assertEqual(device.screen.count("\x1bc"), 2) @@ -790,9 +961,8 @@ class ThePortMasterPort(unittest.TestCase): device.assert_console_restored() def test_offline_says_so_at_once_and_bounds_the_detection(self): - """The file-list download is three tries of 30 s: the detection run - is pointed at a closed loopback port and stopped after - RB_DETECT_LIMIT, with a line on the console from the start.""" + """A detection run that hangs is stopped after RB_DETECT_LIMIT, with + a line on the console from the start.""" device = self.device(FAKE_CURL="28") pack = self.gamedir / "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip" pack.write_bytes(b"") @@ -802,12 +972,35 @@ class ThePortMasterPort(unittest.TestCase): screen = device.screen self.assertIn("Checking the connection", screen) self.assertIn("No connection. Looking for the BIOS folder", screen) - [check] = [c for c in device.calls() if "--check" in c["args"]] - self.assertEqual(check["base_url"], "http://127.0.0.1:9") [call] = device.tui_calls() # The detection was stopped: no platform, the first pack. self.assertEqual(call["args"], ["--tui", "--from-pack", str(pack)]) - self.assertEqual(call["base_url"], "") + log = (self.gamedir / "log.txt").read_text() + self.assertIn("the detection run took over 2 s and was stopped", log) + + def test_the_detection_is_told_to_make_no_request(self): + """RETROBIOS_INSTALL_OFFLINE: install.py detects, then stops where + the file list would download, at once and with reason offline + (install.py _network_allowed). The interface run does not inherit + it, and no address is borrowed for the detection.""" + device = self.device(FAKE_CURL="7") + pack = self.gamedir / "RetroArch_Lakka_v1.22.2_BIOS_Pack.zip" + pack.write_bytes(b"") + proc = device.run(["bash", self.script]) + self.assertEqual(proc.returncode, 0, proc.stderr.decode()) + self.assertLess(device.elapsed, 15) + [check] = [c for c in device.calls() if "--check" in c["args"]] + self.assertEqual(check["args"], ["--check", "--events"]) + self.assertEqual((check["offline"], check["base_url"]), ("1", "")) + detected = (self.gamedir / "run" / "detect.jsonl").read_text().splitlines() + self.assertEqual([json.loads(line)["event"] for line in detected], + ["start", "detect", "error", "end"]) + self.assertEqual(json.loads(detected[2])["reason"], "offline") + [call] = device.tui_calls() + self.assertEqual(call["args"], ["--tui", "--from-pack", str(pack), + "--platform", "retroarch"]) + self.assertEqual((call["offline"], call["base_url"]), ("", "")) + self.assertNotIn("took over", (self.gamedir / "log.txt").read_text()) def test_a_certificate_failure_is_not_offline(self): # curl 60: the network answered, install.py reports the certificate. @@ -955,9 +1148,11 @@ class ThePortMasterPort(unittest.TestCase): [call] = device.tui_calls() self.assertEqual(call["args"], ["--tui"]) self.assertEqual(device.tools(), [f"muxterm --gl -- bash {self.gamedir}/tui.txt here"]) - self.assertEqual(device.lines("termapp.sdl"), ["0x0001/0x0001"]) - self.assertEqual(device.mapper_args()[2:], ["-c", str(self.gamedir / "retrobios.gptk")]) + [hint] = device.lines("termapp.sdl") + assert_muxterm_hides_every_pad(self, hint) + assert_muxterm_mapping(self, device.mapper_args(), self.gamedir / "retrobios.gptk") self.assertEqual(call["pad"], "gptokeyb") + self.assertEqual(call["idle_exit"], "300") self.assertEqual(self.dialog(device), ["FINISH"]) def test_on_muos_without_muxterm_the_log_says_why_the_screens_open(self): @@ -1027,13 +1222,73 @@ class ThePortMasterPort(unittest.TestCase): """Nothing moves the interface once gptokeyb is gone: it is asked to stop instead of waiting for a key forever.""" device = self.device() - proc = device.spawn(["bash", self.script], FAKE_HOLD="1") + proc = device.spawn(["bash", self.script], FAKE_HOLD="1", RB_DEAD_WAIT="1", + RB_MESSAGE_WAIT="300") device.wait_for("ui.ready") os.kill(int((device.state / "mapper.pid").read_text()), signal.SIGKILL) self.assertEqual(proc.wait(timeout=20), 143) self.assertEqual(device.lines("ui.log"), ["signal 15"]) + # The interface disappears: the console says why, for RB_DEAD_WAIT. + screen = " ".join(device._drain().split()) + self.assertIn("gptokeyb, which turns the buttons into keys, stopped during the run", + screen) device.assert_console_restored() + def test_start_select_with_an_interface_that_ignores_sigterm(self): + """An interface that does not stop within RB_STOP_GRACE is killed, + and the engine it started in its own session with it.""" + device = self.device() + proc = device.run(["bash", self.script], FAKE_HOLD="1", FAKE_ENGINE="1", + FAKE_IGNORE_TERM="1", FAKE_PRESS="1", RB_STOP_GRACE="1", + RB_MESSAGE_WAIT="300") + self.assertEqual(proc.returncode, 137, proc.stderr.decode()) + self.assertLess(device.elapsed, 15) + engine = int((device.state / "engine.pid").read_text()) + wait_until(self, lambda: not Path(f"/proc/{engine}").exists(), 5, + "the engine outlived the interface") + log = (self.gamedir / "log.txt").read_text() + self.assertIn("the interface did not stop within 1 s", log) + device.assert_console_restored() + + def test_an_interface_that_ignores_sigterm_without_pkill(self): + """With no working pkill, the engines cannot be found: the interface + itself is still killed after RB_STOP_GRACE.""" + device = self.device() + device.tool("pkill", "#!/bin/sh\necho 'pkill: applet not found' >&2\nexit 127\n") + proc = device.run(["bash", self.script], FAKE_HOLD="1", FAKE_IGNORE_TERM="1", + FAKE_PRESS="1", RB_STOP_GRACE="1", RB_MESSAGE_WAIT="300") + self.assertEqual(proc.returncode, 137, proc.stderr.decode()) + self.assertLess(device.elapsed, 15) + ui = int((device.state / "ui.pid").read_text()) + self.assertFalse(Path(f"/proc/{ui}").exists()) + + def test_the_sentinel_ends_when_the_launcher_is_killed(self): + """SIGKILL leaves the launcher no trap: the sentinel ends by itself + once the launcher is gone.""" + device = self.device() + proc = device.spawn(["bash", self.script], FAKE_HOLD="1") + device.wait_for("ui.ready") + token = device.mapper_args()[1] + self.assertEqual(len(sentinels(token)), 1) + proc.kill() + proc.wait(timeout=5) + wait_until(self, lambda: not sentinels(token), 10, "the sentinel outlived the launcher") + + def test_a_closed_terminal_ends_the_run(self): + """The terminal app closed (muxterm's Menu, foot's window): the run + inside it stops the interface and ends the run, and no other route + or screen opens after it.""" + device = self.device(WAYLAND_DISPLAY="wayland-1") + device.tool("foot", FAKE_TERMAPP) + proc = device.spawn(["bash", self.script], FAKE_HOLD="1") + device.wait_for("ui.ready") + # FAKE_TERMAPP execs its command: its pid is the run inside it. + os.kill(int((device.state / "termapp.pid").read_text()), signal.SIGHUP) + self.assertEqual(proc.wait(timeout=20), 143) + self.assertEqual(device.lines("ui.log"), ["signal 15"]) + self.assertEqual(self.dialog(device), ["FINISH"]) + device.assert_mapper_stopped() + def test_sigterm_to_the_launcher_stops_the_interface_and_restores(self): """A firmware that signals the launcher alone: the trap stops the interface, the mapper and puts the console back.""" @@ -1086,10 +1341,12 @@ class ThePortMasterPort(unittest.TestCase): runtime = device.root / "runtime" runtime.mkdir(mode=0o555) self.addCleanup(runtime.chmod, 0o755) - proc = device.run(["bash", self.script]) + proc = device.run(["bash", self.script], RB_DEAD_WAIT="2") self.assertEqual(proc.returncode, 1) self.assertEqual(device.calls(), []) self.assertIn("the SD card may be read-only or full", " ".join(device.screen.split())) + # Left on the console long enough to read before the menu redraws. + self.assertGreaterEqual(device.elapsed, 2) def test_the_previous_log_is_kept(self): device = self.device() @@ -1125,6 +1382,7 @@ class TheBatoceraPort(unittest.TestCase): # evmapy reads RetroBIOS.sh.keys: no gptokeyb is started. self.assertEqual(device.mapper_args(), []) self.assertEqual(call["pad"], "evmapy") + self.assertEqual(call["idle_exit"], "300") def test_the_x_cookie_goes_with_the_display(self): """S31emulationstation starts X with XAUTHORITY=/var/lib/.Xauthority: @@ -1218,13 +1476,15 @@ class TheMuosApp(unittest.TestCase): f"muxterm --gl -- bash {app}/tui.txt here"]) # muxterm leaves the pad to gptokeyb2, which sends each button once # with the interface's mapping: Y is Tab (Options), not muxterm's Space. - self.assertEqual(device.lines("termapp.sdl"), ["0x0001/0x0001"]) + [hint] = device.lines("termapp.sdl") + assert_muxterm_hides_every_pad(self, hint) args = device.mapper_args() self.assertEqual(args[0], "-X") self.assertRegex(args[1], r"^retrobios-stop-\d+x$") - self.assertEqual(args[2:], ["-c", str(app / "retrobios.gptk")]) - self.assertEqual(read_gptk(app / "retrobios.gptk")["y"], ["tab"]) + assert_muxterm_mapping(self, args, app / "retrobios.gptk") + self.assertEqual(read_gptk(Path(args[-1]))["y"], ["tab"]) self.assertEqual(call["pad"], "gptokeyb") + self.assertEqual(call["idle_exit"], "300") device.assert_mapper_stopped() def test_muxterm_alone_without_gptokeyb2(self): @@ -1233,6 +1493,7 @@ class TheMuosApp(unittest.TestCase): self.assertEqual(proc.returncode, 0, proc.stderr.decode()) [call] = device.tui_calls() self.assertEqual(call["pad"], "muxterm") + self.assertEqual(call["idle_exit"], "300") # muxterm then reads the pad itself. self.assertEqual(device.lines("termapp.sdl"), [""]) self.assertEqual(device.mapper_args(), []) @@ -1257,6 +1518,54 @@ class TheMuosApp(unittest.TestCase): self.assertEqual(device.lines("ui.log"), ["signal 15"]) device.assert_console_restored() + def test_start_select_with_a_gptokeyb2_that_keeps_running(self): + """gptokeyb2 before 2026-10-05 keeps running after its -X kill: only + the sentinel's end says Start + Select was pressed.""" + device, app = self.device(muxterm=True) + proc = device.run([str(app / "mux_launch.sh"), str(app)], FAKE_HOLD="1", + FAKE_PRESS="1", FAKE_MAPPER_STAYS="1", RB_MESSAGE_WAIT="300") + self.assertEqual(proc.returncode, 143, proc.stderr.decode()) + self.assertLess(device.elapsed, 10) + self.assertEqual(device.lines("ui.log"), ["signal 15"]) + self.assertIn("Start + Select", (app / "log.txt").read_text()) + device.assert_mapper_stopped() + + def test_a_gptokeyb2_that_cannot_start_is_named(self): + device, app = self.device(muxterm=False) + proc = device.run([str(app / "mux_launch.sh"), str(app)], FAKE_MAPPER_DIES="1", + RB_CONSOLE=device.console, RB_DEAD_WAIT="1") + self.assertEqual(proc.returncode, 1) + self.assertEqual(device.tui_calls(), []) + screen = " ".join(device.screen.split()) + self.assertIn("gptokeyb2, which turns the buttons into keys, stopped as soon as it " + "started", screen) + + def test_an_arrow_at_the_question_leaves_nothing_behind(self): + """An arrow is three bytes: the first answers the question, the rest + must not reach the interface as two more keys.""" + device, app = self.device(muxterm=False) + _sd1, sd2, _pack = self.two_cards(device) + proc = device.run([str(app / "mux_launch.sh"), str(app)], keys="\x1b[B", + FAKE_CURL="7", FAKE_DRAIN="1", RB_CONSOLE=device.console, + RB_BIOS_DEST=str(sd2)) + self.assertEqual(proc.returncode, 0, proc.stderr.decode()) + [call] = device.tui_calls() + self.assertNotIn("--from", call["args"]) + self.assertEqual(device.lines("ui.input"), [""]) + + def test_start_select_at_the_question_starts_no_interface(self): + device, app = self.device(muxterm=False) + _sd1, sd2, _pack = self.two_cards(device) + proc = device.run([str(app / "mux_launch.sh"), str(app)], FAKE_CURL="7", + FAKE_PRESS="1", FAKE_PRESS_AT="calls.jsonl", RB_MESSAGE_WAIT="300", + RB_CONSOLE=device.console, RB_BIOS_DEST=str(sd2)) + self.assertEqual(proc.returncode, 143, proc.stderr.decode()) + self.assertLess(device.elapsed, 10) + self.assertEqual(device.tui_calls(), []) + # Never started: an interface started then stopped at once would + # end the same way, before Python records anything. + self.assertFalse((app / "run" / "tui.err").exists()) + def test_on_the_console_without_muxterm(self): device, app = self.device(muxterm=False) proc = device.run([str(app / "mux_launch.sh"), str(app)], RB_CONSOLE=device.console) @@ -1376,6 +1685,7 @@ class TheToolsEntry(unittest.TestCase): [call] = device.tui_calls() self.assertEqual((call["stdin"], call["term"]), (device.console, "linux")) self.assertEqual(call["pad"], "gptokeyb") + self.assertEqual(call["idle_exit"], "300") args = device.mapper_args() self.assertEqual(args[0], "-1") self.assertRegex(args[1], r"^retrobios-stop-\d+x$") @@ -1454,26 +1764,11 @@ class TheToolsEntry(unittest.TestCase): etc = device.root / "fs" / "etc" etc.mkdir(parents=True) (etc / "os-release").write_text('NAME="ROCKNIX"\n') + device.tool("sudo", ROCKNIX_SUDO) - def test_rocknix_runs_in_the_terminal_tools_opens(self): - device, tools = self.device() - self.rocknix(device) - proc = device.run(["bash", str(tools / "RetroBIOS.sh")], stdio_tty=True, TERM="xterm") - self.assertEqual(proc.returncode, 0, proc.stderr.decode()) - [call] = device.tui_calls() - self.assertEqual((call["stdin"], call["stdout"]), (device.console, device.console)) - self.assertEqual(call["term"], "xterm") - self.assertEqual(call["pad"], "gptokeyb") - # The terminal app owns its window: no console reset, no font. - self.assertNotIn("\x1bc", device.screen) - self.assertEqual([t for t in device.tools() if t.startswith("setfont")], []) - - def test_rocknix_loads_the_controller_list_its_own_tools_load(self): - """control-gen_init.sh writes control.ini, which names the pad's - controller list and the GPTOKEYB to start (ROCKNIX fex-emu's Start - FEXConfig.sh). Its ESUDO is not this launcher's.""" - device, tools = self.device() - self.rocknix(device) + def control_ini(self, device: Device) -> None: + """control-gen_init.sh writes control.ini as control-gen.cpp:31 + does: `ESUDOKILL="-sudokill"`, and GPTOKEYB with it.""" folder = device.root / "fs" / "storage" / ".config" / "gptokeyb" shutil.copy(device.bin / "gptokeyb", device.root / "gptokeyb-rocknix") device.tool("control-gen_init.sh", f'''#!/bin/sh @@ -1490,15 +1785,57 @@ param_device="fake pad" GPTOKEYB="$controlfolder/gptokeyb-rocknix $ESUDOKILL" INI ''') + + def test_rocknix_runs_in_the_terminal_tools_opens(self): + device, tools = self.device() + self.rocknix(device) proc = device.run(["bash", str(tools / "RetroBIOS.sh")], stdio_tty=True, TERM="xterm") self.assertEqual(proc.returncode, 0, proc.stderr.decode()) - args = device.mapper_args() - self.assertEqual(args[0], "-sudokill") - self.assertRegex(args[1], r"^retrobios-stop-\d+x$") + [call] = device.tui_calls() + self.assertEqual((call["stdin"], call["stdout"]), (device.console, device.console)) + self.assertEqual(call["term"], "xterm") + self.assertEqual(call["pad"], "gptokeyb") + self.assertEqual(call["idle_exit"], "300") + # The terminal app owns its window: no console reset, no font. + self.assertNotIn("\x1bc", device.screen) + self.assertEqual([t for t in device.tools() if t.startswith("setfont")], []) + + def test_rocknix_loads_the_controller_list_its_own_tools_load(self): + """control-gen_init.sh writes control.ini, which names the pad's + controller list and the GPTOKEYB to start (ROCKNIX fex-emu's Start + FEXConfig.sh). Its ESUDO is not this launcher's.""" + device, tools = self.device() + self.rocknix(device) + self.control_ini(device) + proc = device.run(["bash", str(tools / "RetroBIOS.sh")], stdio_tty=True, TERM="xterm") + self.assertEqual(proc.returncode, 0, proc.stderr.decode()) + self.assertEqual(device.lines("mapper.args")[0], "-1") + self.assertRegex(device.mapper_args()[1], r"^retrobios-stop-\d+x$") self.assertEqual(device.lines("mapper.db"), [f"{device.root}/gamecontrollerdb.txt"]) + started = [t for t in device.tools() if t.startswith("sudo")] + self.assertEqual(started, []) # Stopped without the sudo control.ini names. device.assert_mapper_stopped() + def test_rocknix_start_select_with_its_controller_setup(self): + """control.ini's `-sudokill` runs the kill through sudo, which on + ROCKNIX runs nothing: the mapper is started with -1, and Start + + Select ends the run at once instead of 3 s later through the + liveness watch.""" + device, tools = self.device() + self.rocknix(device) + self.control_ini(device) + proc = device.run(["bash", str(tools / "RetroBIOS.sh")], stdio_tty=True, TERM="xterm", + FAKE_HOLD="1", FAKE_PRESS="1", RB_MESSAGE_WAIT="300") + self.assertEqual(proc.returncode, 143, proc.stderr.decode()) + self.assertLess(device.elapsed, 10) + self.assertEqual(device.lines("ui.log"), ["signal 15"]) + log = (tools / "retrobios" / "log.txt").read_text() + self.assertIn("Start + Select: the interface is asked to stop", log) + self.assertNotIn("the controller mapper stopped", log) + self.assertNotIn("You are 'root'", log) + device.assert_mapper_gone() + def test_rocknix_offline_names_no_pack_the_releases_lack(self): """No release pack spells ROCKNIX, and install.py refuses one named for another system: the RetroArch pack on the card is not offered, @@ -1527,6 +1864,28 @@ INI device.assert_mapper_gone() +class TheLiveness(unittest.TestCase): + def test_a_zombie_is_not_running(self): + """A process that exited and that its parent has not reaped still + has /proc/PID: rb_alive reads its state.""" + pid = os.fork() + if pid == 0: + os._exit(0) + self.addCleanup(os.waitpid, pid, 0) + wait_until(self, lambda: Path(f"/proc/{pid}/stat").read_text().split(") ")[1][0] == "Z", + 5, "no zombie") + proc = subprocess.run( + ["bash", "-c", 'source "$1"; rb_alive "$2" && echo alive || echo gone', "bash", + str(PORTS / "shared" / "tui.txt"), str(pid)], + capture_output=True, text=True, check=False) + self.assertEqual(proc.stdout.strip(), "gone", proc.stderr) + proc = subprocess.run( + ["bash", "-c", 'source "$1"; rb_alive "$$" && echo alive || echo gone', "bash", + str(PORTS / "shared" / "tui.txt")], + capture_output=True, text=True, check=False) + self.assertEqual(proc.stdout.strip(), "alive", proc.stderr) + + class TheDocs(unittest.TestCase): def test_the_offline_packs_named_are_the_ones_that_serve(self): """The wiki names the pack each firmware takes offline: the release @@ -1552,6 +1911,27 @@ class TheDocs(unittest.TestCase): self.assertIn(f"| `{value}` |", text) self.assertIn(value, tui) + def handheld(self) -> str: + text = (REPO_ROOT / "wiki" / "installer.md").read_text(encoding="utf-8") + start = text.index("## On a handheld") + end = text.find("\n## ", start + 1) + return text[start:end if end != -1 else len(text)] + + def test_the_idle_limit_is_documented(self): + """Every launcher sets RETROBIOS_IDLE_EXIT for the interface: a + mapper that runs but never opens the pad leaves no button to quit.""" + self.assertIn("`RETROBIOS_IDLE_EXIT`", self.handheld()) + self.assertIn("RETROBIOS_IDLE_EXIT", (PORTS / "shared" / "tui.txt").read_text()) + + def test_no_page_buttons_are_promised_in_muxterm(self): + """muxterm keeps PgUp and PgDn for its scrollback (T/input.c:352-360).""" + readme = " ".join((PORTS / "muos" / "README.md").read_text().split()) + self.assertNotIn("L1 and R1 to page", readme) + self.assertIn("L1 and R1 do nothing in muxterm", readme) + rows = [line for line in self.handheld().splitlines() if line.startswith("| L1, R1 |")] + self.assertEqual(len(rows), 1) + self.assertIn("not in muxterm", rows[0]) + def test_muos_paths_are_the_card_muos_mounts(self): """muOS mounts the first card at /mnt/mmc (MustardOS internal mux_launch.sh, GET_VAR storage/rom/mount).""" diff --git a/wiki/installer.md b/wiki/installer.md index 826692fcc..57b499a4f 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1060,7 +1060,7 @@ muxterm, as the muOS app does. | B, Select | Back, and quit on the first screen | | X | Tick or untick a system | | Y | Options | -| L1, R1 | Page up, page down | +| L1, R1 | Page up, page down (not in muxterm, which keeps them for its own scrollback) | | Hold Select, then Start | Quit at once (PortMaster port, muOS, Tools) | Start pressed first also chooses the line in focus: hold Select first. A @@ -1068,6 +1068,11 @@ running install is then stopped the way B stops it, and the last screen says what it wrote. On muOS without PortMaster, muxterm reads the pad itself: Y ticks a system, Select shows its keyboard, and Menu quits. +Each launcher also sets `RETROBIOS_IDLE_EXIT` to 300: with no button pressed +on its first screen for 5 minutes, the interface closes by itself. A pad the +system has no mapping for sends nothing, and would otherwise leave the +console stuck on the interface with no way out. + Each launcher tells the interface which buttons reach it, in `RETROBIOS_PAD`, so the hints can name them: From ade4e2b48b541b9ec3bd86916dd93d6af2ceaf28 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:34:31 +0200 Subject: [PATCH 19/19] docs: say a mounted card is found by itself --- ports/shared/tui.txt | 2 +- wiki/installer.md | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ports/shared/tui.txt b/ports/shared/tui.txt index 0b087674e..112be1f70 100644 --- a/ports/shared/tui.txt +++ b/ports/shared/tui.txt @@ -62,7 +62,7 @@ RB_MAPPER_NEEDED="${RB_MAPPER_NEEDED:-1}" RB_PAD="${RB_PAD:-}" RB_RUNTIME="python_3.11" RB_RELEASES="github.com/Abdess/retrobios/releases/latest" -RB_PC_ROUTE="On a computer, with this SD card mounted, the same installer fills the card: install.py --platform NAME --dest BIOS-FOLDER-ON-THE-CARD. Instructions: abdess.github.io/retrobios/wiki/installer" +RB_PC_ROUTE="On a computer, with this SD card mounted, the same installer fills the card: install.py finds a muOS, Knulli or ROCKNIX card by itself, and any other with install.py --platform NAME --dest BIOS-FOLDER-ON-THE-CARD. Instructions: abdess.github.io/retrobios/wiki/installer" RB_PYTHON="${RB_PYTHON:-}" RB_PYTHONHOME="${RB_PYTHONHOME:-}" RB_PYTHON_PROBLEM="${RB_PYTHON_PROBLEM:-}" diff --git a/wiki/installer.md b/wiki/installer.md index d2a14e4f6..ee12ed43b 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1131,9 +1131,10 @@ names each route it tried, and `log.prev.txt` keeps the run before. **From a computer.** Onion, MinUI, NextUI, Allium, GarlicOS and the Switch have no Python 3 to run the installer. Put the card in a computer and run -the [one-line command](#one-line) there with -`--platform --dest `. This works for -every system above too. +the [one-line command](#one-line) there: it finds a mounted Onion, +spruceOS, MinUI, muOS, Knulli or ROCKNIX card by itself. For a card it does +not recognise, add `--platform --dest `. +This works for every system above too. ## Where the files come from