From cf3b61a17baf60ae90ccf8add5b88fdcc06ba755 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:22:42 +0200 Subject: [PATCH 01/17] fix: exit on the window's signal when it closes A closed window under a changing screen failed the draw (EIO, or a termios.error from the SIGCONT that comes with SIGHUP) before the signal was read, and the run ended on 1 with a crash log kept. --- install.ps1 | 2 +- install.py | 41 ++++++++++++++++++++++++++++++++------- install.sh | 2 +- tests/test_install_tui.py | 34 ++++++++++++++++++++++++++++---- 4 files changed, 66 insertions(+), 13 deletions(-) diff --git a/install.ps1 b/install.ps1 index 70f6b4e5..9a1ae914 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "e2cc5f073a8226fe26b0513ebd25f00de6f3f613d8c9851473c2d30b53356f48" + $defaultInstallSha256 = "c2a66e6b26004e96ef5f59a7fb5a2ed65bcaa9afc7602bd3e0d71f50cc561056" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index b0eca90e..ee80f792 100644 --- a/install.py +++ b/install.py @@ -12500,13 +12500,14 @@ def run_tui(args: argparse.Namespace) -> int: if terminating["signal"] and state.screen == "result": # The result is printed in the terminal on the way out. state.exit_code = state.code if state.code is not None else 1 - except Exception: # noqa: BLE001 - # A traceback is where every reporter of #71, #73 and #74 stopped: - # it goes to a file, and the terminal says one sentence. - crash = work / "error.log" - crash.write_text(traceback.format_exc(), encoding="utf-8") - state.log, state.code, state.exit_code = str(crash), 1, 1 - state.stack = ["crashed"] + except Exception as exc: # noqa: BLE001 + if flags["term"] or _closed_terminal(exc): + # The window closed under a screen that was changing, or the + # SIGCONT that comes with its SIGHUP put the screen back: the + # write failed before the signal was read. That is its exit. + state.exit_code = 128 + (flags["term"] or getattr(signal, "SIGHUP", 1)) + else: + _tui_crashed(state, work) finally: # Each step on its own: a write the closed window refuses, or a # second signal while the install writes its summary, must not @@ -12560,6 +12561,32 @@ def _tui_startup(args: argparse.Namespace) -> tuple[TuiState, list[tuple]] | Non return tui_start(args, os_type) +# What a write to a closed window or a closed pipe answers. +_CLOSED_TERMINAL = frozenset({errno.EIO, errno.EPIPE, errno.ENXIO}) + + +def _closed_terminal(exc: BaseException) -> bool: + """Whether exc is the terminal gone: an OSError, or a termios.error, + whose first argument is the errno, answering EIO, EPIPE or ENXIO.""" + if isinstance(exc, OSError): + return exc.errno in _CLOSED_TERMINAL + if type(exc).__name__ == "error" and type(exc).__module__ == "termios": + return bool(exc.args) and exc.args[0] in _CLOSED_TERMINAL + return False + + +def _tui_crashed(state: TuiState, work: Path) -> None: + """A traceback is where every reporter of #71, #73 and #74 stopped: it + goes to a file, and the terminal says one sentence.""" + crash = work / "error.log" + try: + crash.write_text(traceback.format_exc(), encoding="utf-8") + except OSError as exc: + print(f"The error could not be written to {crash}: {exc}", file=sys.stderr) + state.log, state.code, state.exit_code = str(crash), 1, 1 + state.stack = ["crashed"] + + def _restore_handlers(handlers: dict) -> None: for number, previous in handlers.items(): signal.signal(number, previous) diff --git a/install.sh b/install.sh index 539267b4..6c8d516c 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="e2cc5f073a8226fe26b0513ebd25f00de6f3f613d8c9851473c2d30b53356f48" +DEFAULT_INSTALL_SHA256="c2a66e6b26004e96ef5f59a7fb5a2ed65bcaa9afc7602bd3e0d71f50cc561056" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 392ff1b9..8fe485e1 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -1444,13 +1444,14 @@ class CleanupSurvivesABrokenTerminal(unittest.TestCase): mock.patch.object(sys, "stderr", out): code = install.run_tui(_args()) self.assertIs(sys.stdout, out) - self.assertEqual(code, 1) + # A window that answers EIO is a closed window: its signal's code, + # not a crash. + self.assertEqual(code, 129) # The farewell found no terminal: what still flushes at exit goes to # the null device, and the exit code stays the one returned. self.assertEqual(os.fstat(screen_fd).st_rdev, os.stat(os.devnull).st_rdev) - # The crash keeps its log, marked kept, and nothing else of the run. - self.assertEqual(sorted(path.name for path in work.iterdir()), - ["error.log", "kept", "owner"]) + # No crash log is kept, and nothing else of the run. + self.assertFalse(work.exists()) self.assertEqual(termios.tcgetattr(slave), before) for number, handler in handlers.items(): self.assertEqual(signal.getsignal(number), handler) @@ -2859,6 +2860,31 @@ class EndToEnd(unittest.TestCase): self.assertTrue(os.WIFEXITED(status)) self.assertEqual(term.status, 129) + def test_closing_the_window_while_it_installs(self): + """The changing screen fails to draw on the closed window: that is + the window's signal, not a crash with a log kept.""" + repo = self.repo(GameDataRepository, slow=True) + scratch = repo.root / "scratch" + scratch.mkdir() + self.env["TMPDIR"] = str(scratch) + term = self.start("--platform", "retroarch", "--dest", str(repo.dest), "--jobs", "1") + term.wait("> Install") + term.send("\r") + term.wait("1 of 6 files") + os.close(term.fd) + end = time.monotonic() + 20 + while time.monotonic() < end: + pid, status = os.waitpid(term.pid, os.WNOHANG) + if pid: + break + time.sleep(0.05) + else: + os.kill(term.pid, 9) + self.fail("still running once its window closed") + term.status = os.WEXITSTATUS(status) + self.assertEqual(term.status, 129) + self.assertEqual(list(scratch.glob("retrobios-tui-*/error.log")), []) + def test_a_desktop_names_its_keys_and_a_pad_its_buttons(self): self.repo() for env, bar in (({"DISPLAY": ":0"}, "Enter Continue Esc Quit ? Help"), From 5869da409bc56ba8cd37b3cf2750de06308cc504 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:24:40 +0200 Subject: [PATCH 02/17] fix: open the interface whatever owner files hold An owner file holding a number past a pid raised OverflowError before the screen opened, and a full temp disk stopped the run on the owner write. 0 and negative numbers are unknown owners, not process groups. --- install.ps1 | 2 +- install.py | 30 ++++++++++++++++++++++++------ install.sh | 2 +- tests/test_install_tui.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 8 deletions(-) diff --git a/install.ps1 b/install.ps1 index 9a1ae914..f6295a5b 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "c2a66e6b26004e96ef5f59a7fb5a2ed65bcaa9afc7602bd3e0d71f50cc561056" + $defaultInstallSha256 = "b23c6da4ee131f898c25540bd53677ea4112964fd00b209e4fe85dfdba8427d1" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index ee80f792..0bcd62db 100644 --- a/install.py +++ b/install.py @@ -12279,8 +12279,9 @@ def _owner_alive(folder: Path) -> bool | None: pid = int((folder / TUI_OWNER).read_text(encoding="utf-8").strip()) except (OSError, ValueError): return None - if os.name == "nt": - # os.kill(pid, 0) ends a process on Windows instead of asking. + if os.name == "nt" or pid <= 0: + # os.kill(pid, 0) ends a process on Windows instead of asking, and + # 0 or a negative number names a process group, not the run. return None try: os.kill(pid, 0) @@ -12288,7 +12289,8 @@ def _owner_alive(folder: Path) -> bool | None: return False except PermissionError: return True - except OSError: + except (OSError, OverflowError): + # A number past what a pid holds is no process this run can ask. return None return True @@ -12375,9 +12377,20 @@ def run_tui(args: argparse.Namespace) -> int: state, effects = started if flags["term"]: state.exit_code = 128 + flags["term"] - work = Path(tempfile.mkdtemp(prefix=TUI_WORK_PREFIX)) - (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") - _prune_work_folders(work.parent, time.time()) + try: + work = Path(tempfile.mkdtemp(prefix=TUI_WORK_PREFIX)) + except OSError as exc: + _restore_handlers(handlers) + print(f"Error: no work folder could be made in {tempfile.gettempdir()}: {exc}. " + "Free some space there, or point TMPDIR (TEMP on Windows) at another folder.", file=sys.stderr) + return 1 + try: + (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") + except OSError as exc: + # A full temp disk: the folder has no owner, and a later run leaves + # it a week like any unknown one. The logs fail as they come. + print(f"The work folder {work} could not be marked as this run's: {exc}", + file=sys.stderr) script = str(Path(__file__).resolve()) inbox: queue.Queue = queue.Queue() current: list[_EngineRun] = [] @@ -12445,6 +12458,11 @@ def run_tui(args: argparse.Namespace) -> int: if number is not None: handlers[number] = signal.signal(number, handler) sys.stdout = sys.stderr = buffer + try: + _prune_work_folders(work.parent, time.time()) + except OSError as exc: + # What earlier runs left stays until a later run: this one opens. + print(f"Earlier work folders could not be removed: {exc}", file=sys.stderr) if state.exit_code is None: # Stopped while it started: the screen never opens. terminal.enter() diff --git a/install.sh b/install.sh index 6c8d516c..e2217f75 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="c2a66e6b26004e96ef5f59a7fb5a2ed65bcaa9afc7602bd3e0d71f50cc561056" +DEFAULT_INSTALL_SHA256="b23c6da4ee131f898c25540bd53677ea4112964fd00b209e4fe85dfdba8427d1" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 8fe485e1..ef7c2f67 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -10,6 +10,7 @@ installs it). from __future__ import annotations import argparse +import errno import fcntl import functools import hashlib @@ -1801,6 +1802,31 @@ class WorkFolders(unittest.TestCase): work, out = self.run_to(130) self.assertFalse(work.exists()) + def test_a_full_temp_disk_or_a_failing_pruner_still_opens(self): + real_write = Path.write_text + + def full(path, *args, **kwargs): + if path.name == install.TUI_OWNER: + raise OSError(errno.ENOSPC, "No space left on device") + return real_write(path, *args, **kwargs) + + with mock.patch.object(Path, "write_text", full): + work, out = self.run_to(0) + self.assertFalse(work.exists()) + with mock.patch.object(install, "_prune_work_folders", + side_effect=OSError(errno.EACCES, "denied")): + work, out = self.run_to(0) + self.assertFalse(work.exists()) + err = io.StringIO() + with mock.patch.object(install.tempfile, "mkdtemp", + side_effect=OSError(errno.ENOSPC, "No space left")), \ + mock.patch.object(install, "tui_start", return_value=(fresh_state(), [])), \ + mock.patch.object(install, "_open_local_sources"), \ + mock.patch.object(install, "detect_os", return_value="linux"), \ + mock.patch.object(sys, "stderr", err): + self.assertEqual(install.run_tui(_args()), 1) + self.assertIn("no work folder could be made", err.getvalue()) + def test_a_stop_while_it_starts_ends_on_the_farewell(self): """Start+Select during the detection: the signal is the interface's, never the default action that ended Python without a word.""" @@ -2456,8 +2482,17 @@ class AuditRoundThree(unittest.TestCase): unknown_old = folder("unknown-old", None, False, 2 * week) other = root / "something-else" other.mkdir() + # A stray or hostile owner file is an unknown owner: a number past + # what a pid holds raised OverflowError, and 0 or -1 name a group. + hostile = [folder(f"hostile{index}", None, False, 60) + for index in range(3)] + for path, text in zip(hostile, ("9" * 40, "0", "-1")): + (path / install.TUI_OWNER).write_text(text) + self.assertIsNone(install._owner_alive(path)) removed = install._prune_work_folders(root, now) self.assertEqual(sorted(removed), sorted([killed, failed_old, unknown_old])) + for path in hostile: + shutil.rmtree(path) self.assertEqual(sorted(path.name for path in root.iterdir()), sorted(path.name for path in (running, failed, unknown, other))) From ea48a54329c92e8e4151bb93efbd9aca0a349137 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:36:53 +0200 Subject: [PATCH 03/17] fix: judge a declared slot by the platform's copy --declared-only settled each slot on its first declaration before the cores narrowed the list, and the declared event judged the core copy served in its place. Both now read the declaration whose systems remain, as the platform checks it. --- install.ps1 | 2 +- install.py | 95 ++++++++++++++++++++++++++++++------- install.sh | 2 +- tests/test_install_cores.py | 50 +++++++++++++++++++ 4 files changed, 129 insertions(+), 20 deletions(-) diff --git a/install.ps1 b/install.ps1 index f6295a5b..ab608c8f 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "b23c6da4ee131f898c25540bd53677ea4112964fd00b209e4fe85dfdba8427d1" + $defaultInstallSha256 = "5fedfbf308ba04a174ce0682380a28150b6fe5eab514a605e3b318193ce37c7a" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 0bcd62db..8f2464c7 100644 --- a/install.py +++ b/install.py @@ -665,17 +665,45 @@ def _emit_check( bytes_needed=sum(entry.get("size", 0) for entry in needed), data_missing=data_missing, not_installable=not_installable, ) - own = [entry for entry in files if entry.get("cores") is None] - own_needed = [entry for entry in own if id(entry) not in present] - own_wrong = sum(1 for entry in own if id(entry) in wrong) + # The copy the platform itself checks at each of its slots, which is + # what --declared-only checks: a core's copy served in its place is + # judged against the platform's. + own_states = [] + for entry in files: + if entry.get("cores") is not None: + continue + copy = declared_copy(entry) + if copy is entry or copy.get("sha1") == entry.get("sha1"): + state = ("present" if id(entry) in present + else "wrong_hash" if id(entry) in wrong else "missing") + else: + state = _declared_state(copy, entry, bios_path) + own_states.append((entry["dest"], copy.get("size", 0), state)) EVENTS.emit( - "declared", platform=plat, files=len(own), present=len(own) - len(own_needed), - missing=len(own_needed) - own_wrong, wrong_hash=own_wrong, - bytes_needed=sum(entry.get("size", 0) for entry in own_needed), - dests=[entry["dest"] for entry in own], + "declared", platform=plat, files=len(own_states), + present=sum(1 for _d, _s, state in own_states if state == "present"), + missing=sum(1 for _d, _s, state in own_states if state == "missing"), + wrong_hash=sum(1 for _d, _s, state in own_states if state == "wrong_hash"), + bytes_needed=sum(size for _d, size, state in own_states if state != "present"), + dests=[dest for dest, _s, _state in own_states], ) +def _declared_state(copy: dict, entry: dict, bios_path: Path) -> str: + """present, missing or wrong_hash: the file at entry's place against copy.""" + try: + path = _entry_path(bios_path, entry) + except ValueError: + return "missing" + if not path.is_file(): + return "missing" if not path.exists() else "wrong_hash" + try: + verdict = _classify_local(copy, path) + except OSError: + return "wrong_hash" + return "present" if verdict == "up_to_date" else "wrong_hash" + + def leave_out_game_data( files: list[dict], omitted: list[dict] ) -> tuple[list[dict], list[dict], list[dict]]: @@ -697,17 +725,42 @@ def keep_declared(files: list[dict], omitted: list[dict]) -> tuple[list[dict], l A slot a core contradicts gets the platform's copy, and no core copy stays behind as an alternative: narrowing by cores afterwards would - otherwise put a core's file back where a dropped system's was. + otherwise put a core's file back where a dropped system's was. Which of + the platform's declarations answers is left to that narrowing, as + without --declared-only: the first whose systems remain. """ def own(entries: list[dict]) -> list[dict]: - return [ - {k: v for k, v in _settle_slot(entry, set()).items() if k != "alternatives"} - for entry in entries if entry.get("cores") is None - ] + result = [] + for entry in entries: + if entry.get("cores") is not None: + continue + kept = {k: v for k, v in _settle_slot(entry, set()).items() + if k not in ("alternatives", "declared_copy")} + declared = entry.get("declared_choices") or [] + if len(declared) > 1: + kept["declared_choices"] = declared + result.append(kept) + return result return own(files), own(omitted) +def declared_copy(entry: dict) -> dict: + """The copy the platform's own list checks at a platform file's slot. + + A narrowing that serves a core's copy there records the platform's + answer beside it, and an entry not yet settled answers by its first + declaration. Otherwise the entry is the platform's copy itself. + """ + recorded = entry.get("declared_copy") + if recorded is not None: + return recorded + declared = entry.get("declared_choices") + if declared: + return declared[0] + return entry + + def _left_out_lines(left_out: list[dict], every: bool) -> list[str]: """What --no-game-data left out, named per category, ten each unless every.""" lines = [] @@ -3428,22 +3481,28 @@ def _settle_slot( declared = entry.get("declared_choices") if declared is None: return entry + platform_copy = next( + (c for c in declared + if kept_system is None or any(kept_system(s) for s in c["systems"])), + declared[0], + ) copy = next( (c for c in entry.get("slot_choices", []) if chosen.intersection(c["cores"])), None, ) if copy is None or "sha1" not in copy: - copy = next( - (c for c in declared - if kept_system is None or any(kept_system(s) for s in c["systems"])), - declared[0], - ) + copy = platform_copy kept = { k: v for k, v in entry.items() - if k not in _COPY_FIELDS and k not in ("slot_choices", "declared_choices") + if k not in _COPY_FIELDS + and k not in ("slot_choices", "declared_choices", "declared_copy") } kept.update((k, copy[k]) for k in _COPY_FIELDS if k in copy and k != "cores") kept["cores"] = None + if copy is not platform_copy and copy.get("sha1") != platform_copy.get("sha1"): + # What the platform checks here, for the `declared` event. + kept["declared_copy"] = {k: platform_copy[k] for k in _COPY_FIELDS + if k in platform_copy and k != "cores"} return kept diff --git a/install.sh b/install.sh index e2217f75..47b0c4da 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="b23c6da4ee131f898c25540bd53677ea4112964fd00b209e4fe85dfdba8427d1" +DEFAULT_INSTALL_SHA256="5fedfbf308ba04a174ce0682380a28150b6fe5eab514a605e3b318193ce37c7a" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_cores.py b/tests/test_install_cores.py index 3d104654..7a855642 100644 --- a/tests/test_install_cores.py +++ b/tests/test_install_cores.py @@ -288,6 +288,56 @@ class APlatformFileSettledByCoresAndSystems(unittest.TestCase): self.assertEqual(self.served(["dsi"]), "c") self.assertEqual(self.served(["nds"]), "b") + def test_declared_only_keeps_the_declaration_whose_system_remains(self): + """--declared-only settled the slot on the first declaration before + the cores narrowed it: on a target without melonds it checked the + nintendo-dsi copy, the platform the nintendo-ds one.""" + manifest = self.manifest() + for names, letter in ((["nds"], "b"), (["dsi"], "c"), (["melonds"], "c"), + (["desmume"], "b")): + with self.subTest(names=names): + files, omitted = install.keep_declared(manifest["files"], []) + files, _ = install.narrow_to_cores(manifest, names, files, omitted) + self.assertEqual(files[0]["sha1"][0], letter) + + def test_the_declared_event_judges_the_copy_the_platform_checks(self): + """melonds' copy is served at the slot, the platform checks its own: + the event and a --declared-only check give the same answer.""" + import hashlib + + tmp = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, tmp, True) + payloads = {letter: letter.encode() * 3 for letter in "abc"} + manifest = self.manifest() + entry = manifest["files"][0] + + def real(copy: dict, letter: str) -> None: + copy.update(sha1=hashlib.sha1(payloads[letter]).hexdigest(), + sha256=hashlib.sha256(payloads[letter]).hexdigest(), size=3) + + real(entry, "a") + real(entry["slot_choices"][1], "a") + real(entry["declared_choices"][0], "c") + real(entry["declared_choices"][1], "b") + (tmp / "dsi_bios9.bin").write_bytes(payloads["c"]) + + def declared_event(files: list[dict]) -> dict: + _todo, fine, wrong = install.check_local(files, tmp) + with mock.patch.object(install.EVENTS, "emit") as emit: + install._emit_check("retroarch", manifest, tmp, files, fine, wrong, 0, 0) + return next(call.kwargs for call in emit.call_args_list + if call.args[0] == "declared") + + for names in (None, ["melonds"], ["desmume", "melonds"]): + with self.subTest(names=names): + served = manifest["files"] + own, _ = install.keep_declared(manifest["files"], []) + if names is not None: + served, _ = install.narrow_to_cores(manifest, names) + own, _ = install.narrow_to_cores(manifest, names, own, []) + self.assertEqual(declared_event(served), declared_event(own)) + self.assertEqual(declared_event(own)["present"], 1) + def test_the_boundary_checks_every_copy(self): install._validate_manifest(self.manifest(), "retroarch") bad_choices = ( From 4b8e2f9de2e8a5734f71f8670c3125e9c70fa2a5 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:37:03 +0200 Subject: [PATCH 04/17] fix: check system_titles at the boundary A list or a number among the titles passed validation and raised later, in --system matching or the interface. --- install.ps1 | 2 +- install.py | 9 +++++++++ install.sh | 2 +- tests/test_install_selection.py | 13 +++++++++++++ 4 files changed, 24 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index ab608c8f..1ddca47e 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "5fedfbf308ba04a174ce0682380a28150b6fe5eab514a605e3b318193ce37c7a" + $defaultInstallSha256 = "6d8af0dbca6697ba71692e5e482c8a16d9e8b9896652fec722d9c84f23bba047" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 8f2464c7..cb583f7f 100644 --- a/install.py +++ b/install.py @@ -3069,6 +3069,15 @@ def _validate_manifest(data: object, plat: str) -> dict: _check_tags(mapping, key, field) if mapping[key] is None: raise ValueError(f"invalid {field} for {key}") + titles = data.get("system_titles") + if titles is not None: + if not isinstance(titles, dict) or len(titles) > MAX_MANIFEST_FILES: + raise ValueError("invalid system_titles") + for key, title in titles.items(): + if not isinstance(key, str) or not 0 < len(key) <= 256: + raise ValueError("invalid system_titles key") + if not isinstance(title, str) or not 0 < len(title) <= 256: + raise ValueError(f"invalid system_titles for {key}") copies = data.get("standalone_copies", []) if not isinstance(copies, list) or len(copies) > 10_000: diff --git a/install.sh b/install.sh index 47b0c4da..e85ff322 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="5fedfbf308ba04a174ce0682380a28150b6fe5eab514a605e3b318193ce37c7a" +DEFAULT_INSTALL_SHA256="6d8af0dbca6697ba71692e5e482c8a16d9e8b9896652fec722d9c84f23bba047" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 97b969b3..5cf8f481 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -493,6 +493,19 @@ class TheBoundaryChecksTheNewFields(unittest.TestCase): with self.subTest(field=field, value=bad), self.assertRaises(ValueError): install._validate_manifest(manifest, "retroarch") + def test_the_system_titles(self): + """A title is read as text by --system and the interface: a list or + a number there raised mid-run instead of at the boundary.""" + good = self._manifest() + good["system_titles"] = {"sony-playstation": "Sony PlayStation"} + install._validate_manifest(good, "retroarch") + for bad in (["x"], {"s": ["Sony"]}, {"s": None}, {"s": 5}, {"s": ""}, + {"": "Sony"}, {"s": "x" * 257}, {"s" * 257: "Sony"}): + manifest = self._manifest() + manifest["system_titles"] = bad + with self.subTest(value=bad), self.assertRaises(ValueError): + install._validate_manifest(manifest, "retroarch") + class _QuietHandler(http.server.SimpleHTTPRequestHandler): def log_message(self, format, *args): # noqa: A002 - stdlib signature From 3203c766fc09faa6a6d102d4d5d697ee532b7e35 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:37:14 +0200 Subject: [PATCH 05/17] fix: name a folder or unreadable file in place A folder or a file the user cannot read at a destination ended --check and the install on a traceback from the hashing pool. --check now gives the reason per file, and the install fails that file alone, before any download for a folder. --- install.ps1 | 2 +- install.py | 21 ++++++++++++++++++--- install.sh | 2 +- tests/test_install_selection.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 5 deletions(-) diff --git a/install.ps1 b/install.ps1 index 1ddca47e..c0e93dca 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "6d8af0dbca6697ba71692e5e482c8a16d9e8b9896652fec722d9c84f23bba047" + $defaultInstallSha256 = "c361b511d9550b2115d918a0233115cd9f42920bce881f41395c3d7614be534d" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index cb583f7f..4568f8e5 100644 --- a/install.py +++ b/install.py @@ -4046,9 +4046,18 @@ def _classify_local(entry: dict, dest: Path) -> str: for name in ("sha256", "sha1") if entry.get(name) } + if not dest.is_file(): + # A folder at the file's place: --check says so, an install fails + # on that file alone. + return "mismatched" if not expected: return "up_to_date" - actual = _digest_file(dest, tuple(expected)) + try: + actual = _digest_file(dest, tuple(expected)) + except OSError: + # Unreadable (no permission, a read error on a failing card): its + # contents are not known to be the right ones. + return "mismatched" if all(actual[name] == value for name, value in expected.items()): return "up_to_date" return "mismatched" @@ -4444,6 +4453,9 @@ def _download_one( if _write_refused(exc): return f["dest"], False, NOT_WRITABLE return f["dest"], False, exc.strerror or str(exc) + if dest.is_dir(): + # Fetched, it failed only at the rename, on an errno and two paths. + return f["dest"], False, FOLDER_IN_PLACE if copy is not None: result = _copy_local(f, bios_path, dest, copy, progress) @@ -4834,6 +4846,7 @@ READ_ONLY = "file in place is read-only" # A file in place whose content differs, which could not be moved aside: it # is left as it is rather than lost. NOT_KEPT = "file in place could not be kept aside" +FOLDER_IN_PLACE = "a folder sits where this file goes: move it away, then run again" def _clear_read_only(dest: Path) -> None: @@ -7539,12 +7552,14 @@ def _wrong_contents( owners: dict[str, list[dict]] | None = None, ) -> str: """Why the file at an entry's place is not the one the list installs.""" + if path.is_dir(): + return FOLDER_IN_PLACE try: size = path.stat().st_size with open(path, "rb") as handle: head = handle.read(4) - except OSError: - return "" + except OSError as exc: + return f"could not be read ({exc.strerror or exc}): its contents are unknown" sha1 = index.sha1(path) owner = _content_owner(sha1, entry, files, owners) if owner: diff --git a/install.sh b/install.sh index e85ff322..34c27f9d 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="6d8af0dbca6697ba71692e5e482c8a16d9e8b9896652fec722d9c84f23bba047" +DEFAULT_INSTALL_SHA256="c361b511d9550b2115d918a0233115cd9f42920bce881f41395c3d7614be534d" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_selection.py b/tests/test_install_selection.py index 5cf8f481..3b4e2353 100644 --- a/tests/test_install_selection.py +++ b/tests/test_install_selection.py @@ -1501,5 +1501,38 @@ class TheTitleASetupPrints(unittest.TestCase): self.assertEqual(install.system_aliases(manifest)["Sony PlayStation"], "arcade") +class SomethingElseAtADestination(unittest.TestCase): + """A folder, or a file this user cannot read, at a destination ended + --check and the install on an IsADirectoryError traceback.""" + + def prepare(self, repo: FakeRepository) -> None: + (repo.dest / "scph5500.bin").mkdir(parents=True) + unreadable = repo.dest / "scph5501.bin" + unreadable.write_bytes(b"x") + unreadable.chmod(0) + + @unittest.skipIf(os.name == "nt" or (hasattr(os, "geteuid") and os.geteuid() == 0), + "a mode of 0 stops no read on Windows or as root") + def test_each_says_why_in_the_check_and_the_install(self): + with FakeRepository() as repo: + self.prepare(repo) + proc = repo.run("--check") + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertNotIn("Traceback", proc.stderr) + self.assertIn(" wrong hash: scph5500.bin\n a folder sits where this " + "file goes: move it away, then run again", proc.stdout) + self.assertIn(" wrong hash: scph5501.bin\n could not be read " + "(Permission denied)", proc.stdout) + proc = repo.run("--yes") + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertNotIn("Traceback", proc.stderr) + self.assertIn("failed: scph5500.bin (a folder sits where this file goes", + proc.stdout) + self.assertIn("failed: scph5501.bin (file in place could not be kept aside", + proc.stdout) + self.assertTrue((repo.dest / "scph5500.bin").is_dir()) + self.assertIn("gba_bios.bin", repo.installed()) + + if __name__ == "__main__": unittest.main() From 3523e46595506e31ca193744fc6f8efea09ab86b Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:40:57 +0200 Subject: [PATCH 06/17] fix: open each archive of a folder once Each ZIP member read reopened its archive and parsed its directory again: 4000 members of one size made 4000 parses. A folder index keeps the archives it reads open, 16 at most, read under a lock from the copy threads, and every index is closed at the end of the run. Archives the list installs are still opened and closed at once. --- install.ps1 | 2 +- install.py | 80 +++++++++++++++++++++++++++++++++++--- install.sh | 2 +- tests/test_install_from.py | 49 +++++++++++++++++++++++ 4 files changed, 126 insertions(+), 7 deletions(-) diff --git a/install.ps1 b/install.ps1 index c0e93dca..561ddaed 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "c361b511d9550b2115d918a0233115cd9f42920bce881f41395c3d7614be534d" + $defaultInstallSha256 = "5a7dc2fe2aa2abed511a94b3711f1846461a306ea185cd34407efac6c526cab0" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 4568f8e5..43d1dca0 100644 --- a/install.py +++ b/install.py @@ -45,6 +45,7 @@ import unicodedata import urllib.error import urllib.parse import urllib.request +import weakref import xml.etree.ElementTree as ET import zipfile import zlib @@ -6766,6 +6767,8 @@ class LocalSources: self.pack: PackSource | None = None # The folders named with --from, read once for every platform. self.folders: list[FolderIndex] = [] + # Every folder index of the run still in use, closed at its end. + self.indexes: weakref.WeakSet[FolderIndex] = weakref.WeakSet() @property def offline(self) -> bool: @@ -6775,10 +6778,16 @@ class LocalSources: def reset(self) -> None: if self.pack is not None: self.pack.close() + self.close_indexes() self.manifest = None self.pack = None self.folders = [] + def close_indexes(self) -> None: + for index in list(self.indexes): + index.close() + self.indexes = weakref.WeakSet() + LOCAL = LocalSources() NOT_IN_PACK = "not in the pack" @@ -6860,7 +6869,7 @@ def _open_copy(copy: LocalCopy) -> Iterator[BinaryIO]: with open(copy.path, "rb") as handle: yield handle return - with zipfile.ZipFile(copy.path) as archive, archive.open(copy.member) as handle: + with copy.index.open_member(copy.path, copy.member) as handle: yield handle @@ -7014,6 +7023,10 @@ class LocalCopy(NamedTuple): return f"{self.member} inside {self.path}" if self.member else str(self.path) +# The archives a FolderIndex keeps open at once. +OPEN_ARCHIVES = 16 + + class FolderIndex: """The files under a folder, by name and by size, hashed on demand. @@ -7032,6 +7045,7 @@ class FolderIndex: self.archives: list[Path] = [] # Name a ._ file stands beside (casefolded) -> the ._ files. self.apple_double: dict[str, list[Path]] = {} + LOCAL.indexes.add(self) # The list's destinations, casefolded (an archive among them is not # unpacked for its members) and as the list writes them. self.sealed: set[str] = set() @@ -7050,6 +7064,10 @@ class FolderIndex: self._member_named: dict[int, dict[str, list[tuple[Path, str]]]] = {} self._scans: dict[tuple[str, int, str], _HashScan] = {} self._member_size: dict[tuple[Path, str], int] = {} + # The archives read so far, open, the last read at the end: a ZIP + # was opened again, its directory parsed again, for each member. + self._zips: dict[Path, zipfile.ZipFile] = {} + self._zip_lock = threading.Lock() # (path, member or "") -> (its size or None when unknown, why) of each # file, member or folder below root that could not be read. self.unreadable: dict[tuple[Path, str], tuple[int | None, str]] = {} @@ -7121,6 +7139,50 @@ class FolderIndex: return False return True + def _archive(self, path: Path) -> zipfile.ZipFile: + """The archive at path, opened once. Called with _zip_lock held. + + A few stay open, the least recently read closed first: a card of + MAME sets would otherwise hold a descriptor per archive. Closing + one leaves a member being read open until it is closed itself. + Only archives the list does not install are read this way. + """ + archive = self._zips.pop(path, None) + if archive is None: + archive = zipfile.ZipFile(path) + while len(self._zips) >= OPEN_ARCHIVES: + self._zips.pop(next(iter(self._zips))).close() + self._zips[path] = archive + return archive + + def infolist(self, path: Path) -> list[zipfile.ZipInfo]: + with self._zip_lock: + return self._archive(path).infolist() + + @contextlib.contextmanager + def open_member(self, path: Path, member: str) -> Iterator[BinaryIO]: + """A member of an archive under the folder, read from any thread.""" + with self._zip_lock: + handle = self._archive(path).open(member) + try: + yield handle + finally: + # The archive counts its open members: the count moves under + # the lock, as when the member was opened. + with self._zip_lock: + handle.close() + + def open_archives(self) -> int: + with self._zip_lock: + return len(self._zips) + + def close(self) -> None: + """Close the archives the index holds open.""" + with self._zip_lock: + for archive in self._zips.values(): + archive.close() + self._zips.clear() + def seed(self, path: Path, sha1: str) -> None: """A SHA-1 already known, so the file is not read again.""" self._sha1[path] = sha1 @@ -7147,7 +7209,7 @@ class FolderIndex: if key not in self._digests: try: if member: - with zipfile.ZipFile(path) as archive, archive.open(member) as handle: + with self.open_member(path, member) as handle: value = _digest_handle(handle, (algorithm,))[algorithm] else: value = _digest_file(path, (algorithm,))[algorithm] @@ -7192,6 +7254,8 @@ class FolderIndex: def zip_members(self, path: Path) -> list[str]: if path not in self._members: try: + # Read once and closed: the archive may be a file of the + # list, which Windows will not replace while it is open. with zipfile.ZipFile(path) as archive: self._members[path] = [n.rsplit("/", 1)[-1].casefold() for n in archive.namelist()] @@ -7209,8 +7273,7 @@ class FolderIndex: or self.relative(path).casefold() in self.sealed): continue try: - with zipfile.ZipFile(path) as archive: - infos = archive.infolist() + infos = self.infolist(path) except OSError as exc: # A ZIP that will not open may hold anything: named. A # damaged one is a download cut short, not a copy. @@ -8332,7 +8395,14 @@ def main() -> None: def _run(args: argparse.Namespace) -> None: - """The run, its text quiet or not.""" + """The run, its text quiet or not, the archives it read closed after.""" + try: + _run_text(args) + finally: + LOCAL.close_indexes() + + +def _run_text(args: argparse.Namespace) -> None: if not args.quiet: _install(args, sys.stdout) return diff --git a/install.sh b/install.sh index 34c27f9d..a5588df7 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="c361b511d9550b2115d918a0233115cd9f42920bce881f41395c3d7614be534d" +DEFAULT_INSTALL_SHA256="5a7dc2fe2aa2abed511a94b3711f1846461a306ea185cd34407efac6c526cab0" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_from.py b/tests/test_install_from.py index fafc353e..ccc91de6 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -18,6 +18,7 @@ import io import json import os import subprocess +import threading import sys import time import unittest @@ -843,6 +844,54 @@ class ManySameSizedFiles(FromFolder): self.assertEqual(len(copies), 3) self.assertLess(elapsed, 30, f"{elapsed:.1f} s") + def test_one_archive_of_thousands_of_members_is_opened_once(self): + """Each member read reopened its ZIP and parsed its directory again: + 4000 members of one size made 4000 parses of 4000 entries.""" + count = 4000 + _zip(self.old / "romset.zip", {f"m{n:04}.bin": b"m%015d" % n for n in range(count)}) + wanted = [_entry(f"z{n}.bin", b"m%015d" % n, ["x"]) for n in (count - 1, 3, 1234)] + index = install.FolderIndex(self.old) + opened = [] + real = zipfile.ZipFile + + def counting(*args, **kwargs): + opened.append(args[0]) + return real(*args, **kwargs) + + start = time.monotonic() + with mock.patch.object(install.zipfile, "ZipFile", side_effect=counting): + copies = install.plan_local_copies(wanted, self.dest, None, [index]) + self.assertEqual(len(copies), 3) + # The copy step reads members of the archive the plan opened, + # from threads. + results = [] + threads = [threading.Thread(target=lambda e=e: results.append( + install._copy_local(e, self.dest, self.dest / e["dest"], copies[id(e)], None))) + for e in wanted] + self.dest.mkdir(parents=True, exist_ok=True) + for thread in threads: + thread.start() + for thread in threads: + thread.join() + elapsed = time.monotonic() - start + self.assertEqual(sorted(ok for _d, ok, _r in results), [True] * 3, results) + self.assertEqual(len(opened), 1) + self.assertLess(elapsed, 20, f"{elapsed:.1f} s") + index.close() + self.assertEqual(index.open_archives(), 0) + + def test_a_card_of_many_archives_holds_few_open(self): + count = install.OPEN_ARCHIVES * 3 + for n in range(count): + _zip(self.old / f"sets/s{n:03}.zip", {f"m{n}.bin": b"n%015d" % n}) + wanted = [_entry(f"z{n}.bin", b"n%015d" % n, ["x"]) for n in range(count)] + index = install.FolderIndex(self.old) + copies = install.plan_local_copies(wanted, self.dest, None, [index]) + self.assertEqual(len(copies), count) + self.assertLessEqual(index.open_archives(), install.OPEN_ARCHIVES) + install.LOCAL.close_indexes() + self.assertEqual(index.open_archives(), 0) + def test_a_file_of_the_same_name_comes_first(self): self.keep("a/other.bin", US) self.keep("b/scph5501.bin", US) From 246b6c6efc77612b87111f3b232876d944ffd12c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:42:55 +0200 Subject: [PATCH 07/17] fix: print found paths as the terminal shows them The line output printed the --from folder, the files found there and the BIOS folder paths as they are: a name holding ESC drove the terminal. screen_text moves out of the interface section and cleans every line of the check, the copy plan and the unreadable list. --- install.ps1 | 2 +- install.py | 62 +++++++++++++++++++++----------------- install.sh | 2 +- tests/test_install_from.py | 23 ++++++++++++++ 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/install.ps1 b/install.ps1 index 561ddaed..cf6f068a 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "5a7dc2fe2aa2abed511a94b3711f1846461a306ea185cd34407efac6c526cab0" + $defaultInstallSha256 = "1097dfe33e56238089af275bfea662d29613ff592f68ecdb427a86b5bcbc3e81" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 43d1dca0..4ce73ef1 100644 --- a/install.py +++ b/install.py @@ -7003,6 +7003,24 @@ _MEMBER_ERRORS = (OSError, EOFError, RuntimeError, NotImplementedError, zipfile.BadZipFile, zlib.error) +# C0 and C1 controls, DEL and lone surrogates: a file, folder or core name +# holding ESC would drive the terminal (clear it, or set the clipboard with +# OSC 52) instead of being shown. +_CONTROLS = re.compile("[\x00-\x1f\x7f-\x9f\ud800-\udfff]") + + +def screen_text(text: str) -> str: + """Text as a terminal shows it, every control character a "?".""" + return _CONTROLS.sub("?", text) + + +def shown_lines(lines: list[str]) -> list[str]: + """Lines for the terminal, a blank line opening one kept as it is: the + paths in them come from folders anyone may have filled.""" + return [("\n" + screen_text(line[1:])) if line.startswith("\n") else screen_text(line) + for line in lines] + + class LocalCopy(NamedTuple): """A file found on this machine with the contents an entry wants.""" index: FolderIndex @@ -7016,11 +7034,12 @@ class LocalCopy(NamedTuple): def where(self) -> str: """The copy as the folder it was found in names it.""" rel = self.index.relative(self.path) - return f"{rel}, inside as {self.member}" if self.member else rel + return screen_text(f"{rel}, inside as {self.member}" if self.member else rel) def named(self) -> str: """The copy by its full path, the member first when it is one.""" - return f"{self.member} inside {self.path}" if self.member else str(self.path) + return screen_text(f"{self.member} inside {self.path}" if self.member + else str(self.path)) # The archives a FolderIndex keeps open at once. @@ -7727,7 +7746,7 @@ def check_lines( if len(shown) < len(plain): lines.append(f" ... and {len(plain) - len(shown)} more missing " "(--verbose lists them all)") - return lines + return shown_lines(lines) def bios_folder_index( @@ -7829,7 +7848,7 @@ def _check_copy_lines( them = "it" if count == 1 else "them" lines.append(f" {label} holds {count} of these files: a run without --check " f"copies {them}.") - return lines + return shown_lines(lines) def _counts_by_folder( @@ -7885,7 +7904,7 @@ def copy_plan_lines( lines.append(f" {entry['dest']}: {why}" if why else f" {entry['dest']}") if len(chosen) > 10 and not every: lines.append(f" ... and {len(chosen) - 10} more (--verbose lists them all)") - return lines + return shown_lines(lines) def emit_copies( @@ -7914,7 +7933,7 @@ class Unreadable(NamedTuple): def line(self) -> str: where = f"{self.member} inside {self.path}" if self.member else str(self.path) - return f"{where} could not be read: {self.why}" + return screen_text(f"{where} could not be read: {self.why}") def unreadable_copies( @@ -7936,7 +7955,7 @@ def unreadable_lines(found: list[Unreadable], every: bool) -> list[str]: if len(found) > len(lines): lines.append(f" ... and {len(found) - len(lines)} more could not be read " "(--verbose lists them all)") - return lines + return shown_lines(lines) def emit_unreadable(plat: str, found: list[Unreadable]) -> None: @@ -7951,7 +7970,7 @@ def sources_line( ) -> str: """Where the files of a run that copied some came from, in words: "38 files copied from E:\\, 4 downloaded, 2 still missing." """ - parts = [(count, f"copied from {label}") for label, count in folders] + parts = [(count, f"copied from {screen_text(label)}") for label, count in folders] parts.append((fixed, "fixed from copies already in the BIOS folder")) parts.append((shared, "copied from another place of the BIOS folder")) if not any(count for count, _what in parts): @@ -8515,17 +8534,17 @@ def _open_from_folders(folders: list[Path]) -> None: reason, what = problem message = f"{label} {what}: --from names the folder to copy BIOS files from" EVENTS.emit("error", stage="from", reason=reason, platform="", message=message) - print(f"Error: {message}.", file=sys.stderr) + print(f"Error: {screen_text(message)}.", file=sys.stderr) sys.exit(1) index = FolderIndex(Path(os.path.expanduser(label)), label) LOCAL.folders.append(index) - print(f" Copying BIOS files from: {label}") + print(f" Copying BIOS files from: {screen_text(label)}") if index.truncated: - print(f" Read the first {MAX_SCANNED_FILES:,} files of {label} only. Name " - "the folder that holds the BIOS files to search it all.") + print(f" Read the first {MAX_SCANNED_FILES:,} files of {screen_text(label)} " + "only. Name the folder that holds the BIOS files to search it all.") for drive in index.other_drives: - print(f" Did not go into {drive}: it is another drive. Name it with --from " - "to read it too.") + print(f" Did not go into {screen_text(str(drive))}: it is another drive. Name " + "it with --from to read it too.") EVENTS.emit("scanned", folder=label, files=index.files, truncated=index.truncated, other_drives=[str(drive) for drive in index.other_drives]) @@ -9701,17 +9720,6 @@ def cell_width(text: str) -> int: return sum(_char_width(char) for char in text) -# C0 and C1 controls, DEL and lone surrogates: a file, folder or core name -# holding ESC would drive the terminal (clear it, or set the clipboard with -# OSC 52) instead of being shown. -_CONTROLS = re.compile("[\x00-\x1f\x7f-\x9f\ud800-\udfff]") - - -def screen_text(text: str) -> str: - """Text as a terminal shows it, every control character a "?".""" - return _CONTROLS.sub("?", text) - - def ascii_text(text: str) -> str: """The text in ASCII, accents dropped (Edition) and any other character as one ? per cell it took, so the columns stay where they were.""" @@ -12726,8 +12734,8 @@ def _tui_startup(args: argparse.Namespace) -> tuple[TuiState, list[tuple]] | Non for folder in folders: problem = from_folder_problem(str(folder)) if problem is not None: - print(f"Error: {folder} {problem[1]}: --from names the folder to copy BIOS " - "files from.", file=sys.stderr) + print(f"Error: {screen_text(str(folder))} {problem[1]}: --from names the folder " + "to copy BIOS files from.", file=sys.stderr) return None return tui_start(args, os_type) diff --git a/install.sh b/install.sh index a5588df7..2d1e27a9 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="5a7dc2fe2aa2abed511a94b3711f1846461a306ea185cd34407efac6c526cab0" +DEFAULT_INSTALL_SHA256="1097dfe33e56238089af275bfea662d29613ff592f68ecdb427a86b5bcbc3e81" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_from.py b/tests/test_install_from.py index ccc91de6..b4aef6ca 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -17,6 +17,7 @@ import errno import io import json import os +import re import subprocess import threading import sys @@ -597,6 +598,28 @@ class UnreadableCopies(FromFolder): self.assertIn(line, proc.stdout) self.assertNotIn("notes.txt", proc.stdout) + def test_names_on_the_card_cannot_drive_the_terminal(self): + """The plain output printed the paths it found as they are: a name + holding ESC cleared the screen or set the clipboard (OSC 52).""" + hostile = "x\x1b]52;c;eA==\x07\x1b[2J" + self.locked(f"{hostile}/scph5501.bin", US) + self.keep(f"{hostile}/gba_bios.bin", GBA) + (self.dest / f"pcsx{hostile}").mkdir(parents=True) + (self.dest / f"pcsx{hostile}" / "extra.bin").write_bytes(EXTRA) + for options in (("--check",), ("--dry-run",), ("--check", "--verbose"), ()): + with self.subTest(options=options): + proc = self.install(*options) + self.assertNotIn("Traceback", proc.stderr) + for stream in (proc.stdout, proc.stderr): + self.assertIsNone(re.search(r"[\x00-\x09\x0b-\x1f\x7f-\x9f]", stream), + stream) + self.assertIn("x?]52;c;eA==??[2J", proc.stdout) + proc = self.run_installer("--manifest", str(self.list), "--no-input", + "--from", str(self.root / hostile)) + self.assertEqual(proc.returncode, 1) + self.assertNotIn("\x1b", proc.stderr) + self.assertIn("x?]52;c;eA==??[2J", proc.stderr) + def test_an_unreadable_copy_is_an_event(self): path = self.locked("psx/scph5501.bin", US).resolve() proc = self.install("--check", "--events") From ccc8af7a47ad71f8e50139a6ec9a36b656ff797d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:43:38 +0200 Subject: [PATCH 08/17] fix: compare a card's BIOS folder by folders --dest NOTMUOS/bios ended with MUOS/bios as text and passed as a muOS card's folder. A card two systems claim now says so in one line, and the system named among them decides the folder. --- install.ps1 | 2 +- install.py | 34 ++++++++++++++++++++++++++++------ install.sh | 2 +- tests/test_install_tui.py | 28 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 8 deletions(-) diff --git a/install.ps1 b/install.ps1 index cf6f068a..11bffe1d 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "1097dfe33e56238089af275bfea662d29613ff592f68ecdb427a86b5bcbc3e81" + $defaultInstallSha256 = "2103f380bac5e2dc6ab91183283d268cccce81971da89f2963bab752d1518ef2" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 4ce73ef1..8ddbcfc1 100644 --- a/install.py +++ b/install.py @@ -8667,16 +8667,29 @@ def card_system(root: Path) -> tuple[str, Path, str] | None: plat = card_platform(root) if plat: return plat, root, display_name(plat) - matched = [rule for rule in _card_rules() if _card_rule_matches(rule, root)] - bases = {rule.base for rule in matched if rule.base} - matched = [rule for rule in matched if rule.name not in bases] - targets = {(rule.manifest, rule.bios) for rule in matched} - if len(targets) != 1: + matched = _card_matches(root) + if len({(rule.manifest, rule.bios) for rule in matched}) != 1: return None rule = matched[0] return rule.manifest, root / rule.bios, rule.name +def _card_matches(root: Path) -> list[_OsRule]: + """The card rules root matches, a system another one is built on left + out for it.""" + matched = [rule for rule in _card_rules() if _card_rule_matches(rule, root)] + bases = {rule.base for rule in matched if rule.base} + return [rule for rule in matched if rule.name not in bases] + + +def _ends_with(path: Path, relative: str) -> bool: + """Whether path's last folders are relative's, compared folder by folder + and without case: NOTMUOS/bios does not end with MUOS/bios.""" + tail = [part.casefold() for part in PurePosixPath(relative).parts] + parts = [part.casefold() for part in path.parts] + return len(parts) >= len(tail) and parts[len(parts) - len(tail):] == tail + + def card_dest(named: str, plat: str, dest: Path) -> tuple[str, Path]: """The file list and folder for a --dest that names a handheld's card. @@ -8694,12 +8707,21 @@ def card_dest(named: str, plat: str, dest: Path) -> tuple[str, Path]: key = _name_key(named) rule = next((rule for rule in _card_rules() if key and key == _name_key(rule.name) and key != _name_key(display_name(rule.manifest))), None) + matched = [] if found else _card_matches(dest) if rule is None or found: + if len({(match.manifest, match.bios) for match in matched}) > 1: + names = " and ".join(sorted({match.name for match in matched})) + print(f" {screen_text(str(dest))} holds the files of {names}: installing into " + "it as named. Name the card's BIOS folder with --dest to write there.") return plat, dest # Its folder on the card, or one its device reads (/run/muos/storage/bios). own = {device.bios for device in _EMBEDDED_RULES if device.name == rule.name} - if dest.as_posix() in own or dest.as_posix().casefold().endswith(rule.bios.casefold()): + if dest.as_posix() in own or _ends_with(dest, rule.bios): return plat, dest + if any(match.name == rule.name for match in matched): + # A card two systems claim, the one named among them. + print(f" {rule.name} card: {display_name(plat)} files, in {dest / rule.bios}") + return plat, dest / rule.bios marker = rule.values[0].split("/")[0] print(f"Error: {dest} is not a {rule.name} card: it holds no {marker}. " f"{rule.name} reads its BIOS files from {rule.bios} on its card: name the card's " diff --git a/install.sh b/install.sh index 2d1e27a9..b0c2495d 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="1097dfe33e56238089af275bfea662d29613ff592f68ecdb427a86b5bcbc3e81" +DEFAULT_INSTALL_SHA256="2103f380bac5e2dc6ab91183283d268cccce81971da89f2963bab752d1518ef2" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index ef7c2f67..73eee5ea 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -10,6 +10,7 @@ installs it). from __future__ import annotations import argparse +import contextlib import errno import fcntl import functools @@ -2122,6 +2123,33 @@ class AuditRoundThree(unittest.TestCase): self.assertEqual(install.card_dest("retroarch", "retroarch", root), ("retroarch", root)) + def test_p1_1_the_folder_named_is_compared_by_folders(self): + """NOTMUOS/bios ended with MUOS/bios as text and was taken for a + muOS card's folder.""" + root = self.card({"roms/": ""}) + stray = root / "NOTMUOS" / "bios" + with self.assertRaises(SystemExit), \ + contextlib.redirect_stderr(io.StringIO()) as err: + install.card_dest("muos", "retroarch", stray) + self.assertIn("is not a muOS card", err.getvalue()) + self.assertEqual(install.card_dest("muos", "retroarch", root / "muos" / "BIOS"), + ("retroarch", root / "muos" / "BIOS")) + + def test_p1_1_a_card_two_systems_claim_says_so(self): + root = self.card({"MUOS/": "", "spruce/": ""}) + self.assertIsNone(install.card_system(root)) + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(install.card_dest("retroarch", "retroarch", root), + ("retroarch", root)) + lines = out.getvalue().splitlines() + self.assertEqual(len(lines), 1, lines) + self.assertIn("holds the files of muOS and spruceOS", lines[0]) + # The system named among them decides. + with contextlib.redirect_stdout(io.StringIO()) as out: + self.assertEqual(install.card_dest("muos", "retroarch", root), + ("retroarch", root / "MUOS/bios")) + self.assertIn("muOS card", out.getvalue()) + # P1-2: Space is X on PortMaster and Y on muxterm. def test_p1_2_space_on_install_opens_the_confirm_screen(self): From abba99d352f2925444ceb37c67697d8f3e49f02c Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:47:36 +0200 Subject: [PATCH 09/17] fix: find the save directory for a named folder A run given --dest detected nothing, so retroarch.cfg was never read: the full screen, which names every folder, sent Azahar's keys.txt to a guessed RetroArch/saves or dropped it without a word, where line mode wrote it to savefile_directory. The configs are now read once when a save directory is asked for, and the plan and the result say what was left out. --- install.ps1 | 2 +- install.py | 43 +++++++++++++++++- install.sh | 2 +- tests/test_install_tui.py | 96 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 140 insertions(+), 3 deletions(-) diff --git a/install.ps1 b/install.ps1 index 11bffe1d..2bdb4e8d 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "2103f380bac5e2dc6ab91183283d268cccce81971da89f2963bab752d1518ef2" + $defaultInstallSha256 = "7065a9d52aa9016066c273ec8a5907b09fcce7eaad5a423583c79515f09edb99" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 8ddbcfc1..33955194 100644 --- a/install.py +++ b/install.py @@ -3949,6 +3949,27 @@ def _cfg_true(value: str | None) -> bool: return (value or "").strip().lower() == "true" +_CONFIGS_READ: list[bool] = [] + + +def _retroarch_config_for(bios_path: Path) -> tuple[Path, Path] | None: + """The retroarch.cfg whose system_directory is bios_path, and the + folder ':' expands to there. + + A run given --dest detects nothing, so the configs are read here, once + and silently, the first time a save directory is asked for: the + interface names every folder with --dest, and its runs sent Azahar's + keys.txt to a guessed folder where line mode read savefile_directory. + """ + found = _RETROARCH_CONFIGS.get(bios_path) + if found is not None or _CONFIGS_READ: + return found + _CONFIGS_READ.append(True) + with contextlib.redirect_stdout(io.StringIO()), contextlib.redirect_stderr(io.StringIO()): + detect_platforms(_INVOCATION.os_type or detect_os()) + return _RETROARCH_CONFIGS.get(bios_path) + + def save_directory_for( bios_path: Path, manifest: dict, override: Path | None = None, ) -> SavePlace | str: @@ -3975,7 +3996,7 @@ def save_directory_for( folder = layout["folder"] listed = bool(layout["by_core"]) by_core = listed - found = None if layout.get("by_system") else _RETROARCH_CONFIGS.get(bios_path) + found = None if layout.get("by_system") else _retroarch_config_for(bios_path) if found is not None: cfg_path, app_dir = found for key in ("savefiles_in_content_dir", "sort_savefiles_by_content_enable"): @@ -9920,6 +9941,8 @@ class TuiState: self.dry: dict | None = None self.dry_files: list = [] self.dry_summary: dict | None = None + # The check run's summary: what it left out and why. + self.check_summary: dict | None = None # The confirm screen waits for A even when the plan is unchanged: # it was opened by Space, which never installs by itself. self.must_confirm: bool = False @@ -10427,6 +10450,7 @@ def _start_run(state: TuiState, run: str) -> list[tuple]: # dropped the --system the command line gave. state.check, state.files, state.systems, state.reading = None, [], [], None state.found, state.scanned, state.unreadable = [], [], [] + state.check_summary = None state.checked_flags = tui_flags(state) elif run in ("confirm", "preview"): state.dry, state.dry_files, state.dry_summary = None, [], None @@ -11060,6 +11084,8 @@ def _take_event(state: TuiState, event: dict) -> None: state.files.append(event) elif kind == "system": state.systems.append(event) + elif kind == "summary": + state.check_summary = event elif state.run in ("confirm", "preview"): if kind == "check": state.dry = event @@ -11413,9 +11439,23 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: lines += _scan_lines(state) if check.get("not_installable"): lines += _unavailable_lines(state) + if check is state.check: + lines += _unplaced_lines(state.check_summary, state) return lines +def _unplaced_lines(summary: dict | None, state: TuiState) -> list[str]: + """The files read from the frontend's save directory that the run left + out, that directory being unknown: said, never silent.""" + count = sum(group.get("files", 0) for group in (summary or {}).get("skipped") or [] + if group.get("reason") == "no_save_directory") + if not count: + return [] + name = display_name(state.platform) if state.platform else "the frontend" + return [f"{_plural(count, 'file')} read from {name}'s save directory left out: " + "its place is not known. Name it with --save-dir."] + + def _view_setup(state: TuiState, width: int, height: int) -> tuple: rows = tui_rows(state) focus = rows[min(state.focus.get("setup", 0), len(rows) - 1)] @@ -11895,6 +11935,7 @@ def tui_outcome(state: TuiState) -> tuple[str, list[str]]: lines += _unavailable_lines(state) for group in summary.get("left_out") or []: lines.append(f"Left out as {group['label']}: {_plural(group['files'], 'file')}.") + lines += _unplaced_lines(summary, state) lines += ["Same choice without questions:", f" {tui_command(state)}"] if tui_keeps_log(state): lines.append(f"Details: {state.log}") diff --git a/install.sh b/install.sh index b0c2495d..559077c8 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="2103f380bac5e2dc6ab91183283d268cccce81971da89f2963bab752d1518ef2" +DEFAULT_INSTALL_SHA256="7065a9d52aa9016066c273ec8a5907b09fcce7eaad5a423583c79515f09edb99" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 73eee5ea..f9bd298d 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -2525,6 +2525,36 @@ class AuditRoundThree(unittest.TestCase): sorted(path.name for path in (running, failed, unknown, other))) +class AuditRoundFour(unittest.TestCase): + """tui_audit_3.md, each finding held where it was found.""" + + UNPLACED = {"reason": "no_save_directory", "label": "no save directory", "files": 2, + "required": 0} + + def test_p1_1_files_left_out_of_the_save_directory_are_said(self): + state = fresh_state() + state.stack = ["setup"] + with mock.patch.object(install.Path, "exists", return_value=True): + install.tui_update(state, Key("enter")) + install.tui_receive(state, ("manifest", "retroarch", MANIFEST)) + summary = {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 0, + "pending": 5, "failed": [], "skipped": [self.UNPLACED], "left_out": [], + "kept": [], "interrupted": False} + for event in [*FILES, *SYSTEMS, CHECK, summary]: + install.tui_receive(state, ("event", state.run_id, event)) + with mock.patch.object(install, "_tui_folder_facts", return_value={ + "free": 50 * 1024 ** 3, "writable": True, "cores": []}): + install.tui_receive(state, ("ended", state.run_id, 1, [])) + line = ("2 files read from RetroArch's save directory left out: its place is " + "not known. Name it with --save-dir.") + plan = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn(line, plan) + state.stack.append("result") + state.code = 0 + state.summary = {**summary, "mode": "install", "installed": 5} + self.assertIn(line, install.tui_outcome(state)[1]) + + class Crash(unittest.TestCase): """An unexpected error ends on one sentence and a log, never a traceback.""" @@ -2684,6 +2714,29 @@ class GameDataRepository(FakeRepository): return self +class SaveDirectoryRepository(FakeRepository): + """The fake repository with one file read from RetroArch's save + directory, as Azahar's keys.txt and 105 others of the real list.""" + + KEYS = b"azahar keys" + + def __enter__(self) -> SaveDirectoryRepository: + super().__enter__() + path = self.root / "install" / "retroarch.json" + manifest = json.loads(path.read_text()) + (self.root / "bios" / "keys.txt").write_bytes(self.KEYS) + manifest["files"].append({ + "dest": "Azahar/Azahar/sysdata/keys.txt", "root": "save", + "core_folder": "Azahar", "sha1": hashlib.sha1(self.KEYS).hexdigest(), + "sha256": hashlib.sha256(self.KEYS).hexdigest(), "size": len(self.KEYS), + "repo_path": "bios/keys.txt", "cores": ["azahar"], + "systems": ["nintendo-3ds"]}) + manifest["save_directory"] = {"folder": "saves", "by_core": True} + manifest["base_destination"] = "system" + path.write_text(json.dumps(manifest)) + return self + + @unittest.skipUnless(pyte and pty and hasattr(pty, "fork"), "screen tests need pyte (pip install pyte==0.8.2) and a pty") class EndToEnd(unittest.TestCase): @@ -2735,6 +2788,49 @@ class EndToEnd(unittest.TestCase): # The cursor and the main screen come back before the result is printed. self.assertIn(b"\x1b[?25h\x1b[?1049l", term.raw) + def test_the_save_directory_is_the_one_line_mode_finds(self): + """The interface named the BIOS folder with --dest, so its runs never + read retroarch.cfg: keys.txt went to a guessed RetroArch/saves, or + was dropped without a word, where line mode writes it to the + savefile_directory RetroArch reads.""" + import subprocess + + def written(repo, saves: Path) -> list[str]: + return sorted([*repo.installed(), *( + "saves/" + p.relative_to(saves).as_posix() + for p in saves.rglob("*") if p.is_file())]) + + results = [] + for mode in ("line", "interface"): + with self.subTest(mode=mode): + repo = self.repo(SaveDirectoryRepository) + home = Path(self.env["HOME"]) + saves = home / "MySaves" + saves.mkdir() + cfg = home / ".config" / "retroarch" / "retroarch.cfg" + cfg.write_text(f'system_directory = "{repo.dest}"\n' + f'savefile_directory = "{saves}"\n') + if mode == "line": + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", + "retroarch", "--yes"], env=self.env, capture_output=True, + text=True, timeout=60, check=False, stdin=subprocess.DEVNULL) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("6 files installed", proc.stdout) + else: + term = self.start() + term.wait("Where do you play?") + term.send("\r") + screen = term.wait("files to download") + self.assertIn("6 files to download", screen) + term.send("\r") + term.wait("6 files installed.") + term.send("\r") + self.assertEqual(term.close(), 0) + results.append(written(repo, saves)) + self.assertIn("saves/Azahar/Azahar/sysdata/keys.txt", results[-1]) + self.assertEqual(results[0], results[1]) + def test_one_system_from_the_checklist(self): repo = self.repo() term = self.start("--platform", "retroarch", "--dest", str(repo.dest), cols=40, rows=15) From a296bbfdffde903bb04ccbd505e84086b6fa205f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:49:23 +0200 Subject: [PATCH 10/17] fix: keep why a data tree or a copy failed A damaged member of a data tree's archive (zlib.error, EOFError) ended the run on a traceback, and a failed write left its .part. --offline refusing a data tree, or the download after a --from copy failed, lost its kind, so the advice named the network instead of --offline. --- install.ps1 | 2 +- install.py | 84 ++++++++++++++++++++++++-------------- install.sh | 2 +- tests/test_install.py | 35 ++++++++++++++++ tests/test_install_from.py | 12 ++++++ 5 files changed, 103 insertions(+), 32 deletions(-) diff --git a/install.ps1 b/install.ps1 index 2bdb4e8d..3224f28a 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "7065a9d52aa9016066c273ec8a5907b09fcce7eaad5a423583c79515f09edb99" + $defaultInstallSha256 = "079fcba3b83003932af8f5ed513e556675832ab20e55bbbbc356581b2daf1a14" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 33955194..de751a65 100644 --- a/install.py +++ b/install.py @@ -4497,7 +4497,9 @@ def _download_one( or why.startswith(NOT_KEPT)): return fetched both = f"{result[2]}, and {why}" - return fetched[0], False, NetworkReason(both) if isinstance(why, NetworkReason) else both + # The fetch's kind of failure stays, so the advice is the right one. + kind = type(why) if isinstance(why, (NetworkReason, OfflineReason)) else str + return fetched[0], False, kind(both) if LOCAL.pack is not None: return _extract_from_pack(f, bios_path, dest, progress) @@ -5599,40 +5601,62 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict: continue target = _destination_path(root, rel) target.parent.mkdir(parents=True, exist_ok=True) - digest = hashlib.sha1() - size = 0 - with opener() as src, tempfile.NamedTemporaryFile( - dir=target.parent, prefix=f".{target.name}.", suffix=".part", - delete=False, - ) as out: - while size <= meta["size"]: - chunk = src.read(min(1024 * 1024, meta["size"] + 1 - size)) - if not chunk: - break - digest.update(chunk) - size += len(chunk) - out.write(chunk) - if digest.hexdigest() == meta["sha1"] and size == meta["size"]: - slot = f"{entry['dest']}/{rel}" if entry.get("dest") else rel - try: - KEPT.keep(*_kept_slot(bios_path, slot), target, - {"sha1": meta["sha1"]}) - except _NotKept as exc: - Path(out.name).unlink(missing_ok=True) - counts["not_kept"].append((rel, str(exc))) - continue - os.replace(out.name, target) - counts["installed"] += 1 - else: - Path(out.name).unlink(missing_ok=True) - counts["changed"] += 1 - except (urllib.error.URLError, OSError, ValueError, zipfile.BadZipFile, - tarfile.TarError) as exc: + outcome = _install_data_member( + opener, meta, target, bios_path, + f"{entry['dest']}/{rel}" if entry.get("dest") else rel, + rel, counts["not_kept"]) + if outcome: + counts[outcome] += 1 + except _Offline as exc: + # Kept apart, so the advice is the offline one, not the network's. + counts["error"] = OfflineReason(str(exc)) + except (urllib.error.URLError, OSError, ValueError, tarfile.TarError) as exc: counts["error"] = str(getattr(exc, "reason", None) or exc) + except _MEMBER_ERRORS as exc: + # A damaged, encrypted or unsupported member, in words. + counts["error"] = unreadable_words(exc) counts["absent"] = len(needed) return counts +def _install_data_member( + opener: Callable[[], ContextManager[BinaryIO]], meta: dict, target: Path, + bios_path: Path, slot: str, rel: str, not_kept: list[tuple[str, str]], +) -> str: + """Write one member of a data tree's archive at target once its bytes + are the recorded ones: "installed", "changed" when they are not, or "" + when the file in place could not be moved aside (named in not_kept). + A read or write error is raised, and leaves no .part behind.""" + scratch: Path | None = None + try: + digest = hashlib.sha1() + size = 0 + with opener() as src, tempfile.NamedTemporaryFile( + dir=target.parent, prefix=f".{target.name}.", suffix=".part", delete=False, + ) as out: + scratch = Path(out.name) + while size <= meta["size"]: + chunk = src.read(min(1024 * 1024, meta["size"] + 1 - size)) + if not chunk: + break + digest.update(chunk) + size += len(chunk) + out.write(chunk) + if digest.hexdigest() != meta["sha1"] or size != meta["size"]: + return "changed" + try: + KEPT.keep(*_kept_slot(bios_path, slot), target, {"sha1": meta["sha1"]}) + except _NotKept as exc: + not_kept.append((rel, str(exc))) + return "" + os.replace(scratch, target) + scratch = None + return "installed" + finally: + if scratch is not None: + scratch.unlink(missing_ok=True) + + def select_data_directories(manifest: dict, files: list[dict], narrowed: bool) -> list[dict]: """The data trees this install keeps: all, or those of a kept system.""" entries = manifest.get("data_directories") or [] diff --git a/install.sh b/install.sh index 559077c8..1cc244ce 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="7065a9d52aa9016066c273ec8a5907b09fcce7eaad5a423583c79515f09edb99" +DEFAULT_INSTALL_SHA256="079fcba3b83003932af8f5ed513e556675832ab20e55bbbbc356581b2daf1a14" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 4da52996..b7cb3285 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3248,6 +3248,41 @@ class DataDirectoriesAreInstalled(unittest.TestCase): self.assertEqual(counts["changed"], 1) self.assertLess(Endless.served, 64 * 1024) + def test_a_damaged_member_is_a_reason_and_leaves_no_part(self): + """zlib.error and EOFError from a damaged member ended the run on a + traceback, and a failed write left its .part beside the tree.""" + import zlib + + work = Path(_tempdir(self)) + entry = self._entry() + entry["files"] = {"Sys/GC/font.bin": {"sha1": "1" * 40, "size": 4}} + for error in (zlib.error("invalid stored block lengths"), EOFError(), + OSError(28, "No space left on device")): + with self.subTest(error=type(error).__name__): + class Broken(io.RawIOBase): + def readable(self): + return True + + def readinto(self, buffer, error=error): + raise error + + def members(_archive, _entry): + yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Broken()) + + with unittest.mock.patch.object(install, "_fetch_archive", + lambda url, d: Path(d) / "a"), \ + unittest.mock.patch.object(install, "_archive_members", members): + counts = install.install_data_directory(entry, work) + self.assertTrue(counts["error"]) + self.assertEqual(counts["installed"], 0) + self.assertEqual([p.name for p in work.rglob("*.part")], []) + + def test_offline_keeps_its_reason(self): + work = Path(_tempdir(self)) + with unittest.mock.patch.dict(os.environ, {install.OFFLINE_VARIABLE: "1"}): + counts = install.install_data_directory(self._entry(), work) + self.assertIsInstance(counts["error"], install.OfflineReason) + def test_a_tarball_maps_through_source_path(self): import io as _io import tarfile diff --git a/tests/test_install_from.py b/tests/test_install_from.py index b4aef6ca..2f5f2880 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -520,6 +520,18 @@ class ACopyThatFails(FromFolder): self.assertNotIn(install.NOT_WRITABLE, reason) self.assertIsInstance(reason, install.NetworkReason) + def test_offline_the_fallback_keeps_the_offline_reason(self): + """The copy failed, the download was refused by --offline: the + joined reason lost its kind, and the advice named neither.""" + _unprivileged(self) + self.source.chmod(0) + self.addCleanup(self.source.chmod, 0o644) + with mock.patch.dict(os.environ, {install.OFFLINE_VARIABLE: "1"}): + _dest, ok, reason = install._download_one(self.entry, self.dest, copy=self.copy) + self.assertFalse(ok) + self.assertIsInstance(reason, install.OfflineReason) + self.assertIn("could not be read: Permission denied, and not fetched", reason) + def test_a_read_error_is_the_source_and_not_the_folder(self): class Failing(io.BytesIO): def read(self, *_args): From a23e9a09b87d07b40baa3c92a722cb36d1eaed89 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:50:24 +0200 Subject: [PATCH 11/17] feat: close the first screen when no button comes RETROBIOS_IDLE_EXIT=SECONDS: with no key on the first screen for that long, before any choice, the interface exits 0 with one line. A mapper that never opens the pad no longer freezes a handheld there. --- install.ps1 | 2 +- install.py | 29 +++++++++++++++++++++++++++++ install.sh | 2 +- tests/test_install_tui.py | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 2 deletions(-) diff --git a/install.ps1 b/install.ps1 index 3224f28a..d9cc53e2 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "079fcba3b83003932af8f5ed513e556675832ab20e55bbbbc356581b2daf1a14" + $defaultInstallSha256 = "4d0a09cb5df35b5ccac1addbcd31feaf1ae00153694c11c7a1894e73bc063833" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index de751a65..a02b083f 100644 --- a/install.py +++ b/install.py @@ -9965,6 +9965,8 @@ class TuiState: self.dry: dict | None = None self.dry_files: list = [] self.dry_summary: dict | None = None + # Closed by RETROBIOS_IDLE_EXIT, no key ever pressed. + self.idle_closed = False # The check run's summary: what it left out and why. self.check_summary: dict | None = None # The confirm screen waits for A even when the plan is unchanged: @@ -10021,6 +10023,24 @@ TUI_PAD_ROUTES = ("gptokeyb", "muxterm", "evmapy", "miyoo") _STEAM_MARKERS = ("SteamGamepadUI", "SteamDeck", "SteamGameId", "GAMESCOPE_WAYLAND_DISPLAY") +# Seconds the first screen waits for a first key before it closes, set by a +# launcher whose mapper may never open the pad: the screen would otherwise +# hold a handheld that has no other way out. +TUI_IDLE_ENV = "RETROBIOS_IDLE_EXIT" +TUI_IDLE_WORDS = "No button pressed, closing." + + +def tui_idle_exit(env: Mapping[str, str] | None = None) -> float | None: + """The seconds RETROBIOS_IDLE_EXIT names, None when unset or not a + positive number.""" + env = os.environ if env is None else env + try: + seconds = float(env.get(TUI_IDLE_ENV, "").strip()) + except ValueError: + return None + return seconds if 0 < seconds < float("inf") else None + + def tui_pad_route(os_type: str, env: Mapping[str, str] | None = None) -> str: """The keys the hints name: a launcher's pad route, "keys" at a desktop, "both" (button and key) where nothing says which one is held.""" @@ -12580,6 +12600,8 @@ def tui_farewell(state: TuiState) -> list[str]: def _farewell_lines(state: TuiState) -> list[str]: + if state.idle_closed: + return [f"RetroBIOS: {TUI_IDLE_WORDS}"] if state.screen != "result" and state.download is not None: # An install ran: what it wrote is said, whatever ended it. title, lines = tui_outcome(state) @@ -12705,6 +12727,8 @@ def run_tui(args: argparse.Namespace) -> int: number = getattr(signal, name, None) if number is not None: handlers[number] = signal.signal(number, handler) + idle = tui_idle_exit() + first_screen, opened, pressed = state.screen, time.monotonic(), False sys.stdout = sys.stderr = buffer try: _prune_work_folders(work.parent, time.time()) @@ -12735,10 +12759,15 @@ def run_tui(args: argparse.Namespace) -> int: state.now = time.monotonic() terminal.draw(tui_view(state, cols, rows), cols) for key in terminal.keys(TUI_FRAME): + pressed = True state.now = time.monotonic() perform(tui_update(state, key)) if state.exit_code is not None: break + if (idle is not None and not pressed and state.exit_code is None + and state.screen == first_screen + and time.monotonic() - opened >= idle): + state.idle_closed, state.exit_code = True, 0 if flags["interrupt"]: flags["interrupt"] = False if time.monotonic() >= ignore_interrupt_until[0]: diff --git a/install.sh b/install.sh index 1cc244ce..bb28d346 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="079fcba3b83003932af8f5ed513e556675832ab20e55bbbbc356581b2daf1a14" +DEFAULT_INSTALL_SHA256="4d0a09cb5df35b5ccac1addbcd31feaf1ae00153694c11c7a1894e73bc063833" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index f9bd298d..64c06276 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -2525,6 +2525,22 @@ class AuditRoundThree(unittest.TestCase): sorted(path.name for path in (running, failed, unknown, other))) +class IdleExit(unittest.TestCase): + def test_the_variable(self): + for value, seconds in (("30", 30.0), (" 2.5 ", 2.5), ("", None), ("0", None), + ("-4", None), ("soon", None), ("inf", None), ("nan", None)): + with self.subTest(value=value): + self.assertEqual(install.tui_idle_exit({install.TUI_IDLE_ENV: value}), + seconds) + self.assertIsNone(install.tui_idle_exit({})) + + def test_the_farewell(self): + state = fresh_state() + state.idle_closed, state.exit_code = True, 0 + self.assertEqual(install.tui_farewell(state), + ["RetroBIOS: No button pressed, closing."]) + + class AuditRoundFour(unittest.TestCase): """tui_audit_3.md, each finding held where it was found.""" @@ -3044,6 +3060,26 @@ class EndToEnd(unittest.TestCase): self.assertEqual(term.status, 129) self.assertEqual(list(scratch.glob("retrobios-tui-*/error.log")), []) + def test_no_key_on_the_first_screen_closes_it(self): + """A mapper that never opens the pad left the first screen up for + good: a handheld then has no way out but the power button.""" + self.repo() + self.env[install.TUI_IDLE_ENV] = "1" + term = self.start() + term.wait("Where do you play?") + self.assertEqual(term.close(), 0) + self.assertIn("No button pressed, closing.", term.after()) + # A key pressed, the screen stays: the person is there. + term = self.start() + term.wait("Where do you play?") + term.send(DOWN) + end = time.monotonic() + 2.5 + while time.monotonic() < end: + term._pump(0.05) + self.assertEqual(os.waitpid(term.pid, os.WNOHANG), (0, 0)) + term.send("q") + self.assertEqual(term.close(), 130) + def test_a_desktop_names_its_keys_and_a_pad_its_buttons(self): self.repo() for env, bar in (({"DISPLAY": ":0"}, "Enter Continue Esc Quit ? Help"), From 36f4b338e165e5ea3b5a52c7174cd534bfd372a1 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:50:59 +0200 Subject: [PATCH 12/17] fix: offer no paging keys on muxterm muxterm keeps PgUp and PgDn for its scrollback and its L1 and R1 send nothing: the help named a page key that never reaches the interface. --- install.ps1 | 2 +- install.py | 7 +++++-- install.sh | 2 +- tests/test_install_tui.py | 7 +++++++ 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/install.ps1 b/install.ps1 index d9cc53e2..7bb0e43c 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "4d0a09cb5df35b5ccac1addbcd31feaf1ae00153694c11c7a1894e73bc063833" + $defaultInstallSha256 = "3d36f9aaf0bfd17cc2eeff61d0f99d35ff60d7cf967572ec146b166681334e5d" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index a02b083f..c91ddc7a 100644 --- a/install.py +++ b/install.py @@ -12044,7 +12044,9 @@ _HELP_KEYS: dict[str, dict[str, str | None]] = { "tick": "X, Space, Right/Left", "options": "Y, Tab, o", "back": "B, Select, Esc", "stop": "B, Esc, Ctrl-C", "kill": None, "enter": "A, Enter", "esc": "B, Esc"}, - "muxterm": {"move": "D-pad, Up/Down", "page": "PgUp/PgDn", "ok": "Start, Enter", + # muxterm keeps PgUp and PgDn for its own scrollback, and its L1 and R1 + # send nothing: no key pages a list there. + "muxterm": {"move": "D-pad, Up/Down", "page": None, "ok": "Start, Enter", "tick": "Y, Space, Right/Left", "options": "o, Tab", "back": "B, Esc, Backspace", "stop": "B, Esc, Ctrl-C", "kill": None, "enter": "Start, Enter", "esc": "Esc"}, @@ -12080,7 +12082,8 @@ def _help_lines(state: TuiState) -> list[tuple[str, str]]: *_route_extras(state)] keys = [(names["move"], "Move")] if len(tui_rows(state)) > 9 or screen in _PAGERS: - keys += [(names["page"], "Page"), ("Home/End, g/G", "First, last")] + keys += [*([(names["page"], "Page")] if names["page"] else []), + ("Home/End, g/G", "First, last")] primary = _primary(state) or _TUI_VIEWS[screen](state, 80, 24)[2][0] ok = names["ok"] if state.pad == "miyoo" and screen in _CHECKLISTS: diff --git a/install.sh b/install.sh index bb28d346..0a7fce9a 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="4d0a09cb5df35b5ccac1addbcd31feaf1ae00153694c11c7a1894e73bc063833" +DEFAULT_INSTALL_SHA256="3d36f9aaf0bfd17cc2eeff61d0f99d35ff60d7cf967572ec146b166681334e5d" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 64c06276..1f46b96d 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -1097,6 +1097,13 @@ class PadRoutes(unittest.TestCase): "Close the terminal"): self.assertIn(words, muxterm) self.assertNotIn("Start+Select", muxterm) + # muxterm keeps PgUp and PgDn for its scrollback, and L1 and R1 + # send nothing: no paging is offered there, on a list or a pager. + for screen in ("systems", "result", "plan"): + with self.subTest(screen=screen): + text = help_text("muxterm", screen) + for words in ("L1", "R1", "PgUp", "PgDn", "Page"): + self.assertNotIn(words, text) miyoo = help_text("miyoo", "systems") for words in ("Start, Enter", "A, Space, Right/Left", "R2, Esc, Backspace", "L2, Tab, o", "Sends no key here"): From ba4db2ab57eacc797ff230f6c4d2ea8bacd54415 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:53:07 +0200 Subject: [PATCH 13/17] fix: keep system names whole at 40 columns The size column took half the line and cut every name to eleven letters: Game Boy, Game Boy Advance and Game Boy Color read alike. A system row now carries shorter size forms, and the longest one that leaves the name whole is shown, none when that is what keeps it. --- install.ps1 | 2 +- install.py | 26 +++++++++++++++++++++++--- install.sh | 2 +- tests/test_install_tui.py | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/install.ps1 b/install.ps1 index 7bb0e43c..38b7ee9e 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "3d36f9aaf0bfd17cc2eeff61d0f99d35ff60d7cf967572ec146b166681334e5d" + $defaultInstallSha256 = "eaa912af47dc6f29e4436ec200a1b431b8ddd8006ca08e57c8c73d3a05717818" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index c91ddc7a..d7987b3a 100644 --- a/install.py +++ b/install.py @@ -10178,6 +10178,9 @@ class TuiRow(NamedTuple): right: str = "" value: object = None style: str = "" + # Shorter forms of right, longest first, for a narrow screen: the label + # keeps its room before the right-hand text does. + narrow: tuple = () # Screens that scroll text instead of listing rows. @@ -10358,16 +10361,21 @@ def _system_rows(state: TuiState) -> list[TuiRow]: need = _needed(row) count, size = shared.get(row["system"], (0, 0)) own, own_size = need - count, row["bytes_needed"] - size + narrow: tuple = () if not need: right = "in place" elif not count: right = f"{_plural(need, 'file')}, {format_size(row['bytes_needed'])}" + narrow = (format_size(row["bytes_needed"]),) elif not own: right = f"{_plural(need, 'file')}, {format_size(size)}, all shared" + narrow = (f"{format_size(size)}, shared", format_size(size)) else: right = f"{_plural(own, 'file')}, {format_size(own_size)} + {count} shared" + narrow = (f"{format_size(own_size)} + {count} shared", + f"{format_size(own_size)} + {count} sh.", format_size(own_size)) rows.append(TuiRow("system", f"{box} {label}", right, row["system"], - "" if need else "dim")) + "" if need else "dim", narrow)) return rows @@ -11292,8 +11300,20 @@ _ROW_VERBS = { def _row_line(row: TuiRow, focused: bool, width: int) -> TuiLine: - right = middle(row.right, max(12, width // 2)) if row.right else "" - text = _columns(("> " if focused else " ") + row.label, right, width) + left = ("> " if focused else " ") + row.label + if row.narrow: + # The name first: the longest right-hand form that leaves it whole, + # else none at all when that is what keeps it whole. Nintendo Game + # Boy, Game Boy Advance and Game Boy Color read alike once cut. + whole = width - cell_width(left) - 2 + right = next((form for form in (row.right, *row.narrow) + if cell_width(form) <= whole), "") + if not right and cell_width(left) > width: + right = row.narrow[-1] + text = _columns(left, right, width) + else: + right = middle(row.right, max(12, width // 2)) if row.right else "" + text = _columns(left, right, width) return _styled(text, "focus" if focused else row.style) diff --git a/install.sh b/install.sh index 0a7fce9a..f64306ff 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="3d36f9aaf0bfd17cc2eeff61d0f99d35ff60d7cf967572ec146b166681334e5d" +DEFAULT_INSTALL_SHA256="eaa912af47dc6f29e4436ec200a1b431b8ddd8006ca08e57c8c73d3a05717818" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 1f46b96d..f8ba0aa8 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -2578,6 +2578,39 @@ class AuditRoundFour(unittest.TestCase): self.assertIn(line, install.tui_outcome(state)[1]) +class NarrowSystemNames(unittest.TestCase): + """At 40 columns the size column took half the line: Game Boy, Game Boy + Advance and Game Boy Color all read "Nintendo Gam".""" + + def test_the_name_comes_before_the_size(self): + state = checked_state() + names = {"nintendo-gb": "Nintendo Game Boy", "nintendo-gba": "Nintendo Game Boy Advance", + "nintendo-gbc": "Nintendo Game Boy Color", + "videoton-tvc": "Videoton TV Computer"} + state.manifest = {**state.manifest, "system_titles": names} + state.systems = [ + {"event": "system", "platform": "retroarch", "system": system, "name": system, + "files": 30, "present": 0, "missing": 30, "wrong_hash": 0, + "bytes_needed": 30 * 1024 * 1024} + for system in names] + state.files = [{"event": "file", "platform": "retroarch", "dest": f"s{n}.bin", + "systems": list(names), "size": 1024, "state": "missing"} + for n in range(27)] + state.ticked = set(names) + state.stack.append("systems") + for cols, rows in ((40, 15), (40, 12), (52, 15)): + with self.subTest(size=f"{cols}x{rows}"): + text = "\n".join(install.tui_view(state, cols, rows)) + for name in names.values(): + self.assertIn(name, text) + self.assertNotIn("...", text) + for line in text.splitlines(): + self.assertLessEqual(install.cell_width(line), cols) + # Wide, the full form stays. + self.assertIn("3 files, 30.0 MB + 27 shared", + "\n".join(install.tui_view(state, 120, 40))) + + class Crash(unittest.TestCase): """An unexpected error ends on one sentence and a log, never a traceback.""" From 812d5d157e6e28d7967a6b6ff8557907ccd61e8d Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:56:37 +0200 Subject: [PATCH 14/17] fix: tell offline from a lost connection on screen With --offline and no list, the full screen said No connection and offered a Try again that could never work. It now names offline mode and the two ways a run gets its list without the network. With a list, the plan counted downloads the run forbids: it says what this machine can supply and what needs the internet, and the printed command keeps --offline. --- install.ps1 | 2 +- install.py | 46 ++++++++++++++++++++++++++++++++++--- install.sh | 2 +- tests/test_install_tui.py | 48 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 5 deletions(-) diff --git a/install.ps1 b/install.ps1 index 38b7ee9e..b00b2a73 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "eaa912af47dc6f29e4436ec200a1b431b8ddd8006ca08e57c8c73d3a05717818" + $defaultInstallSha256 = "1f6b90760303d04cb6beef7fd294371b2e1d4ba733578c59a95e1b0395c2f376" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index d7987b3a..874efc26 100644 --- a/install.py +++ b/install.py @@ -10140,6 +10140,8 @@ def tui_flags(state: TuiState) -> list[tuple[str, str | None]]: if state.jobs != 8: flags.append(("--jobs", str(state.jobs))) flags += [("--from", str(folder)) for folder in state.from_folders] + if offline_requested() and not any(flag == "--offline" for flag, _v in flags): + flags.append(("--offline", None)) return flags @@ -10312,7 +10314,8 @@ def tui_rows(state: TuiState) -> list[TuiRow]: return [TuiRow("install", "Install"), TuiRow("back", "Back")] if screen == "error": rows = ([TuiRow("retry", "Try again")] - if state.run == "check" and state.error_title else []) + if state.run == "check" and state.error_title + and state.error_title != TUI_OFFLINE_TITLE else []) return rows + [TuiRow("back", "Choose another setup or folder" if state.run == "check" else "Back"), TuiRow("quit", "Quit")] @@ -10328,6 +10331,10 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: size = check.get("bytes_needed", 0) label = f"Install {format_size(size)}" if size >= CONFIRM_FROM_BYTES else "Install" rows = [TuiRow("install", label)] + if _offline_only() and not _plan_copies(state, check): + # Nothing here to copy, and the network refused: Install would + # only fail every file. + rows = [] if state.facts.get("writable") is False or _short_of_space(state, check): # Every file would fail alike, or the run would stop before the # first one: the way out is another folder. @@ -10343,6 +10350,18 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: return rows + [TuiRow("options", "Options"), TuiRow("quit", "Quit")] +def _offline_only() -> bool: + """--offline without a pack: what is not on this machine cannot come.""" + return offline_requested() and LOCAL.pack is None + + +def _plan_copies(state: TuiState, check: dict) -> list[str]: + """The files to install that this machine holds, by destination.""" + local = {event["dest"] for event in state.found} if check is state.check else set() + sizes = {event["dest"] for event in state.files} + return [dest for dest in local if dest in sizes] + + def _short_of_space(state: TuiState, check: dict) -> bool: free = state.facts.get("free") return free is not None and check.get("bytes_needed", 0) > free @@ -11443,13 +11462,18 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: fetch = max(0, need - len(copied)) fetch_bytes = max(0, check["bytes_needed"] - sum(sizes[dest_] for dest_ in copied)) elsewhere = _tui_found_elsewhere(state) if check is state.check else {} + offline = _offline_only() if elsewhere: # The folder by its last name: the whole path pushed the BIOS # folder off a 40x15 screen. The Options row spells it out. where = ", ".join(f"{len(events)} found in {_last_name(folder)}" for folder, events in elsewhere.items()) - headline = (f"{where}, {fetch} to download ({format_size(fetch_bytes)})" + headline = (f"{where}, {fetch} need the internet (--offline)" if fetch and offline + else f"{where}, {fetch} to download ({format_size(fetch_bytes)})" if fetch else f"{where}, nothing to download") + elif need and fetch and offline: + headline = (f"{_plural(fetch, 'file')} {'needs' if fetch == 1 else 'need'} the " + "internet, which --offline does not use") elif need and fetch: headline = f"{_plural(fetch, 'file')} to download, {format_size(fetch_bytes)}" elif need: @@ -11461,6 +11485,9 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: headline = (f"{_plural(data, 'data file')} to fetch" if data else "Everything is in place") wrong = check["wrong_hash"] + offline_lines = (["Copy them from your old card or a folder, or run the installer " + "again with a pack (--from-pack) or without --offline."] + if offline and fetch else []) # The folder comes second: on a 40x12 screen it is what is left once # the rows are drawn, and nothing is written before it is seen. lines: list = [ @@ -11505,7 +11532,7 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: lines += _unavailable_lines(state) if check is state.check: lines += _unplaced_lines(state.check_summary, state) - return lines + return lines[:2] + offline_lines + lines[2:] def _unplaced_lines(summary: dict | None, state: TuiState) -> list[str]: @@ -12379,6 +12406,9 @@ class _EngineRun: TUI_FETCH_TIMEOUT = 15 +TUI_OFFLINE_TITLE = "No internet: offline mode" + + def _tui_fetch(plat: str, inbox: queue.Queue, log: io.StringIO) -> None: """The file list, fetched once in this process, or what went wrong.""" if LOCAL.manifest is not None or LOCAL.pack is not None: @@ -12407,6 +12437,16 @@ def _tui_fetch(plat: str, inbox: queue.Queue, log: io.StringIO) -> None: if exc.code == 404 else f"The server answered HTTP {exc.code}."] except (urllib.error.URLError, OSError) as exc: code, words = network_words(exc) + if code == "offline": + # Asked for, not a failure: no Try again, and the two ways a + # run without the network gets its list. + inbox.put(("manifest_failed", plat, TUI_OFFLINE_TITLE, [ + "This run makes no network request (--offline), and the list of " + "BIOS files comes from the network.", + "Run the installer again with the list saved as a file " + "(--manifest FILE, a copy of install/.json) or with a pack " + "(--from-pack PACK), or without --offline."], 1)) + return title = {"clock": "The clock is wrong", "no_ca": "No trusted certificates"}.get(code, "No connection") # The screen's first row is Try again: "then run again" was a second way. diff --git a/install.sh b/install.sh index f64306ff..07943fb3 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="eaa912af47dc6f29e4436ec200a1b431b8ddd8006ca08e57c8c73d3a05717818" +DEFAULT_INSTALL_SHA256="1f6b90760303d04cb6beef7fd294371b2e1d4ba733578c59a95e1b0395c2f376" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index f8ba0aa8..d9a9fac8 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -19,6 +19,7 @@ import http.server import io import json import os +import queue import re import select import shutil @@ -2578,6 +2579,53 @@ class AuditRoundFour(unittest.TestCase): self.assertIn(line, install.tui_outcome(state)[1]) +class OfflineInTheInterface(unittest.TestCase): + """--offline in the full screen: a "No connection" error offering a Try + again that can never work, and a plan counting downloads the run + forbids.""" + + def setUp(self): + patcher = mock.patch.dict(os.environ, {install.OFFLINE_VARIABLE: "1"}) + patcher.start() + self.addCleanup(patcher.stop) + + def test_no_list_names_the_ways_that_work(self): + inbox = queue.Queue() + install._tui_fetch("retroarch", inbox, io.StringIO()) + kind, plat, title, lines, code = inbox.get_nowait() + self.assertEqual((kind, title, code), ("manifest_failed", + "No internet: offline mode", 1)) + text = " ".join(lines) + for words in ("--manifest", "--from-pack", "without --offline"): + self.assertIn(words, text) + state = fresh_state() + state.stack = ["setup"] + with mock.patch.object(install.Path, "exists", return_value=True): + install.tui_update(state, Key("enter")) + install.tui_receive(state, (kind, plat, title, lines, code)) + self.assertEqual(state.screen, "error") + self.assertNotIn("retry", [row.action for row in install.tui_rows(state)]) + view = "\n".join(install.tui_view(state, 80, 24)) + self.assertIn("No internet: offline mode", view) + self.assertNotIn("Try again", view) + + def test_the_plan_offers_no_download(self): + state = checked_state() + text = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertNotIn("to download", text) + self.assertIn("need the internet, which --offline does not use", text) + self.assertNotIn("install", [row.action for row in install.tui_rows(state)]) + # A file found on this machine can still be copied. + state.found = [{"event": "found", "platform": "retroarch", "dest": FILES[0]["dest"], + "folder": "/media/old", "path": "/media/old/scph5500.bin", + "member": "", "reason": ""}] + text = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn("1 found in old,", text) + self.assertIn("need the internet (--offline)", text) + self.assertEqual(install.tui_rows(state)[0].action, "install") + self.assertIn("--offline", install.tui_command(state)) + + class NarrowSystemNames(unittest.TestCase): """At 40 columns the size column took half the line: Game Boy, Game Boy Advance and Game Boy Color all read "Nintendo Gam".""" From d297cc3ff6e32350bf610d8eac13c971576a1da4 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 20:58:19 +0200 Subject: [PATCH 15/17] fix: bound the wait for a silent list server A server that took the connection and never answered held line mode 93 s, three waits of 30 s for a first byte. A file list now waits 15 s per answer and asks a silent server twice. Downloads keep their own timeout between chunks. --- install.ps1 | 2 +- install.py | 30 ++++++++++++++++++++++++++---- install.sh | 2 +- tests/test_install.py | 43 +++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 6 deletions(-) diff --git a/install.ps1 b/install.ps1 index b00b2a73..fdba5838 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "1f6b90760303d04cb6beef7fd294371b2e1d4ba733578c59a95e1b0395c2f376" + $defaultInstallSha256 = "aef82904ca3356ded55f7b43bc03fe80dc30da178a28bc598caef8367b3f9766" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 874efc26..76b83f59 100644 --- a/install.py +++ b/install.py @@ -3322,16 +3322,34 @@ def install_bounded_opener() -> None: urllib.request.install_opener(urllib.request.build_opener(*handlers)) +# The wait for each answer of a file list, its first byte included: a +# server that takes the connection and never answers (a captive portal, a +# stuck proxy) held the run 93 s on three waits of 30 s. A list is a few +# MB, never a slow stream. +LIST_TIMEOUT = 15.0 +# Attempts after a server that took the connection went silent: once more, +# a stall is rarely a blip. +STALLED_ATTEMPTS = 2 + + +def _stalled(exc: BaseException) -> bool: + """Whether exc is a wait that ran out, the connection made.""" + reason = getattr(exc, "reason", None) + return isinstance(exc, socket.timeout) or isinstance(reason, socket.timeout) + + def _fetch_json( - url: str, limit: int, label: str, attempts: int = MAX_RETRIES, timeout: float = 30, + url: str, limit: int, label: str, attempts: int = MAX_RETRIES, + timeout: float | None = None, ) -> object: """A JSON document, tried MAX_RETRIES times like every file download. A file list fetched once ended the run on one timeout over flaky Wi-Fi. A 4xx answer or an invalid document is final: asking again changes - nothing. + nothing, and a server gone silent is asked once more only. """ _network_allowed(label) + timeout = LIST_TIMEOUT if timeout is None else timeout for attempt in range(1, attempts + 1): try: with urllib.request.urlopen(url, timeout=timeout) as resp: @@ -3344,9 +3362,12 @@ def _fetch_json( except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: code, problem = network_words(exc) # A wrong clock or no certificate list fails every attempt alike. - if attempt == attempts or code != "network": + if (attempt == attempts or code != "network" + or (_stalled(exc) and attempt >= STALLED_ATTEMPTS)): if isinstance(exc, http.client.HTTPException): raise ConnectionError(f"connection cut: {exc!r}") from exc + # How many were made, for the line that says so. + exc.attempts = attempt # type: ignore[union-attr] raise delay = _retry_delay(attempt, None) # Said, or three 30-second attempts read as a frozen program. @@ -3416,7 +3437,8 @@ def fetch_manifest(plat: str) -> dict: print(f" Could not fetch the file list for {plat}.", file=sys.stderr) print(f" {words}.", file=sys.stderr) sys.exit(1) - print(f" Could not fetch the file list for {plat} after {MAX_RETRIES} " + tries = getattr(exc, "attempts", MAX_RETRIES) + print(f" Could not fetch the file list for {plat} after {tries} " f"attempts: {words}", file=sys.stderr) for line in _address_lines(url): print(line, file=sys.stderr) diff --git a/install.sh b/install.sh index 07943fb3..e1e8a74a 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="1f6b90760303d04cb6beef7fd294371b2e1d4ba733578c59a95e1b0395c2f376" +DEFAULT_INSTALL_SHA256="aef82904ca3356ded55f7b43bc03fe80dc30da178a28bc598caef8367b3f9766" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index b7cb3285..1748ab29 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -17,6 +17,7 @@ import subprocess import sys import tempfile import threading +import time import unittest from types import SimpleNamespace import unittest.mock @@ -2369,6 +2370,48 @@ class FileListsAreRetried(unittest.TestCase): self.assertIn(f"after {install.MAX_RETRIES} attempts", err.getvalue()) +class ASilentServerIsBounded(unittest.TestCase): + """A server that takes the connection and never answers held line mode + 93 s: three waits of 30 s for a first byte that never came.""" + + def test_two_short_waits_then_the_reason(self): + import socket + import threading + + listener = socket.socket() + listener.bind(("127.0.0.1", 0)) + listener.listen(8) + self.addCleanup(listener.close) + held = [] + + def accept(): + while True: + try: + conn, _addr = listener.accept() + except OSError: + return + held.append(conn) + + threading.Thread(target=accept, daemon=True).start() + self.addCleanup(lambda: [conn.close() for conn in held]) + port = listener.getsockname()[1] + err = io.StringIO() + start = time.monotonic() + with unittest.mock.patch.object(install, "LIST_TIMEOUT", 0.5), \ + unittest.mock.patch.object(install, "_retry_delay", return_value=0), \ + unittest.mock.patch.object(install, "MANIFEST_URL", + f"http://127.0.0.1:{port}/{{platform}}.json"), \ + contextlib.redirect_stderr(err): + with self.assertRaises(SystemExit): + install.fetch_manifest("retroarch") + elapsed = time.monotonic() - start + self.assertLess(elapsed, 5) + self.assertEqual(len(held), install.STALLED_ATTEMPTS) + self.assertIn(f"after {install.STALLED_ATTEMPTS} attempts", err.getvalue()) + # A download keeps its own, longer wait between chunks. + self.assertGreater(install.LIST_TIMEOUT, 1) + + class TargetListFailureIsNotAnAnswer(unittest.TestCase): """A timeout is not 'no targets': it sent users to the full pack.""" From 67a6ad3e32a52f5fde44bce7bfecda8b130aac25 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:11:45 +0200 Subject: [PATCH 16/17] chore: wrap the work folder error line --- install.ps1 | 2 +- install.py | 3 ++- install.sh | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/install.ps1 b/install.ps1 index fdba5838..821c2dc5 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "aef82904ca3356ded55f7b43bc03fe80dc30da178a28bc598caef8367b3f9766" + $defaultInstallSha256 = "563ed14fea5ea54f5d6cb98311932293e714e9581014ea60c195325c20a587eb" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 76b83f59..5d744967 100644 --- a/install.py +++ b/install.py @@ -12737,7 +12737,8 @@ def run_tui(args: argparse.Namespace) -> int: except OSError as exc: _restore_handlers(handlers) print(f"Error: no work folder could be made in {tempfile.gettempdir()}: {exc}. " - "Free some space there, or point TMPDIR (TEMP on Windows) at another folder.", file=sys.stderr) + "Free some space there, or point TMPDIR (TEMP on Windows) at another " + "folder.", file=sys.stderr) return 1 try: (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") diff --git a/install.sh b/install.sh index e1e8a74a..8959bba1 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="aef82904ca3356ded55f7b43bc03fe80dc30da178a28bc598caef8367b3f9766" +DEFAULT_INSTALL_SHA256="563ed14fea5ea54f5d6cb98311932293e714e9581014ea60c195325c20a587eb" MAX_INSTALLER_BYTES=2097152 cleanup() { From 49b2a69567639ab2d983f8e5192a4ebb553ed024 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 21:11:45 +0200 Subject: [PATCH 17/17] docs: exit 129, offline refusals and pad variables The exit status names 129 for a closed window and the --offline refusals of --target and --list-targets, and the events say a crash leaves no end line. RETROBIOS_PAD and RETROBIOS_IDLE_EXIT join the environment table, muxterm's missing paging and the save directory found under --dest are said, and the options at a glance list --declared-only and --offline. The first install page no longer promises a controller on a computer, and gives the RetroPie command for Windows too. --- wiki/first-install.md | 10 +++++++--- wiki/installer.md | 37 +++++++++++++++++++++++++++---------- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/wiki/first-install.md b/wiki/first-install.md index 33a97475..1537570c 100644 --- a/wiki/first-install.md +++ b/wiki/first-install.md @@ -83,7 +83,7 @@ It opens a menu: - **Wrong folder?** Choose **Another setup or folder**, then browse to yours. - **A handheld's SD card in a reader?** It is listed, as **muOS SD card**. -Arrows move, Enter chooses, Esc goes back. A controller's A and B do the same. +Arrows move, Enter chooses, Esc goes back. A handheld's A and B do the same. ## 3. Install @@ -142,8 +142,12 @@ The message shows the command that installs Python. Run it, then run the installer again. A RetroPie 4.8 image has Python 3.7.3, and its update command finds nothing -newer. Run the command on a computer, with -`--platform retropie --dest \\RETROPIE\bios\`. +newer. Run it on a computer, the `bios` share open: + + +```sh +curl -fsSL https://raw.githubusercontent.com/Abdess/retrobios/main/install.sh | sh -s -- --platform retropie --dest +``` ### Error: the downloaded installer does not match its expected fingerprint. diff --git a/wiki/installer.md b/wiki/installer.md index 826692fc..becbca57 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -26,9 +26,9 @@ How the command protects your machine is under Each links to its row in the [Options](#options) table. - **Where and for what**: [`--platform`](#option-platform), [`--dest`](#option-dest), [`--target`](#option-target), [`--every-board`](#option-every-board), [`--save-dir`](#option-save-dir), [`--romm-firmware`](#option-romm-firmware), [`--standalone-copies`](#option-standalone-copies) -- **Fewer files**: [`--system`](#option-system), [`--core`](#option-core), [`--emulator`](#option-emulator), [`--region`](#option-region), [`--no-game-data`](#option-no-game-data), [`--installed-cores`](#option-installed-cores), [`--cores-dir`](#option-cores-dir) +- **Fewer files**: [`--system`](#option-system), [`--core`](#option-core), [`--emulator`](#option-emulator), [`--region`](#option-region), [`--no-game-data`](#option-no-game-data), [`--installed-cores`](#option-installed-cores), [`--cores-dir`](#option-cores-dir), [`--declared-only`](#option-declared-only) - **Look first**: [`--check`](#option-check), [`--dry-run`](#option-dry-run), [`--list-platforms`](#option-list-platforms), [`--list-targets`](#option-list-targets), [`--list-systems`](#option-list-systems), [`--list-cores`](#option-list-cores), [`--list-emulators`](#option-list-emulators) -- **From files you have**: [`--from`](#option-from), [`--from-pack`](#option-from-pack), [`--manifest`](#option-manifest), [`--restore`](#option-restore) +- **From files you have**: [`--from`](#option-from), [`--from-pack`](#option-from-pack), [`--manifest`](#option-manifest), [`--offline`](#option-offline), [`--restore`](#option-restore) - **How the run talks**: [`--tui`](#option-tui), [`--plain`](#option-plain), [`--no-input`](#option-no-input), [`--yes`](#option-yes), [`--quiet`](#option-quiet), [`--verbose`](#option-verbose), [`--events`](#option-events), [`--jobs`](#option-jobs), [`--version`](#option-version), [`--help`](#option-help) ## One line @@ -232,7 +232,8 @@ below the BIOS folder: Azahar's keys, shared font and NAND files, Citra's `sysdata`, Dolphin's `User` tree. The file list marks them `root: save`, and the run writes them below the save directory, which it names before checking files (`5 files read from RetroArch's save directory go to ...`). For a -RetroArch found by its `retroarch.cfg`, that is `savefile_directory` as +RetroArch found by its `retroarch.cfg` (with `--dest` too, when the folder is +the config's `system_directory`, as in the full screen), that is `savefile_directory` as RetroArch expands it (`:` for its own folder, `~` for the home), `saves/` beside the config when the value is `default`, empty, or names no folder, and the core's own folder below it when `sort_savefiles_enable` is on, as it is by @@ -242,7 +243,7 @@ the save directory beside it (`saves`, `/storage/savefiles` on Lakka). applies, or when `savefiles_in_content_dir` or `sort_savefiles_by_content_enable` keeps saves beside each game, those files are left out and the run says why, like the other entries the list cannot -install. +install. The full screen says so on its plan and its result. Batocera, KNULLI, Recalbox, RetroBat and RetroDECK write the save directory at each launch, one folder per system (`by_system` in the file list). Their lists @@ -1066,7 +1067,9 @@ muxterm, as the muOS app does. Start pressed first also chooses the line in focus: hold Select first. A running install is then stopped the way B stops it, and the last screen says what it wrote. On muOS without PortMaster, muxterm reads the pad itself: Y -ticks a system, Select shows its keyboard, and Menu quits. +ticks a system, Select shows its keyboard, and Menu quits. L1 and R1 do not +page there: muxterm keeps PgUp and PgDn for its own scrollback, so a list +moves with the D-pad. Each launcher tells the interface which buttons reach it, in `RETROBIOS_PAD`, so the hints can name them: @@ -1078,7 +1081,12 @@ Each launcher tells the interface which buttons reach it, in | `evmapy` | `RetroBIOS.sh.keys`, the same buttons as the table | the Batocera and Knulli port | | `miyoo` | buttons that are already keys: A as Space, Start as Enter, L1 and R1 as E and T, L2 as Tab, R2 as Backspace | no launcher yet | -A launcher with no mapper at all (a keyboard) leaves it unset. On a Miyoo, B, +A launcher with no mapper at all (a keyboard) leaves it unset. +A launcher can also set `RETROBIOS_IDLE_EXIT` to a number of seconds: when +no button reaches the first screen for that long, before any choice, the +interface closes with `No button pressed, closing.` and exit status 0. A +mapper that never opened the pad would otherwise hold the device on that +screen. On a Miyoo, B, X, Y and Select are Left Ctrl, Shift, Alt and Right Ctrl: pressed alone, a modifier sends nothing to a terminal, so the interface goes back with R2 instead, and the bottom line says so. @@ -1235,13 +1243,15 @@ fields, the installer refuses to narrow rather than install everything. |------|---------| | `0` | everything asked for is installed or already up to date | | `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, or `--restore` could not put a file back | -| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | +| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`, `--target` or `--list-targets` with `--offline` (or `RETROBIOS_INSTALL_OFFLINE` set), which read a core list from the network. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | | `130` | stopped with Ctrl+C, or with `q`, Ctrl+C or Ctrl+D at a question. Run the same command again to carry on | | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) | The full screen ended by SIGTERM (Start+Select on a handheld) before any install exits `143`. During an install it stops the install first, and exits -`130`. +`130`. Its window or terminal closed (SIGHUP) exits `129`, during an install +too: the install is stopped as Ctrl+C stops it, and the last lines go +nowhere, no one being left to read them. `install.ps1` passes the installer's code on unchanged: run as a file it exits with it, run through `irm | iex` it sets `$LASTEXITCODE` (an `exit` there @@ -1293,7 +1303,9 @@ other schemas under `api/v1/schemas/`. Schema 1 can gain a kind (`found` came with `--from`, `declared` with `--declared-only`), never a key of an existing kind: a reader skips a kind it does not know. An option refused before the run starts (`install.py: error:`, code 2) writes -no event at all. A program reads a missing `end` as that case. +no event at all. A program reads a missing `end` as that case, or as a +crash: an error the installer did not expect ends the process on a Python +traceback, on standard error, without an `end` line. Five lines of a run of `--check --events --system sony-playstation` on a RetroArch folder that holds only `scph5501.bin`, in the order written: @@ -1322,6 +1334,8 @@ RetroArch folder that holds only `scph5501.bin`, in the order written: | `FORCE_COLOR`, `CLICOLOR_FORCE` | Colour even when the output is a pipe or a file, set to any value but `0` or an empty one. `NO_COLOR` wins over both, and the progress line stays one line per file | | `SSL_CERT_FILE`, `SSL_CERT_DIR` | The trusted certificates to check servers with, for a Python that finds none of its own | | `RETROBIOS_PLAIN`, `ACCESSIBLE` | Set to anything but `0`: the questions come line by line, as with `--plain`, instead of the full-screen interface | +| `RETROBIOS_PAD` | The buttons a handheld launcher's mapper sends to the full screen (`gptokeyb`, `muxterm`, `evmapy`, `miyoo`), so its hints name them. See [On a handheld](#on-a-handheld-the-retrobios-port) | +| `RETROBIOS_IDLE_EXIT` | Seconds the full screen's first screen waits for a first button. With none pressed, it closes with exit status 0 | | `HTTPS_PROXY`, `NO_PROXY` | Honoured for every download through the standard library's default proxy handling: one names the proxy, the other the hosts to reach directly | ## When something goes wrong @@ -1336,7 +1350,10 @@ The installer fetches its manifest, then downloads only the repository or `https://github.com` opens. Behind a proxy, set `HTTPS_PROXY` in the environment. A connection that does not open within 4 seconds counts as none, so a dropped network gives its error in about 15 seconds. Once a -connection is open, a slow transfer keeps the full timeout. A file that fails three attempts is counted and reported at the +connection is open, a slow transfer keeps the full timeout. The file list +is the exception: a server that takes the connection and then sends nothing +(a Wi-Fi sign-in page, a stuck proxy) gets 15 seconds per wait and two +attempts, about 30 seconds in all. A file that fails three attempts is counted and reported at the end, and the next run fetches only what is still missing. The reason is said in words: no internet connection or a name that could not