From cffd9eeb029a17536b9972374d2251ce3d8be093 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:43:18 +0200 Subject: [PATCH] feat: read the engine's transfer and scan events The speed is bytes_network over time, the interface's own guess at local bytes is gone. The plan says when a --from folder was read only in part or holds another drive, and how many files on the old card could not be read, with a list of each reason. The --from folders are only checked before the screen opens, never indexed there. --- install.ps1 | 2 +- install.py | 85 ++++++++++++++++++++++++++++++--------- install.sh | 2 +- tests/test_install_tui.py | 71 +++++++++++++++++++++++++++++++- 4 files changed, 138 insertions(+), 22 deletions(-) diff --git a/install.ps1 b/install.ps1 index 1dcc2992..9f56ce58 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "e0a65cc08918a2f53032f63d08974995542a0beabb7528ac3de35d0fdb698e36" + $defaultInstallSha256 = "1a26e1066a699413ab3d19f50ee552185ee2e9cbf74ee44548d3c9ea66f1b512" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 09f75b88..f2daef16 100644 --- a/install.py +++ b/install.py @@ -9468,7 +9468,10 @@ class TuiState: # Files an install copies from this machine, whose bytes are no # download: dest to (folder, reason), reason set for a misplaced one. self.local_dests: dict = {} - self.local_done: int = 0 + # Per --from folder, its scanned event, and the files on those + # folders that could not be read (unreadable events). + self.scanned: list = [] + self.unreadable: list = [] # The child run in progress: check, confirm, preview or install. self.run: str = "" self.running: bool = False @@ -9498,9 +9501,11 @@ class TuiState: # The install. self.download: dict | None = None self.progress: dict | None = None - # (time, bytes done) of the last progress events: a partial copy - # resumed counts in the first one and never in a speed. + # (time, bytes from the network) of the last transfer events: what + # came from this machine (a copy, the pack, a partial file resumed) + # is never in a speed. self.samples: list = [] + self.network: int = 0 self.finished: list = [] self.summary: dict | None = None self.code: int | None = None @@ -9644,7 +9649,7 @@ class TuiRow(NamedTuple): # Screens that scroll text instead of listing rows. -_PAGERS = ("preview", "command", "result", "misplaced") +_PAGERS = ("preview", "command", "result", "misplaced", "unreadable") _CHECKLISTS = ("systems", "cores") @@ -9795,6 +9800,8 @@ def _plan_rows(state: TuiState) -> list[TuiRow]: rows.append(TuiRow("fromcard", "Copy the BIOS files from your old card or a folder")) if _tui_misplaced(state): rows.append(TuiRow("misplaced", "Show the files in the wrong place")) + if state.unreadable: + rows.append(TuiRow("unreadable", "Show the files that could not be read")) return rows + [TuiRow("options", "Options"), TuiRow("quit", "Quit")] @@ -9951,13 +9958,13 @@ def _start_run(state: TuiState, run: str) -> list[tuple]: # The systems of an earlier check go first: ticks read against them # dropped the --system the command line gave. state.check, state.files, state.systems, state.reading = None, [], [], None - state.found = [] + state.found, state.scanned, state.unreadable = [], [], [] state.checked_flags = tui_flags(state) elif run in ("confirm", "preview"): state.dry, state.dry_files, state.dry_summary = None, [], None else: state.download = state.progress = state.reading = None - state.samples, state.local_dests, state.local_done = [], {}, 0 + state.samples, state.local_dests, state.network = [], {}, 0 state.finished, state.summary, state.code, state.stopping = [], None, None, False return [("run", run)] @@ -10174,8 +10181,8 @@ def _activate(state: TuiState, row: TuiRow) -> list[tuple]: return _open_folder_screen(state, state.dest) if action == "fromcard": return _open_from_browser(state) - if action == "misplaced": - return _push(state, "misplaced") + if action in ("misplaced", "unreadable"): + return _push(state, action) # On the first system, so A continues and the D-pad ticks. return _push(state, action, 3 if action == "systems" else 0) if screen == "systems": @@ -10547,6 +10554,10 @@ def _take_event(state: TuiState, event: dict) -> None: state.reading = (event["files_done"], event["files_total"]) elif kind == "found" and state.run == "check": state.found.append(event) + elif kind == "scanned" and state.run == "check": + state.scanned.append(event) + elif kind == "unreadable" and state.run == "check": + state.unreadable.append(event) elif kind == "found" and state.run == "install": state.local_dests[event["dest"]] = (event["folder"], event["reason"]) elif state.run == "check": @@ -10566,15 +10577,12 @@ def _take_event(state: TuiState, event: dict) -> None: elif state.run == "install": if kind == "download": state.download = event + elif kind == "transfer": + state.network = event["bytes_network"] + state.samples = [sample for sample in state.samples + if sample[0] >= state.now - 10] + [(state.now, state.network)] elif kind in ("progress", "downloaded"): state.progress = event - if kind == "downloaded" and event["dest"] in state.local_dests: - # A copy from an old card or a folder: its bytes come from - # this machine, and a speed counting them was not the network's. - state.local_done += event["size"] - state.samples = [sample for sample in state.samples - if sample[0] >= state.now - 10] + [ - (state.now, event["bytes_done"] - state.local_done)] if kind == "downloaded": state.finished.append((event["dest"], event["ok"], event["reason"])) elif kind == "summary": @@ -10667,6 +10675,7 @@ _ROW_VERBS = { "preview": "Open", "folder": "Choose", "game_data": "Change", "installed": "Change", "jobs": "Change", "retry": "Try again", "back": "Back", "stay": "Stay", "fromcard": "Choose", "from": "Add a folder", "misplaced": "Show", "zip": "Copy from it", + "unreadable": "Show", } @@ -10772,6 +10781,20 @@ def _tui_found_elsewhere(state: TuiState) -> dict[str, list[dict]]: return found +def _scan_lines(state: TuiState) -> list[str]: + """What the reading of the --from folders left out, one line each.""" + lines = [] + for scan in state.scanned: + if scan["truncated"]: + lines.append(f"Read the first {MAX_SCANNED_FILES:,} files of {scan['folder']}.") + lines += [f"{drive} is another drive: add it to copy from it too." + for drive in scan["other_drives"]] + if state.unreadable: + count = len(state.unreadable) + lines.append(f"{_plural(count, 'file')} on your old card could not be read.") + return lines + + def _last_name(folder: str) -> str: """A folder or ZIP by its last name, a drive by itself (E:\\).""" return Path(folder).name or folder @@ -10844,6 +10867,8 @@ def _plan_lines(check: dict, dest: Path | None, state: TuiState) -> list: hint.os_type = state.os_type lines.append("Or run it as root, this folder named: " + hint.command(state.platform, dest, elevated=True)) + if check is state.check: + lines += _scan_lines(state) if check.get("not_installable"): lines += _unavailable_lines(state) return lines @@ -11073,6 +11098,9 @@ def _pager_items(state: TuiState) -> list: """The text of a screen that scrolls, the same for its view and its keys.""" if state.screen == "misplaced": return [f"{event['dest']}: {event['reason']}" for event in _tui_misplaced(state)] + if state.screen == "unreadable": + return [f"{event['path']}" + (f", {event['member']}" if event["member"] else "") + + f": {event['reason']}" for event in state.unreadable] if state.screen == "result": return tui_outcome(state)[1] if state.screen == "command": @@ -11099,6 +11127,12 @@ def _view_preview(state: TuiState, width: int, height: int) -> tuple: ("A/Enter Back", "Up/Down Scroll")) +def _view_unreadable(state: TuiState, width: int, height: int) -> tuple: + return ("Files that could not be read", + _pager(state, width, height, _pager_items(state)), + ("A/Enter Back", "Up/Down Scroll")) + + def _view_misplaced(state: TuiState, width: int, height: int) -> tuple: return ("Files in the wrong place", _pager(state, width, height, _pager_items(state)), ("A/Enter Back", "Up/Down Scroll")) @@ -11136,10 +11170,9 @@ def _view_installing(state: TuiState, width: int, height: int) -> tuple: lines.append(_bar(min(done / total, 1.0) if total else 1.0, width)) lines.append(f"{format_size(done)} of {format_size(total)}") window = [sample for sample in state.samples if sample[0] >= state.now - 5] - fetched = done - state.local_done - start, base = window[0] if window else (state.now, fetched) + start, base = window[0] if window else (state.now, state.network) seconds = state.now - start - speed = (fetched - base) / seconds if seconds >= 1 else 0 + speed = (state.network - base) / seconds if seconds >= 1 else 0 if speed > 0: lines.append(f"{format_size(int(speed))}/s, " f"{format_duration((total - done) / speed)} left") @@ -11351,6 +11384,7 @@ _TUI_VIEWS: dict[str, Callable[[TuiState, int, int], tuple]] = { "preview": _view_preview, "command": _view_command, "confirm": _view_confirm, "installing": _view_installing, "result": _view_result, "error": _view_error, "quit": _view_quit, "cardroot": _view_cardroot, "misplaced": _view_misplaced, + "unreadable": _view_unreadable, } @@ -11893,7 +11927,20 @@ def run_tui(args: argparse.Namespace) -> int: os_type = _INVOCATION.os_type or detect_os() if os_type == "wsl": _enter_wsl() - _open_local_sources(args) + # The --from folders are read by each check, not here: indexing a large + # card held the first frame back. Only whether each can be read is asked. + folders = getattr(args, "from") or [] + setattr(args, "from", None) + try: + _open_local_sources(args) + finally: + setattr(args, "from", folders) + for folder in folders: + problem = from_folder_problem(str(folder)) + if problem is not None: + print(f"Error: {folder} {problem[1]}: --from names the folder to copy BIOS " + "files from.", file=sys.stderr) + return 1 state, effects = tui_start(args, os_type) work = Path(tempfile.mkdtemp(prefix=TUI_WORK_PREFIX)) (work / TUI_OWNER).write_text(str(os.getpid()), encoding="utf-8") diff --git a/install.sh b/install.sh index d2ad8edf..5a10ae72 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="e0a65cc08918a2f53032f63d08974995542a0beabb7528ac3de35d0fdb698e36" +DEFAULT_INSTALL_SHA256="1a26e1066a699413ab3d19f50ee552185ee2e9cbf74ee44548d3c9ea66f1b512" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 7feade1c..6a907996 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -883,6 +883,7 @@ def golden_states() -> dict: installing.run, installing.running = "install", True installing.download = {"event": "download", "files": 4, "bytes": 1064964} installing.samples, installing.now = [(100.0, 12672), (104.0, 540672)], 104.0 + installing.network = 540672 installing.progress = {"files_done": 2, "files_total": 4, "bytes_done": 540672, "bytes_total": 1064964} installing.finished = [("gba_bios.bin", True, ""), ("scph5500.bin", False, "HTTP 404")] @@ -1103,6 +1104,10 @@ class SmallFixes(unittest.TestCase): install.tui_receive(state, ("event", state.run_id, { "event": "progress", "files_done": 0, "files_total": 2, "bytes_done": done, "bytes_total": 1000 * mega})) + # The partial copy resumed is counted local by the engine. + install.tui_receive(state, ("event", state.run_id, { + "event": "transfer", "bytes_local": 500 * mega, + "bytes_network": done - 500 * mega})) text = "\n".join(install.tui_view(state, 80, 24)) self.assertIn("1.0 MB/s, 8:18 left", text) @@ -1713,10 +1718,17 @@ class CopyFromAnOldCard(unittest.TestCase): (10.0, {"event": "downloaded", "dest": "big.bin", "ok": True, "reason": "", "size": 200 * mega, "files_done": 1, "files_total": 2, "bytes_done": 200 * mega, "bytes_total": 300 * mega}), + # The engine says which bytes came from this machine. + (10.0, {"event": "transfer", "bytes_local": 200 * mega, + "bytes_network": 0}), (11.0, {"event": "progress", "files_done": 1, "files_total": 2, "bytes_done": 201 * mega, "bytes_total": 300 * mega}), + (11.0, {"event": "transfer", "bytes_local": 200 * mega, + "bytes_network": mega}), (12.0, {"event": "progress", "files_done": 1, "files_total": 2, - "bytes_done": 202 * mega, "bytes_total": 300 * mega})] + "bytes_done": 202 * mega, "bytes_total": 300 * mega}), + (12.0, {"event": "transfer", "bytes_local": 200 * mega, + "bytes_network": 2 * mega})] for when, event in events: state.now = when install.tui_receive(state, ("event", state.run_id, event)) @@ -2119,6 +2131,63 @@ class AuditRoundThree(unittest.TestCase): text = "\n".join(install.tui_view(checked_state(), 80, 24)) self.assertIn("Named by ~/.config/retroarch/retroarch.cfg", text) + # The engine's own events (main, after the second audit). + + def test_the_speed_is_the_network_bytes(self): + state = checked_state() + state.stack.append("installing") + state.run, state.running = "install", True + install.tui_receive(state, ("event", state.run_id, + {"event": "download", "files": 4, "bytes": 4 << 20})) + for now, done, network in ((100.0, 1 << 20, 0), (102.0, 3 << 20, 1 << 20)): + state.now = now + install.tui_receive(state, ("event", state.run_id, { + "event": "progress", "files_done": 1, "files_total": 4, "bytes_done": done, + "bytes_total": 4 << 20})) + install.tui_receive(state, ("event", state.run_id, { + "event": "transfer", "bytes_local": done - network, "bytes_network": network})) + text = "\n".join(install.tui_view(state, 80, 24)) + # 1 MB from the network in 2 s, whatever came from the old card. + self.assertIn("512.0 KB/s, 0:02 left", text) + + def test_the_plan_says_what_the_scan_left_out(self): + state = checked_state() + state.from_folders = [Path("E:\\")] + install._start_check(state) + events = [{"event": "scanned", "folder": "E:\\", "files": 200000, "truncated": True, + "other_drives": ["E:\\D"]}, + {"event": "unreadable", "platform": "retroarch", "folder": "E:\\", + "path": "E:\\bios\\scph5501.bin", "member": "", "reason": "Permission denied"}, + {"event": "unreadable", "platform": "retroarch", "folder": "E:\\", + "path": "E:\\backup.zip", "member": "gba_bios.bin", + "reason": "it is encrypted, a password is needed"}] + for event in [*FILES, *SYSTEMS, *events, CHECK]: + install.tui_receive(state, ("event", state.run_id, event)) + with mock.patch.object(install, "_tui_folder_facts", return_value={ + "free": 50 * 1024 ** 3, "writable": True, "cores": []}): + install.tui_receive(state, ("ended", state.run_id, 1, [])) + text = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn("Read the first 200,000 files of E:\\.", text) + self.assertIn("E:\\D is another drive: add it to copy from it too.", text) + self.assertIn("2 files on your old card could not be read.", text) + rows = install.tui_rows(state) + state.focus["plan"] = next(i for i, row in enumerate(rows) if row.action == "unreadable") + press(state, "enter") + self.assertEqual(state.screen, "unreadable") + self.assertEqual(install._pager_items(state), [ + "E:\\bios\\scph5501.bin: Permission denied", + "E:\\backup.zip, gba_bios.bin: it is encrypted, a password is needed"]) + + def test_the_interface_does_not_index_the_folders_itself(self): + missing = REPO_ROOT / "tmp" / "no-such-old-card" + err = io.StringIO() + with mock.patch.object(install, "FolderIndex", side_effect=AssertionError("indexed")), \ + mock.patch.object(install, "detect_os", return_value="linux"), \ + mock.patch.object(install, "tui_start", side_effect=AssertionError("opened")), \ + mock.patch.object(sys, "stderr", err): + self.assertEqual(install.run_tui(_args(**{"from": [missing]})), 1) + self.assertIn(f"Error: {missing} does not exist: --from names the folder", err.getvalue()) + # Work folders: only a kill no handler sees leaves one, and the next # run takes it away.