From a5e025b7eed02d17e6991b91a3abc2d3f8cbf228 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 18:23:22 +0200 Subject: [PATCH] feat: transfer event splits local and network The engine counts the bytes a copy, the pack or a resumed partial copy supplied, and those the network brought. A transfer event carries both after each progress event and once when the downloads end, so a screen shows a download speed without guessing. --- install.ps1 | 2 +- install.py | 28 ++++++++++++++++++--- install.sh | 2 +- tests/test_install.py | 5 +++- tests/test_install_from.py | 50 ++++++++++++++++++++++++++++++++++++++ wiki/installer.md | 1 + 6 files changed, 82 insertions(+), 6 deletions(-) diff --git a/install.ps1 b/install.ps1 index 573da24e..96a8a487 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "06fde26aa660b6e28c05010dd4a1de8f1ff1ab891d6ebbe460b670b30b71befd" + $defaultInstallSha256 = "c90676dcc433061bd97aca94930cd7acd74295624840cb5c85a22897f9c1dd12" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 297cf2d0..d7d51353 100644 --- a/install.py +++ b/install.py @@ -392,6 +392,9 @@ class Progress: self.files = 0 self.bytes = 0 self.resumed = 0 + # Bytes that came from this machine and not the network: a copy + # found here, the pack, a resumed partial copy. + self.local = 0 # Files written to their place, which a stopped run counts: the # queue's length said "118 files installed" over 8 on disk. self.installed = 0 @@ -400,11 +403,16 @@ class Progress: self._evented = self.start self._lock = threading.Lock() - def add(self, count: int, resumed: bool = False) -> None: + def add(self, count: int, resumed: bool = False, network: bool | None = None) -> None: + """count bytes written, resumed ones left out of the speed. network + is whether they came over the network, by default unless resumed.""" with self._lock: self.bytes += count if resumed: self.resumed += count + fetched = not resumed if network is None else network + if not fetched: + self.local += count now = time.monotonic() if self.term.live and not self.term.quiet and now - self._drawn >= 0.1: self._drawn = now @@ -413,10 +421,18 @@ class Progress: self._evented = now EVENTS.emit("progress", files_done=self.files, files_total=self.total_files, bytes_done=self.bytes, bytes_total=self.expected_bytes) + self.emit_transfer() + + def emit_transfer(self) -> None: + """The transfer event: of the bytes done, those this machine held + and those the network brought.""" + EVENTS.emit("transfer", bytes_local=self.local, + bytes_network=self.bytes - self.local) def forget_resumed(self, count: int) -> None: with self._lock: self.resumed -= count + self.local -= count def transferred(self) -> int: """Bytes fetched by this run, a resumed partial copy excluded.""" @@ -521,6 +537,11 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { "unreadable": ("platform", "folder", "path", "member", "reason"), "download": ("platform", "files", "bytes"), "progress": ("files_done", "files_total", "bytes_done", "bytes_total"), + # Right after each progress event, and once when the files of a + # download event are done: bytes_done split into the bytes this machine + # held (a copy found here, the pack, a resumed partial copy) and the + # bytes the network brought. A speed is bytes_network over time. + "transfer": ("bytes_local", "bytes_network"), "downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total", "bytes_done", "bytes_total"), # failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files", @@ -5172,6 +5193,7 @@ def download_files( if not future.cancelled() and future.exception() is None and future.result()[1] ) progress.finish() + progress.emit_transfer() return failed, interrupted @@ -6634,7 +6656,7 @@ def _write_verified( raise ValueError(f"{held_in} holds more bytes than the list records") out.write(chunk) if progress is not None: - progress.add(len(chunk), resumed=local) + progress.add(len(chunk), resumed=local, network=False) counted += len(chunk) if size != f["size"]: raise ValueError(f"{held_in} holds {size} bytes, the list records {f['size']}") @@ -6654,7 +6676,7 @@ def _write_verified( return f["dest"], False, f"{NOT_KEPT} ({exc})" except (_SourceUnreadable, ValueError, *_MEMBER_ERRORS) as exc: if progress is not None and counted: - progress.add(-counted, resumed=local) + progress.add(-counted, resumed=local, network=False) if isinstance(exc, OSError) and _write_refused(exc): return f["dest"], False, READ_ONLY if _read_only_file(dest) else NOT_WRITABLE return f["dest"], False, str(exc) diff --git a/install.sh b/install.sh index aa93f114..cb584208 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="06fde26aa660b6e28c05010dd4a1de8f1ff1ab891d6ebbe460b670b30b71befd" +DEFAULT_INSTALL_SHA256="c90676dcc433061bd97aca94930cd7acd74295624840cb5c85a22897f9c1dd12" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index d5103312..a7877ea2 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -5102,7 +5102,10 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree): "bytes_total": 245} for n, (dest, size, total) in enumerate( (("c.bin", 70, 70), ("d.bin", 25, 95), ("b.bin", 150, 245)), 1)] - self.assertEqual([e for e in events if e["event"] != "progress"], + # The last transfer: every byte came over the network. + self.assertEqual([e for e in events if e["event"] == "transfer"][-1], + {"event": "transfer", "bytes_local": 0, "bytes_network": 245}) + self.assertEqual([e for e in events if e["event"] not in ("progress", "transfer")], self._head("install", installer) + [ {"event": "download", "platform": "retroarch", "files": 3, "bytes": 245}, *done, diff --git a/tests/test_install_from.py b/tests/test_install_from.py index 9c04d1af..a0c74408 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -748,6 +748,56 @@ class WhereTheSearchGoes(FromFolder): self.assertLess(kinds.index("scanned"), kinds.index("detect")) +class LocalAndNetworkBytes(unittest.TestCase): + """The engine knows which bytes came from this machine. The interface + guessed it to show a download speed: the transfer event tells it.""" + + def test_the_counts_follow_each_source_and_each_rollback(self): + out = io.StringIO() + install.EVENTS.open(out) + self.addCleanup(setattr, install.EVENTS, "stream", None) + term = install.Terminal() + term.live, term.quiet = False, True + with mock.patch.object(install, "EVENT_PROGRESS_INTERVAL", 0): + progress = install.Progress(3, 100, term) + progress.add(10, resumed=True) # a partial copy resumed + progress.add(20) # from the network + progress.add(30, resumed=True, network=False) # a copy found here + progress.add(15, network=False) # the pack + progress.add(-30, resumed=True, network=False) + progress.add(-30) # a download dropped, + progress.forget_resumed(10) # its resumed part too + self.assertEqual((progress.bytes, progress.local), (15, 15)) + events = [json.loads(line) for line in out.getvalue().splitlines()] + transfers = [e for e in events if e["event"] == "transfer"] + progresses = [e for e in events if e["event"] == "progress"] + self.assertEqual(len(transfers), len(progresses)) + self.assertEqual([(t["bytes_local"], t["bytes_network"]) for t in transfers][:4], + [(10, 0), (10, 20), (40, 20), (55, 20)]) + for transfer, update in zip(transfers, progresses): + self.assertEqual(transfer["bytes_local"] + transfer["bytes_network"], + update["bytes_done"]) + + def test_an_install_from_a_folder_and_the_network_splits_them(self): + with FakeRepository() as repo: + old = repo.root / "old" + payloads = FakeRepository.PAYLOADS + old.mkdir() + (old / "a.bin").write_bytes(payloads["scph5500.bin"][0]) + (old / "b.bin").write_bytes(payloads["gba_bios.bin"][0]) + proc = repo.run("--from", str(old), "--no-input", "--events") + self.assertEqual(proc.returncode, 0, proc.stderr) + events = _events(proc) + local = len(payloads["scph5500.bin"][0]) + len(payloads["gba_bios.bin"][0]) + total = sum(len(p[0]) for p in payloads.values()) + kinds = [e["event"] for e in events] + last = len(kinds) - 1 - kinds[::-1].index("transfer") + self.assertEqual(events[last], {"event": "transfer", "bytes_local": local, + "bytes_network": total - local}) + self.assertLess(kinds.index("download"), last) + self.assertLess(last, kinds.index("summary")) + + class AFolderAroundTheBiosFolder(FromFolder): """--from names a folder holding the BIOS folder: an archive this run replaces was a source of members while it was being written.""" diff --git a/wiki/installer.md b/wiki/installer.md index be9c62bd..ea072e16 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1202,6 +1202,7 @@ kind, and each kind has exactly the keys below, in this order. The schema is | `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) | | `download` | `platform`, `files`, `bytes` | Before the downloads start. Counts the files copied from this machine too | | `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive | +| `transfer` | `bytes_local`, `bytes_network` | Right after each `progress` event, and once when the files of a `download` event are done. It splits `bytes_done`: `bytes_local` came from this machine (a copy found by `--from` or in the BIOS folder, the pack, a partial copy resumed), `bytes_network` from the network. A download speed is `bytes_network` over the time elapsed | | `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file, downloaded or copied from where a `found` event named it. `reason` says why one failed, and is empty otherwise | | `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `kept`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. `kept` lists `{folder, files, bytes}`, one per backup folder, for the files in place the run moved aside before replacing them (with `--dry-run`, would move), and is empty when none. Not written for a listing | | `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) |