diff --git a/install.ps1 b/install.ps1 index 5fb3dea2e..1a6afab37 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "e6f1825bd04698851d98e30ad9ff016d3fbf28ea5db2d65e5d7e25820026cd4a" + $defaultInstallSha256 = "7940e0785cce9f2bc1665c691ec0d5a122d43e01d60113e6220364d1c0b256ba" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index b367f5906..ac825668b 100644 --- a/install.py +++ b/install.py @@ -7377,6 +7377,22 @@ def data_directory_state(entry: dict, bios_path: Path, return root, needed, present +def data_space_bytes(data_states: list[tuple[dict, tuple[Path, dict[str, dict], int]]]) -> int: + """What the data trees take on the drive while they are rebuilt: every + file they lack, plus the largest source archive, which sits in its tree + until that tree is read. Only the BIOS files were held against the free + space: 18.8 KB passed on a card with 20 MB left, and 81.8 MB followed. + Nothing is written for them offline without a pack.""" + if LOCAL.pack is None and _offline_only(): + return 0 + files = sum(meta.get("size", 0) for _entry, (_root, needed, _present) in data_states + for meta in needed.values()) + if LOCAL.pack is not None: + return files + return files + max((data_fetch_bytes(entry, state[1]) for entry, state in data_states + if state[1]), default=0) + + def data_fetch_bytes(entry: dict, needed: dict[str, dict]) -> int: """What getting a tree's missing files costs: its source archive, which comes whole for one file, or from a pack the files themselves. A list @@ -15997,6 +16013,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: print(line) continue + space_checked = False if to_download: fetched = [f for f in to_download if id(f) not in copies] if _offline_only() and fetched: @@ -16093,9 +16110,11 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: print(" Press Ctrl+C to stop. Running the same command again " "carries on from here.") # Checked before the folder is created: a run stopped for space - # left an empty BIOS folder behind. - _check_free_space(bios_path, dl_size, plat_name, 1 if total_downloaded else 2, - on_its_card(plat_name, bios_path)) + # left an empty BIOS folder behind. The data trees count too, + # written after the files on the same drive. + _check_free_space(bios_path, dl_size + data_space_bytes(data_states), plat_name, + 1 if total_downloaded else 2, on_its_card(plat_name, bios_path)) + space_checked = True created = not bios_path.exists() try: bios_path.mkdir(parents=True, exist_ok=True) @@ -16173,6 +16192,12 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: total_errors += 1 else: writable.append((entry, state)) + if writable and not space_checked: + # With every BIOS file in place, the trees alone: checked + # before the first archive is written. + _check_free_space(bios_path, data_space_bytes(writable), plat_name, + 1 if total_downloaded else 2, + on_its_card(plat_name, bios_path)) if writable: print("\nData directories:") for entry, state in writable: diff --git a/install.sh b/install.sh index 0a873ae9d..1b1b80aeb 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="e6f1825bd04698851d98e30ad9ff016d3fbf28ea5db2d65e5d7e25820026cd4a" +DEFAULT_INSTALL_SHA256="7940e0785cce9f2bc1665c691ec0d5a122d43e01d60113e6220364d1c0b256ba" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index f1dfad8b6..f7e3b9375 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1118187, - "sha256": "e6f1825bd04698851d98e30ad9ff016d3fbf28ea5db2d65e5d7e25820026cd4a", + "size": 1119606, + "sha256": "7940e0785cce9f2bc1665c691ec0d5a122d43e01d60113e6220364d1c0b256ba", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1171814, - "sha256": "848576a731e09308fb897e760acf62c566523b881b1eca4f2818bacc259747ad" + "size": 1173233, + "sha256": "43ff8a0cfeefb5b044aee1d31ebbbfcd75ba679f9f67bf96603e706202fb1397" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1118187, - "sha256": "e6f1825bd04698851d98e30ad9ff016d3fbf28ea5db2d65e5d7e25820026cd4a", + "size": 1119606, + "sha256": "7940e0785cce9f2bc1665c691ec0d5a122d43e01d60113e6220364d1c0b256ba", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1118187, - "sha256": "e6f1825bd04698851d98e30ad9ff016d3fbf28ea5db2d65e5d7e25820026cd4a", + "size": 1119606, + "sha256": "7940e0785cce9f2bc1665c691ec0d5a122d43e01d60113e6220364d1c0b256ba", "executable": false }, { diff --git a/tests/test_install.py b/tests/test_install.py index 4304e6bd1..b2b24689e 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -411,6 +411,84 @@ class TestFreeSpaceGuard(unittest.TestCase): self.assertLess(check, source.index("bios_path.mkdir(parents=True, exist_ok=True)")) +class TheDataTreesCountInTheFreeSpace(unittest.TestCase): + """The space check held only the BIOS files: 18.8 KB passed on a card + with 20 MB free, then 55.4 MB of archives came and 81.8 MB were written, + against the page's promise to refuse before writing anything.""" + + def _run(self, *frees_after_bios: int) -> list[tuple[int, str, list]]: + """One run per free size, in one folder: the second finds the BIOS + files the first installed.""" + from types import SimpleNamespace + + from test_install_selection import FakeRepository # noqa: PLC0415 + + member = os.urandom(4096) + results = [] + with FakeRepository() as repo, unittest.mock.patch.dict(os.environ): + listed = json.loads((repo.root / "install" / "retroarch.json").read_text()) + listed["data_directories"] = [{ + "ref": "tree", "dest": "tree", "type": "zip", "archive_size": 50_000_000, + "url": "https://example.invalid/tree.zip", "systems": ["nintendo-gba"], + "files": {"Sys/a.bin": {"sha1": hashlib.sha1(member).hexdigest(), + "size": len(member)}}}] + (repo.root / "install" / "retroarch.json").write_text(json.dumps(listed)) + bios = sum(len(payload) for payload, *_ in FakeRepository.PAYLOADS.values()) + base = repo.env()["RETROBIOS_BASE_URL"] + for after in frees_after_bios: + out = io.StringIO() + fetched: list = [] + free = bios + after + with unittest.mock.patch.object( + sys, "argv", ["install.py", "--platform", "retroarch", "--dest", + str(repo.dest), "--no-input"]), \ + unittest.mock.patch.object(install, "MANIFEST_URL", + base + "/install/{platform}.json"), \ + unittest.mock.patch.object(install, "RAW_FILE_URL", base + "/{path}"), \ + unittest.mock.patch.object(install.shutil, "disk_usage", + lambda _p, free=free: SimpleNamespace(free=free)), \ + unittest.mock.patch.object( + install, "install_data_directory", + lambda entry, *a, fetched=fetched, **k: fetched.append(entry["ref"]) or { + "present": 0, "installed": 1, "changed": 0, "absent": 0, + "error": "", "not_kept": []}), \ + contextlib.redirect_stdout(out), contextlib.redirect_stderr(out): + try: + install.main() + code = 0 + except SystemExit as exc: + code = exc.code + results.append((code, out.getvalue(), repo.installed() + fetched)) + return results + + def test_a_card_short_for_the_trees_is_refused_before_any_write(self): + ((code, said, written),) = self._run(20_000_000) + self.assertEqual(code, 2, said) + self.assertIn("Not enough space", said) + self.assertEqual(written, []) + + def test_room_for_the_trees_goes_on(self): + ((code, said, written),) = self._run(60_000_000) + self.assertEqual(code, 0, said) + self.assertIn("tree", written) + + def test_the_trees_alone_are_checked_too(self): + """Every BIOS file in place: the trees were written unchecked.""" + _first, (code, said, written) = self._run(60_000_000, -1) + self.assertEqual(code, 2, said) + self.assertIn("Not enough space", said) + self.assertNotIn("tree", written) + + def test_the_bytes_counted(self): + state = (Path("x"), {"a": {"size": 10}, "b": {"size": 5}}, 0) + entry = {"archive_size": 1000} + with unittest.mock.patch.object(install.LOCAL, "pack", None), \ + unittest.mock.patch.object(install, "_offline_only", return_value=False): + self.assertEqual(install.data_space_bytes([(entry, state), (entry, state)]), 1030) + with unittest.mock.patch.object(install.LOCAL, "pack", object()): + self.assertEqual(install.data_space_bytes([(entry, state)]), 15) + + class AMissingDestIsSaid(unittest.TestCase): """--dest ~/.config/retroarh/system (a typo) was created with every missing parent and reported done, and --check on it said only diff --git a/wiki/installer.md b/wiki/installer.md index 934e92a68..dbb48e664 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -445,7 +445,8 @@ The figures change with each release. The size depends on the platform and changes with each release. The run prints it before downloading, and asks first when the download reaches 1 GB. The installer also measures the free space and refuses before writing anything if -there is not enough. +there is not enough. The count holds the BIOS files, every data file still to +come, and the largest source archive, which sits in its tree while it is read. `curl` or `wget`, and Python 3.8 or newer, are checked by the command itself: it stops with a message naming what is absent. PowerShell's execution policy