From 97d0a478d2fd3a1d62cdbee62ed911c372df5712 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:18:59 +0100 Subject: [PATCH 1/7] fix: read RetroArch's config under XDG_CONFIG_HOME Native RetroArch on Linux keeps retroarch.cfg under $XDG_CONFIG_HOME/retroarch when the variable is set, and falls back to ~/.retroarch.cfg (file_path_special.c fill_pathname_application_data, configuration.c open_default_config_file at e98e1da0). Detection read ~/.config/retroarch only. --- install.ps1 | 2 +- install.py | 31 +++++++++++++++++------- install.sh | 2 +- install/launchers/index.json | 16 ++++++------- install/retroarch.json | 12 ++++++++++ platforms/_registry.yml | 13 ++++++++++ tests/test_install.py | 46 ++++++++++++++++++++++++++++++++++++ 7 files changed, 104 insertions(+), 18 deletions(-) diff --git a/install.ps1 b/install.ps1 index 896cdaf76..e41f7f32d 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858" + $defaultInstallSha256 = "f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 01c0b9425..05a057480 100644 --- a/install.py +++ b/install.py @@ -291,7 +291,6 @@ def _os_default_dests(os_type: str) -> dict[str, Path]: the command line but not found must not land in ~/bios, which none of them reads. """ - home = Path.home() if os_type == "windows": return { # RetroArch's system directory defaults to \system @@ -307,13 +306,13 @@ def _os_default_dests(os_type: str) -> dict[str, Path]: # RetroArch may be the Linux one inside the distribution, while RetroBat # only exists on the Windows side, reached through /mnt/c. return { - "retroarch": home / ".config" / "retroarch" / "system", + "retroarch": _unix_retroarch_base() / "system", "retrobat": _host_path("C:/RetroBat/bios"), } if os_type == "darwin": return {"retroarch": _macos_retroarch_base() / "system"} return { - "retroarch": home / ".config" / "retroarch" / "system", + "retroarch": _unix_retroarch_base() / "system", } @@ -1385,6 +1384,17 @@ def _macos_retroarch_base() -> Path: return Path.home() / "Documents" / "RetroArch" +def _unix_retroarch_base() -> Path: + """RetroArch's base folder on Linux: $XDG_CONFIG_HOME/retroarch when the + variable is set, else ~/.config/retroarch (fill_pathname_application_data + in file_path_special.c, frontend_unix_get_env in platform_unix.c). The + code tests only that the variable exists, so an empty one counts.""" + xdg = os.environ.get("XDG_CONFIG_HOME") + if xdg is not None: + return Path(xdg) / "retroarch" + return Path.home() / ".config" / "retroarch" + + # System directory -> the retroarch.cfg it was read from and the directory # ':' expands to there, for finding the cores of the same installation. _RETROARCH_CONFIGS: dict[Path, tuple[Path, Path]] = {} @@ -2773,11 +2783,16 @@ def detect_platforms(os_type: str) -> list[tuple[str, Path]]: if ra_dir: found.append(("retroarch", ra_dir)) - # RetroArch native - native_cfg = home / ".config" / "retroarch" / "retroarch.cfg" - ra_dir = _parse_retroarch_system_dir(native_cfg) - if ra_dir: - found.append(("retroarch", ra_dir)) + # RetroArch native: the config under its base folder, then + # ~/.retroarch.cfg when that one is missing (open_default_config_file + # in configuration.c). Either way the system folder defaults to the + # base folder's. + native_base = _unix_retroarch_base() + for native_cfg in (native_base / "retroarch.cfg", home / ".retroarch.cfg"): + ra_dir = _parse_retroarch_system_dir(native_cfg, native_base) + if ra_dir: + found.append(("retroarch", ra_dir)) + break if os_type == "darwin": # open_default_config_file reads /RetroArch/config diff --git a/install.sh b/install.sh index b085a4cbb..26c45e4c6 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858" +DEFAULT_INSTALL_SHA256="f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index 358ee41c9..5ee293708 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1103894, + "sha256": "f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1156738, - "sha256": "35c0920008baadca997d0f778e99ce46d702bbe3d3f0d33104d7158422b292aa" + "size": 1157521, + "sha256": "0f9b070334a530a0ed5a6b6b773928d92f5a08aa0d0af18fde7b6cb4a16e731d" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1103894, + "sha256": "f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1103111, - "sha256": "34fd006cc0456c61b9a621e52f35cee6d3fd55ff14bbc6cea207ddc7bc57d858", + "size": 1103894, + "sha256": "f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0", "executable": false }, { diff --git a/install/retroarch.json b/install/retroarch.json index 024fb6396..166d197d3 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -21,12 +21,24 @@ "config": "$HOME/snap/retroarch/current/.config/retroarch/retroarch.cfg", "parse_key": "system_directory" }, + { + "os": "linux", + "method": "config_file", + "config": "$XDG_CONFIG_HOME/retroarch/retroarch.cfg", + "parse_key": "system_directory" + }, { "os": "linux", "method": "config_file", "config": "$HOME/.config/retroarch/retroarch.cfg", "parse_key": "system_directory" }, + { + "os": "linux", + "method": "config_file", + "config": "$HOME/.retroarch.cfg", + "parse_key": "system_directory" + }, { "os": "darwin", "method": "config_file", diff --git a/platforms/_registry.yml b/platforms/_registry.yml index 891b01a95..a2da8d55c 100644 --- a/platforms/_registry.yml +++ b/platforms/_registry.yml @@ -219,10 +219,23 @@ platforms: method: config_file config: $HOME/snap/retroarch/current/.config/retroarch/retroarch.cfg parse_key: system_directory + # Native: $XDG_CONFIG_HOME/retroarch when the variable is set, else + # $HOME/.config/retroarch, then $HOME/.retroarch.cfg when that file is + # missing (file_path_special.c fill_pathname_application_data, + # configuration.c open_default_config_file). A value left at default + # is the system folder of the first of the two folders. + - os: linux + method: config_file + config: $XDG_CONFIG_HOME/retroarch/retroarch.cfg + parse_key: system_directory - os: linux method: config_file config: $HOME/.config/retroarch/retroarch.cfg parse_key: system_directory + - os: linux + method: config_file + config: $HOME/.retroarch.cfg + parse_key: system_directory # configuration.c reads /RetroArch/config, and a value # left at default is ~/Documents/RetroArch/system (platform_darwin.m). # Older builds kept the file one level up. diff --git a/tests/test_install.py b/tests/test_install.py index f36389a1e..dbef10dd9 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -2646,6 +2646,52 @@ class RetroArchWhereItReallyLives(unittest.TestCase): self.assertIn(("retroarch", support / "system"), found) +class LinuxRetroArchFollowsXdgConfigHome(unittest.TestCase): + """RetroArch on Linux keeps its config under $XDG_CONFIG_HOME/retroarch + when the variable is set, then falls back to ~/.retroarch.cfg + (file_path_special.c fill_pathname_application_data, configuration.c + open_default_config_file). Only ~/.config/retroarch was read: a home + with its own XDG layout detected nothing.""" + + def _env(self) -> tuple[Path, Path]: + home = Path(_tempdir(self)) + xdg = Path(_tempdir(self)) + patcher = unittest.mock.patch.dict(os.environ, {"HOME": str(home)}, clear=True) + patcher.start() + self.addCleanup(patcher.stop) + return home, xdg + + def test_the_config_under_xdg_config_home_is_read(self): + home, xdg = self._env() + (xdg / "retroarch").mkdir() + (xdg / "retroarch" / "retroarch.cfg").write_text('system_directory = "/srv/bios"\n') + # A config under ~/.config is not the one this RetroArch opens. + (home / ".config" / "retroarch").mkdir(parents=True) + (home / ".config" / "retroarch" / "retroarch.cfg").write_text( + 'system_directory = "/elsewhere"\n') + os.environ["XDG_CONFIG_HOME"] = str(xdg) + found = install.detect_platforms("linux") + self.assertIn(("retroarch", Path("/srv/bios")), found) + self.assertNotIn(("retroarch", Path("/elsewhere")), found) + self.assertEqual(install._os_default_dests("linux")["retroarch"], + xdg / "retroarch" / "system") + + def test_home_retroarch_cfg_is_the_fallback(self): + home, xdg = self._env() + os.environ["XDG_CONFIG_HOME"] = str(xdg) + (home / ".retroarch.cfg").write_text('system_directory = "default"\n') + found = install.detect_platforms("linux") + self.assertIn(("retroarch", xdg / "retroarch" / "system"), found) + + def test_without_the_variable_the_folder_is_under_dot_config(self): + home, _ = self._env() + (home / ".config" / "retroarch").mkdir(parents=True) + (home / ".config" / "retroarch" / "retroarch.cfg").write_text( + 'system_directory = "default"\n') + found = install.detect_platforms("linux") + self.assertIn(("retroarch", home / ".config" / "retroarch" / "system"), found) + + class NoPythonNamesThisSystemsCommand(unittest.TestCase): """Without Python the bootstrap said `sudo apt install python3` on every system: Termux has no sudo and installs with `pkg install python`, and From f696d4974fb4a545afc98fef323ca15593b1a987 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:22:18 +0100 Subject: [PATCH 2/7] fix: name a --dest that is a file A BIOS folder that is a file, or lies under one, failed at mkdir with EEXIST or ENOTDIR, and the run blamed permissions and offered sudo. It is now refused before anything is checked, with an error event whose reason is not_a_folder. --- install.ps1 | 2 +- install.py | 28 ++++++++++++++++++++++++++-- install.sh | 2 +- install/launchers/index.json | 16 ++++++++-------- tests/test_install.py | 34 ++++++++++++++++++++++++++++++++++ wiki/installer.md | 4 ++-- 6 files changed, 72 insertions(+), 14 deletions(-) diff --git a/install.ps1 b/install.ps1 index e41f7f32d..57aca4f16 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0" + $defaultInstallSha256 = "482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 05a057480..baa21de99 100644 --- a/install.py +++ b/install.py @@ -680,8 +680,9 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { # found nowhere and no folder to assume) or from (a folder named with --from, # reason: not_found, not_a_folder, unreadable, or system for the root # of the system and /proc, /sys, /dev) or dest (reason: unreadable, a - # folder above it this user cannot open, or not_writable, the BIOS - # folder cannot be created, a summary following) or option (reason: + # folder above it this user cannot open, not_a_folder, the BIOS folder + # is a file or lies under one, or not_writable, the BIOS folder cannot + # be created, a summary following) or option (reason: # unknown, a platform, target, system, core or emulator the run cannot # use, refused before any download). "error": ("stage", "reason", "platform", "message"), @@ -13773,6 +13774,27 @@ def _refuse_closed(dest: Path, named: str) -> None: sys.exit(1) +def _refuse_file(dest: Path, named: str) -> None: + """End the run on a BIOS folder that is a file, or lies under one. + + mkdir then failed on EEXIST or ENOTDIR, and the run blamed permissions + and offered sudo, which fails the same way. + """ + for folder in (dest, *dest.parents): + if _exists(folder): + break + else: + return + if _is_dir(folder): + return + where = (f"{shown_path(dest)} is a file, not a folder" if folder == dest + else f"{shown_path(dest)} cannot be a folder: {shown_path(folder)} is a file") + EVENTS.emit("error", stage="dest", reason="not_a_folder", platform="", message=where) + print(f"Error: {screen_text(where)}. Name the folder the BIOS files go in, not " + f"the file {named}.", file=sys.stderr) + sys.exit(1) + + def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest", again: bool = False) -> bool: """Say that a --dest folder does not exist before anything goes there. @@ -13787,6 +13809,7 @@ def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest", a card's folder ended the run where it meant "not that card". """ _refuse_closed(dest, named) + _refuse_file(dest, named) if dest.exists(): return True nearest = dest.absolute() @@ -15114,6 +15137,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: else f"{found_in[0]} names" if found_in else "detected here") # One setup by now: two found without questions stopped above. _refuse_closed(path, f"{named} (name another with --dest)") + _refuse_file(path, f"{named} (name another with --dest)") total_downloaded = 0 total_up_to_date = 0 diff --git a/install.sh b/install.sh index 26c45e4c6..4f1d02bbf 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0" +DEFAULT_INSTALL_SHA256="482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index 5ee293708..6f8f8ff49 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1103894, - "sha256": "f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0", + "size": 1104874, + "sha256": "482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1157521, - "sha256": "0f9b070334a530a0ed5a6b6b773928d92f5a08aa0d0af18fde7b6cb4a16e731d" + "size": 1158501, + "sha256": "0ecf443039c136286322a57b33f1374dd52de0b5f2a772fbae93c1f98c19b62d" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1103894, - "sha256": "f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0", + "size": 1104874, + "sha256": "482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1103894, - "sha256": "f48287847612b616bcaf607bd50207353bef5648e650a732c2d9595a64726ab0", + "size": 1104874, + "sha256": "482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610", "executable": false }, { diff --git a/tests/test_install.py b/tests/test_install.py index dbef10dd9..7efea9660 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -448,6 +448,40 @@ class AMissingDestIsSaid(unittest.TestCase): self.assertIn("does not exist", out) +class ADestThatIsAFileIsNamed(unittest.TestCase): + """--dest naming a file, or a path under one, failed at mkdir with + EEXIST or ENOTDIR, and the run said this user cannot write there and + offered a sudo command that fails the same way.""" + + def _run(self, dest: Path) -> subprocess.CompletedProcess: + env = dict(os.environ, RETROBIOS_OS="linux") + return subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", str(dest), "--manifest", str(REPO_ROOT / "install" / "retroarch.json"), + "--system", "nintendo-gba", "--yes", "--events", "--offline"], + env=env, capture_output=True, text=True, timeout=120, check=False, + stdin=subprocess.DEVNULL) + + def test_the_file_is_named_and_sudo_is_not_offered(self): + root = Path(_tempdir(self)) + (root / "scph5501.bin").write_bytes(b"x") + for dest, words in ( + (root / "scph5501.bin", f"{root / 'scph5501.bin'} is a file, not a folder"), + (root / "scph5501.bin" / "sub", + f"cannot be a folder: {root / 'scph5501.bin'} is a file"), + ): + with self.subTest(dest=dest): + proc = self._run(dest) + self.assertEqual(proc.returncode, 1, proc.stderr) + self.assertIn(words, proc.stderr) + self.assertNotIn("sudo", proc.stderr) + errors = [json.loads(line) for line in proc.stdout.splitlines() + if line.startswith("{") and '"error"' in line] + self.assertEqual([(e["stage"], e["reason"]) for e in errors], + [("dest", "not_a_folder")]) + self.assertEqual((root / "scph5501.bin").read_bytes(), b"x") + + class AFolderThatRefusesWritesIsNamed(unittest.TestCase): """A read-only BIOS folder ended the run on a traceback (a subfolder to create), or on three retries per file and "once that is resolved", and a diff --git a/wiki/installer.md b/wiki/installer.md index 73719bf16..88da4c7b0 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1738,7 +1738,7 @@ fields, the installer refuses to narrow rather than install everything. | Code | Meaning | |------|---------| | `0` | everything asked for is installed or already in place | -| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 with its ssl, unicodedata and zlib modules (nothing was run), an unusable base URL, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed or were left out by `--offline` while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, a `--dest` that is not the card of the system named (another system's card, a folder inside one, a drive with none of its files where its card always has them, or a folder that does not exist), `--restore` could not put a file back, or `--add-app` could not put every file of the app on the card | +| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 with its ssl, unicodedata and zlib modules (nothing was run), an unusable base URL, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed or were left out by `--offline` while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, a BIOS folder that is a file or lies under one, a `--dest` that is not the card of the system named (another system's card, a folder inside one, a drive with none of its files where its card always has them, or a folder that does not exist), `--restore` could not put a file back, or `--add-app` could not put every file of the app on the card | | `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), or `--offline` left every file out, the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board`, `--target` with `--from-pack` or `--offline` (or `RETROBIOS_INSTALL_OFFLINE` set) when no core list per board is at hand (the pack carries none and none is beside `--manifest` as `targets/.json`), or for a platform that publishes none, which is said to have no hardware targets, `--list-targets` with `--offline` and neither, `--add-app` with `--check`, `--dry-run`, a listing, `--from-pack` or `--offline`. The message then starts with `install.py: error:`. Also an unknown platform, target, system, core or emulator, said in an `Error:` or `Unknown platform` line and an `error` event whose `stage` is `option`. Also `--add-app` for a folder on no card that takes the app, said in an `Error:` line and an `app` event before anything is written. With `--restore`, a folder that is not a dated folder inside `retrobios-backup`. Also a quit chosen before the install, as `gh` ends a cancelled prompt: `q` at a question, `q` or **Quit** on the full screen, which then keeps `RetroBIOS: quit. Nothing was installed.`, and the full screen closed with no button pressed (`RETROBIOS_IDLE_EXIT`). No code but `0` closes a Windows Terminal tab started on the installer itself, so such a tab stays open on a quit | | `130` | stopped with Ctrl+C, or with Ctrl+C or Ctrl+D at a question, the terminal keeping `RetroBIOS: stopped. Nothing was installed.` before an install. Run the same command again to carry on | | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) | @@ -1785,7 +1785,7 @@ other schemas under `api/v1/schemas/`. | `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` | | `unsearched` | `folder`, `reason`, `message` | Before `detect`, once for each folder the search for setups could not look in, this user being refused it or a read failing there (a failing card, a link that loops). What it would hold counts as absent, and the other places are still searched: `detect` then lists what they hold, and an empty `platforms` after an `unsearched` line means nothing was found where the installer could look. `reason` says why in words (`Permission denied`), `message` says it in a sentence | | `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found | -| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), `no_ssl` when this Python was built without its `ssl` module and cannot use HTTPS, or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). The BIOS folder sits under a folder this user cannot open (`stage` is `dest`, `reason` `unreadable`), and the run ends with code 1 before anything is written. A platform, target, system, core or emulator named on the command line is unknown or cannot be used here (`stage` is `option`, `reason` `unknown`, `platform` the platform it was checked against or the name typed), refused before any download with code 2. The BIOS folder cannot be created (`stage` is `dest`, `reason` `not_writable`): a `summary` follows with what the run installed before it. Two systems are detected on one machine and no `--dest` names the folder (`stage` is `detect`, `reason` `conflict`). The platform named with `--platform` is found nowhere and no folder can be assumed for it: a card system with no card named, a portable emulator, Batocera or Recalbox named on a computer (their folder is the `bios` folder of their SHARE drive), or no usual folder on this system or one set aside behind a folder this user cannot open (`stage` is `detect`, `reason` `not_found`). So is a platform named by a file list or a pack, in a run without questions where it is found nowhere here. A `--dest` refused as the card of the system named carries the same `stage` and `reason`: another system's card, a folder inside a card, a drive with none of the files its card always has, or a folder that does not exist. `message` says it in words, and is the only thing that tells these cases apart | +| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one), `no_ssl` when this Python was built without its `ssl` module and cannot use HTTPS, or `offline` when `--offline` forbids the request. A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). The BIOS folder sits under a folder this user cannot open (`stage` is `dest`, `reason` `unreadable`), or is a file or lies under one (`reason` `not_a_folder`), and the run ends with code 1 before anything is written. A platform, target, system, core or emulator named on the command line is unknown or cannot be used here (`stage` is `option`, `reason` `unknown`, `platform` the platform it was checked against or the name typed), refused before any download with code 2. The BIOS folder cannot be created (`stage` is `dest`, `reason` `not_writable`): a `summary` follows with what the run installed before it. Two systems are detected on one machine and no `--dest` names the folder (`stage` is `detect`, `reason` `conflict`). The platform named with `--platform` is found nowhere and no folder can be assumed for it: a card system with no card named, a portable emulator, Batocera or Recalbox named on a computer (their folder is the `bios` folder of their SHARE drive), or no usual folder on this system or one set aside behind a folder this user cannot open (`stage` is `detect`, `reason` `not_found`). So is a platform named by a file list or a pack, in a run without questions where it is found nowhere here. A `--dest` refused as the card of the system named carries the same `stage` and `reason`: another system's card, a folder inside a card, a drive with none of the files its card always has, or a folder that does not exist. `message` says it in words, and is the only thing that tells these cases apart | | `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered | | `version` | `platform`, `list_version`, `installed_version`, `message` | Before `choices`, when the system here runs another version than the one the list was built for: `list_version` is the list's, `installed_version` the one the system's version file states, and `message` says it and what it means. Not written when either version is unknown. The run goes on | | `board` | `platform`, `board`, `applied`, `message` | Before `choices`, with no `--target` and no `--every-board`, when the system names its board or a Batocera or KNULLI card mounted here does (`boot/batocera.board` or `boot/knulli.board` on its boot partition, `board` then being the board its image maps to): `applied` is true when the run keeps that board's files only, false for a board the platform's list does not name, which keeps every board's files. `message` says it | From b3f8f64395654073fc8e588aee064d736a546494 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:33:01 +0100 Subject: [PATCH 3/7] fix: refuse options a listing does not read Two listing options printed one list, and --list-platforms, --list-targets and --list-emulators printed the same output with --system, --region or --core as without, code 0. A run now prints one list, and an option a listing ignores is refused. --- install.ps1 | 2 +- install.py | 25 +++++++++++++++++++- install.sh | 2 +- install/launchers/index.json | 16 ++++++------- tests/test_install.py | 44 +++++++++++++++++++++++++++++++++++ tests/test_install_offline.py | 7 +++++- wiki/installer.md | 10 ++++---- 7 files changed, 89 insertions(+), 17 deletions(-) diff --git a/install.ps1 b/install.ps1 index 57aca4f16..dab7b3426 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610" + $defaultInstallSha256 = "7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index baa21de99..429dffa35 100644 --- a/install.py +++ b/install.py @@ -13429,6 +13429,29 @@ def main() -> None: if (args.list_platforms or args.list_targets or args.list_systems or args.list_cores or args.list_emulators): parser.error("--for-games narrows an install, not a listing") + # Applied or refused, never ignored: a run prints one list, and the + # narrowing options apply to --list-systems and --list-cores only, past + # --target and --installed-cores, which --list-emulators also reads. + listings = [flag for flag, given in ( + ("--list-platforms", args.list_platforms), ("--list-targets", args.list_targets), + ("--list-systems", args.list_systems), ("--list-cores", args.list_cores), + ("--list-emulators", args.list_emulators)) if given] + if len(listings) > 1: + parser.error(f"{listings[0]} and {listings[1]} each print a list: run them " + "one at a time") + if listings and listings[0] in ("--list-platforms", "--list-targets", + "--list-emulators"): + unread = (("--system", args.system), ("--core", args.core), + ("--region", args.region), ("--emulator", args.emulator)) + if listings[0] != "--list-emulators": + unread += (("--target", args.target), ("--every-board", args.every_board), + ("--installed-cores", args.installed_cores)) + if listings[0] == "--list-platforms": + unread += (("--platform", args.platform),) + given = next((flag for flag, value in unread if value), "") + if given: + parser.error(f"{listings[0]} prints the whole list, which {given} " + "does not narrow") if args.declared_only: # Applied or refused, never ignored. for given, reason in ( @@ -15176,7 +15199,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # --installed-cores is below: EmuDeck installed 3.2 GB and only then was # steamos refused for RetroArch, with no summary of what had landed. target_lists: dict[str, dict] = {} - if args.target and not args.list_targets: + if args.target: for plat_name, _bios_path in platforms: targets = fetch_targets(plat_name) if targets is None: diff --git a/install.sh b/install.sh index 4f1d02bbf..cbdaed395 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610" +DEFAULT_INSTALL_SHA256="7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index 6f8f8ff49..2be9a4f89 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1104874, - "sha256": "482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610", + "size": 1106280, + "sha256": "7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1158501, - "sha256": "0ecf443039c136286322a57b33f1374dd52de0b5f2a772fbae93c1f98c19b62d" + "size": 1159907, + "sha256": "6460af3bc1a9d27c7373b42cf45f19bf8619b8e55daac9892af9f57df21b258e" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1104874, - "sha256": "482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610", + "size": 1106280, + "sha256": "7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1104874, - "sha256": "482a7710467d988f3878a5d8ebe5f885dfee35c3a1aac0df05077b6b062a7610", + "size": 1106280, + "sha256": "7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300", "executable": false }, { diff --git a/tests/test_install.py b/tests/test_install.py index 7efea9660..658385670 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -4475,6 +4475,50 @@ def _install_run(env: dict, *argv: str, **kwargs) -> subprocess.CompletedProcess ) +class AListingAppliesItsOptionsOrRefusesThem(unittest.TestCase): + """--list-systems --list-cores printed the systems alone, and + --list-platforms, --list-targets and --list-emulators printed the same + list with --system, --region or --core as without, code 0.""" + + def test_two_listings_or_an_option_a_listing_ignores_are_refused(self): + env = dict(os.environ, RETROBIOS_OS="linux") + given = ["--platform", "retroarch", "--offline", "--no-input", + "--manifest", "install/retroarch.json"] + for argv, words in ( + ([*given, "--list-systems", "--list-cores"], + "--list-systems and --list-cores each print a list"), + ([*given, "--list-cores", "--list-emulators"], + "--list-cores and --list-emulators each print a list"), + (["--list-platforms", "--list-systems", "--platform", "retroarch"], + "--list-platforms and --list-systems each print a list"), + (["--list-platforms", "--region", "us"], + "--list-platforms prints the whole list, which --region does not narrow"), + (["--list-platforms", "--platform", "retroarch"], + "--list-platforms prints the whole list, which --platform does not narrow"), + ([*given, "--list-targets", "--system", "sony-playstation"], + "--list-targets prints the whole list, which --system does not narrow"), + ([*given, "--list-targets", "--target", "switch"], + "--list-targets prints the whole list, which --target does not narrow"), + ([*given, "--list-emulators", "--system", "sony-playstation"], + "--list-emulators prints the whole list, which --system does not narrow"), + ([*given, "--list-emulators", "--core", "beetle_psx"], + "--list-emulators prints the whole list, which --core does not narrow"), + ): + with self.subTest(argv=argv): + proc = _install_run(env, *argv) + self.assertEqual(proc.returncode, 2, proc.stdout + proc.stderr) + self.assertIn(words, proc.stderr) + + def test_the_options_a_listing_reads_still_apply(self): + env = dict(os.environ, RETROBIOS_OS="linux") + given = ["--platform", "retroarch", "--offline", "--no-input", + "--manifest", "install/retroarch.json"] + whole = _install_run(env, *given, "--list-cores") + narrowed = _install_run(env, *given, "--list-cores", "--system", "nintendo-gba") + self.assertEqual((whole.returncode, narrowed.returncode), (0, 0), narrowed.stderr) + self.assertLess(len(narrowed.stdout), len(whole.stdout)) + + class AMissingPackNamesTheOptionAndThePacksBeside(unittest.TestCase): """A typo in --from-pack said "Error: X does not exist": not the option, not the pack sitting next to the name typed.""" diff --git a/tests/test_install_offline.py b/tests/test_install_offline.py index b46ea29be..08d9194d2 100644 --- a/tests/test_install_offline.py +++ b/tests/test_install_offline.py @@ -778,7 +778,12 @@ class OfflineMakesNoRequest(Scratch): self.assertIn("--offline", proc.stderr) def test_a_listing_and_a_pack_need_no_network_anyway(self): - proc = self.run_installer("--offline", "--list-platforms") + # --platform alone with it is refused: the listing names every one. + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--offline", "--list-platforms"], + env=_offline_env(), capture_output=True, text=True, timeout=120, + check=False, stdin=subprocess.DEVNULL, + ) self.assertEqual(proc.returncode, 0, proc.stderr) diff --git a/wiki/installer.md b/wiki/installer.md index 88da4c7b0..f66a8eb54 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -976,9 +976,9 @@ what the options name, everything by default. | `--required-only` | With `--for-games`, only the files the games cannot start without | | `--no-game-data` | Leave out game data (a game's own files, not firmware) and install the firmware only. What it leaves out is named. See [Firmware without game data](#firmware-without-game-data) | | `--declared-only` | Keep only the files the platform's own list names (`cores: null` in the manifest), not the extra files cores load. With `--check`, the answer is about the list the platform itself checks. Refused with `--core`, the listings it does not change, `--standalone-copies` and `--tui` | -| `--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart | -| `--list-cores` | Print the cores whose extra files a platform installs | -| `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for | +| `--list-systems` | Print a platform's systems with the files and size each holds, the files other systems share too counted apart. `--system`, `--core`, `--region`, `--target` and `--installed-cores` narrow it, as they narrow an install. A run prints one list: two listing options are refused | +| `--list-cores` | Print the cores whose extra files a platform installs. The options that narrow `--list-systems` narrow it too | +| `--list-emulators` | Print every name `--emulator` accepts, each buildbot name beside the emulator it stands for. `--target` and `--installed-cores` narrow it, and `--system`, `--core`, `--region` and `--emulator` are refused | | `--romm-firmware TEMPLATE` | RomM: where each platform's firmware goes in the library, as `filesystem.structure.firmware` in RomM's `config.yml` says (`{platform}/bios` for a library laid out as `{platform}/roms`). Read from `config.yml` beside the library when it is there. See [RomM firmware layout](#romm-firmware-layout) | | `--no-input` | Never ask a question, even at a terminal. A run whose output goes to a file or a pipe asks nothing either, and says so on stderr. It finds the setups and the mounted cards the screen finds: one is taken, as Enter takes it, two stop the run with an `error` event (`stage` `detect`, `reason` `conflict`) naming each with its `--platform` and `--dest`, and none stops it too, with `reason` `not_found` when a file list or a pack names the platform | | `--check` | Report and exit without writing | @@ -994,8 +994,8 @@ what the options name, everything by default. | `--tui` | Open the full-screen interface, or stop with the reason it cannot open here (code 2) instead of falling back to the questions. Wins over `RETROBIOS_PLAIN` and `ACCESSIBLE`. See [The full-screen interface](#the-full-screen-interface) | | `--plain` | Ask the questions line by line instead of the full-screen interface, the mode for a screen reader. See [The full-screen interface](#the-full-screen-interface) | | `--version` | Print the installer's identity, the first 12 hex digits of the SHA-256 the bootstraps pin | -| `--list-platforms` | Print the supported platforms and what was detected here | -| `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries | +| `--list-platforms` | Print the supported platforms and what was detected here. An option that narrows a list (`--platform`, `--system`, `--core`, `--region`, `--emulator`, `--target`, `--every-board`, `--installed-cores`) is refused | +| `--list-targets` | Print the hardware targets a platform publishes, with the core count each carries. `--system`, `--core`, `--region`, `--emulator`, `--target`, `--every-board` and `--installed-cores` are refused | | `--jobs N`, `-j N` | Parallel downloads, 1 to 32, default 8 | | `--verbose`, `-v` | Print per-attempt failures and hash mismatches | | `--standalone-copies` | Also copy into detected standalone emulator directories | From d1d94f597910abb73baa5a5c551070f91abf9c08 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:44:22 +0100 Subject: [PATCH 4/7] fix: keep card and closed-folder tests apart The --dest refusal for a path under a file stopped two fixtures: muOS's MUOS marker was written as a file with the BIOS folder below it, and the folder that cannot be created was a path under a file. The marker is now a folder and that case uses a folder this user cannot write. --- tests/test_install_tui.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 24bbd3485..7e0b2e0b2 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -5898,7 +5898,10 @@ def _card_from_rules(root: Path, setup: install.CardSetup) -> Path: (root / "bios").mkdir(exist_ok=True) for value in rule.values: (root / value).parent.mkdir(parents=True, exist_ok=True) - if rule.method == "games_part" and "." not in PurePosixPath(value).name: + # A marker the BIOS folder sits under is a folder on the card + # (muOS's MUOS/bios): written as a file, no folder fits below it. + if (rule.method == "games_part" and "." not in PurePosixPath(value).name + or str(setup.below).startswith(f"{value}/")): (root / value).mkdir(exist_ok=True) else: (root / value).write_text("", encoding="utf-8") @@ -9763,14 +9766,21 @@ class ScreenRowsAndFilters(unittest.TestCase): from test_install_offline import PAYLOAD, _offline_env, _pack - blocker = self.root / "file" - blocker.write_text("not a folder") + if os.name == "nt" or os.geteuid() == 0: + self.skipTest("a folder that refuses writes is POSIX, and root writes anyway") + # A file in the way is refused before the plan, as not_a_folder. + blocker = self.root / "closed" + blocker.mkdir() pack = _pack(self.root, "RetroArch_v1_BIOS_Pack.zip", PAYLOAD) - proc = subprocess.run( - [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", - "--dest", str(blocker / "system"), "--from-pack", str(pack), "--events"], - env=_offline_env(), capture_output=True, text=True, timeout=120, check=False, - stdin=subprocess.DEVNULL) + os.chmod(blocker, 0o500) + try: + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", str(blocker / "system"), "--from-pack", str(pack), "--events"], + env=_offline_env(), capture_output=True, text=True, timeout=120, + check=False, stdin=subprocess.DEVNULL) + finally: + os.chmod(blocker, 0o700) events = [json.loads(line) for line in proc.stdout.splitlines()] kinds = [event["event"] for event in events] self.assertEqual(kinds[-3:], ["error", "summary", "end"], proc.stderr) From eb99131a6efa9419049393c75fbf52b15e6f116f Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:45:03 +0100 Subject: [PATCH 5/7] fix: stop advising --save-dir a list cannot serve RetroPie, ROCKNIX, AmberELEC, EmuELEC and RomM keep their save directory in no fixed place, and their lists leave those files out with no hash or address. The run said to name the folder with --save-dir, and said it again when --save-dir was given. It now says the list holds none of them. --- install.ps1 | 2 +- install.py | 20 +++++++++++++++++-- install.sh | 2 +- install/launchers/index.json | 16 +++++++-------- tests/test_install.py | 24 +++++++++++++++++++++++ tests/test_install_tui.py | 38 +++++++++++++++++++++++++++++++++++- wiki/installer.md | 5 ++++- 7 files changed, 93 insertions(+), 14 deletions(-) diff --git a/install.ps1 b/install.ps1 index dab7b3426..412a6456e 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300" + $defaultInstallSha256 = "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 429dffa35..feabcc09e 100644 --- a/install.py +++ b/install.py @@ -6456,7 +6456,8 @@ def _report_outcome( kept = _kept_lines() + ([ f"{_plural(len(unplaced), 'file')} read from " f"{unplaced[0].get('frontend') or 'the frontend'}'s save directory left out: " - f"its place is not known. {LINE_SAVE_REMEDY}"] if unplaced else []) + + (f"{unplaced[0]['save_why']}." if unplaced[0].get("save_why") + else f"its place is not known. {LINE_SAVE_REMEDY}")] if unplaced else []) kept += list(notes or []) if interrupted: say(f"\nStopped. {_plural(downloaded, 'file')} installed.") @@ -8636,6 +8637,13 @@ def saves_line(display: str, count: int, where: str) -> str: f"{shown_path(folder)}") +def unlisted_saves_words(display: str) -> str: + """Why a file list holds none of the files read from the save + directory: the platform keeps it in no fixed place, and the list leaves + them out with no hash or address, which --save-dir cannot change.""" + return f"{display} keeps it in no fixed place, and its file list holds none of them" + + def unplaced_line(display: str, count: int, remedy: str) -> str: """The files read from an unknown save directory, left out.""" return (f"{_plural(count, 'file')} read from {display}'s save directory left out: " @@ -15395,6 +15403,10 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: display = display_name(plat_name, manifest) files, unplaced = place_save_entries( files, bios_path, manifest, args.save_dir, display, say=False) + if not manifest.get("save_directory"): + # Its own list leaves these files out: naming the folder with + # --save-dir installs none of them, and was advised in a loop. + _SAVE_WHY[bios_path] = unlisted_saves_words(display) # The games of the card, read before the choices: where it holds # some, what they need is the menu's first choice, and --for-games # narrows the list here, before the systems are counted. @@ -15597,7 +15609,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: total_omitted += len(omitted_files) # Named for the end of the run: ROCKNIX's own list leaves them out, # and the end said "the frontend's save directory". - omitted_seen.extend({**entry, "frontend": display} + omitted_seen.extend({**entry, "frontend": display, + "save_why": unlisted_saves_words(display)} if entry.get("reason") == "no_save_directory" and not entry.get("frontend") else entry for entry in omitted_files) @@ -21212,6 +21225,9 @@ def _unplaced_lines(summary: dict | None, state: TuiState, if not count: return [] name = display_name(state.platform) if state.platform else "the frontend" + if state.manifest and not state.manifest.get("save_directory"): + return [f"{_plural(count, 'file')} read from {name}'s save directory left out: " + f"{unlisted_saves_words(name)}."] return [f"{_plural(count, 'file')} read from {name}'s save directory left out: " f"its place is not known. {remedy or TUI_SAVE_REMEDY}"] diff --git a/install.sh b/install.sh index cbdaed395..c8cba304a 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300" +DEFAULT_INSTALL_SHA256="3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index 2be9a4f89..cd32da514 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1106280, - "sha256": "7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300", + "size": 1107258, + "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1159907, - "sha256": "6460af3bc1a9d27c7373b42cf45f19bf8619b8e55daac9892af9f57df21b258e" + "size": 1160885, + "sha256": "5ccc4ec3df34a1fef96caea9e84218de1fb92331664047c36d6863c11973dc3d" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1106280, - "sha256": "7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300", + "size": 1107258, + "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1106280, - "sha256": "7622c5e1a9ec7a76bf3c325b1fd356a22b53aefe0b625d6c12447afee9414300", + "size": 1107258, + "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", "executable": false }, { diff --git a/tests/test_install.py b/tests/test_install.py index 658385670..b8c72ae82 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -482,6 +482,30 @@ class ADestThatIsAFileIsNamed(unittest.TestCase): self.assertEqual((root / "scph5501.bin").read_bytes(), b"x") +class SaveFilesAListHoldsNoneOfAreNotPromised(unittest.TestCase): + """RetroPie, ROCKNIX, AmberELEC, EmuELEC and RomM keep their save + directory in no fixed place: their lists leave those files out with no + hash or address. The run said "Name it with --save-dir", and said it + again with --save-dir given, which installs none of them.""" + + def test_the_reason_is_said_and_no_option_is_offered(self): + manifest = json.loads((REPO_ROOT / "install" / "retropie.json").read_text()) + self.assertIsNone(manifest.get("save_directory")) + root = Path(_tempdir(self)) + env = dict(os.environ, RETROBIOS_OS="linux") + for extra in ([], ["--save-dir", str(root / "saves")]): + with self.subTest(extra=extra): + proc = _install_run( + env, "--platform", "retropie", "--dest", str(root / "bios"), + "--manifest", "install/retropie.json", "--offline", "--no-input", + "--yes", "--emulator", "azahar", *extra) + out = proc.stdout + proc.stderr + self.assertIn("read from RetroPie's save directory left out: RetroPie " + "keeps it in no fixed place, and its file list holds none " + "of them.", out) + self.assertNotIn("Name it with --save-dir", out) + + class AFolderThatRefusesWritesIsNamed(unittest.TestCase): """A read-only BIOS folder ended the run on a traceback (a subfolder to create), or on three retries per file and "once that is resolved", and a diff --git a/tests/test_install_tui.py b/tests/test_install_tui.py index 7e0b2e0b2..988b7234e 100644 --- a/tests/test_install_tui.py +++ b/tests/test_install_tui.py @@ -2967,12 +2967,44 @@ class SaveDirectoryLeftOut(unittest.TestCase): UNPLACED = {"reason": "no_save_directory", "label": "no save directory", "files": 2, "required": 0} + def _ended(self, manifest: dict) -> tuple: + state = fresh_state() + state.stack = ["setup"] + with mock.patch.object(install.Path, "exists", return_value=True): + install.tui_update(state, Key("enter")) + install.tui_receive(state, ("manifest", "retroarch", manifest)) + summary = {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 0, + "pending": 5, "failed": [], "skipped": [self.UNPLACED], "left_out": [], + "kept": [], "interrupted": False} + for event in [*FILES, *SYSTEMS, CHECK, summary]: + install.tui_receive(state, ("event", state.run_id, event)) + with mock.patch.object(install, "_tui_folder_facts", return_value={ + "free": 50 * 1024 ** 3, "writable": True, "cores": []}): + install.tui_receive(state, ("ended", state.run_id, 1, [])) + return state, summary + + def test_a_list_that_holds_none_of_them_offers_no_folder(self): + """RetroPie's list leaves its save files out with no hash or address: + --save-dir installs none of them, and was advised again each run.""" + state, summary = self._ended(MANIFEST) + line = ("2 files read from RetroArch's save directory left out: RetroArch keeps " + "it in no fixed place, and its file list holds none of them.") + plan = " ".join(" ".join(install.tui_view(state, 120, 40)).split()) + self.assertIn(line, plan) + self.assertNotIn("--save-dir", plan) + state.stack.append("result") + state.code = 0 + state.summary = {**summary, "mode": "install", "installed": 5} + self.assertEqual([text for text in install.tui_outcome(state)[1] if "save" in text], + [line]) + def test_files_left_out_of_the_save_directory_are_said(self): state = fresh_state() state.stack = ["setup"] with mock.patch.object(install.Path, "exists", return_value=True): install.tui_update(state, Key("enter")) - install.tui_receive(state, ("manifest", "retroarch", MANIFEST)) + install.tui_receive(state, ("manifest", "retroarch", { + **MANIFEST, "save_directory": {"folder": "saves", "by_core": True}})) summary = {"event": "summary", "mode": "check", "installed": 0, "up_to_date": 0, "pending": 5, "failed": [], "skipped": [self.UNPLACED], "left_out": [], "kept": [], "interrupted": False} @@ -3834,6 +3866,8 @@ class BackTicksAndSaveFolder(unittest.TestCase): def test_the_save_folder_remedy_is_the_same_on_both_screens(self): summary = {"skipped": [{"reason": "no_save_directory", "files": 1}]} state = checked_state() + state.manifest = {**state.manifest, "save_directory": {"folder": "saves", + "by_core": True}} lines = install._unplaced_lines(summary, state) self.assertIn("Options, Save folder (--save-dir)", lines[0]) @@ -11846,6 +11880,8 @@ class OneChoicePerRun(unittest.TestCase): def test_the_farewell_names_the_option_not_the_screen(self): state = _installed_state(SYSTEM_DIR) + state.manifest = {**state.manifest, "save_directory": {"folder": "saves", + "by_core": True}} state.summary = {**state.summary, "skipped": [ {"reason": "no_save_directory", "label": "", "files": 1, "required": False}]} farewell = " ".join(install.tui_farewell(state)) diff --git a/wiki/installer.md b/wiki/installer.md index f66a8eb54..0d6e5141b 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -386,7 +386,10 @@ the save directory beside it (`saves`, `/storage/savefiles` on Lakka). applies, or when `savefiles_in_content_dir` or `sort_savefiles_by_content_enable` keeps saves beside each game, those files are left out and the run says why, like the other entries the list cannot -install. The full screen says so on its plan and its result. +install. The full screen says so on its plan and its result. RetroPie, ROCKNIX, +AmberELEC, EmuELEC and RomM keep the save directory in no fixed place: their +lists hold none of these files, so `--save-dir` installs none, and the run says +that instead of naming the option. Batocera, KNULLI, Recalbox, RetroBat and RetroDECK write the save directory at each launch, one folder per system (`by_system` in the file list). Their lists From 725325f0a159a39a7dea64102edb8c37659814ac Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 08:59:10 +0100 Subject: [PATCH 6/7] fix: announce a data tree at its archive size A tarball tree was announced at its extracted size: yaps2 at 6.5 MB for a 21.2 MB download, Cemu's game profiles at 9.9 KB for 21.4 MB. refresh_data_dirs records the size of each archive it downloads (a tree current from before is measured, not refetched), the install lists carry it as archive_size, and the plan, the menu and the data event count it. --- install.ps1 | 2 +- install.py | 32 ++++++++++--- install.sh | 2 +- install/amberelec.json | 12 +++-- install/batocera.json | 6 ++- install/emudeck.json | 27 +++++++---- install/emuelec.json | 15 ++++--- install/knulli.json | 12 +++-- install/lakka.json | 21 ++++++--- install/launchers/index.json | 16 +++---- install/myminui.json | 14 +++--- install/nextui.json | 11 +++-- install/recalbox.json | 6 ++- install/retroarch.json | 30 ++++++++----- install/retrobat.json | 9 ++-- install/retrodeck.json | 15 ++++--- install/retropie.json | 18 +++++--- install/rocknix.json | 30 ++++++++----- schemas/install-manifest.schema.json | 5 +++ scripts/generate_pack.py | 16 ++++++- scripts/refresh_data_dirs.py | 67 +++++++++++++++++++++++++--- tests/test_install.py | 31 +++++++++++++ tests/test_refresh_data_dirs.py | 49 +++++++++++++++++++- wiki/installer.md | 9 ++-- 24 files changed, 352 insertions(+), 103 deletions(-) diff --git a/install.ps1 b/install.ps1 index 412a6456e..7225221c0 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51" + $defaultInstallSha256 = "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index feabcc09e..8a4f6cb3c 100644 --- a/install.py +++ b/install.py @@ -3708,6 +3708,9 @@ def _check_data_directory(entry: object, where: str) -> None: strip = entry.get("strip_components", 0) if not isinstance(strip, int) or isinstance(strip, bool) or not 0 <= strip <= 16: raise ValueError(f"{where}.strip_components is invalid") + size = entry.get("archive_size", 1) + if not isinstance(size, int) or isinstance(size, bool) or size < 1: + raise ValueError(f"{where}.archive_size is invalid") if kind == "tarball" and ( not isinstance(entry.get("source_path"), str) or not entry["source_path"] ): @@ -6873,11 +6876,12 @@ def _print_plan( # files, and 2636 data files" read as two kinds of one thing. The kinds # follow, adding up to it. total = counts.files + tree_files - parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_counts.size)})"] + tree_bytes = sum(data_fetch_bytes(entry, needed) for entry, needed in trees) + parts = ([f"{_plural(total, 'file')} ({format_size(counts.size + tree_bytes)})"] if total else []) split = parts_words(counts if (manifest or {}).get("categorized") else tally({"size": e.get("size")} for e in to_download), - tree_files, tree_counts.size) + tree_files, tree_bytes) if split and trees: split += f", the data files extracted from {_plural(len(trees), 'archive')}" # A pack is copied from, not downloaded: "Would download 23 files" was @@ -6952,8 +6956,11 @@ def _print_plan( "every system and its files.") for entry, needed in trees: source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"] + fetched = data_fetch_bytes(entry, needed) + archive = ("" if fetched == sum(meta.get("size", 0) for meta in needed.values()) + else f" (a {format_size(fetched)} archive)") print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, " - f"from {source}") + f"from {source}{archive}") for folder, (count, size) in sorted((kept or {}).items()): print(f"\n Would keep {_plural(count, 'file')} you have ({format_size(size)}), " f"whose contents differ, in {folder}") @@ -7184,6 +7191,19 @@ def data_directory_state(entry: dict, bios_path: Path, return root, needed, present +def data_fetch_bytes(entry: dict, needed: dict[str, dict]) -> int: + """What getting a tree's missing files costs: its source archive, which + comes whole for one file, or from a pack the files themselves. A list + that records no archive size gives the files' own sizes: a tarball whose + tree is 10 KB was announced as 10 KB and was a download of 21 MB.""" + if not needed: + return 0 + size = entry.get("archive_size") + if LOCAL.pack is None and isinstance(size, int) and size > 0: + return size + return sum(meta.get("size", 0) for meta in needed.values()) + + def install_data_directory(entry: dict, bios_path: Path, state: tuple[Path, dict[str, dict], int] | None = None, progress: Progress | None = None) -> dict: @@ -8963,7 +8983,7 @@ def found_head(files: list[dict], trees: list[dict], bios_path: Path) -> str: for tree in trees: _root, wanted, _present = data_directory_state(tree, bios_path, remember=True) data += len(wanted) - data_bytes += sum(meta.get("size", 0) for meta in wanted.values()) + data_bytes += data_fetch_bytes(tree, wanted) left = [entry for entry in needed if id(entry) not in copies] rest = len(left) + data where = ", ".join(f"{count} found in {_last_name(label)}" @@ -15683,8 +15703,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: plat_name, manifest, bios_path, files, up_to_date, mismatched, sum(len(state[1]) for _entry, state in data_states), len(omitted_files), chosen=selection[0] if selection else None, - data_bytes=sum(meta.get("size", 0) for _entry, state in data_states - for meta in state[1].values()), + data_bytes=sum(data_fetch_bytes(entry, state[1]) + for entry, state in data_states), ) emit_copies(plat_name, to_download, copies, bios_index, bios_path, files) emit_unreadable(plat_name, unread) diff --git a/install.sh b/install.sh index c8cba304a..fe3d4fd6e 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51" +DEFAULT_INSTALL_SHA256="567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/amberelec.json b/install/amberelec.json index 745e5792b..51f5a9028 100644 --- a/install/amberelec.json +++ b/install/amberelec.json @@ -63489,7 +63489,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "fbneo-cheats", @@ -77218,7 +77219,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -77451,7 +77453,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -77823,7 +77826,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ] } \ No newline at end of file diff --git a/install/batocera.json b/install/batocera.json index 8c24297c0..a5ecfcb19 100644 --- a/install/batocera.json +++ b/install/batocera.json @@ -225291,7 +225291,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -225524,7 +225525,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ], "save_directory": { diff --git a/install/emudeck.json b/install/emudeck.json index c1d09d585..32f4ed6a2 100644 --- a/install/emudeck.json +++ b/install/emudeck.json @@ -100486,7 +100486,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -100934,7 +100935,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -100990,7 +100992,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -109666,7 +109669,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -123395,7 +123399,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -123628,7 +123633,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -124000,7 +124006,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "lrps2", @@ -124016,7 +124023,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "yaps2-resources", @@ -124476,7 +124484,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/emuelec.json b/install/emuelec.json index 8135201bd..e9709f639 100644 --- a/install/emuelec.json +++ b/install/emuelec.json @@ -69309,7 +69309,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -69757,7 +69758,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "fbneo-cheats", @@ -83486,7 +83488,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -83719,7 +83722,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -84091,7 +84095,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ] } \ No newline at end of file diff --git a/install/knulli.json b/install/knulli.json index 25b3ccce4..4e70dab7f 100644 --- a/install/knulli.json +++ b/install/knulli.json @@ -157933,7 +157933,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -158166,7 +158167,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "lrps2", @@ -158182,7 +158184,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "ppsspp-assets", @@ -158870,7 +158873,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 } ], "save_directory": { diff --git a/install/lakka.json b/install/lakka.json index 093e07996..5866fe5db 100644 --- a/install/lakka.json +++ b/install/lakka.json @@ -115932,7 +115932,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "cemu-gameprofiles", @@ -116888,7 +116889,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dirksimple", @@ -116944,7 +116946,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "fbneo-cheats", @@ -130673,7 +130676,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -130906,7 +130910,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -131278,7 +131283,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -131738,7 +131744,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/launchers/index.json b/install/launchers/index.json index cd32da514..acfb61130 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1107258, - "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", + "size": 1108265, + "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1160885, - "sha256": "5ccc4ec3df34a1fef96caea9e84218de1fb92331664047c36d6863c11973dc3d" + "size": 1161892, + "sha256": "1031750dcd599e0c4dd2e6c39296b3bcada68494a4837ad442150da64b67a55c" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1107258, - "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", + "size": 1108265, + "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1107258, - "sha256": "3aa392c9c4fd50ab2a8474bb71ef11be45945964fb147b2ade45296461c87f51", + "size": 1108265, + "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", "executable": false }, { diff --git a/install/myminui.json b/install/myminui.json index 919ecf4b1..85de7b9bc 100644 --- a/install/myminui.json +++ b/install/myminui.json @@ -6,7 +6,7 @@ "platform": "myminui", "display_name": "MyMinUI", "version": "2026-08-09", - "generated": "2026-11-05T18:33:47Z", + "generated": "2026-11-05T23:59:37Z", "base_destination": "", "detect": [ { @@ -18957,7 +18957,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-cheats", @@ -32686,7 +32687,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -32919,7 +32921,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "fbneo-samples", @@ -33152,7 +33155,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ] } \ No newline at end of file diff --git a/install/nextui.json b/install/nextui.json index d38175219..4518d3d5c 100644 --- a/install/nextui.json +++ b/install/nextui.json @@ -6,7 +6,7 @@ "platform": "nextui", "display_name": "NextUI", "version": "2026-10-20", - "generated": "2026-11-05T18:33:47Z", + "generated": "2026-11-05T23:59:37Z", "base_destination": "", "detect": [ { @@ -5635,7 +5635,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "fbneo-cheats", @@ -19364,7 +19365,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -19597,7 +19599,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ] } \ No newline at end of file diff --git a/install/recalbox.json b/install/recalbox.json index b947571f5..9b97c9d50 100644 --- a/install/recalbox.json +++ b/install/recalbox.json @@ -62833,7 +62833,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -63066,7 +63067,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 } ], "save_directory": { diff --git a/install/retroarch.json b/install/retroarch.json index 166d197d3..11b60b965 100644 --- a/install/retroarch.json +++ b/install/retroarch.json @@ -116923,7 +116923,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "cemu-gameprofiles", @@ -117879,7 +117880,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dinothawr", @@ -118327,7 +118329,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -118383,7 +118386,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -127054,7 +127058,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -140783,7 +140788,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -141016,7 +141022,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -141388,7 +141395,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "ppsspp-assets", @@ -142076,7 +142084,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 }, { "ref": "yaps2-resources", @@ -142536,7 +142545,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/retrobat.json b/install/retrobat.json index 3a2e41f89..ca7459771 100644 --- a/install/retrobat.json +++ b/install/retrobat.json @@ -89075,7 +89075,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -89308,7 +89309,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -89680,7 +89682,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 } ], "save_directory": { diff --git a/install/retrodeck.json b/install/retrodeck.json index b70189e03..e94e2c442 100644 --- a/install/retrodeck.json +++ b/install/retrodeck.json @@ -137751,7 +137751,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "fbneo-cheats", @@ -151480,7 +151481,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -151713,7 +151715,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -152085,7 +152088,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -152549,7 +152553,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ], "save_directory": { diff --git a/install/retropie.json b/install/retropie.json index c255b9d1f..3db3da56e 100644 --- a/install/retropie.json +++ b/install/retropie.json @@ -124071,7 +124071,8 @@ "size": 36 } }, - "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default" + "source_path": "Cemu-3310f3b8b184d64a62b89fd59088c799432badf5/bin/gameProfiles/default", + "archive_size": 21360579 }, { "ref": "dolphin-sys", @@ -132742,7 +132743,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -146471,7 +146473,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -146704,7 +146707,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -147076,7 +147080,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "yaps2-resources", @@ -147536,7 +147541,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ] } \ No newline at end of file diff --git a/install/rocknix.json b/install/rocknix.json index 793ca6670..f06bd3144 100644 --- a/install/rocknix.json +++ b/install/rocknix.json @@ -86538,7 +86538,8 @@ "size": 344 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 4465709 }, { "ref": "dinothawr", @@ -86986,7 +86987,8 @@ "size": 105 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 5752250 }, { "ref": "dirksimple", @@ -87042,7 +87044,8 @@ "size": 12988 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 175384 }, { "ref": "dolphin-sys", @@ -95719,7 +95722,8 @@ "strip_components": 2, "exclude": [ "Themes" - ] + ], + "archive_size": 3197914 }, { "ref": "fbneo-cheats", @@ -109448,7 +109452,8 @@ "size": 10054 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 5083767 }, { "ref": "fbneo-samples", @@ -109681,7 +109686,8 @@ "size": 551776 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 53006540 }, { "ref": "freeintv-overlays", @@ -110053,7 +110059,8 @@ "size": 248636 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 21311170 }, { "ref": "lrps2", @@ -110069,7 +110076,8 @@ "size": 1814032 } }, - "strip_components": 0 + "strip_components": 0, + "archive_size": 318896 }, { "ref": "ppsspp-assets", @@ -110757,7 +110765,8 @@ "size": 15005 } }, - "strip_components": 1 + "strip_components": 1, + "archive_size": 9844388 }, { "ref": "yaps2-resources", @@ -111217,7 +111226,8 @@ "size": 637 } }, - "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources" + "source_path": "yaps2-57ea93c0ce8306063b31fe6658dab6b492bbedfe/bin/resources", + "archive_size": 21181421 } ] } \ No newline at end of file diff --git a/schemas/install-manifest.schema.json b/schemas/install-manifest.schema.json index 4e7d1ec20..264c08558 100644 --- a/schemas/install-manifest.schema.json +++ b/schemas/install-manifest.schema.json @@ -429,6 +429,11 @@ "minimum": 0, "description": "With type zip: leading path parts dropped from each member." }, + "archive_size": { + "type": "integer", + "minimum": 1, + "description": "Bytes of the source archive when it was last downloaded: what fetching the tree costs. Absent when not recorded." + }, "source_path": { "type": "string", "description": "With type tarball: only members below this path are kept, relative to it." diff --git a/scripts/generate_pack.py b/scripts/generate_pack.py index 354964481..8a62186a3 100644 --- a/scripts/generate_pack.py +++ b/scripts/generate_pack.py @@ -4949,9 +4949,12 @@ def _manifest_data_directories( (strip_components for a ZIP, source_path for a tarball, exclude) come from _data_dirs.yml, the same refresh_data_dirs applies. Every file carries the sha1 and size the cache holds, so a member the source - changed since is refused rather than installed. + changed since is refused rather than installed. archive_size is the + size of the archive refresh_data_dirs last downloaded, what fetching + the tree costs: a tarball whose tree is 10 KB is a download of 21 MB. """ grouped: dict[tuple[str, str], dict] = {} + versions: dict[str, dict[str, dict]] = {} for src, full in members: ref, systems, prefix = refs[full] entry = registry.get(ref, {}) @@ -4976,6 +4979,17 @@ def _manifest_data_directories( group["source_path"] = str(entry.get("source_path", "")).format(version=version) if entry.get("exclude"): group["exclude"] = [str(e) for e in entry["exclude"]] + cache = entry.get("local_cache") + if cache: + recorded = Path(cache).parent / ".versions.json" + if str(recorded) not in versions: + try: + versions[str(recorded)] = json.loads(recorded.read_text()) + except FileNotFoundError: + versions[str(recorded)] = {} + size = versions[str(recorded)].get(ref, {}).get("archive_size") + if isinstance(size, int) and size > 0: + group["archive_size"] = size group["systems"].extend(s for s in systems if s not in group["systems"]) rel = full[len(prefix):].lstrip("/") if prefix else full hashes = compute_hashes(src, frozenset({"sha1"})) diff --git a/scripts/refresh_data_dirs.py b/scripts/refresh_data_dirs.py index 7b74f7be1..7c3d63562 100644 --- a/scripts/refresh_data_dirs.py +++ b/scripts/refresh_data_dirs.py @@ -24,7 +24,7 @@ import time import urllib.error import urllib.request import zipfile -from collections.abc import Iterator +from collections.abc import Callable, Iterator from pathlib import Path from artifacts import file_lock as _file_lock, set_plain_mode from common import InputMissing, github_token, platform_arg, require_platform, suggest_names, yaml_load @@ -270,10 +270,12 @@ def _download_and_extract( source_path: str, local_cache: str, exclude: list[str] | None = None, + measured: Callable[[int], None] | None = None, ) -> int: """Download tarball, extract source_path subtree to local_cache. - Returns the number of files extracted. + Returns the number of files extracted. measured receives the size of + the archive downloaded. """ exclude = exclude or [] cache_dir = Path(local_cache) @@ -290,6 +292,8 @@ def _download_and_extract( if not chunk: break f.write(chunk) + if measured is not None: + measured(tarball_path.stat().st_size) log.info("extracting %s -> %s", source_path, local_cache) @@ -347,6 +351,7 @@ def _download_and_extract_zip( local_cache: str, exclude: list[str] | None = None, strip_components: int = 0, + measured: Callable[[int], None] | None = None, ) -> int: """Download ZIP, extract to local_cache. Returns file count. @@ -369,6 +374,8 @@ def _download_and_extract_zip( if not chunk: break f.write(chunk) + if measured is not None: + measured(zip_path.stat().st_size) extract_dir = Path(tmpdir) / "extract" extract_dir.mkdir() @@ -408,6 +415,46 @@ def _download_and_extract_zip( return file_count +def _measure_archive(source_url: str) -> int | None: + """The size of the archive at source_url, read by streaming it: a + tarball GitHub builds on request is sent without a Content-Length.""" + req = urllib.request.Request(source_url, headers={"User-Agent": USER_AGENT}) + size = 0 + try: + with urllib.request.urlopen(req, timeout=DOWNLOAD_TIMEOUT) as resp: + while chunk := resp.read(65536): + size += len(chunk) + except _TRANSIENT as exc: + log.warning("could not measure %s: %s", source_url, exc) + return None + return size or None + + +def _record_archive_size(key: str, entry: dict, versions_path: str) -> None: + """Record the archive size of a tree current since before it was kept. + + The installer announces a tree's download as that size: a tarball + whose tree is 10 KB is a download of 21 MB. Measured, not refetched, + so the tree the packs carry stays the one recorded. + """ + path = Path(versions_path) + with _file_lock(path.with_name(f".{path.name}.lock")): + versions = _load_versions(versions_path) + record = versions.get(key) + if record is None or "archive_size" in record: + return + version = entry.get("version", "master") + size = _measure_archive(entry["source_url"].format(version=version)) + if size is None: + return + with _file_lock(path.with_name(f".{path.name}.lock")): + versions = _load_versions(versions_path) + if key in versions and "archive_size" not in versions[key]: + versions[key]["archive_size"] = size + _save_versions(versions, versions_path) + log.info("[%s] archive size recorded: %d bytes", key, size) + + def _get_remote_etag(source_url: str) -> str | None: """HEAD request to get ETag or Last-Modified for freshness check.""" try: @@ -456,6 +503,7 @@ def refresh_entry( return None if not needs: log.info("[%s] up to date (tag: %s)", key, _short_tag(cached_tag)) + _record_archive_size(key, entry, versions_path) return False with cache_lock(entry["local_cache"]): return _refresh_entry(key, entry, force, dry_run, versions_path) @@ -516,13 +564,19 @@ def _refresh_entry( else: remote_tag = get_remote_sha(entry["source_url"], version) + archive_sizes: list[int] = [] + def fetch() -> int: + archive_sizes.clear() if source_type == "zip": return _download_and_extract_zip( - source_url, local_cache, exclude, entry.get("strip_components", 0) + source_url, local_cache, exclude, entry.get("strip_components", 0), + measured=archive_sizes.append, ) source_path = entry["source_path"].format(version=version) - return _download_and_extract(source_url, source_path, local_cache, exclude) + return _download_and_extract( + source_url, source_path, local_cache, exclude, measured=archive_sizes.append + ) # An archive that reads but holds nothing under the path is the same on # every try: only a failed or unreadable download is tried again. @@ -540,7 +594,10 @@ def _refresh_entry( log.warning("[%s] download failed (%s), trying again in %d s", key, exc, wait) time.sleep(wait) - _record_version(key, {"sha": remote_tag or "", "version": version}, versions_path) + record = {"sha": remote_tag or "", "version": version} + if archive_sizes: + record["archive_size"] = archive_sizes[-1] + _record_version(key, record, versions_path) log.info("[%s] refreshed: %d files extracted to %s", key, file_count, local_cache) return True diff --git a/tests/test_install.py b/tests/test_install.py index b8c72ae82..a918d7d06 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3888,6 +3888,37 @@ class DataDirectoriesAreInstalled(unittest.TestCase): self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) self.assertIn(f"{install.format_size(bios + archive)} downloaded in", proc.stdout) + def test_the_plan_announces_the_archive_not_the_tree(self): + """yaps2's tarball is 21.2 MB for a 6.5 MB tree, Cemu's 21.4 MB for + 9.9 KB: the plan announced the tree, and the run downloaded far more.""" + from test_install_selection import FakeRepository # noqa: PLC0415 + + member = b"profile" + archive_size = 21_360_579 + with FakeRepository() as repo: + listed = json.loads((repo.root / "install" / "retroarch.json").read_text()) + listed["data_directories"] = [{ + "ref": "cemu-gameprofiles", "dest": "cemu", "type": "tarball", + "url": "https://example.invalid/Cemu.tar.gz", "source_path": "Cemu-1/bin", + "systems": ["nintendo-gba"], "archive_size": archive_size, + "files": {"gameProfiles/a.ini": {"sha1": hashlib.sha1(member).hexdigest(), + "size": len(member)}}}] + (repo.root / "install" / "retroarch.json").write_text(json.dumps(listed)) + proc = repo.run("--no-input", "--dry-run") + bios = sum(len(payload) for payload, *_ in FakeRepository.PAYLOADS.values()) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn(f"({install.format_size(bios + archive_size)})", proc.stdout) + self.assertIn(f"(a {install.format_size(archive_size)} archive)", proc.stdout) + + def test_a_pack_or_an_old_list_counts_the_files_themselves(self): + needed = {"a": {"size": 7}} + entry = {"archive_size": 21_360_579} + self.assertEqual(install.data_fetch_bytes(entry, needed), 21_360_579) + self.assertEqual(install.data_fetch_bytes({}, needed), 7) + self.assertEqual(install.data_fetch_bytes(entry, {}), 0) + with unittest.mock.patch.object(install.LOCAL, "pack", object()): + self.assertEqual(install.data_fetch_bytes(entry, needed), 7) + def test_verified_members_land_the_rest_are_said(self): work = Path(_tempdir(self)) archive = self._zip(work) diff --git a/tests/test_refresh_data_dirs.py b/tests/test_refresh_data_dirs.py index 42f21d122..0a4af04fd 100644 --- a/tests/test_refresh_data_dirs.py +++ b/tests/test_refresh_data_dirs.py @@ -93,6 +93,26 @@ class RefreshConcurrency(unittest.TestCase): ) self.assertEqual(seen, [True]) + def test_the_archive_size_is_recorded_without_replacing_the_tree(self): + """The installer announces a tree's download as the archive's size: + a tarball extracted to 10 KB is a download of 21 MB. A tree current + before the size was kept is measured, its files left as they are.""" + payload = _zip_bytes({"a.bin": b"x" * 100}) + entry = {"source_type": "zip", "source_url": "https://x/pak.zip", + "local_cache": str(self.cache)} + with mock.patch.object(rdd.urllib.request, "urlopen", + lambda *a, **k: _Response(payload)), \ + mock.patch.object(rdd, "_get_remote_etag", return_value="e1"): + self.assertTrue(rdd.refresh_entry("pak", entry, versions_path=self.versions)) + self.assertEqual(rdd._load_versions(self.versions)["pak"]["archive_size"], + len(payload)) + rdd._record_version("pak", {"sha": "e1"}, self.versions) + (self.cache / "a.bin").write_bytes(b"kept") + self.assertFalse(rdd.refresh_entry("pak", entry, versions_path=self.versions)) + self.assertEqual(rdd._load_versions(self.versions)["pak"], + {"sha": "e1", "archive_size": len(payload)}) + self.assertEqual((self.cache / "a.bin").read_bytes(), b"kept") + def test_a_current_tree_is_not_waited_for(self): """A pipeline reads every tree for its whole run. A build beside it waited on the exclusive lock before asking the remote, which would @@ -100,7 +120,7 @@ class RefreshConcurrency(unittest.TestCase): import fcntl # noqa: PLC0415 self.cache.mkdir(parents=True) - rdd._record_version("pak", {"sha": "e1"}, self.versions) + rdd._record_version("pak", {"sha": "e1", "archive_size": 1}, self.versions) entry = {"source_type": "zip", "source_url": "https://x/pak.zip", "local_cache": str(self.cache)} result: list = [] @@ -229,7 +249,7 @@ class DryRunReport(unittest.TestCase): def test_the_remote_tag_is_shown_beside_the_cached_one(self): with tempfile.TemporaryDirectory() as tmp: versions = Path(tmp) / "versions.json" - versions.write_text(json.dumps({"k": {"etag": '"2725165595"'}})) + versions.write_text(json.dumps({"k": {"etag": '"2725165595"', "archive_size": 1}})) entry = {"source_type": "zip", "source_url": "https://x.invalid/a.zip", "local_cache": tmp} with mock.patch.object(rdd, "_get_remote_etag", return_value='"2685251384"'), \ @@ -249,5 +269,30 @@ class DryRunReport(unittest.TestCase): ) +class TheManifestCarriesTheArchiveSize(unittest.TestCase): + """The size refresh_data_dirs recorded reaches the install list, which + the installer announces as the tree's download.""" + + def test_recorded_size_is_written_and_an_unrecorded_one_left_out(self): + from generate_pack import _manifest_data_directories # noqa: PLC0415 + + (REPO_ROOT / "tmp").mkdir(exist_ok=True) + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + data = Path(tmp) / "data" + for ref in ("pak", "old"): + (data / ref).mkdir(parents=True) + (data / ref / "a.bin").write_bytes(b"abc") + (data / ".versions.json").write_text(json.dumps( + {"pak": {"sha": "e1", "archive_size": 21_360_579}, "old": {"sha": "e2"}})) + registry = {ref: {"source_type": "zip", "source_url": f"https://x/{ref}.zip", + "local_cache": str(data / ref)} for ref in ("pak", "old")} + members = [(str(data / ref / "a.bin"), f"{ref}/a.bin") for ref in ("pak", "old")] + refs = {f"{ref}/a.bin": (ref, ("s",), ref) for ref in ("pak", "old")} + listed = {entry["ref"]: entry for entry in + _manifest_data_directories(members, refs, registry, "")} + self.assertEqual(listed["pak"]["archive_size"], 21_360_579) + self.assertNotIn("archive_size", listed["old"]) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index 0d6e5141b..796938117 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -1620,8 +1620,11 @@ built (`strip_components` for a ZIP, `source_path` for a tarball, `exclude`), and writes a file only when its hash matches the list. A file the source has changed since is left out and counted, and the pack still holds it. `--check` counts the data files missing, `--dry-run` names each tree it would fetch, and -narrowing with `--system`, `--core` or `--target` keeps only the trees of the -systems that stay. +the size announced for a tree is its archive's (`archive_size`, recorded when +`refresh_data_dirs.py` last downloaded it): one missing file fetches the whole +archive, and a tarball can be a thousand times its tree. Narrowing with +`--system`, `--core` or `--target` keeps only the trees of the systems that +stay. ## What the bootstrap does @@ -1802,7 +1805,7 @@ other schemas under `api/v1/schemas/`. | `games_unavailable` | `platform`, `reason`, `message` | With `--for-games`, what the games need could not be read (`offline`, `network`, `invalid`, `stale`, `no_sets`, `no_folder`): the run keeps every file of the list, except for a `FOLDER` that does not exist, where it stops | | `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply | | `declared` | `platform`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `dests` | Right after `check`, the same counts over the files the platform's own list names (`cores: null`), and their `dests` in the order of the `file` events. A setup that checks its own list reads its answer here, with or without `--declared-only` | -| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes they write once in place | +| `data` | `platform`, `files`, `bytes` | Right after `declared`: the data-tree files to fetch (the `data_missing` of `check`) and the bytes fetching them costs: each tree's archive, or the files themselves from a pack or a list that records no archive size | | `found` | `platform`, `dest`, `folder`, `path`, `member`, `reason` | Each file to install that this machine already holds, after `check` and before anything is written, in `--check` and `--dry-run` too. `folder` is the `--from` folder as given, or the `bios_dir` of the check when the BIOS folder holds the file at another place, and then `reason` says where in words (`nested folder: ...` for a misplaced file, `the same file is at ...` for one the list installs there too). `path` is the file, and `member` the ZIP member holding the bytes when `path` is an archive, an empty string otherwise. Its copy is reported by a `downloaded` event | | `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) | | `saves` | `platform`, `files`, `folder` | After `choices` and before `checking`, when the files the run installs (once `--system` and the other choices narrowed the list) hold some read from the frontend's save directory and that directory is known: how many, and the `folder` they go to, where `` and `` stand for one folder per system or per core (the screen and `--plain` say it in words: "a folder per core under /sdcard/RetroArch/saves"). When it is not known, the files are counted in the summary's `skipped` instead (`no_save_directory`) | From 333f1ac3cabd8317007f7cd58821789e8d029784 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 6 Nov 2026 09:04:52 +0100 Subject: [PATCH 7/7] fix: resume a cut data archive A data archive cut mid-download was deleted on every failure: the retry and the next run started from zero, while the run said nothing was lost. The archive is now kept beside its tree with the validator its server sent, and the rest is asked for with Range and If-Range. Checked against the release: fbneo-samples stopped at 16 MB, the next run downloaded 34.6 MB of 50.5. --- install.ps1 | 2 +- install.py | 114 ++++++++++++++++++++++++++++++----- install.sh | 2 +- install/launchers/index.json | 16 ++--- tests/test_install.py | 104 ++++++++++++++++++++++++++++++-- wiki/installer.md | 6 +- 6 files changed, 210 insertions(+), 34 deletions(-) diff --git a/install.ps1 b/install.ps1 index 7225221c0..c8d2c3399 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f" + $defaultInstallSha256 = "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 8a4f6cb3c..16c011ea6 100644 --- a/install.py +++ b/install.py @@ -7086,20 +7086,72 @@ def download_files( return failed, interrupted -def _fetch_archive(url: str, directory: Path, progress: Progress | None = None) -> Path: +def _archive_validator(headers: object) -> str: + """What names this revision of an archive to its server: an If-Range + with it gets the rest of the same bytes, or the whole new archive.""" + if not headers: + return "" + etag = headers.get("ETag", "") or "" + # A weak tag never answers a range (RFC 9110 13.1.5). + return etag if etag and not etag.startswith("W/") else headers.get("Last-Modified", "") or "" + + +def _fetch_archive(url: str, directory: Path, progress: Progress | None = None, + keep: Path | None = None) -> Path: """A data directory's source archive, in *directory*, retried like files. - progress draws its bytes as they come, an attempt that failed taken back.""" + progress draws its bytes as they come, an attempt that failed taken back. + + With keep, the archive is written there and kept when the transfer is + cut, beside a note of its address and the validator its server sent: + the next attempt, or the next run, asks for the rest with Range and + If-Range, and a server that changed the archive sends it whole. A cut + 50 MB archive started again from zero, and a link that drops more often + than one archive takes never installed the tree. + """ _network_allowed(url) + note = keep.with_name(keep.name + ".source") if keep is not None else None reason: Exception | None = None for attempt in range(1, MAX_RETRIES + 1): - handle = tempfile.NamedTemporaryFile(dir=directory, suffix=".archive", delete=False) + offset = 0 + validator = "" + if keep is not None: + try: + recorded = note.read_text(encoding="utf-8").split("\n") + if recorded[0] == url and len(recorded) > 1 and recorded[1]: + validator = recorded[1] + offset = keep.stat().st_size + except (OSError, UnicodeError): + offset = 0 + request = urllib.request.Request(url) + if offset: + request.add_header("Range", f"bytes={offset}-") + request.add_header("If-Range", validator) + handle = (open(keep, "r+b" if offset else "wb") if keep is not None else + tempfile.NamedTemporaryFile(dir=directory, suffix=".archive", delete=False)) received = 0 + resumed = 0 try: - with handle, urllib.request.urlopen(url, timeout=60) as resp: + with handle, urllib.request.urlopen(request, timeout=60) as resp: + headers = resp.headers or {} + ranged = (offset and getattr(resp, "status", 200) == 206 and str( + headers.get("Content-Range", "")).startswith(f"bytes {offset}-")) + if not ranged: + offset = 0 + handle.seek(offset) + handle.truncate() + if keep is not None and not offset: + # Written before the first byte: a cut transfer is then + # resumed only against the revision it holds. + note.write_text(f"{url}\n{_archive_validator(headers)}\n", + encoding="utf-8") if progress is not None: - length = resp.headers.get("Content-Length", "") + length = headers.get("Content-Length", "") if length.isdigit(): - progress.expect(int(length)) + progress.expect(offset + int(length)) + if offset: + progress.add(offset, resumed=True) + resumed = offset + total = offset while True: if _stop.is_set(): raise _Interrupted() @@ -7107,30 +7159,51 @@ def _fetch_archive(url: str, directory: Path, progress: Progress | None = None) if not chunk: break received += len(chunk) - if received > MAX_DATA_ARCHIVE_SIZE: + total += len(chunk) + if total > MAX_DATA_ARCHIVE_SIZE: raise ValueError("archive exceeds the size limit") handle.write(chunk) if progress is not None: progress.add(len(chunk)) - return Path(handle.name) + return keep if keep is not None else Path(handle.name) except urllib.error.HTTPError as exc: - Path(handle.name).unlink(missing_ok=True) - if exc.code < 500 and exc.code != 429: + if keep is None or exc.code == 416 or exc.code < 500 and exc.code != 429: + # Refused, or a range past an archive that shrank: no + # attempt resumes it. + _drop_archive(Path(handle.name), note) + if exc.code < 500 and exc.code not in (416, 429): raise reason = exc except (urllib.error.URLError, OSError, http.client.HTTPException) as exc: - Path(handle.name).unlink(missing_ok=True) + if keep is None: + Path(handle.name).unlink(missing_ok=True) reason = exc if isinstance(exc, OSError) else ConnectionError(repr(exc)) - except BaseException: - Path(handle.name).unlink(missing_ok=True) + except ValueError: + _drop_archive(Path(handle.name), note) raise - if progress is not None and received: - progress.add(-received) + except BaseException: + # Stopped: a kept archive waits for the next run. + if keep is None: + Path(handle.name).unlink(missing_ok=True) + raise + if progress is not None: + if received: + progress.add(-received) + if resumed: + progress.add(-resumed) + progress.forget_resumed(resumed) if attempt < MAX_RETRIES: time.sleep(_retry_delay(attempt, None)) raise reason if reason else OSError("download failed") +def _drop_archive(path: Path, note: Path | None) -> None: + """Remove an archive and the note of where it came from.""" + path.unlink(missing_ok=True) + if note is not None: + note.unlink(missing_ok=True) + + def _archive_members( archive: Path, entry: dict ) -> Iterator[tuple[str, int, Callable[[], BinaryIO]]]: @@ -7223,6 +7296,7 @@ def install_data_directory(entry: dict, bios_path: Path, if not needed: return counts _writing.set() + fetched: Path | None = None try: root.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=root) as scratch: @@ -7230,7 +7304,11 @@ def install_data_directory(entry: dict, bios_path: Path, # The pack holds the tree where it extracts, already mapped. members = LOCAL.pack.tree(entry.get("dest", "")) else: - archive = _fetch_archive(entry["url"], Path(scratch), progress) + # Beside the tree, outside the scratch folder: a transfer + # cut short is resumed by the next attempt or the next run. + kept = root / f".{entry['ref']}.archive.part" + archive = _fetch_archive(entry["url"], Path(scratch), progress, kept) + fetched = archive members = _archive_members(archive, entry) for rel, declared, opener in members: meta = needed.pop(rel, None) @@ -7264,6 +7342,10 @@ def install_data_directory(entry: dict, bios_path: Path, except _MEMBER_ERRORS as exc: # A damaged, encrypted or unsupported member, in words. counts["error"] = unreadable_words(exc) + finally: + if fetched is not None: + # Read whole: the next run fetches the archive afresh. + _drop_archive(fetched, fetched.with_name(fetched.name + ".source")) counts["absent"] = len(needed) return counts diff --git a/install.sh b/install.sh index fe3d4fd6e..4e6c1560d 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f" +DEFAULT_INSTALL_SHA256="fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index acfb61130..2f435fcb1 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1108265, - "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", + "size": 1112094, + "sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1161892, - "sha256": "1031750dcd599e0c4dd2e6c39296b3bcada68494a4837ad442150da64b67a55c" + "size": 1165721, + "sha256": "1b6ea28a5fc2f413aa7444ddb9ebe00b4b89c29ef639d62e3e1cf4d10793ed16" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1108265, - "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", + "size": 1112094, + "sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1108265, - "sha256": "567d810f5c4f5cbb6793d6f0c96a7492d90b53664d74f045aae4ef75d6058a0f", + "size": 1112094, + "sha256": "fdccf00cb56c4caf3ddf916e126d538539e0d9efa527fb808cde954b6b0d29b1", "executable": false }, { diff --git a/tests/test_install.py b/tests/test_install.py index a918d7d06..13deeaa5c 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -3867,6 +3867,96 @@ class DataDirectoriesAreInstalled(unittest.TestCase): (len(body), len(body))) emit.assert_not_called() + def test_a_cut_archive_resumes_where_it_stopped(self): + """A data archive cut at 90% was deleted and fetched again from + zero, on the retry and on the next run, while the run said that + nothing was lost.""" + work = Path(_tempdir(self)) + body = self._zip(work).read_bytes() + keep = work / ".dolphin-sys.archive.part" + requests: list[dict] = [] + + class Response(io.BytesIO): + def __init__(self, data, status, headers): + super().__init__(data) + self.status, self.headers = status, headers + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + def urlopen(request, timeout=None): + sent = dict(request.header_items()) + requests.append(sent) + if "Range" in sent: + start = int(sent["Range"][len("bytes="):-1]) + if sent.get("If-range") != '"v1"': + return Response(body, 200, {"ETag": '"v2"'}) + return Response(body[start:], 206, { + "Content-Range": f"bytes {start}-{len(body) - 1}/{len(body)}", + "Content-Length": str(len(body) - start), "ETag": '"v1"'}) + cut = len(body) * 9 // 10 + + class Cut(Response): + def read(self, size=-1): + if self.tell() >= cut: + raise ConnectionResetError("cut") + return super().read(min(size, cut - self.tell())) + return Cut(body, 200, {"Content-Length": str(len(body)), "ETag": '"v1"'}) + + with unittest.mock.patch.object(install.urllib.request, "urlopen", urlopen), \ + unittest.mock.patch.object(install, "_network_allowed", lambda url: None), \ + unittest.mock.patch.object(install, "_retry_delay", lambda *a: 0): + # The next run, after a stop: what the cut attempt wrote is kept. + with unittest.mock.patch.object(install, "MAX_RETRIES", 1), \ + self.assertRaises(ConnectionResetError): + install._fetch_archive("https://example.invalid/D.zip", work, None, keep) + self.assertEqual(keep.stat().st_size, len(body) * 9 // 10) + path = install._fetch_archive("https://example.invalid/D.zip", work, None, keep) + self.assertEqual(path.read_bytes(), body) + self.assertEqual(requests[-1]["Range"], f"bytes={len(body) * 9 // 10}-") + self.assertEqual(requests[-1]["If-range"], '"v1"') + + def test_a_changed_archive_is_fetched_whole(self): + work = Path(_tempdir(self)) + body = self._zip(work).read_bytes() + keep = work / ".dolphin-sys.archive.part" + keep.write_bytes(b"older revision") + keep.with_name(keep.name + ".source").write_text( + 'https://example.invalid/D.zip\n"v0"\n') + + class Whole(io.BytesIO): + status, headers = 200, {"ETag": '"v1"'} + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + with unittest.mock.patch.object(install.urllib.request, "urlopen", + lambda request, timeout=None: Whole(body)), \ + unittest.mock.patch.object(install, "_network_allowed", lambda url: None): + path = install._fetch_archive("https://example.invalid/D.zip", work, None, keep) + self.assertEqual(path.read_bytes(), body) + + def test_a_tree_read_whole_leaves_no_archive(self): + work = Path(_tempdir(self)) + archive = self._zip(work) + bios = work / "system" + bios.mkdir() + + def fetch(url, directory, progress=None, keep=None): + shutil.copy(archive, keep) + keep.with_name(keep.name + ".source").write_text(f"{url}\n\"v1\"\n") + return keep + + with unittest.mock.patch.object(install, "_fetch_archive", fetch): + install.install_data_directory(self._entry(), bios) + self.assertEqual([p.name for p in (bios / "dolphin-emu").iterdir()], ["Sys"]) + def test_the_run_ends_on_the_archive_bytes_too(self): from test_install_selection import FakeRepository # noqa: PLC0415 @@ -3925,7 +4015,7 @@ class DataDirectoriesAreInstalled(unittest.TestCase): bios = work / "system" bios.mkdir() - def fetch(url, directory, progress=None): + def fetch(url, directory, progress=None, keep=None): copy = Path(directory) / "a.zip" shutil.copy(archive, copy) return copy @@ -3961,8 +4051,9 @@ class DataDirectoriesAreInstalled(unittest.TestCase): def members(_archive, _entry): yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Endless()) - with unittest.mock.patch.object(install, "_fetch_archive", - lambda url, d, progress=None: Path(d) / "a"), \ + with unittest.mock.patch.object( + install, "_fetch_archive", + lambda url, d, progress=None, keep=None: Path(d) / "a"), \ unittest.mock.patch.object(install, "_archive_members", members): counts = install.install_data_directory(entry, work) self.assertEqual(counts["changed"], 1) @@ -3989,8 +4080,9 @@ class DataDirectoriesAreInstalled(unittest.TestCase): def members(_archive, _entry): yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Broken()) - with unittest.mock.patch.object(install, "_fetch_archive", - lambda url, d, progress=None: Path(d) / "a"), \ + with unittest.mock.patch.object( + install, "_fetch_archive", + lambda url, d, progress=None, keep=None: Path(d) / "a"), \ unittest.mock.patch.object(install, "_archive_members", members): counts = install.install_data_directory(entry, work) self.assertTrue(counts["error"]) @@ -6714,7 +6806,7 @@ class AFileYouHadIsKept(unittest.TestCase): "type": "zip", "files": {"Sys/a.ini": { "sha1": hashlib.sha1(self.NEW).hexdigest(), "size": len(self.NEW)}}} - def fetch(url, directory, progress=None): + def fetch(url, directory, progress=None, keep=None): copy = Path(directory) / "a.zip" shutil.copy(archive, copy) return copy diff --git a/wiki/installer.md b/wiki/installer.md index 796938117..1aa89e133 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -352,8 +352,10 @@ file list is downloaded once and handed to each run. times, after 1, 2 then 4 seconds or the delay a `Retry-After` header asks. A file of 16 MB or more keeps its partial copy (`..part`) when a transfer stops, and the next attempt or the next run requests only the - rest. A file already there with other contents is moved aside first, see - [Files you already had](#files-you-already-had). + rest. A data tree's archive does the same beside the tree + (`..archive.part`), asking with `If-Range` so that a server whose + archive changed sends it whole. A file already there with other contents + is moved aside first, see [Files you already had](#files-you-already-had). 8. Copies into standalone emulator directories, only with `--standalone-copies`. The file list names the version of the platform it was built for (Batocera