From c33114714e840509f08f638db80a6f0baa84e1de Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Mon, 9 Nov 2026 05:47:43 +0100 Subject: [PATCH] feat: leave out names a FAT card refuses FAT, exFAT and Windows refuse < > : " | ? * in a name (fs/fat/namei_vfat.c:201-207, fs/exfat/nls.c:396-403). RetroDECK's component manifest names zxs:48.rom, zxs:if1-2.rom and snes:st018.data.rom: on an exFAT card each write failed with EINVAL on every run. The installer now probes the BIOS folder as keeps_case does, leaves those entries out and names them. --- install.ps1 | 2 +- install.py | 52 ++++++++++++++++++++++ install.sh | 2 +- install/launchers/index.json | 16 +++---- tests/test_install_filesystems.py | 71 +++++++++++++++++++++++++++++++ wiki/installer.md | 6 +++ wiki/troubleshooting.md | 2 + 7 files changed, 141 insertions(+), 10 deletions(-) diff --git a/install.ps1 b/install.ps1 index f20e65a2c..47488df21 100644 --- a/install.ps1 +++ b/install.ps1 @@ -16,7 +16,7 @@ # and Invoke-WebRequest throws on a cut transfer before the hash is read: # the site's build rewrites this line there (scripts/sitebootstrap.py). $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." - $defaultInstallSha256 = "192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b" + $defaultInstallSha256 = "1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 4e24fd57b..4ffe2f285 100644 --- a/install.py +++ b/install.py @@ -3132,6 +3132,49 @@ def fold_case_twins(files: list[dict]) -> tuple[list[dict], list[str]]: return kept, folded +# The characters FAT and exFAT refuse in a name, as Windows does on any drive +# (Linux 7475ea8b, fs/fat/namei_vfat.c:201-207 and fs/exfat/nls.c:396-403; +# "Naming Files, Paths, and Namespaces" in the Win32 documentation). The +# slash, the backslash and the controls never reach a destination +# (_safe_relative_path). RetroDECK names three files zxs:48.rom, +# zxs:if1-2.rom and snes:st018.data.rom (its retroarch component manifest): +# on an exFAT card each write failed, on every run. +FAT_REFUSED = re.compile(r'[<>:"|?*]') +FAT_REFUSED_SHOWN = '< > : " | ? *' + + +def holds_fat_refused_names(directory: Path) -> bool: + """Whether a name holding a colon can be created there. + + Probed on the nearest existing directory, as keeps_case is: an ext4 + card on a Steam Deck holds one, an exFAT card refuses it with EINVAL. + Windows refuses it on every drive. A probe refused for another cause + (a folder this user cannot write) decides nothing. + """ + if os.name == "nt": + return False + probe_dir = directory + while not probe_dir.is_dir() and probe_dir != probe_dir.parent: + probe_dir = probe_dir.parent + try: + handle, name = tempfile.mkstemp(prefix=".retrobios-name-", suffix=":", + dir=str(probe_dir)) + except OSError as exc: + return exc.errno != errno.EINVAL + os.close(handle) + os.unlink(name) + return True + + +def split_fat_refused(files: list[dict]) -> tuple[list[dict], list[str]]: + """The entries whose destination a FAT, exFAT or Windows folder can + hold, and the destinations it cannot.""" + kept = [entry for entry in files if not FAT_REFUSED.search(str(entry.get("dest", "")))] + refused = [str(entry.get("dest", "")) for entry in files + if FAT_REFUSED.search(str(entry.get("dest", "")))] + return kept, refused + + # Android's media scanner marks a folder holding a .nomedia file hidden, and # every folder below it (MediaProvider a183e2b9, FileUtils.java:1587-1629 # isDirectoryHidden, ModernMediaScanner.java:845-846 and 1017-1020). Without @@ -15616,6 +15659,15 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: f" {_plural(len(folded), 'file')} left out: {bios_path} does not keep " f"case and holds another file under the same name ({shown})" ) + if any(FAT_REFUSED.search(str(entry.get("dest", ""))) for entry in files) \ + and not holds_fat_refused_names(bios_path): + files, refused = split_fat_refused(files) + shown = ", ".join(refused[:3]) + (" ..." if len(refused) > 3 else "") + print( + f" {_plural(len(refused), 'file')} left out: the file system of " + f"{shown_path(bios_path)} cannot hold a name with any of " + f"{FAT_REFUSED_SHOWN} ({screen_text(shown)})" + ) game_data_cut = 0 if args.no_game_data and not manifest.get("categorized"): diff --git a/install.sh b/install.sh index 3eb6c581b..c626a4b20 100755 --- a/install.sh +++ b/install.sh @@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins # curl or wget stop on a cut transfer before the hash is read: the site's # build rewrites this line there (scripts/sitebootstrap.py). MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that." -DEFAULT_INSTALL_SHA256="192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b" +DEFAULT_INSTALL_SHA256="1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/install/launchers/index.json b/install/launchers/index.json index 1aaa624f4..8f666970b 100644 --- a/install/launchers/index.json +++ b/install/launchers/index.json @@ -27,8 +27,8 @@ { "dest": "roms/ports/retrobios/install.py", "repo_path": "install.py", - "size": 1126407, - "sha256": "192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b", + "size": 1128787, + "sha256": "1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4", "executable": false }, { @@ -71,8 +71,8 @@ "muos": { "archive": { "dest": "ARCHIVE/RetroBIOS.muxapp", - "size": 1180034, - "sha256": "14c4e4b762ec84fe30d2cf0baed58d1022c3423e05118b43dbccfbaff5e4d84a" + "size": 1182414, + "sha256": "2c2d797483fa6c64031d55fd4b5dde327cb64b163c506eace2593e90f88b58e3" }, "members": [ { @@ -92,8 +92,8 @@ { "name": "RetroBIOS/install.py", "repo_path": "install.py", - "size": 1126407, - "sha256": "192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b", + "size": 1128787, + "sha256": "1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4", "executable": false }, { @@ -166,8 +166,8 @@ { "dest": ".config/modules/retrobios/install.py", "repo_path": "install.py", - "size": 1126407, - "sha256": "192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b", + "size": 1128787, + "sha256": "1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4", "executable": false }, { diff --git a/tests/test_install_filesystems.py b/tests/test_install_filesystems.py index 010011134..4eb66519b 100644 --- a/tests/test_install_filesystems.py +++ b/tests/test_install_filesystems.py @@ -4,15 +4,25 @@ from __future__ import annotations import tests # noqa: F401 offline, scratch under tmp/, the suite's own cache +import contextlib +import errno +import hashlib +import io +import json +import os +import shutil import subprocess import sys +import tempfile import unittest from pathlib import Path +from unittest import mock REPO_ROOT = Path(__file__).resolve().parent.parent sys.path.insert(0, str(REPO_ROOT)) sys.path.insert(0, str(REPO_ROOT / "tests")) +import install # noqa: E402 from test_install_selection import FakeRepository # noqa: E402 @@ -60,5 +70,66 @@ class AndroidMediaScanner(unittest.TestCase): self.assertFalse((repo.dest / ".nomedia").exists()) +class NamesAFatCardCannotHold(unittest.TestCase): + """RetroDECK names three files with a colon (zxs:48.rom): on an exFAT + card each write failed with EINVAL, on every run.""" + + def test_the_probe_reads_einval_as_a_refusal(self): + with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp: + refused = OSError(errno.EINVAL, "Invalid argument") + closed = PermissionError(errno.EACCES, "Permission denied") + with mock.patch.object(install.os, "name", "posix"): + self.assertTrue(install.holds_fat_refused_names(Path(tmp) / "a" / "b")) + with mock.patch.object(install.tempfile, "mkstemp", side_effect=refused): + self.assertFalse(install.holds_fat_refused_names(Path(tmp))) + with mock.patch.object(install.tempfile, "mkstemp", side_effect=closed): + self.assertTrue(install.holds_fat_refused_names(Path(tmp))) + self.assertEqual(os.listdir(tmp), []) + + def test_windows_refuses_them_on_any_drive(self): + with mock.patch.object(install.os, "name", "nt"): + self.assertFalse(install.holds_fat_refused_names(Path("."))) + + def _install(self, holds: bool) -> tuple[str, list[str]]: + root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp")) + self.addCleanup(shutil.rmtree, root, True) + old, dest = root / "old", root / "dest" + old.mkdir() + files = [] + for name, payload in (("zxs:48.rom", b"zx 48"), ("gba_bios.bin", b"gba")): + (old / name.replace(":", "_")).write_bytes(payload) + files.append({"dest": name, "sha1": hashlib.sha1(payload).hexdigest(), + "sha256": hashlib.sha256(payload).hexdigest(), + "size": len(payload), "repo_path": f"bios/{name}"}) + listed = root / "list.json" + listed.write_text(json.dumps({"manifest_version": 2, "platform": "retrodeck", + "files": files})) + out = io.StringIO() + argv = ["install.py", "--platform", "retrodeck", "--dest", str(dest), + "--manifest", str(listed), "--from", str(old), "--no-input"] + with mock.patch.object(install, "holds_fat_refused_names", return_value=holds), \ + mock.patch.object(install, "detect_os", return_value="linux"), \ + mock.patch("sys.argv", argv), contextlib.redirect_stdout(out), \ + contextlib.redirect_stderr(io.StringIO()): + try: + install.main() + except SystemExit as exc: + self.assertIn(exc.code, (0, None), out.getvalue()) + self.addCleanup(install.LOCAL.reset) + return (" ".join(out.getvalue().split()), + sorted(p.name for p in dest.iterdir()) if dest.is_dir() else []) + + def test_the_names_are_left_out_and_said(self): + said, installed = self._install(holds=False) + self.assertEqual(installed, ["gba_bios.bin"]) + self.assertIn("1 file left out: the file system of", said) + self.assertIn('cannot hold a name with any of < > : " | ? * (zxs:48.rom)', said) + + def test_a_folder_that_holds_them_gets_them(self): + said, installed = self._install(holds=True) + self.assertEqual(installed, ["gba_bios.bin", "zxs:48.rom"]) + self.assertNotIn("left out", said) + + if __name__ == "__main__": unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index c9bc26a51..886ec39c1 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -2005,6 +2005,12 @@ opened from Windows, an exFAT card), one overwrites the other. The installer checks the folder first, keeps one file per name there and lists the ones it left out. +FAT, exFAT and every Windows drive also refuse `< > : " | ? *` in a name +(Linux `fs/fat/namei_vfat.c:201-207`, `fs/exfat/nls.c:396-403`). RetroDECK's +list names three files with a colon, like `zxs:48.rom`. Before writing such a +name, the installer creates one in the BIOS folder: on an ext4 card it goes +in, on an exFAT card the installer leaves those files out and names them. + ### Packs and their parts `scripts/download.py --list` and `scripts/download.sh --list` print the packs of diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md index 4452074a6..8056d77b1 100644 --- a/wiki/troubleshooting.md +++ b/wiki/troubleshooting.md @@ -50,6 +50,8 @@ Without `--check`, it adds missing files and replaces wrong ones. folder. Emulators add their own, like `dc/` for Flycast. - **A ZIP in parts, like `Pack.part1of2.zip`?** Unzip all parts into one folder. +- **"left out" on an exFAT card?** The card refuses a `:` in a name. An + ext4 card holds these files.