feat: leave out names a FAT card refuses

FAT, exFAT and Windows refuse < > : " | ? * in a name (fs/fat/namei_vfat.c:201-207, fs/exfat/nls.c:396-403). RetroDECK's component manifest names zxs:48.rom, zxs:if1-2.rom and snes:st018.data.rom: on an exFAT card each write failed with EINVAL on every run. The installer now probes the BIOS folder as keeps_case does, leaves those entries out and names them.

install.ps1

@@ -16,7 +16,7 @@
     # and Invoke-WebRequest throws on a cut transfer before the hash is read:
     # the site's build rewrites this line there (scripts/sitebootstrap.py).
     $mismatchAdvice = "The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that."
-    $defaultInstallSha256 = "192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b"
+    $defaultInstallSha256 = "1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4"
     $maximumInstallerBytes = 2MB
     $installerExit = 0
     $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }

install.py

@@ -3132,6 +3132,49 @@ def fold_case_twins(files: list[dict]) -> tuple[list[dict], list[str]]:
     return kept, folded
 
 
+# The characters FAT and exFAT refuse in a name, as Windows does on any drive
+# (Linux 7475ea8b, fs/fat/namei_vfat.c:201-207 and fs/exfat/nls.c:396-403;
+# "Naming Files, Paths, and Namespaces" in the Win32 documentation). The
+# slash, the backslash and the controls never reach a destination
+# (_safe_relative_path). RetroDECK names three files zxs:48.rom,
+# zxs:if1-2.rom and snes:st018.data.rom (its retroarch component manifest):
+# on an exFAT card each write failed, on every run.
+FAT_REFUSED = re.compile(r'[<>:"|?*]')
+FAT_REFUSED_SHOWN = '< > : " | ? *'
+
+
+def holds_fat_refused_names(directory: Path) -> bool:
+    """Whether a name holding a colon can be created there.
+
+    Probed on the nearest existing directory, as keeps_case is: an ext4
+    card on a Steam Deck holds one, an exFAT card refuses it with EINVAL.
+    Windows refuses it on every drive. A probe refused for another cause
+    (a folder this user cannot write) decides nothing.
+    """
+    if os.name == "nt":
+        return False
+    probe_dir = directory
+    while not probe_dir.is_dir() and probe_dir != probe_dir.parent:
+        probe_dir = probe_dir.parent
+    try:
+        handle, name = tempfile.mkstemp(prefix=".retrobios-name-", suffix=":",
+                                        dir=str(probe_dir))
+    except OSError as exc:
+        return exc.errno != errno.EINVAL
+    os.close(handle)
+    os.unlink(name)
+    return True
+
+
+def split_fat_refused(files: list[dict]) -> tuple[list[dict], list[str]]:
+    """The entries whose destination a FAT, exFAT or Windows folder can
+    hold, and the destinations it cannot."""
+    kept = [entry for entry in files if not FAT_REFUSED.search(str(entry.get("dest", "")))]
+    refused = [str(entry.get("dest", "")) for entry in files
+               if FAT_REFUSED.search(str(entry.get("dest", "")))]
+    return kept, refused
+
+
 # Android's media scanner marks a folder holding a .nomedia file hidden, and
 # every folder below it (MediaProvider a183e2b9, FileUtils.java:1587-1629
 # isDirectoryHidden, ModernMediaScanner.java:845-846 and 1017-1020). Without
@@ -15616,6 +15659,15 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
                     f"  {_plural(len(folded), 'file')} left out: {bios_path} does not keep "
                     f"case and holds another file under the same name ({shown})"
                 )
+        if any(FAT_REFUSED.search(str(entry.get("dest", ""))) for entry in files) \
+                and not holds_fat_refused_names(bios_path):
+            files, refused = split_fat_refused(files)
+            shown = ", ".join(refused[:3]) + (" ..." if len(refused) > 3 else "")
+            print(
+                f"  {_plural(len(refused), 'file')} left out: the file system of "
+                f"{shown_path(bios_path)} cannot hold a name with any of "
+                f"{FAT_REFUSED_SHOWN} ({screen_text(shown)})"
+            )
 
         game_data_cut = 0
         if args.no_game_data and not manifest.get("categorized"):

install.sh

@@ -23,7 +23,7 @@ DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/ins
 # curl or wget stop on a cut transfer before the hash is read: the site's
 # build rewrites this line there (scripts/sitebootstrap.py).
 MISMATCH_ADVICE="The download was most likely cut short: try again. Within minutes of an update of this project the two files can come from caches of different ages. Waiting five minutes settles that."
-DEFAULT_INSTALL_SHA256="192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b"
+DEFAULT_INSTALL_SHA256="1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4"
 MAX_INSTALLER_BYTES=2097152
 
 cleanup() {

install/launchers/index.json

@@ -27,8 +27,8 @@
         {
           "dest": "roms/ports/retrobios/install.py",
           "repo_path": "install.py",
-          "size": 1126407,
-          "sha256": "192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b",
+          "size": 1128787,
+          "sha256": "1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4",
           "executable": false
         },
         {
@@ -71,8 +71,8 @@
     "muos": {
       "archive": {
         "dest": "ARCHIVE/RetroBIOS.muxapp",
-        "size": 1180034,
-        "sha256": "14c4e4b762ec84fe30d2cf0baed58d1022c3423e05118b43dbccfbaff5e4d84a"
+        "size": 1182414,
+        "sha256": "2c2d797483fa6c64031d55fd4b5dde327cb64b163c506eace2593e90f88b58e3"
       },
       "members": [
         {
@@ -92,8 +92,8 @@
         {
           "name": "RetroBIOS/install.py",
           "repo_path": "install.py",
-          "size": 1126407,
-          "sha256": "192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b",
+          "size": 1128787,
+          "sha256": "1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4",
           "executable": false
         },
         {
@@ -166,8 +166,8 @@
         {
           "dest": ".config/modules/retrobios/install.py",
           "repo_path": "install.py",
-          "size": 1126407,
-          "sha256": "192ecdd781447b69967d5d335911fb34887757ee1bf9b6c000ad68a62b17de4b",
+          "size": 1128787,
+          "sha256": "1282e441fb6562b4413f60504976763b5fc337170334dc3836ef70914b67c4f4",
           "executable": false
         },
         {

tests/test_install_filesystems.py

@@ -4,15 +4,25 @@ from __future__ import annotations
 
 import tests  # noqa: F401  offline, scratch under tmp/, the suite's own cache
 
+import contextlib
+import errno
+import hashlib
+import io
+import json
+import os
+import shutil
 import subprocess
 import sys
+import tempfile
 import unittest
 from pathlib import Path
+from unittest import mock
 
 REPO_ROOT = Path(__file__).resolve().parent.parent
 sys.path.insert(0, str(REPO_ROOT))
 sys.path.insert(0, str(REPO_ROOT / "tests"))
 
+import install  # noqa: E402
 from test_install_selection import FakeRepository  # noqa: E402
 
 
@@ -60,5 +70,66 @@ class AndroidMediaScanner(unittest.TestCase):
         self.assertFalse((repo.dest / ".nomedia").exists())
 
 
+class NamesAFatCardCannotHold(unittest.TestCase):
+    """RetroDECK names three files with a colon (zxs:48.rom): on an exFAT
+    card each write failed with EINVAL, on every run."""
+
+    def test_the_probe_reads_einval_as_a_refusal(self):
+        with tempfile.TemporaryDirectory(dir=REPO_ROOT / "tmp") as tmp:
+            refused = OSError(errno.EINVAL, "Invalid argument")
+            closed = PermissionError(errno.EACCES, "Permission denied")
+            with mock.patch.object(install.os, "name", "posix"):
+                self.assertTrue(install.holds_fat_refused_names(Path(tmp) / "a" / "b"))
+                with mock.patch.object(install.tempfile, "mkstemp", side_effect=refused):
+                    self.assertFalse(install.holds_fat_refused_names(Path(tmp)))
+                with mock.patch.object(install.tempfile, "mkstemp", side_effect=closed):
+                    self.assertTrue(install.holds_fat_refused_names(Path(tmp)))
+            self.assertEqual(os.listdir(tmp), [])
+
+    def test_windows_refuses_them_on_any_drive(self):
+        with mock.patch.object(install.os, "name", "nt"):
+            self.assertFalse(install.holds_fat_refused_names(Path(".")))
+
+    def _install(self, holds: bool) -> tuple[str, list[str]]:
+        root = Path(tempfile.mkdtemp(dir=REPO_ROOT / "tmp"))
+        self.addCleanup(shutil.rmtree, root, True)
+        old, dest = root / "old", root / "dest"
+        old.mkdir()
+        files = []
+        for name, payload in (("zxs:48.rom", b"zx 48"), ("gba_bios.bin", b"gba")):
+            (old / name.replace(":", "_")).write_bytes(payload)
+            files.append({"dest": name, "sha1": hashlib.sha1(payload).hexdigest(),
+                          "sha256": hashlib.sha256(payload).hexdigest(),
+                          "size": len(payload), "repo_path": f"bios/{name}"})
+        listed = root / "list.json"
+        listed.write_text(json.dumps({"manifest_version": 2, "platform": "retrodeck",
+                                      "files": files}))
+        out = io.StringIO()
+        argv = ["install.py", "--platform", "retrodeck", "--dest", str(dest),
+                "--manifest", str(listed), "--from", str(old), "--no-input"]
+        with mock.patch.object(install, "holds_fat_refused_names", return_value=holds), \
+                mock.patch.object(install, "detect_os", return_value="linux"), \
+                mock.patch("sys.argv", argv), contextlib.redirect_stdout(out), \
+                contextlib.redirect_stderr(io.StringIO()):
+            try:
+                install.main()
+            except SystemExit as exc:
+                self.assertIn(exc.code, (0, None), out.getvalue())
+        self.addCleanup(install.LOCAL.reset)
+        return (" ".join(out.getvalue().split()),
+                sorted(p.name for p in dest.iterdir()) if dest.is_dir() else [])
+
+    def test_the_names_are_left_out_and_said(self):
+        said, installed = self._install(holds=False)
+        self.assertEqual(installed, ["gba_bios.bin"])
+        self.assertIn("1 file left out: the file system of", said)
+        self.assertIn('cannot hold a name with any of < > : " | ? * (zxs:48.rom)', said)
+
+    def test_a_folder_that_holds_them_gets_them(self):
+        said, installed = self._install(holds=True)
+        self.assertEqual(installed, ["gba_bios.bin", "zxs:48.rom"])
+        self.assertNotIn("left out", said)
+
+
 if __name__ == "__main__":
     unittest.main()

wiki/installer.md

@@ -2005,6 +2005,12 @@ opened from Windows, an exFAT card), one overwrites the other. The installer
 checks the folder first, keeps one file per name there and lists the ones it
 left out.
 
+FAT, exFAT and every Windows drive also refuse `< > : " | ? *` in a name
+(Linux `fs/fat/namei_vfat.c:201-207`, `fs/exfat/nls.c:396-403`). RetroDECK's
+list names three files with a colon, like `zxs:48.rom`. Before writing such a
+name, the installer creates one in the BIOS folder: on an ext4 card it goes
+in, on an exFAT card the installer leaves those files out and names them.
+
 ### Packs and their parts
 
 `scripts/download.py --list` and `scripts/download.sh --list` print the packs of

wiki/troubleshooting.md

@@ -50,6 +50,8 @@ Without `--check`, it adds missing files and replaces wrong ones.
   folder. Emulators add their own, like `dc/` for Flycast.
 - **A ZIP in parts, like `Pack.part1of2.zip`?** Unzip all parts into one
   folder.
+- **"left out" on an exFAT card?** The card refuses a `:` in a name. An
+  ext4 card holds these files.
 
 <!-- Flycast's libretro port appends "dc/" to the system directory:
 shell/libretro/libretro.cpp:2252-2253, emulators/flycast.yml. -->