diff --git a/install.ps1 b/install.ps1 index cb079f36..573da24e 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "4b658dc29528e94460995eaf844b1fc1cca9040ffa10cd71ac6656b4db5f1b34" + $defaultInstallSha256 = "06fde26aa660b6e28c05010dd4a1de8f1ff1ab891d6ebbe460b670b30b71befd" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 5e17f22a..297cf2d0 100644 --- a/install.py +++ b/install.py @@ -494,9 +494,14 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = { # stage: manifest (reason: network, not_found, invalid, clock for a # refused certificate under a clock set in the past, or no_ca for no # trusted certificates on the machine) or detect (reason: conflict, two - # systems found on one machine) or from (reason: not_found or - # not_a_folder, a folder named with --from). + # systems found on one machine) or from (a folder named with --from, + # reason: not_found, not_a_folder, unreadable, or system for the root + # of the system and /proc, /sys, /dev). "error": ("stage", "reason", "platform", "message"), + # A folder named with --from, once read: the files it holds (counted + # up to MAX_SCANNED_FILES, truncated past them), and the folders below + # it on another drive, not entered. + "scanned": ("folder", "files", "truncated", "other_drives"), # state: present (hash verified), wrong_hash or missing. "file": ("platform", "dest", "systems", "size", "state"), "system": ("platform", "system", "name", "files", "present", "missing", @@ -6675,6 +6680,10 @@ SYSTEM_FOLDERS = frozenset(name.casefold() for name in ( # A Mac copying to a FAT or exFAT card writes ._name beside every file: its # Finder data, 4 KB, never the file. APPLE_DOUBLE = "._" +# Folders the kernel makes up on the fly: endless, and never a copy. +PSEUDO_FOLDERS = frozenset() if os.name == "nt" else frozenset(("/proc", "/sys", "/dev")) +# FILE_ATTRIBUTE_REPARSE_POINT: a junction, a mount point or a link on Windows. +_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) # Errors a member of someone's ZIP can raise on reading: damaged, encrypted # (RuntimeError), or packed with a method zipfile lacks. _MEMBER_ERRORS = (OSError, EOFError, RuntimeError, NotImplementedError, @@ -6740,18 +6749,28 @@ class FolderIndex: # (path, member or "") -> (its size or None when unknown, why) of each # file, member or folder below root that could not be read. self.unreadable: dict[tuple[Path, str], tuple[int | None, str]] = {} + # Folders below root on another drive, not entered. + self.other_drives: list[Path] = [] + self.files = 0 if root.is_file(): if root.name.casefold().endswith(".zip"): self.archives.append(root) + self.files = 1 return seen = 0 + try: + device = os.stat(root).st_dev + except OSError as exc: + self._unread(root, "", None, exc) + return def unlisted(exc: OSError) -> None: self._unread(Path(exc.filename or root), "", None, exc) for folder, dirs, names in os.walk(root, onerror=unlisted): dirs[:] = sorted(d for d in dirs - if d != BACKUP_DIR and d.casefold() not in SYSTEM_FOLDERS) + if d != BACKUP_DIR and d.casefold() not in SYSTEM_FOLDERS + and self._enters(Path(folder) / d, device)) for name in sorted(names): path = Path(folder) / name if name.startswith(APPLE_DOUBLE): @@ -6774,10 +6793,30 @@ class FolderIndex: if name.casefold().endswith(_ARCHIVE_SUFFIXES): self.archives.append(path) seen += 1 + self.files = seen if seen >= MAX_SCANNED_FILES: self.truncated = True return + def _enters(self, path: Path, device: int) -> bool: + """Whether the walk goes into a folder: not a pseudo file system, a + Windows junction or reparse point (a link the walk cannot tell from + a folder, which led out of the folder named), nor another drive + mounted below the root, which is named instead.""" + if str(path) in PSEUDO_FOLDERS: + return False + try: + held = os.lstat(path) + except OSError as exc: + self._unread(path, "", None, exc) + return False + if getattr(held, "st_file_attributes", 0) & _REPARSE_POINT: + return False + if held.st_dev != device: + self.other_drives.append(path) + return False + return True + def seed(self, path: Path, sha1: str) -> None: """A SHA-1 already known, so the file is not read again.""" self._sha1[path] = sha1 @@ -8004,26 +8043,71 @@ def _open_local_sources(args: argparse.Namespace) -> None: sys.exit(1) +def _system_folder(folder: Path) -> str: + """Why a --from folder is one of the system's own, in words, or "".""" + try: + resolved = folder.resolve() + except OSError: + return "" + text = str(resolved) + if any(text == pseudo or text.startswith(pseudo + "/") for pseudo in PSEUDO_FOLDERS): + return "is a folder the system makes up, it holds no copy" + system_drive = os.environ.get("SystemDrive", "C:") if os.name == "nt" else "" + if resolved == Path(resolved.anchor) and ( + os.name != "nt" or resolved.drive.casefold() == system_drive.casefold()): + return "is the root of the whole system" + return "" + + +def from_folder_problem(label: str) -> tuple[str, str] | None: + """Why a folder named with --from cannot be read, without reading what + it holds: (reason, words), or None when it can. + + reason is the one the error event carries: not_found, not_a_folder (a + file that is no ZIP), unreadable, or system (the root of the system, + /proc, /sys or /dev). words follow the folder as given: "E:\\old does + not exist". Cheap enough for a screen to call before indexing.""" + folder = Path(os.path.expanduser(label)) + zipped = folder.is_file() and folder.name.casefold().endswith(".zip") + if not (folder.is_dir() or zipped): + if folder.exists(): + return "not_a_folder", "is neither a folder nor a ZIP" + return "not_found", "does not exist" + try: + if zipped: + with open(folder, "rb"): + pass + else: + with os.scandir(folder) as listing: + next(listing, None) + except OSError as exc: + return "unreadable", f"could not be read: {unreadable_words(exc)}" + system = "" if zipped else _system_folder(folder) + return ("system", system) if system else None + + def _open_from_folders(folders: list[Path]) -> None: - """Read each folder named with --from, or stop on one that is not there.""" + """Read each folder named with --from, or stop on one that cannot be.""" for folder in folders: label = str(folder) - folder = Path(os.path.expanduser(label)) - zipped = folder.is_file() and folder.name.casefold().endswith(".zip") - if not (folder.is_dir() or zipped): - reason = "not_a_folder" if folder.exists() else "not_found" - what = ("does not exist" if reason == "not_found" - else "is neither a folder nor a ZIP") + problem = from_folder_problem(label) + if problem is not None: + reason, what = problem message = f"{label} {what}: --from names the folder to copy BIOS files from" EVENTS.emit("error", stage="from", reason=reason, platform="", message=message) print(f"Error: {message}.", file=sys.stderr) sys.exit(1) - index = FolderIndex(folder, label) + index = FolderIndex(Path(os.path.expanduser(label)), label) LOCAL.folders.append(index) print(f" Copying BIOS files from: {label}") if index.truncated: print(f" Read the first {MAX_SCANNED_FILES:,} files of {label} only. Name " "the folder that holds the BIOS files to search it all.") + for drive in index.other_drives: + print(f" Did not go into {drive}: it is another drive. Name it with --from " + "to read it too.") + EVENTS.emit("scanned", folder=label, files=index.files, truncated=index.truncated, + other_drives=[str(drive) for drive in index.other_drives]) def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest") -> None: diff --git a/install.sh b/install.sh index d9ca30f3..aa93f114 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="4b658dc29528e94460995eaf844b1fc1cca9040ffa10cd71ac6656b4db5f1b34" +DEFAULT_INSTALL_SHA256="06fde26aa660b6e28c05010dd4a1de8f1ff1ab891d6ebbe460b670b30b71befd" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_from.py b/tests/test_install_from.py index ebca9a29..d8f7d129 100644 --- a/tests/test_install_from.py +++ b/tests/test_install_from.py @@ -650,6 +650,104 @@ class UnreadableCopies(FromFolder): "Input/output error"]) +class WhereTheSearchGoes(FromFolder): + """--from / walked the whole system, /proc and every drive mounted in + it. The search stays on the drive it starts on, and the folder is + checked before it is read, cheaply enough for a screen to do it.""" + + def test_a_folder_is_checked_without_being_read(self): + archive = _zip(self.root / "backup.zip", {"a": b"a"}) + (self.root / "notes.txt").write_text("x") + self.assertIsNone(install.from_folder_problem(str(self.old))) + self.assertIsNone(install.from_folder_problem(str(archive))) + self.assertEqual(install.from_folder_problem(str(self.root / "nowhere")), + ("not_found", "does not exist")) + self.assertEqual(install.from_folder_problem(str(self.root / "notes.txt")), + ("not_a_folder", "is neither a folder nor a ZIP")) + if os.name == "posix": + self.assertEqual(install.from_folder_problem("/"), + ("system", "is the root of the whole system")) + self.assertEqual(install.from_folder_problem("/proc")[0], "system") + + def test_a_folder_that_cannot_be_listed_is_refused(self): + _unprivileged(self) + self.old.chmod(0) + self.addCleanup(self.old.chmod, 0o755) + self.assertEqual(install.from_folder_problem(str(self.old)), + ("unreadable", "could not be read: Permission denied")) + proc = self.install("--events") + error = next(e for e in _events(proc) if e["event"] == "error") + self.assertEqual((error["stage"], error["reason"]), ("from", "unreadable")) + self.assertEqual(proc.returncode, 1) + + @unittest.skipUnless(os.name == "posix", "a POSIX root") + def test_the_root_of_the_system_is_refused(self): + proc = self.run_installer("--manifest", str(self.list), "--from", "/", "--events") + error = next(e for e in _events(proc) if e["event"] == "error") + self.assertEqual((error["stage"], error["reason"]), ("from", "system")) + self.assertIn("/ is the root of the whole system", proc.stderr) + self.assertEqual(proc.returncode, 1) + + def lstat_saying(self, folder: str, **fields): + real = os.lstat + + def lstat(path, *args, **kwargs): + held = real(path, *args, **kwargs) + if Path(path).name != folder: + return held + values = {"st_dev": held.st_dev, "st_file_attributes": 0, **fields} + return mock.Mock(**values) + + return mock.patch.object(install.os, "lstat", side_effect=lstat) + + def test_another_drive_inside_is_named_and_not_entered(self): + self.keep("mnt/scph5501.bin", US) + self.keep("scph5500.bin", JP) + with self.lstat_saying("mnt", st_dev=os.lstat(self.old).st_dev + 1): + index = install.FolderIndex(self.old) + self.assertEqual(index.other_drives, [self.old.resolve() / "mnt"]) + self.assertEqual(sorted(p.name for p in index.size), ["scph5500.bin"]) + + def test_a_junction_is_not_entered(self): + self.keep("junction/scph5501.bin", US) + with self.lstat_saying("junction", st_file_attributes=0x400): + index = install.FolderIndex(self.old) + self.assertEqual(index.size, {}) + self.assertEqual(index.other_drives, []) + + def test_pseudo_file_systems_are_never_entered(self): + if not install.PSEUDO_FOLDERS: + self.skipTest("no pseudo file system here") + index = install.FolderIndex(self.old) + for pseudo in sorted(install.PSEUDO_FOLDERS): + self.assertFalse(index._enters(Path(pseudo), os.lstat("/").st_dev), pseudo) + + def test_a_search_cut_short_is_an_event(self): + for n in range(3): + self.keep(f"f{n}.bin", b"%d" % n) + out = io.StringIO() + install.EVENTS.open(out) + self.addCleanup(setattr, install.EVENTS, "stream", None) + self.addCleanup(install.LOCAL.reset) + with mock.patch.object(install, "MAX_SCANNED_FILES", 2), \ + contextlib.redirect_stdout(io.StringIO()) as text: + install._open_from_folders([self.old]) + self.assertIn(f"Read the first 2 files of {self.old} only", text.getvalue()) + self.assertEqual([json.loads(line) for line in out.getvalue().splitlines()], [{ + "event": "scanned", "folder": str(self.old), "files": 2, "truncated": True, + "other_drives": []}]) + + def test_each_folder_read_is_an_event_before_detect(self): + self.keep("scph5501.bin", US) + events = _events(self.install("--check", "--events")) + kinds = [e["event"] for e in events] + scanned = next(e for e in events if e["event"] == "scanned") + self.assertEqual(scanned, {"event": "scanned", "folder": str(self.old), "files": 1, + "truncated": False, "other_drives": []}) + self.assertLess(kinds.index("start"), kinds.index("scanned")) + self.assertLess(kinds.index("scanned"), kinds.index("detect")) + + class ManySameSizedFiles(FromFolder): """A card of 16 KB ROMs and a list of 16 KB entries: each entry sorted and walked every file of its size.""" diff --git a/wiki/installer.md b/wiki/installer.md index 3b14d35c..be9c62bd 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -465,6 +465,13 @@ writes, the system folders (`System Volume Information`, `.Spotlight-V100`, `.Trashes`, `$RECYCLE.BIN`) and a link leading out of the folder are skipped. `--from` also takes a ZIP of an old BIOS folder. +The search stays on one drive: a drive mounted inside the folder is named and +not entered (give it its own `--from`), and so are `/proc`, `/sys`, `/dev` and +Windows junctions. The root of the system (`/`, `C:\`) is refused. A file that +cannot be read, and has the size of a file still missing, is named with the +reason (`E:\bios\a.bin could not be read: Permission denied`), as is a ZIP +member locked by a password. + The folder is only read: each file is copied, checked against the list's hashes once written, and the original stays where it is. A file in place with other contents is kept aside first, as a download would replace it (see @@ -1152,7 +1159,7 @@ fields, the installer refuses to narrow rather than install everything. | Code | Meaning | |------|---------| | `0` | everything asked for is installed or already up to date | -| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist, or `--restore` could not put a file back | +| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, or `--restore` could not put a file back | | `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | | `130` | stopped with Ctrl+C, or with `q`, Ctrl+C or Ctrl+D at a question. Run the same command again to carry on | | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) | @@ -1185,7 +1192,8 @@ kind, and each kind has exactly the keys below, in this order. The schema is |------|------|------| | `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` | | `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found | -| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). A folder named with `--from` is not there (`stage` is `from`, `reason` `not_found`, or `not_a_folder` for a file that is no ZIP). `message` says it in words | +| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). `message` says it in words | +| `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered | | `checking` | `files_done`, `files_total` | While the files already in the folder are read and hashed, at most every half second and once at the end. `files_total` counts the files found in place | | `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` | | `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id, and with `--system` only the systems named. Files with no system count under `other`. `name` is the platform's own name for it |