chore: merge the engine review fixes

install.ps1

@@ -11,7 +11,7 @@
 
     $ErrorActionPreference = "Stop"
     $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-    $defaultInstallSha256 = "bcbde4c289b32c02ada78001b75274a87b125c1efd3d8fb1064d7f5482ef3997"
+    $defaultInstallSha256 = "c90676dcc433061bd97aca94930cd7acd74295624840cb5c85a22897f9c1dd12"
     $maximumInstallerBytes = 2MB
     $installerExit = 0
     $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }

install.py

@@ -50,7 +50,7 @@ import zipfile
 import zlib
 from collections.abc import Callable, Collection, Iterator, Mapping
 from pathlib import Path, PurePosixPath
-from typing import BinaryIO, ContextManager, NamedTuple, TextIO
+from typing import Any, BinaryIO, ContextManager, NamedTuple, TextIO
 
 # Manifests are read from the same ref the bootstrap verified this installer
 # against, so a file list and the code reading it always come from one commit.
@@ -392,6 +392,9 @@ class Progress:
         self.files = 0
         self.bytes = 0
         self.resumed = 0
+        # Bytes that came from this machine and not the network: a copy
+        # found here, the pack, a resumed partial copy.
+        self.local = 0
         # Files written to their place, which a stopped run counts: the
         # queue's length said "118 files installed" over 8 on disk.
         self.installed = 0
@@ -400,11 +403,16 @@ class Progress:
         self._evented = self.start
         self._lock = threading.Lock()
 
-    def add(self, count: int, resumed: bool = False) -> None:
+    def add(self, count: int, resumed: bool = False, network: bool | None = None) -> None:
+        """count bytes written, resumed ones left out of the speed. network
+        is whether they came over the network, by default unless resumed."""
         with self._lock:
             self.bytes += count
             if resumed:
                 self.resumed += count
+            fetched = not resumed if network is None else network
+            if not fetched:
+                self.local += count
             now = time.monotonic()
             if self.term.live and not self.term.quiet and now - self._drawn >= 0.1:
                 self._drawn = now
@@ -413,10 +421,18 @@ class Progress:
                 self._evented = now
                 EVENTS.emit("progress", files_done=self.files, files_total=self.total_files,
                             bytes_done=self.bytes, bytes_total=self.expected_bytes)
+                self.emit_transfer()
+
+    def emit_transfer(self) -> None:
+        """The transfer event: of the bytes done, those this machine held
+        and those the network brought."""
+        EVENTS.emit("transfer", bytes_local=self.local,
+                    bytes_network=self.bytes - self.local)
 
     def forget_resumed(self, count: int) -> None:
         with self._lock:
             self.resumed -= count
+            self.local -= count
 
     def transferred(self) -> int:
         """Bytes fetched by this run, a resumed partial copy excluded."""
@@ -494,9 +510,14 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = {
     # stage: manifest (reason: network, not_found, invalid, clock for a
     # refused certificate under a clock set in the past, or no_ca for no
     # trusted certificates on the machine) or detect (reason: conflict, two
-    # systems found on one machine) or from (reason: not_found or
-    # not_a_folder, a folder named with --from).
+    # systems found on one machine) or from (a folder named with --from,
+    # reason: not_found, not_a_folder, unreadable, or system for the root
+    # of the system and /proc, /sys, /dev).
     "error": ("stage", "reason", "platform", "message"),
+    # A folder named with --from, once read: the files it holds (counted
+    # up to MAX_SCANNED_FILES, truncated past them), and the folders below
+    # it on another drive, not entered.
+    "scanned": ("folder", "files", "truncated", "other_drives"),
     # state: present (hash verified), wrong_hash or missing.
     "file": ("platform", "dest", "systems", "size", "state"),
     "system": ("platform", "system", "name", "files", "present", "missing",
@@ -510,8 +531,17 @@ EVENT_KEYS: dict[str, tuple[str, ...]] = {
     # a folder named with --from (reason empty). member names the ZIP member
     # when path is an archive, else "". Its copy is a downloaded event.
     "found": ("platform", "dest", "folder", "path", "member", "reason"),
+    # What a folder named with --from holds that could not be read, of the
+    # size of a file no copy was found for, or of no known size (a folder,
+    # a ZIP that would not open). member as in found, reason in words.
+    "unreadable": ("platform", "folder", "path", "member", "reason"),
     "download": ("platform", "files", "bytes"),
     "progress": ("files_done", "files_total", "bytes_done", "bytes_total"),
+    # Right after each progress event, and once when the files of a
+    # download event are done: bytes_done split into the bytes this machine
+    # held (a copy found here, the pack, a resumed partial copy) and the
+    # bytes the network brought. A speed is bytes_network over time.
+    "transfer": ("bytes_local", "bytes_network"),
     "downloaded": ("dest", "ok", "reason", "size", "files_done", "files_total",
                    "bytes_done", "bytes_total"),
     # failed: [{"dest", "reason"}]. skipped: [{"reason", "label", "files",
@@ -4221,9 +4251,17 @@ def _download_one(
         if result[2] in ("stopped", NOT_WRITABLE, READ_ONLY) or result[2].startswith(NOT_KEPT):
             return result
         # The copy went or changed since it was read (a card pulled out, a
-        # file another run replaced): the file is fetched as if never found.
+        # file another run replaced): the file is fetched as if never found,
+        # and a fetch that fails too names both reasons.
         if verbose:
             print(f"    {f['dest']}: {result[2]}, fetching it instead", file=sys.stderr)
+        fetched = _download_one(f, bios_path, verbose, progress)
+        why = fetched[2]
+        if (fetched[1] or why in ("stopped", NOT_WRITABLE, READ_ONLY)
+                or why.startswith(NOT_KEPT)):
+            return fetched
+        both = f"{result[2]}, and {why}"
+        return fetched[0], False, NetworkReason(both) if isinstance(why, NetworkReason) else both
 
     if LOCAL.pack is not None:
         return _extract_from_pack(f, bios_path, dest, progress)
@@ -5155,6 +5193,7 @@ def download_files(
         if not future.cancelled() and future.exception() is None and future.result()[1]
     )
     progress.finish()
+    progress.emit_transfer()
     return failed, interrupted
 
 
@@ -6529,7 +6568,7 @@ def _extract_from_pack(
         return f["dest"], False, NOT_IN_PACK
     archive, info = held
     return _write_verified(f, bios_path, dest, lambda: archive.open(info), progress,
-                           "the pack")
+                           "the pack", named=f"{info.filename} inside {archive.filename}")
 
 
 @contextlib.contextmanager
@@ -6549,30 +6588,67 @@ def _copy_local(
     """Copy one file found on this machine to its place, checked like a
     download. Its bytes count as already here, not in the download speed."""
     return _write_verified(f, bios_path, dest, lambda: _open_copy(copy), progress,
-                           f"the copy in {copy.index.label}", local=True)
+                           f"the copy in {copy.index.label}", local=True,
+                           named=copy.named())
+
+
+class _SourceUnreadable(Exception):
+    """The bytes to write could not be read: the source's fault, never the
+    folder's, so it falls back like any copy that failed."""
+
+
+def unreadable_words(exc: BaseException) -> str:
+    """Why a file or a ZIP member could not be read, in words."""
+    if isinstance(exc, RuntimeError) and "encrypted" in str(exc):
+        return "it is encrypted, a password is needed"
+    if isinstance(exc, NotImplementedError):
+        return "it is packed with a compression method this installer cannot read"
+    if isinstance(exc, (zipfile.BadZipFile, zlib.error, EOFError)):
+        return "it is damaged"
+    if isinstance(exc, OSError) and exc.strerror:
+        return exc.strerror
+    return str(exc) or type(exc).__name__
 
 
 def _write_verified(
     f: dict, bios_path: Path, dest: Path,
     source: Callable[[], ContextManager[BinaryIO]], progress: Progress | None,
-    held_in: str, local: bool = False,
+    held_in: str, local: bool = False, named: str = "",
 ) -> tuple[str, bool, str]:
     """Write the bytes source opens at dest once they match the list's size
-    and hashes, the file in place kept aside first when it differs."""
+    and hashes, the file in place kept aside first when it differs.
+
+    The source is opened before the temporary file, and its read errors
+    are told apart from the write errors: a copy that could not be read
+    was reported as a folder not writable. named is the source in the
+    reason then, held_in when empty."""
     expected = {name: f[name].lower() for name in ("sha256", "sha1") if f.get(name)}
     tmp_path: Path | None = None
     counted = 0
+
+    def unreadable(exc: BaseException) -> _SourceUnreadable:
+        return _SourceUnreadable(f"{named or held_in} could not be read: "
+                                 f"{unreadable_words(exc)}")
+
     try:
-        with source() as src, tempfile.NamedTemporaryFile(
-            mode="wb", dir=dest.parent, prefix=f".{dest.name}.", suffix=".part",
-            delete=False,
-        ) as out:
+        with contextlib.ExitStack() as stack:
+            try:
+                src = stack.enter_context(source())
+            except _MEMBER_ERRORS as exc:
+                raise unreadable(exc) from exc
+            out = stack.enter_context(tempfile.NamedTemporaryFile(
+                mode="wb", dir=dest.parent, prefix=f".{dest.name}.", suffix=".part",
+                delete=False,
+            ))
             tmp_path = Path(out.name)
             size = 0
             while True:
                 if _stop.is_set():
                     raise _Interrupted()
-                chunk = src.read(min(1024 * 1024, f["size"] + 1 - size))
+                try:
+                    chunk = src.read(min(1024 * 1024, f["size"] + 1 - size))
+                except _MEMBER_ERRORS as exc:
+                    raise unreadable(exc) from exc
                 if not chunk:
                     break
                 size += len(chunk)
@@ -6580,7 +6656,7 @@ def _write_verified(
                     raise ValueError(f"{held_in} holds more bytes than the list records")
                 out.write(chunk)
                 if progress is not None:
-                    progress.add(len(chunk), resumed=local)
+                    progress.add(len(chunk), resumed=local, network=False)
                     counted += len(chunk)
         if size != f["size"]:
             raise ValueError(f"{held_in} holds {size} bytes, the list records {f['size']}")
@@ -6598,9 +6674,9 @@ def _write_verified(
         return f["dest"], False, "stopped"
     except _NotKept as exc:
         return f["dest"], False, f"{NOT_KEPT} ({exc})"
-    except (ValueError, *_MEMBER_ERRORS) as exc:
+    except (_SourceUnreadable, ValueError, *_MEMBER_ERRORS) as exc:
         if progress is not None and counted:
-            progress.add(-counted, resumed=local)
+            progress.add(-counted, resumed=local, network=False)
         if isinstance(exc, OSError) and _write_refused(exc):
             return f["dest"], False, READ_ONLY if _read_only_file(dest) else NOT_WRITABLE
         return f["dest"], False, str(exc)
@@ -6626,6 +6702,10 @@ SYSTEM_FOLDERS = frozenset(name.casefold() for name in (
 # A Mac copying to a FAT or exFAT card writes ._name beside every file: its
 # Finder data, 4 KB, never the file.
 APPLE_DOUBLE = "._"
+# Folders the kernel makes up on the fly: endless, and never a copy.
+PSEUDO_FOLDERS = frozenset() if os.name == "nt" else frozenset(("/proc", "/sys", "/dev"))
+# FILE_ATTRIBUTE_REPARSE_POINT: a junction, a mount point or a link on Windows.
+_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400)
 # Errors a member of someone's ZIP can raise on reading: damaged, encrypted
 # (RuntimeError), or packed with a method zipfile lacks.
 _MEMBER_ERRORS = (OSError, EOFError, RuntimeError, NotImplementedError,
@@ -6647,6 +6727,10 @@ class LocalCopy(NamedTuple):
         rel = self.index.relative(self.path)
         return f"{rel}, inside as {self.member}" if self.member else rel
 
+    def named(self) -> str:
+        """The copy by its full path, the member first when it is one."""
+        return f"{self.member} inside {self.path}" if self.member else str(self.path)
+
 
 class FolderIndex:
     """The files under a folder, by name and by size, hashed on demand.
@@ -6678,14 +6762,37 @@ class FolderIndex:
         self._digests: dict[tuple[Path, str, str], str] = {}
         self._members: dict[Path, list[str]] = {}
         self._member_sizes: dict[int, list[tuple[Path, str]]] | None = None
+        # (size, casefolded name) -> the files and the ZIP members, built
+        # the first time a copy is looked for.
+        self._loose_named: dict[tuple[int, str], list[Path]] | None = None
+        self._member_named: dict[int, dict[str, list[tuple[Path, str]]]] = {}
+        self._scans: dict[tuple[str, int, str], _HashScan] = {}
+        self._member_size: dict[tuple[Path, str], int] = {}
+        # (path, member or "") -> (its size or None when unknown, why) of each
+        # file, member or folder below root that could not be read.
+        self.unreadable: dict[tuple[Path, str], tuple[int | None, str]] = {}
+        # Folders below root on another drive, not entered.
+        self.other_drives: list[Path] = []
+        self.files = 0
         if root.is_file():
             if root.name.casefold().endswith(".zip"):
                 self.archives.append(root)
+                self.files = 1
             return
         seen = 0
-        for folder, dirs, names in os.walk(root):
+        try:
+            device = os.stat(root).st_dev
+        except OSError as exc:
+            self._unread(root, "", None, exc)
+            return
+
+        def unlisted(exc: OSError) -> None:
+            self._unread(Path(exc.filename or root), "", None, exc)
+
+        for folder, dirs, names in os.walk(root, onerror=unlisted):
             dirs[:] = sorted(d for d in dirs
-                             if d != BACKUP_DIR and d.casefold() not in SYSTEM_FOLDERS)
+                             if d != BACKUP_DIR and d.casefold() not in SYSTEM_FOLDERS
+                             and self._enters(Path(folder) / d, device))
             for name in sorted(names):
                 path = Path(folder) / name
                 if name.startswith(APPLE_DOUBLE):
@@ -6699,7 +6806,8 @@ class FolderIndex:
                     if not path.is_file():
                         continue
                     size = path.stat().st_size
-                except OSError:
+                except OSError as exc:
+                    self._unread(path, "", None, exc)
                     continue
                 self.by_name.setdefault(name.casefold(), []).append(path)
                 self.by_size.setdefault(size, []).append(path)
@@ -6707,19 +6815,45 @@ class FolderIndex:
                 if name.casefold().endswith(_ARCHIVE_SUFFIXES):
                     self.archives.append(path)
                 seen += 1
+                self.files = seen
                 if seen >= MAX_SCANNED_FILES:
                     self.truncated = True
                     return
 
+    def _enters(self, path: Path, device: int) -> bool:
+        """Whether the walk goes into a folder: not a pseudo file system, a
+        Windows junction or reparse point (a link the walk cannot tell from
+        a folder, which led out of the folder named), nor another drive
+        mounted below the root, which is named instead."""
+        if str(path) in PSEUDO_FOLDERS:
+            return False
+        try:
+            held = os.lstat(path)
+        except OSError as exc:
+            self._unread(path, "", None, exc)
+            return False
+        if getattr(held, "st_file_attributes", 0) & _REPARSE_POINT:
+            return False
+        if held.st_dev != device:
+            self.other_drives.append(path)
+            return False
+        return True
+
     def seed(self, path: Path, sha1: str) -> None:
         """A SHA-1 already known, so the file is not read again."""
         self._sha1[path] = sha1
 
+    def _unread(self, path: Path, member: str, size: int | None, exc: BaseException) -> None:
+        """Note a file, a member or a folder that could not be read: a copy
+        the run needs may be the one, and it is named instead of dropped."""
+        self.unreadable.setdefault((path, member), (size, unreadable_words(exc)))
+
     def sha1(self, path: Path) -> str:
         if path not in self._sha1:
             try:
                 self._sha1[path] = _sha1_file(path)
-            except OSError:
+            except OSError as exc:
+                self._unread(path, "", self.size.get(path), exc)
                 self._sha1[path] = ""
         return self._sha1[path]
 
@@ -6735,15 +6869,43 @@ class FolderIndex:
                         value = _digest_handle(handle, (algorithm,))[algorithm]
                 else:
                     value = _digest_file(path, (algorithm,))[algorithm]
-            except _MEMBER_ERRORS:
+            except _MEMBER_ERRORS as exc:
+                size = self._member_size.get((path, member)) if member else self.size.get(path)
+                self._unread(path, member, size, exc)
                 value = ""
             self._digests[key] = value
         return self._digests[key]
 
-    def holding(self, sizes: set[int], sha1s: set[str]) -> list[Path]:
-        """Files whose contents are one of sha1s, read only when the size fits."""
-        return [path for size in sorted(sizes) for path in self.by_size.get(size, [])
-                if self.sha1(path) in sha1s]
+    def unreadable_for(self, sizes: Collection[int]) -> list[tuple[Path, str, str]]:
+        """(path, member, why) of what could not be read and may hold a file
+        of one of sizes: its size is one of them, or unknown (a folder, an
+        archive whose list of members could not be read)."""
+        return sorted((path, member, why)
+                      for (path, member), (size, why) in self.unreadable.items()
+                      if size is None or size in sizes)
+
+    def _scan(self, kind: str, size: int, algorithm: str) -> _HashScan:
+        """The files (kind "loose") or ZIP members ("member") of one size,
+        hashed in the order the folder was read, each read once."""
+        key = (kind, size, algorithm)
+        if key not in self._scans:
+            if kind == "loose":
+                self._scans[key] = _HashScan(
+                    self.by_size.get(size, []), lambda path: self.digest(path, algorithm))
+            else:
+                self._scans[key] = _HashScan(
+                    self._members_of_size(size),
+                    lambda item: self.digest(item[0], algorithm, item[1]))
+        return self._scans[key]
+
+    def first_holding(self, sizes: set[int], sha1s: set[str]) -> Path | None:
+        """The first file whose contents are one of sha1s, the smallest size
+        first, read only when the size fits."""
+        for size in sorted(sizes):
+            path = self._scan("loose", size, "sha1").first(sha1s)
+            if path is not None:
+                return path
+        return None
 
     def zip_members(self, path: Path) -> list[str]:
         if path not in self._members:
@@ -6767,12 +6929,18 @@ class FolderIndex:
                 try:
                     with zipfile.ZipFile(path) as archive:
                         infos = archive.infolist()
-                except (OSError, zipfile.BadZipFile):
+                except OSError as exc:
+                    # A ZIP that will not open may hold anything: named. A
+                    # damaged one is a download cut short, not a copy.
+                    self._unread(path, "", None, exc)
+                    continue
+                except zipfile.BadZipFile:
                     continue
                 for info in infos:
                     if not info.is_dir():
                         self._member_sizes.setdefault(info.file_size, []).append(
                             (path, info.filename))
+                        self._member_size[(path, info.filename)] = info.file_size
         return self._member_sizes.get(size, [])
 
     def find(self, entry: dict, skip: Collection[Path] = ()) -> LocalCopy | None:
@@ -6786,15 +6954,27 @@ class FolderIndex:
         if not wanted:
             return None
         name = str(entry.get("dest", "")).rsplit("/", 1)[-1].casefold()
-        loose = sorted(self.by_size.get(size, []), key=lambda p: p.name.casefold() != name)
-        for path in loose:
-            if path not in skip and self.digest(path, algorithm) == wanted:
+        if self._loose_named is None:
+            self._loose_named = {}
+            for path, held in self.size.items():
+                self._loose_named.setdefault((held, path.name.casefold()), []).append(path)
+        for path in self._loose_named.get((size, name), []):
+            if not _being_written(path, skip) and self.digest(path, algorithm) == wanted:
                 return LocalCopy(self, path)
-        members = sorted(self._members_of_size(size),
-                         key=lambda item: item[1].rsplit("/", 1)[-1].casefold() != name)
-        for path, member in members:
-            if self.digest(path, algorithm, member) == wanted:
-                return LocalCopy(self, path, member)
+        path = self._scan("loose", size, algorithm).first({wanted}, skip)
+        if path is not None:
+            return LocalCopy(self, path)
+        if size not in self._member_named:
+            named = self._member_named[size] = {}
+            for item in self._members_of_size(size):
+                named.setdefault(item[1].rsplit("/", 1)[-1].casefold(), []).append(item)
+        for archive, member in self._member_named[size].get(name, []):
+            if (not _being_written(archive, skip)
+                    and self.digest(archive, algorithm, member) == wanted):
+                return LocalCopy(self, archive, member)
+        held = self._scan("member", size, algorithm).first({wanted}, skip)
+        if held is not None:
+            return LocalCopy(self, held[0], held[1])
         return None
 
     def count_copy(self, misplaced: bool = False) -> None:
@@ -6812,6 +6992,98 @@ class FolderIndex:
         return rel if rel != "." else path.name
 
 
+def _being_written(path: Path, skip: Collection[Path]) -> bool:
+    """Whether a file, or the archive holding a member, is one this run
+    writes: its bytes are being replaced, so they are no source."""
+    return bool(skip) and (path in skip or Path(os.path.realpath(path)) in skip)
+
+
+class _HashScan:
+    """Candidates of one size, hashed in order and each once, so finding
+    the first that holds a digest never reads the earlier ones again: 3,000
+    same-sized files and 3,000 entries of that size cost 3,000 reads, not
+    nine million comparisons."""
+
+    def __init__(self, items: list, digest: Callable[[Any], str]) -> None:
+        self.items = items
+        self.digest = digest
+        self.read = 0
+        self.at: dict[str, list[int]] = {}
+
+    def first(self, wanted: Collection[str], skip: Collection[Path] = ()) -> Any:
+        """The first candidate, in the folder's order, whose digest is one of
+        wanted and that skip does not name, or None."""
+        best = None
+        for value in wanted:
+            for position in self.at.get(value, ()):
+                if not _being_written(self._holder(position), skip):
+                    best = position if best is None else min(best, position)
+                    break
+        if best is not None:
+            return self.items[best]
+        while self.read < len(self.items):
+            position = self.read
+            self.read += 1
+            value = self.digest(self.items[position])
+            self.at.setdefault(value, []).append(position)
+            if value in wanted and not _being_written(self._holder(position), skip):
+                return self.items[position]
+        return None
+
+    def _holder(self, position: int) -> Path:
+        item = self.items[position]
+        return item[0] if isinstance(item, tuple) else item
+
+
+class _ZippedLookup:
+    """The archives of a folder not listed as files to install, by the
+    folder holding them: each one's names read once, so telling that a
+    missing file is still zipped costs two lookups and never a walk."""
+
+    def __init__(self, index: FolderIndex, known: set[str]) -> None:
+        self.index = index
+        self._by_parent: dict[Path, list[tuple[int, Path]]] = {}
+        for position, path in enumerate(index.archives):
+            if index.relative(path).casefold() not in known:
+                self._by_parent.setdefault(path.parent, []).append((position, path))
+        # folder -> (member name -> first archive, stem -> first archive).
+        self._maps: dict[Path, tuple[dict[str, tuple[int, Path]],
+                                     dict[str, tuple[int, Path]]]] = {}
+
+    def _folder(self, folder: Path) -> tuple[dict[str, tuple[int, Path]],
+                                             dict[str, tuple[int, Path]]]:
+        if folder not in self._maps:
+            members: dict[str, tuple[int, Path]] = {}
+            stems: dict[str, tuple[int, Path]] = {}
+            for position, path in self._by_parent.get(folder, []):
+                lowered = path.name.casefold()
+                if lowered.endswith(".zip"):
+                    for member in self.index.zip_members(path):
+                        members.setdefault(member, (position, path))
+                stems.setdefault(lowered.rsplit(".", 1)[0], (position, path))
+            self._maps[folder] = (members, stems)
+        return self._maps[folder]
+
+    def reason(self, name: str, folders: Collection[Path]) -> str:
+        """Why name is missing when an archive in folders holds it or carries
+        its name: the first such archive in the order the folder was read,
+        and one holding it before one carrying its name."""
+        lowered = name.casefold()
+        stem = lowered.rsplit(".", 1)[0]
+        best: tuple[int, int, Path] | None = None
+        for folder in folders:
+            members, stems = self._folder(folder)
+            for kind, held in ((0, members.get(lowered)), (1, stems.get(stem))):
+                if held is not None and (best is None or (held[0], kind) < best[:2]):
+                    best = (held[0], kind, held[1])
+        if best is None:
+            return ""
+        where = self.index.relative(best[2])
+        if best[1] == 0:
+            return f"still zipped: it is inside {where}"
+        return f"still zipped: {where} needs extracting"
+
+
 def _inside(path: Path, root: Path) -> bool:
     """Whether a link at path leads to a place under root."""
     try:
@@ -6858,7 +7130,19 @@ def _known_destinations(files: list[dict]) -> set[str]:
     return {str(f.get("dest", "")).casefold() for f in files}
 
 
-def _content_owner(sha1: str, entry: dict, files: list[dict]) -> str:
+def _content_owners(files: list[dict]) -> dict[str, list[dict]]:
+    """SHA-1 -> the entries one of whose copies holds it, in the list's order."""
+    owners: dict[str, list[dict]] = {}
+    for other in files:
+        for sha1 in dict.fromkeys(c.get("sha1") for c in _copies(other) if c.get("sha1")):
+            owners.setdefault(sha1, []).append(other)
+    return owners
+
+
+def _content_owner(
+    sha1: str, entry: dict, files: list[dict],
+    owners: dict[str, list[dict]] | None = None,
+) -> str:
     """Why a file holding sha1 is not the one entry wants, if the list knows."""
     for copy in entry.get("alternatives", []) + entry.get("slot_choices", []):
         if copy.get("sha1") == sha1 and copy.get("cores"):
@@ -6868,15 +7152,16 @@ def _content_owner(sha1: str, entry: dict, files: list[dict]) -> str:
         if copy.get("sha1") == sha1:
             return ("another revision: the one the platform's own list declares, "
                     "not the one the cores read")
-    for other in files:
+    if owners is None:
+        owners = _content_owners(files)
+    for other in owners.get(sha1, []):
         if other is entry or other.get("dest") == entry.get("dest"):
             continue
-        if any(copy.get("sha1") == sha1 for copy in _copies(other)):
-            regions = [r for r in other.get("regions") or [] if r != WORLD_REGION]
-            if regions:
-                return (f"another region: the contents of {other['dest']} "
-                        f"({', '.join(regions)})")
-            return f"another file: the contents of {other['dest']}"
+        regions = [r for r in other.get("regions") or [] if r != WORLD_REGION]
+        if regions:
+            return (f"another region: the contents of {other['dest']} "
+                    f"({', '.join(regions)})")
+        return f"another file: the contents of {other['dest']}"
     return ""
 
 
@@ -6935,7 +7220,7 @@ def _copy_reason(entry: dict, copy: LocalCopy, bios_path: Path, known: set[str])
 
 def _found_elsewhere(
     entry: dict, index: FolderIndex, bios_path: Path, files: list[dict],
-    known: set[str],
+    known: set[str], zipped: _ZippedLookup | None = None,
 ) -> tuple[str, str]:
     """(reason, folder too deep) for an entry missing at its place."""
     dest = str(entry.get("dest", ""))
@@ -6954,28 +7239,25 @@ def _found_elsewhere(
         return "", ""
     same_name = [p for p in index.by_name.get(name.casefold(), []) if p != expected]
     matching = [p for p in same_name if index.size[p] in sizes and index.sha1(p) in sha1s]
+    held = set(matching)
     for path in matching or same_name:
-        contents = "" if path in matching else ", with other contents"
+        contents = "" if path in held else ", with other contents"
         found = _place_reason(entry, path, index, expected, known, contents)
         if found[0]:
             return found
     above = index.root.parent / dest
     if index.root.parent != index.root and above.is_file():
         return f"one folder too high: it is at {above}", ""
-    for path in index.holding(sizes, sha1s):
-        return _place_reason(entry, path, index, expected, known)
+    holder = index.first_holding(sizes, sha1s)
+    if holder is not None:
+        return _place_reason(entry, holder, index, expected, known)
     if name.casefold().endswith(_ARCHIVE_SUFFIXES):
         return "", ""
-    stem = name.rsplit(".", 1)[0].casefold()
-    folders = {expected.parent, index.root}
-    for path in index.archives:
-        if path.parent not in folders or index.relative(path).casefold() in known:
-            continue
-        lowered = path.name.casefold()
-        if lowered.endswith(".zip") and name.casefold() in index.zip_members(path):
-            return f"still zipped: it is inside {index.relative(path)}", ""
-        if lowered.rsplit(".", 1)[0] == stem:
-            return f"still zipped: {index.relative(path)} needs extracting", ""
+    if zipped is None:
+        zipped = _ZippedLookup(index, known)
+    reason = zipped.reason(name, {expected.parent, index.root})
+    if reason:
+        return reason, ""
     for path in index.apple_double.get(name.casefold(), []):
         if path.parent == expected.parent:
             return (f"only a Mac's ._ file: {path.name} holds Finder data, not the "
@@ -6983,7 +7265,10 @@ def _found_elsewhere(
     return "", ""
 
 
-def _wrong_contents(entry: dict, path: Path, index: FolderIndex, files: list[dict]) -> str:
+def _wrong_contents(
+    entry: dict, path: Path, index: FolderIndex, files: list[dict],
+    owners: dict[str, list[dict]] | None = None,
+) -> str:
     """Why the file at an entry's place is not the one the list installs."""
     try:
         size = path.stat().st_size
@@ -6992,7 +7277,7 @@ def _wrong_contents(entry: dict, path: Path, index: FolderIndex, files: list[dic
     except OSError:
         return ""
     sha1 = index.sha1(path)
-    owner = _content_owner(sha1, entry, files)
+    owner = _content_owner(sha1, entry, files, owners)
     if owner:
         return owner
     if head == b"PK\x03\x04" and not str(entry.get("dest", "")).casefold().endswith(".zip"):
@@ -7024,18 +7309,20 @@ def diagnose(
     if index is None:
         index = bios_folder_index(bios_path, files, verified or [])
     known = _known_destinations(files)
+    zipped = _ZippedLookup(index, known)
     for entry in missing:
-        reason, extra = _found_elsewhere(entry, index, bios_path, files, known)
+        reason, extra = _found_elsewhere(entry, index, bios_path, files, known, zipped)
         if reason:
             reasons[id(entry)] = reason
         if extra:
             deep[extra] = deep.get(extra, 0) + 1
+    owners = _content_owners(files) if mismatched else {}
     for entry in mismatched:
         try:
             path = _entry_path(bios_path, entry)
         except ValueError:
             continue
-        reason = _wrong_contents(entry, path, index, files)
+        reason = _wrong_contents(entry, path, index, files, owners)
         if reason:
             reasons[id(entry)] = reason
     return Diagnosis(reasons, deep)
@@ -7270,6 +7557,47 @@ def emit_copies(
                     path=str(copy.path), member=copy.member, reason=reason)
 
 
+class Unreadable(NamedTuple):
+    """A file, ZIP member or folder below a --from folder that could not be
+    read, and may be a file the run still needs."""
+    index: FolderIndex
+    path: Path
+    member: str
+    why: str
+
+    def line(self) -> str:
+        where = f"{self.member} inside {self.path}" if self.member else str(self.path)
+        return f"{where} could not be read: {self.why}"
+
+
+def unreadable_copies(
+    entries: list[dict], copies: dict[int, LocalCopy], folders: list[FolderIndex],
+) -> list[Unreadable]:
+    """What the --from folders hold that could not be read, of the size of
+    a file no copy was found for, or of no known size: dropped silently,
+    a file a user had read as one to download."""
+    sizes = {e["size"] for e in entries
+             if id(e) not in copies and isinstance(e.get("size"), int)}
+    if not sizes:
+        return []
+    return [Unreadable(index, *held) for index in folders
+            for held in index.unreadable_for(sizes)]
+
+
+def unreadable_lines(found: list[Unreadable], every: bool) -> list[str]:
+    lines = [f"  {item.line()}" for item in (found if every else found[:10])]
+    if len(found) > len(lines):
+        lines.append(f"  ... and {len(found) - len(lines)} more could not be read "
+                     "(--verbose lists them all)")
+    return lines
+
+
+def emit_unreadable(plat: str, found: list[Unreadable]) -> None:
+    for item in found:
+        EVENTS.emit("unreadable", platform=plat, folder=item.index.label,
+                    path=str(item.path), member=item.member, reason=item.why)
+
+
 def sources_line(
     folders: list[tuple[str, int]], fixed: int, downloaded: int, missing: int,
     fetched: str = "downloaded", shared: int = 0,
@@ -7737,26 +8065,71 @@ def _open_local_sources(args: argparse.Namespace) -> None:
         sys.exit(1)
 
 
+def _system_folder(folder: Path) -> str:
+    """Why a --from folder is one of the system's own, in words, or ""."""
+    try:
+        resolved = folder.resolve()
+    except OSError:
+        return ""
+    text = str(resolved)
+    if any(text == pseudo or text.startswith(pseudo + "/") for pseudo in PSEUDO_FOLDERS):
+        return "is a folder the system makes up, it holds no copy"
+    system_drive = os.environ.get("SystemDrive", "C:") if os.name == "nt" else ""
+    if resolved == Path(resolved.anchor) and (
+            os.name != "nt" or resolved.drive.casefold() == system_drive.casefold()):
+        return "is the root of the whole system"
+    return ""
+
+
+def from_folder_problem(label: str) -> tuple[str, str] | None:
+    """Why a folder named with --from cannot be read, without reading what
+    it holds: (reason, words), or None when it can.
+
+    reason is the one the error event carries: not_found, not_a_folder (a
+    file that is no ZIP), unreadable, or system (the root of the system,
+    /proc, /sys or /dev). words follow the folder as given: "E:\\old does
+    not exist". Cheap enough for a screen to call before indexing."""
+    folder = Path(os.path.expanduser(label))
+    zipped = folder.is_file() and folder.name.casefold().endswith(".zip")
+    if not (folder.is_dir() or zipped):
+        if folder.exists():
+            return "not_a_folder", "is neither a folder nor a ZIP"
+        return "not_found", "does not exist"
+    try:
+        if zipped:
+            with open(folder, "rb"):
+                pass
+        else:
+            with os.scandir(folder) as listing:
+                next(listing, None)
+    except OSError as exc:
+        return "unreadable", f"could not be read: {unreadable_words(exc)}"
+    system = "" if zipped else _system_folder(folder)
+    return ("system", system) if system else None
+
+
 def _open_from_folders(folders: list[Path]) -> None:
-    """Read each folder named with --from, or stop on one that is not there."""
+    """Read each folder named with --from, or stop on one that cannot be."""
     for folder in folders:
         label = str(folder)
-        folder = Path(os.path.expanduser(label))
-        zipped = folder.is_file() and folder.name.casefold().endswith(".zip")
-        if not (folder.is_dir() or zipped):
-            reason = "not_a_folder" if folder.exists() else "not_found"
-            what = ("does not exist" if reason == "not_found"
-                    else "is neither a folder nor a ZIP")
+        problem = from_folder_problem(label)
+        if problem is not None:
+            reason, what = problem
             message = f"{label} {what}: --from names the folder to copy BIOS files from"
             EVENTS.emit("error", stage="from", reason=reason, platform="", message=message)
             print(f"Error: {message}.", file=sys.stderr)
             sys.exit(1)
-        index = FolderIndex(folder, label)
+        index = FolderIndex(Path(os.path.expanduser(label)), label)
         LOCAL.folders.append(index)
         print(f"  Copying BIOS files from: {label}")
         if index.truncated:
             print(f"  Read the first {MAX_SCANNED_FILES:,} files of {label} only. Name "
                   "the folder that holds the BIOS files to search it all.")
+        for drive in index.other_drives:
+            print(f"  Did not go into {drive}: it is another drive. Name it with --from "
+                  "to read it too.")
+        EVENTS.emit("scanned", folder=label, files=index.files, truncated=index.truncated,
+                    other_drives=[str(drive) for drive in index.other_drives])
 
 
 def _missing_dest(dest: Path, ask: bool, named: str = "named with --dest") -> None:
@@ -8362,6 +8735,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
         if bios_index is not None:
             bios_indexes.append(bios_index)
         copies = plan_local_copies(to_download, bios_path, bios_index, LOCAL.folders)
+        unread = unreadable_copies(to_download, copies, LOCAL.folders)
         hint = [] if LOCAL.pack is not None else bulk_hint(
             to_download, manifest,
             systems_chosen=bool(selection and selection[0]),
@@ -8386,6 +8760,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
                 chosen=selection[0] if selection else None,
             )
             emit_copies(plat_name, to_download, copies, bios_index, bios_path, files)
+            emit_unreadable(plat_name, unread)
         total_verified += len(up_to_date)
 
         if args.check:
@@ -8404,6 +8779,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
                 for line in check_lines(missing, mismatched, files, manifest, bios_path,
                                         args.verbose, up_to_date, bios_index, copies):
                     print(line)
+                for line in unreadable_lines(unread, args.verbose):
+                    print(line)
                 print("  Run the same command without --check to fetch them; "
                       "a file with a wrong hash is replaced, and the one in place "
                       f"kept in {BACKUP_DIR} beside the BIOS folder.")
@@ -8437,6 +8814,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
             _print_plan([f for f in to_download if id(f) not in copies], bios_path, term,
                         trees, kept, copy_plan_lines(to_download, copies, bios_index,
                                                      bios_path, files, args.verbose, True))
+            for line in unreadable_lines(unread, args.verbose):
+                print(line)
             for line in hint:
                 print(line)
             continue
@@ -8462,6 +8841,10 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
                 files_pending += len(to_download)
                 continue
             EVENTS.emit("download", platform=plat_name, files=len(to_download), bytes=dl_size)
+            if unread:
+                print()
+                for line in unreadable_lines(unread, args.verbose):
+                    print(line)
             if copies:
                 print()
                 for line in copy_plan_lines(to_download, copies, bios_index, bios_path,

install.sh

@@ -18,7 +18,7 @@ esac
 TEMP_INSTALLER=""
 TEMP_DIRECTORY=""
 DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-DEFAULT_INSTALL_SHA256="bcbde4c289b32c02ada78001b75274a87b125c1efd3d8fb1064d7f5482ef3997"
+DEFAULT_INSTALL_SHA256="c90676dcc433061bd97aca94930cd7acd74295624840cb5c85a22897f9c1dd12"
 MAX_INSTALLER_BYTES=2097152
 
 cleanup() {

tests/test_install.py

@@ -5102,7 +5102,10 @@ class EventsDescribeTheRun(unittest.TestCase, _SyntheticTree):
                  "bytes_total": 245}
                 for n, (dest, size, total) in enumerate(
                     (("c.bin", 70, 70), ("d.bin", 25, 95), ("b.bin", 150, 245)), 1)]
-        self.assertEqual([e for e in events if e["event"] != "progress"],
+        # The last transfer: every byte came over the network.
+        self.assertEqual([e for e in events if e["event"] == "transfer"][-1],
+                         {"event": "transfer", "bytes_local": 0, "bytes_network": 245})
+        self.assertEqual([e for e in events if e["event"] not in ("progress", "transfer")],
                          self._head("install", installer) + [
             {"event": "download", "platform": "retroarch", "files": 3, "bytes": 245},
             *done,

tests/test_install_from.py

@@ -13,11 +13,15 @@ from __future__ import annotations
 
 import contextlib
 import copy
+import errno
 import io
 import json
+import os
 import subprocess
 import sys
+import time
 import unittest
+import urllib.error
 import zipfile
 from pathlib import Path
 from unittest import mock
@@ -480,6 +484,377 @@ class MisplacedInTheBiosFolder(Scratch):
             self.assertNotIn("copied from", proc.stdout)
 
 
+def _unprivileged(test: unittest.TestCase) -> None:
+    if os.name != "posix" or os.geteuid() == 0:
+        test.skipTest("a file of mode 000 is read here anyway")
+
+
+class ACopyThatFails(FromFolder):
+    """A copy found by the plan that cannot be read when it is copied:
+    it read as "folder not writable", and a download failing after it lost
+    the first reason."""
+
+    def setUp(self):
+        super().setUp()
+        self.entry = json.loads(self.list.read_text())["files"][0]
+        self.source = self.keep("old/scph5501.bin", US)
+        self.copy = install.FolderIndex(self.old).find(self.entry)
+        self.assertIsNotNone(self.copy)
+        self.addCleanup(install.LOCAL.reset)
+        self.dest.mkdir()
+
+    def test_an_unreadable_source_falls_back_and_names_both_reasons(self):
+        _unprivileged(self)
+        self.source.chmod(0)
+        self.addCleanup(self.source.chmod, 0o644)
+        refused = urllib.error.URLError(ConnectionRefusedError(111, "Connection refused"))
+        with mock.patch.object(install, "_retry_delay", return_value=0), \
+                mock.patch.object(install.urllib.request, "urlopen", side_effect=refused):
+            dest, ok, reason = install._download_one(self.entry, self.dest, copy=self.copy)
+        self.assertFalse(ok)
+        self.assertTrue(reason.startswith(
+            f"{self.source.resolve()} could not be read: Permission denied, and "), reason)
+        self.assertNotIn(install.NOT_WRITABLE, reason)
+        self.assertIsInstance(reason, install.NetworkReason)
+
+    def test_a_read_error_is_the_source_and_not_the_folder(self):
+        class Failing(io.BytesIO):
+            def read(self, *_args):
+                raise OSError(errno.EIO, "Input/output error")
+
+        for source, words in (
+                (lambda: Failing(b""), "Input/output error"),
+                (mock.Mock(side_effect=PermissionError(errno.EACCES, "Permission denied")),
+                 "Permission denied")):
+            with self.subTest(words=words):
+                result = install._write_verified(
+                    self.entry, self.dest, self.dest / "scph5501.bin", source, None,
+                    "the copy in E:\\", named="E:\\bios\\a.bin")
+                self.assertEqual(result, ("scph5501.bin", False,
+                                          f"E:\\bios\\a.bin could not be read: {words}"))
+        self.assertEqual(list(self.dest.iterdir()), [], "no temporary file is left")
+
+    def test_a_run_reading_a_pack_falls_back_to_the_pack_only(self):
+        _unprivileged(self)
+        self.source.chmod(0)
+        self.addCleanup(self.source.chmod, 0o644)
+        install.LOCAL.pack = mock.Mock(find=mock.Mock(return_value=None))
+        with mock.patch.object(install.urllib.request, "urlopen",
+                               side_effect=AssertionError("network")):
+            _dest, ok, reason = install._download_one(self.entry, self.dest, copy=self.copy)
+        install.LOCAL.pack = None
+        self.assertFalse(ok)
+        self.assertEqual(reason, f"{self.source.resolve()} could not be read: "
+                                 f"Permission denied, and {install.NOT_IN_PACK}")
+
+
+def _patched_zip(path: Path, name: str, data: bytes, flags: int = 0,
+                 method: int | None = None) -> Path:
+    """A stored ZIP of one member, its headers then given other flags or
+    another compression method: zipfile writes neither encryption nor
+    methods it cannot read."""
+    _zip(path, {})
+    with zipfile.ZipFile(path, "w", zipfile.ZIP_STORED) as archive:
+        archive.writestr(name, data)
+    raw = bytearray(path.read_bytes())
+    for signature, at in ((b"PK\x03\x04", 6), (b"PK\x01\x02", 8)):
+        start = raw.find(signature)
+        raw[start + at] |= flags
+        if method is not None:
+            raw[start + at + 2:start + at + 4] = method.to_bytes(2, "little")
+    path.write_bytes(bytes(raw))
+    return path
+
+
+class UnreadableCopies(FromFolder):
+    """A copy that could not be read was cached as no match and dropped:
+    the user saw a file to download that the old card held."""
+
+    def setUp(self):
+        super().setUp()
+        self.keep("scph5500.bin", JP)
+        self.keep("pcsx/extra.bin", EXTRA)
+        self.keep("gba_bios.bin", GBA)
+
+    def locked(self, relative: str, data: bytes) -> Path:
+        _unprivileged(self)
+        path = self.keep(relative, data)
+        path.chmod(0)
+        self.addCleanup(path.chmod, 0o644)
+        return path
+
+    def test_a_file_of_mode_000_is_named_in_every_mode(self):
+        path = self.locked("psx/scph5501.bin", US).resolve()
+        self.locked("notes.txt", b"not the size of any file")
+        line = f"{path} could not be read: Permission denied"
+        for options in (("--check",), ("--dry-run",), ()):
+            with self.subTest(options=options):
+                proc = self.install(*options)
+                self.assertEqual(proc.returncode, 1 if options != ("--dry-run",) else 0,
+                                 proc.stdout + proc.stderr)
+                self.assertIn(line, proc.stdout)
+                self.assertNotIn("notes.txt", proc.stdout)
+
+    def test_an_unreadable_copy_is_an_event(self):
+        path = self.locked("psx/scph5501.bin", US).resolve()
+        proc = self.install("--check", "--events")
+        events = _events(proc)
+        for event in events:
+            self.assertEqual(list(event), ["event", *install.EVENT_KEYS[event["event"]]])
+        self.assertEqual([e for e in events if e["event"] == "unreadable"], [{
+            "event": "unreadable", "platform": "retroarch", "folder": str(self.old),
+            "path": str(path), "member": "", "reason": "Permission denied"}])
+        kinds = [e["event"] for e in events]
+        self.assertLess(kinds.index("found"), kinds.index("unreadable"))
+
+    def test_a_copy_found_elsewhere_leaves_the_unreadable_one_unnamed(self):
+        self.locked("psx/scph5501.bin", US)
+        self.keep("psx/copy.bin", US)
+        proc = self.install("--check")
+        self.assertNotIn("could not be read", proc.stdout)
+
+    def test_a_folder_of_mode_000_is_named(self):
+        _unprivileged(self)
+        folder = self.old / "psx"
+        folder.mkdir()
+        folder.chmod(0)
+        self.addCleanup(folder.chmod, 0o755)
+        proc = self.install("--check")
+        self.assertIn(f"{folder.resolve()} could not be read: Permission denied", proc.stdout)
+
+    def test_members_that_cannot_be_read_say_why(self):
+        for flags, method, words in (
+                (0x1, None, "it is encrypted, a password is needed"),
+                (0, 6, "it is packed with a compression method this installer "
+                       "cannot read")):
+            with self.subTest(words=words):
+                archive = _patched_zip(self.old / "psx.zip", "scph5501.bin", US,
+                                       flags, method).resolve()
+                proc = self.install("--check")
+                self.assertIn(f"scph5501.bin inside {archive} could not be read: {words}",
+                              proc.stdout)
+
+    def test_an_input_output_error_is_named(self):
+        self.keep("psx/scph5501.bin", US)
+        index = install.FolderIndex(self.old)
+        entry = json.loads(self.list.read_text())["files"][0]
+        failing = OSError(errno.EIO, "Input/output error")
+        with mock.patch.object(install, "_sha1_file", side_effect=failing):
+            self.assertIsNone(index.find(entry))
+        found = install.unreadable_copies([entry], {}, [index])
+        # scph5500.bin holds another region of the same size: read too.
+        self.assertEqual([item.line() for item in found], [
+            f"{(self.old / 'psx' / 'scph5501.bin').resolve()} could not be read: "
+            "Input/output error",
+            f"{(self.old / 'scph5500.bin').resolve()} could not be read: "
+            "Input/output error"])
+
+
+class WhereTheSearchGoes(FromFolder):
+    """--from / walked the whole system, /proc and every drive mounted in
+    it. The search stays on the drive it starts on, and the folder is
+    checked before it is read, cheaply enough for a screen to do it."""
+
+    def test_a_folder_is_checked_without_being_read(self):
+        archive = _zip(self.root / "backup.zip", {"a": b"a"})
+        (self.root / "notes.txt").write_text("x")
+        self.assertIsNone(install.from_folder_problem(str(self.old)))
+        self.assertIsNone(install.from_folder_problem(str(archive)))
+        self.assertEqual(install.from_folder_problem(str(self.root / "nowhere")),
+                         ("not_found", "does not exist"))
+        self.assertEqual(install.from_folder_problem(str(self.root / "notes.txt")),
+                         ("not_a_folder", "is neither a folder nor a ZIP"))
+        if os.name == "posix":
+            self.assertEqual(install.from_folder_problem("/"),
+                             ("system", "is the root of the whole system"))
+            self.assertEqual(install.from_folder_problem("/proc")[0], "system")
+
+    def test_a_folder_that_cannot_be_listed_is_refused(self):
+        _unprivileged(self)
+        self.old.chmod(0)
+        self.addCleanup(self.old.chmod, 0o755)
+        self.assertEqual(install.from_folder_problem(str(self.old)),
+                         ("unreadable", "could not be read: Permission denied"))
+        proc = self.install("--events")
+        error = next(e for e in _events(proc) if e["event"] == "error")
+        self.assertEqual((error["stage"], error["reason"]), ("from", "unreadable"))
+        self.assertEqual(proc.returncode, 1)
+
+    @unittest.skipUnless(os.name == "posix", "a POSIX root")
+    def test_the_root_of_the_system_is_refused(self):
+        proc = self.run_installer("--manifest", str(self.list), "--from", "/", "--events")
+        error = next(e for e in _events(proc) if e["event"] == "error")
+        self.assertEqual((error["stage"], error["reason"]), ("from", "system"))
+        self.assertIn("/ is the root of the whole system", proc.stderr)
+        self.assertEqual(proc.returncode, 1)
+
+    def lstat_saying(self, folder: str, **fields):
+        real = os.lstat
+
+        def lstat(path, *args, **kwargs):
+            held = real(path, *args, **kwargs)
+            if Path(path).name != folder:
+                return held
+            values = {"st_dev": held.st_dev, "st_file_attributes": 0, **fields}
+            return mock.Mock(**values)
+
+        return mock.patch.object(install.os, "lstat", side_effect=lstat)
+
+    def test_another_drive_inside_is_named_and_not_entered(self):
+        self.keep("mnt/scph5501.bin", US)
+        self.keep("scph5500.bin", JP)
+        with self.lstat_saying("mnt", st_dev=os.lstat(self.old).st_dev + 1):
+            index = install.FolderIndex(self.old)
+        self.assertEqual(index.other_drives, [self.old.resolve() / "mnt"])
+        self.assertEqual(sorted(p.name for p in index.size), ["scph5500.bin"])
+
+    def test_a_junction_is_not_entered(self):
+        self.keep("junction/scph5501.bin", US)
+        with self.lstat_saying("junction", st_file_attributes=0x400):
+            index = install.FolderIndex(self.old)
+        self.assertEqual(index.size, {})
+        self.assertEqual(index.other_drives, [])
+
+    def test_pseudo_file_systems_are_never_entered(self):
+        if not install.PSEUDO_FOLDERS:
+            self.skipTest("no pseudo file system here")
+        index = install.FolderIndex(self.old)
+        for pseudo in sorted(install.PSEUDO_FOLDERS):
+            self.assertFalse(index._enters(Path(pseudo), os.lstat("/").st_dev), pseudo)
+
+    def test_a_search_cut_short_is_an_event(self):
+        for n in range(3):
+            self.keep(f"f{n}.bin", b"%d" % n)
+        out = io.StringIO()
+        install.EVENTS.open(out)
+        self.addCleanup(setattr, install.EVENTS, "stream", None)
+        self.addCleanup(install.LOCAL.reset)
+        with mock.patch.object(install, "MAX_SCANNED_FILES", 2), \
+                contextlib.redirect_stdout(io.StringIO()) as text:
+            install._open_from_folders([self.old])
+        self.assertIn(f"Read the first 2 files of {self.old} only", text.getvalue())
+        self.assertEqual([json.loads(line) for line in out.getvalue().splitlines()], [{
+            "event": "scanned", "folder": str(self.old), "files": 2, "truncated": True,
+            "other_drives": []}])
+
+    def test_each_folder_read_is_an_event_before_detect(self):
+        self.keep("scph5501.bin", US)
+        events = _events(self.install("--check", "--events"))
+        kinds = [e["event"] for e in events]
+        scanned = next(e for e in events if e["event"] == "scanned")
+        self.assertEqual(scanned, {"event": "scanned", "folder": str(self.old), "files": 1,
+                                   "truncated": False, "other_drives": []})
+        self.assertLess(kinds.index("start"), kinds.index("scanned"))
+        self.assertLess(kinds.index("scanned"), kinds.index("detect"))
+
+
+class LocalAndNetworkBytes(unittest.TestCase):
+    """The engine knows which bytes came from this machine. The interface
+    guessed it to show a download speed: the transfer event tells it."""
+
+    def test_the_counts_follow_each_source_and_each_rollback(self):
+        out = io.StringIO()
+        install.EVENTS.open(out)
+        self.addCleanup(setattr, install.EVENTS, "stream", None)
+        term = install.Terminal()
+        term.live, term.quiet = False, True
+        with mock.patch.object(install, "EVENT_PROGRESS_INTERVAL", 0):
+            progress = install.Progress(3, 100, term)
+            progress.add(10, resumed=True)            # a partial copy resumed
+            progress.add(20)                          # from the network
+            progress.add(30, resumed=True, network=False)  # a copy found here
+            progress.add(15, network=False)           # the pack
+            progress.add(-30, resumed=True, network=False)
+            progress.add(-30)                         # a download dropped,
+            progress.forget_resumed(10)               # its resumed part too
+        self.assertEqual((progress.bytes, progress.local), (15, 15))
+        events = [json.loads(line) for line in out.getvalue().splitlines()]
+        transfers = [e for e in events if e["event"] == "transfer"]
+        progresses = [e for e in events if e["event"] == "progress"]
+        self.assertEqual(len(transfers), len(progresses))
+        self.assertEqual([(t["bytes_local"], t["bytes_network"]) for t in transfers][:4],
+                         [(10, 0), (10, 20), (40, 20), (55, 20)])
+        for transfer, update in zip(transfers, progresses):
+            self.assertEqual(transfer["bytes_local"] + transfer["bytes_network"],
+                             update["bytes_done"])
+
+    def test_an_install_from_a_folder_and_the_network_splits_them(self):
+        with FakeRepository() as repo:
+            old = repo.root / "old"
+            payloads = FakeRepository.PAYLOADS
+            old.mkdir()
+            (old / "a.bin").write_bytes(payloads["scph5500.bin"][0])
+            (old / "b.bin").write_bytes(payloads["gba_bios.bin"][0])
+            proc = repo.run("--from", str(old), "--no-input", "--events")
+            self.assertEqual(proc.returncode, 0, proc.stderr)
+            events = _events(proc)
+        local = len(payloads["scph5500.bin"][0]) + len(payloads["gba_bios.bin"][0])
+        total = sum(len(p[0]) for p in payloads.values())
+        kinds = [e["event"] for e in events]
+        last = len(kinds) - 1 - kinds[::-1].index("transfer")
+        self.assertEqual(events[last], {"event": "transfer", "bytes_local": local,
+                                        "bytes_network": total - local})
+        self.assertLess(kinds.index("download"), last)
+        self.assertLess(last, kinds.index("summary"))
+
+
+class AFolderAroundTheBiosFolder(FromFolder):
+    """--from names a folder holding the BIOS folder: an archive this run
+    replaces was a source of members while it was being written."""
+
+    def test_an_archive_being_replaced_gives_no_member(self):
+        new_set = b"the archive the list installs" * 4
+        inner = b"a rom inside the old archive" * 3
+        _zip(self.dest / "set.zip", {"inner.bin": inner})
+        files = [_entry("set.zip", new_set, ["x"]), _entry("inner.bin", inner, ["x"])]
+        around = install.FolderIndex(self.root)
+        copies = install.plan_local_copies(files, self.dest, None, [around])
+        self.assertEqual(copies, {})
+        _zip(self.old / "copy.zip", {"inner.bin": inner})
+        copies = install.plan_local_copies(files, self.dest, None,
+                                           [install.FolderIndex(self.root)])
+        self.assertEqual([copies[id(e)].where() for e in files if id(e) in copies],
+                         ["old card/copy.zip, inside as inner.bin"])
+
+
+class ManySameSizedFiles(FromFolder):
+    """A card of 16 KB ROMs and a list of 16 KB entries: each entry sorted
+    and walked every file of its size."""
+
+    def test_each_file_is_read_once_and_the_order_holds(self):
+        for n in range(3000):
+            self.keep(f"roms/r{n:04}.bin", b"%016d" % n)
+            _zip(self.old / f"sets/s{n:04}.zip", {f"m{n}.bin": b"m%015d" % n})
+        entries = [{"dest": f"e{n}.bin", "size": 16, "sha1": f"{n:040}"}
+                   for n in range(3000)]
+        wanted = [_entry("late.bin", b"%016d" % 2999, ["x"]),
+                  _entry("m5.bin", b"m%015d" % 5, ["x"]),
+                  _entry("r0007.bin", b"%016d" % 7, ["x"])]
+        index = install.FolderIndex(self.old)
+        start = time.monotonic()
+        copies = install.plan_local_copies(entries + wanted, self.dest, None, [index])
+        elapsed = time.monotonic() - start
+        self.assertEqual({e["dest"]: copies[id(e)].where() for e in wanted if id(e) in copies}, {
+            "late.bin": "roms/r2999.bin",
+            "m5.bin": "sets/s0005.zip, inside as m5.bin",
+            "r0007.bin": "roms/r0007.bin",
+        })
+        self.assertEqual(len(copies), 3)
+        self.assertLess(elapsed, 30, f"{elapsed:.1f} s")
+
+    def test_a_file_of_the_same_name_comes_first(self):
+        self.keep("a/other.bin", US)
+        self.keep("b/scph5501.bin", US)
+        _zip(self.old / "c.zip", {"x/gba.bin": GBA})
+        _zip(self.old / "d.zip", {"gba_bios.bin": GBA})
+        index = install.FolderIndex(self.old)
+        files = json.loads(self.list.read_text())["files"]
+        copies = install.plan_local_copies(files, self.dest, None, [index])
+        found = {e["dest"]: copies[id(e)].where() for e in files if id(e) in copies}
+        self.assertEqual(found["scph5501.bin"], "b/scph5501.bin")
+        self.assertEqual(found["gba_bios.bin"], "d.zip, inside as gba_bios.bin")
+
+
 class TheSummaryLine(unittest.TestCase):
     def test_the_parts_in_words(self):
         line = install.sources_line([("E:\\", 38)], fixed=0, downloaded=4, missing=2)

tests/test_install_offline.py

@@ -19,6 +19,7 @@ import socket
 import subprocess
 import sys
 import tempfile
+import time
 import unittest
 import zipfile
 from pathlib import Path
@@ -243,6 +244,58 @@ class CheckNamesWhyAFileIsRefused(Scratch):
         self.assertNotIn("opened at", proc.stdout)
 
 
+class ArchivesInTheBiosFolder(Scratch):
+    """The reason an archive gives is the first archive, in the order the
+    folder is read, that holds the file or carries its name."""
+
+    def zipped(self, relative: str, members: dict[str, bytes]) -> None:
+        path = self.dest / relative
+        path.parent.mkdir(parents=True, exist_ok=True)
+        with zipfile.ZipFile(path, "w") as archive:
+            for name, data in members.items():
+                archive.writestr(name, data)
+
+    def test_each_reason_names_the_first_archive_that_answers(self):
+        self.zipped("a.zip", {"x.bin": b"x"})
+        self.zipped("x.zip", {"other": b"o"})
+        self.zipped("b.zip", {"other": b"o"})
+        self.zipped("c.zip", {"bios/b.bin": b"b"})
+        self.place("y.7z", b"7z")
+        self.zipped("sub/z.zip", {"Z.BIN": b"z"})
+        self.zipped("elsewhere/w.zip", {"w.bin": b"w"})
+        self.zipped("listed.zip", {"v.bin": b"v"})
+        entries = {dest: _entry(dest, dest.encode() * 3, ["x"]) for dest in (
+            "x.bin", "b.bin", "y.bin", "sub/z.bin", "w.bin", "v.bin")}
+        files = [*entries.values(), _entry("listed.zip", b"zip", ["x"])]
+        found = install.diagnose(list(entries.values()), [], files, self.dest)
+        self.assertEqual({dest: found.reasons.get(id(e), "") for dest, e in entries.items()}, {
+            "x.bin": "still zipped: it is inside a.zip",
+            "b.bin": "still zipped: b.zip needs extracting",
+            "y.bin": "still zipped: y.7z needs extracting",
+            "sub/z.bin": "still zipped: it is inside sub/z.zip",
+            "w.bin": "",
+            "v.bin": "",
+        })
+
+    def test_many_archives_and_many_missing_files_take_one_pass(self):
+        """1,519 ZIPs and 2,000 missing files took 57 seconds: each missing
+        file walked every archive of the folder."""
+        self.dest.mkdir()
+        for n in range(3000):
+            with zipfile.ZipFile(self.dest / f"set{n:04}.zip", "w") as archive:
+                archive.writestr(f"rom{n}.bin", b"x")
+            (self.dest / f"same{n:04}.bin").write_bytes(b"%016d" % n)
+        missing = [{"dest": f"file{n}.bin", "size": 16 if n % 2 else 1000 + n,
+                    "sha1": f"{n:040}"} for n in range(3000)]
+        missing.append(_entry("rom7.bin", b"r", ["x"]))
+        start = time.monotonic()
+        found = install.diagnose(missing, [], missing, self.dest)
+        elapsed = time.monotonic() - start
+        self.assertEqual(found.reasons, {
+            id(missing[-1]): "still zipped: it is inside set0007.zip"})
+        self.assertLess(elapsed, 30, f"{elapsed:.1f} s")
+
+
 def _pack(directory: Path, name: str, members: dict[str, bytes]) -> Path:
     """A pack as generate_pack writes one: the files and a manifest.json."""
     listed = [{"path": path, "sha1": _sha1(data), "md5": hashlib.md5(data).hexdigest(),

wiki/installer.md

@@ -465,6 +465,13 @@ writes, the system folders (`System Volume Information`, `.Spotlight-V100`,
 `.Trashes`, `$RECYCLE.BIN`) and a link leading out of the folder are skipped.
 `--from` also takes a ZIP of an old BIOS folder.
 
+The search stays on one drive: a drive mounted inside the folder is named and
+not entered (give it its own `--from`), and so are `/proc`, `/sys`, `/dev` and
+Windows junctions. The root of the system (`/`, `C:\`) is refused. A file that
+cannot be read, and has the size of a file still missing, is named with the
+reason (`E:\bios\a.bin could not be read: Permission denied`), as is a ZIP
+member locked by a password.
+
 The folder is only read: each file is copied, checked against the list's
 hashes once written, and the original stays where it is. A file in place with
 other contents is kept aside first, as a download would replace it (see
@@ -1152,7 +1159,7 @@ fields, the installer refuses to narrow rather than install everything.
 | Code | Meaning |
 |------|---------|
 | `0` | everything asked for is installed or already up to date |
-| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist, or `--restore` could not put a file back |
+| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, a `--from` folder that does not exist or cannot be read, or `--restore` could not put a file back |
 | `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a folder that does not exist, named with `--dest` or typed at the folder question, was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` |
 | `130` | stopped with Ctrl+C, or with `q`, Ctrl+C or Ctrl+D at a question. Run the same command again to carry on |
 | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) |
@@ -1185,14 +1192,17 @@ kind, and each kind has exactly the keys below, in this order. The schema is
 |------|------|------|
 | `start` | `schema`, `installer`, `mode` | First line. `schema` is 1, `installer` the identity `--version` prints, `mode` one of `install`, `check`, `dry-run`, `list` |
 | `detect` | `os`, `os_name`, `platforms` | Once the platforms are known. `os_name` comes from `/etc/os-release`. `platforms` lists `{platform, name, bios_dir}`, and is empty when nothing supported was found |
-| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). A folder named with `--from` is not there (`stage` is `from`, `reason` `not_found`, or `not_a_folder` for a file that is no ZIP). `message` says it in words |
+| `error` | `stage`, `reason`, `platform`, `message` | The file list could not be read (`stage` is `manifest`): `reason` is `network`, `not_found` or `invalid`, `clock` when the server's certificate was refused and the system clock reads a date before the installer was written, or `no_ca` when the machine holds no list of trusted certificates (`SSL_CERT_FILE` names one). A folder named with `--from` cannot be read (`stage` is `from`, `reason` `not_found`, `not_a_folder` for a file that is no ZIP, `unreadable`, or `system` for the root of the system, `/proc`, `/sys` or `/dev`). `message` says it in words |
+| `scanned` | `folder`, `files`, `truncated`, `other_drives` | Each folder named with `--from`, once read and before `detect`. `files` counts the files found, `truncated` is true when the search stopped at 200,000 of them, and `other_drives` lists the folders below it on another drive, which were not entered |
 | `checking` | `files_done`, `files_total` | While the files already in the folder are read and hashed, at most every half second and once at the end. `files_total` counts the files found in place |
 | `file` | `platform`, `dest`, `systems`, `size`, `state` | Each file of the list after the check. `state` is `present` (hash verified), `wrong_hash` or `missing` |
 | `system` | `platform`, `system`, `name`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed` | Each system, sorted by id, and with `--system` only the systems named. Files with no system count under `other`. `name` is the platform's own name for it |
 | `check` | `platform`, `name`, `bios_dir`, `files`, `present`, `missing`, `wrong_hash`, `bytes_needed`, `data_missing`, `not_installable` | The platform's totals. `data_missing` counts data-tree files to fetch, `not_installable` the files the collection cannot supply |
 | `found` | `platform`, `dest`, `folder`, `path`, `member`, `reason` | Each file to install that this machine already holds, after `check` and before anything is written, in `--check` and `--dry-run` too. `folder` is the `--from` folder as given, or the `bios_dir` of the check when the BIOS folder holds the file at another place, and then `reason` says where in words (`nested folder: ...` for a misplaced file, `the same file is at ...` for one the list installs there too). `path` is the file, and `member` the ZIP member holding the bytes when `path` is an archive, an empty string otherwise. Its copy is reported by a `downloaded` event |
+| `unreadable` | `platform`, `folder`, `path`, `member`, `reason` | Each file in a `--from` folder that could not be read and may be one the run still needs, right after the `found` events: its size is that of a file no copy was found for, or unknown (a folder that could not be listed, a ZIP that would not open). `member` names the ZIP member, or is empty. `reason` says why in words (`Permission denied`, `it is encrypted, a password is needed`) |
 | `download` | `platform`, `files`, `bytes` | Before the downloads start. Counts the files copied from this machine too |
 | `progress` | `files_done`, `files_total`, `bytes_done`, `bytes_total` | At most every half second while bytes arrive |
+| `transfer` | `bytes_local`, `bytes_network` | Right after each `progress` event, and once when the files of a `download` event are done. It splits `bytes_done`: `bytes_local` came from this machine (a copy found by `--from` or in the BIOS folder, the pack, a partial copy resumed), `bytes_network` from the network. A download speed is `bytes_network` over the time elapsed |
 | `downloaded` | `dest`, `ok`, `reason`, `size`, `files_done`, `files_total`, `bytes_done`, `bytes_total` | Each finished file, downloaded or copied from where a `found` event named it. `reason` says why one failed, and is empty otherwise |
 | `summary` | `mode`, `installed`, `up_to_date`, `pending`, `failed`, `skipped`, `left_out`, `kept`, `interrupted` | The outcome. `pending` counts what a next run would still fetch, `failed` lists `{dest, reason}`, `skipped` lists `{reason, label, files, required}` for the files the collection cannot supply, `left_out` lists `{category, label, files, dests}` for what `--no-game-data` left out, and is empty without it. `kept` lists `{folder, files, bytes}`, one per backup folder, for the files in place the run moved aside before replacing them (with `--dry-run`, would move), and is empty when none. Not written for a listing |
 | `end` | `code` | Last line, with the exit status the process returns (see [Exit status](#exit-status)) |