fix: cap a data archive member at its size

install.ps1

@@ -8,7 +8,7 @@ param(
 
 $ErrorActionPreference = "Stop"
 $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-$defaultInstallSha256 = "8e0c7bb63c34f4264ae040f8f322e1a3ff47db3393979466daba1238de9dd942"
+$defaultInstallSha256 = "ad00604f027af67f44fbb7dbe49513a870b6ba722263cf6a514df53cfe7ec13d"
 $maximumInstallerBytes = 2MB
 $installerExit = 0
 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }

install.py

@@ -2762,8 +2762,9 @@ def _fetch_archive(url: str, directory: Path) -> Path:
 
 def _archive_members(
     archive: Path, entry: dict
-) -> Iterator[tuple[str, Callable[[], BinaryIO]]]:
-    """(path in the tree, opener) for each file the archive maps into it.
+) -> Iterator[tuple[str, int, Callable[[], BinaryIO]]]:
+    """(path in the tree, declared size, opener) for each file the archive
+    maps into it.
 
     The rules refresh_data_dirs applies to build the cache: a ZIP drops
     strip_components leading parts, a tarball keeps what sits under
@@ -2785,7 +2786,7 @@ def _archive_members(
                     parts = parts[strip:]
                 if parts[0] in exclude:
                     continue
-                yield "/".join(parts), (lambda info=info: zf.open(info))
+                yield "/".join(parts), info.file_size, (lambda info=info: zf.open(info))
         return
     prefix = entry["source_path"].rstrip("/") + "/"
     with tarfile.open(archive, "r:gz") as tf:
@@ -2795,7 +2796,7 @@ def _archive_members(
             rel = member.name[len(prefix):]
             if not rel or ".." in rel.split("/") or rel.split("/")[0] in exclude:
                 continue
-            yield rel, (lambda member=member: tf.extractfile(member))
+            yield rel, member.size, (lambda member=member: tf.extractfile(member))
 
 
 def data_directory_state(entry: dict, bios_path: Path) -> tuple[Path, dict[str, dict], int]:
@@ -2831,10 +2832,17 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict:
         root.mkdir(parents=True, exist_ok=True)
         with tempfile.TemporaryDirectory(dir=root) as scratch:
             archive = _fetch_archive(entry["url"], Path(scratch))
-            for rel, opener in _archive_members(archive, entry):
+            for rel, declared, opener in _archive_members(archive, entry):
                 meta = needed.pop(rel, None)
                 if meta is None:
                     continue
+                # Read no more than the list records: the archive's own size
+                # field is checked first, and the stream is cut at the
+                # expected size whatever the member claims, so a member that
+                # inflates without end costs one byte past it.
+                if declared != meta["size"]:
+                    counts["changed"] += 1
+                    continue
                 target = _destination_path(root, rel)
                 target.parent.mkdir(parents=True, exist_ok=True)
                 digest = hashlib.sha1()
@@ -2843,7 +2851,10 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict:
                     dir=target.parent, prefix=f".{target.name}.", suffix=".part",
                     delete=False,
                 ) as out:
-                    for chunk in iter(lambda: src.read(1024 * 1024), b""):
+                    while size <= meta["size"]:
+                        chunk = src.read(min(1024 * 1024, meta["size"] + 1 - size))
+                        if not chunk:
+                            break
                         digest.update(chunk)
                         size += len(chunk)
                         out.write(chunk)

install.sh

@@ -18,7 +18,7 @@ esac
 TEMP_INSTALLER=""
 TEMP_DIRECTORY=""
 DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-DEFAULT_INSTALL_SHA256="8e0c7bb63c34f4264ae040f8f322e1a3ff47db3393979466daba1238de9dd942"
+DEFAULT_INSTALL_SHA256="ad00604f027af67f44fbb7dbe49513a870b6ba722263cf6a514df53cfe7ec13d"
 MAX_INSTALLER_BYTES=2097152
 
 cleanup() {

tests/test_install.py

@@ -2524,6 +2524,34 @@ class DataDirectoriesAreInstalled(unittest.TestCase):
         self.assertEqual(again["present"], 1)
         self.assertEqual(sorted(p.name for p in (bios / "dolphin-emu").iterdir()), ["Sys"])
 
+    def test_a_member_that_inflates_is_cut_at_its_size(self):
+        # A member whose header claims the recorded size and whose stream
+        # never ends: reading stops one byte past it.
+        class Endless(io.RawIOBase):
+            served = 0
+
+            def readable(self):
+                return True
+
+            def readinto(self, buffer):
+                Endless.served += len(buffer)
+                buffer[:] = b"x" * len(buffer)
+                return len(buffer)
+
+        work = Path(_tempdir(self))
+        entry = self._entry()
+        entry["files"] = {"Sys/GC/font.bin": {"sha1": "1" * 40, "size": 4}}
+
+        def members(_archive, _entry):
+            yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Endless())
+
+        with unittest.mock.patch.object(install, "_fetch_archive",
+                                        lambda url, d: Path(d) / "a"), \
+                unittest.mock.patch.object(install, "_archive_members", members):
+            counts = install.install_data_directory(entry, work)
+        self.assertEqual(counts["changed"], 1)
+        self.assertLess(Endless.served, 64 * 1024)
+
     def test_a_tarball_maps_through_source_path(self):
         import io as _io
         import tarfile
@@ -2536,7 +2564,7 @@ class DataDirectoriesAreInstalled(unittest.TestCase):
             info.size = len(data)
             tf.addfile(info, _io.BytesIO(data))
         entry = {"type": "tarball", "source_path": "pcsx2-v1/bin/resources"}
-        names = [rel for rel, _open in install._archive_members(archive, entry)]
+        names = [rel for rel, _size, _open in install._archive_members(archive, entry)]
         self.assertEqual(names, ["GameIndex.yaml"])
 
     def test_the_trust_boundary(self):