diff --git a/install.ps1 b/install.ps1 index e75d549b..37d5a50b 100644 --- a/install.ps1 +++ b/install.ps1 @@ -8,7 +8,7 @@ param( $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -$defaultInstallSha256 = "8e0c7bb63c34f4264ae040f8f322e1a3ff47db3393979466daba1238de9dd942" +$defaultInstallSha256 = "ad00604f027af67f44fbb7dbe49513a870b6ba722263cf6a514df53cfe7ec13d" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index e3205c0e..747c38d2 100644 --- a/install.py +++ b/install.py @@ -2762,8 +2762,9 @@ def _fetch_archive(url: str, directory: Path) -> Path: def _archive_members( archive: Path, entry: dict -) -> Iterator[tuple[str, Callable[[], BinaryIO]]]: - """(path in the tree, opener) for each file the archive maps into it. +) -> Iterator[tuple[str, int, Callable[[], BinaryIO]]]: + """(path in the tree, declared size, opener) for each file the archive + maps into it. The rules refresh_data_dirs applies to build the cache: a ZIP drops strip_components leading parts, a tarball keeps what sits under @@ -2785,7 +2786,7 @@ def _archive_members( parts = parts[strip:] if parts[0] in exclude: continue - yield "/".join(parts), (lambda info=info: zf.open(info)) + yield "/".join(parts), info.file_size, (lambda info=info: zf.open(info)) return prefix = entry["source_path"].rstrip("/") + "/" with tarfile.open(archive, "r:gz") as tf: @@ -2795,7 +2796,7 @@ def _archive_members( rel = member.name[len(prefix):] if not rel or ".." in rel.split("/") or rel.split("/")[0] in exclude: continue - yield rel, (lambda member=member: tf.extractfile(member)) + yield rel, member.size, (lambda member=member: tf.extractfile(member)) def data_directory_state(entry: dict, bios_path: Path) -> tuple[Path, dict[str, dict], int]: @@ -2831,10 +2832,17 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict: root.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=root) as scratch: archive = _fetch_archive(entry["url"], Path(scratch)) - for rel, opener in _archive_members(archive, entry): + for rel, declared, opener in _archive_members(archive, entry): meta = needed.pop(rel, None) if meta is None: continue + # Read no more than the list records: the archive's own size + # field is checked first, and the stream is cut at the + # expected size whatever the member claims, so a member that + # inflates without end costs one byte past it. + if declared != meta["size"]: + counts["changed"] += 1 + continue target = _destination_path(root, rel) target.parent.mkdir(parents=True, exist_ok=True) digest = hashlib.sha1() @@ -2843,7 +2851,10 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict: dir=target.parent, prefix=f".{target.name}.", suffix=".part", delete=False, ) as out: - for chunk in iter(lambda: src.read(1024 * 1024), b""): + while size <= meta["size"]: + chunk = src.read(min(1024 * 1024, meta["size"] + 1 - size)) + if not chunk: + break digest.update(chunk) size += len(chunk) out.write(chunk) diff --git a/install.sh b/install.sh index bdf271d0..f1ffd67c 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="8e0c7bb63c34f4264ae040f8f322e1a3ff47db3393979466daba1238de9dd942" +DEFAULT_INSTALL_SHA256="ad00604f027af67f44fbb7dbe49513a870b6ba722263cf6a514df53cfe7ec13d" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install.py b/tests/test_install.py index 2bbc3118..176b54eb 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -2524,6 +2524,34 @@ class DataDirectoriesAreInstalled(unittest.TestCase): self.assertEqual(again["present"], 1) self.assertEqual(sorted(p.name for p in (bios / "dolphin-emu").iterdir()), ["Sys"]) + def test_a_member_that_inflates_is_cut_at_its_size(self): + # A member whose header claims the recorded size and whose stream + # never ends: reading stops one byte past it. + class Endless(io.RawIOBase): + served = 0 + + def readable(self): + return True + + def readinto(self, buffer): + Endless.served += len(buffer) + buffer[:] = b"x" * len(buffer) + return len(buffer) + + work = Path(_tempdir(self)) + entry = self._entry() + entry["files"] = {"Sys/GC/font.bin": {"sha1": "1" * 40, "size": 4}} + + def members(_archive, _entry): + yield "Sys/GC/font.bin", 4, lambda: io.BufferedReader(Endless()) + + with unittest.mock.patch.object(install, "_fetch_archive", + lambda url, d: Path(d) / "a"), \ + unittest.mock.patch.object(install, "_archive_members", members): + counts = install.install_data_directory(entry, work) + self.assertEqual(counts["changed"], 1) + self.assertLess(Endless.served, 64 * 1024) + def test_a_tarball_maps_through_source_path(self): import io as _io import tarfile @@ -2536,7 +2564,7 @@ class DataDirectoriesAreInstalled(unittest.TestCase): info.size = len(data) tf.addfile(info, _io.BytesIO(data)) entry = {"type": "tarball", "source_path": "pcsx2-v1/bin/resources"} - names = [rel for rel, _open in install._archive_members(archive, entry)] + names = [rel for rel, _size, _open in install._archive_members(archive, entry)] self.assertEqual(names, ["GameIndex.yaml"]) def test_the_trust_boundary(self):