From c2de7c3a8e1fba6e6c1223aeea9fe11f03742e45 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:09:43 +0200 Subject: [PATCH 1/7] feat: mark each pack part with its siblings --- scripts/generate_readme.py | 8 ++- scripts/release_notes.py | 3 +- scripts/release_record.py | 5 +- scripts/split_pack.py | 139 ++++++++++++++++++++++++++++++++----- tests/test_doc_surfaces.py | 9 +++ tests/test_download.py | 11 ++- tests/test_pack_counts.py | 11 ++- tests/test_split_pack.py | 57 +++++++++++++++ 8 files changed, 218 insertions(+), 25 deletions(-) diff --git a/scripts/generate_readme.py b/scripts/generate_readme.py index cb5daaaa..dcfee052 100644 --- a/scripts/generate_readme.py +++ b/scripts/generate_readme.py @@ -236,7 +236,9 @@ def download_links( """Markdown links to the files of a platform's pack in the recorded release. A pack published whole gets one link labelled *label*. A pack in parts - gets one link per part, since every part is needed. *repo* is the + gets one link per part, since every part is needed, after the number of + files to download: a reader who took one link for the pack extracted + half of it. *repo* is the repository URL, absolute or relative, and *attrs* an attribute list put after each link, or *button* the function that draws each link as a button on the site. Without a recorded release the link is the latest @@ -263,8 +265,8 @@ def download_links( for number, url in enumerate(urls, 1) ] if styled: - return " ".join(parts) - return f"{label}: " + ", ".join(parts) + return f"{len(urls)} files to download, all needed: " + " ".join(parts) + return f"{label}, {len(urls)} files, all needed: " + ", ".join(parts) def extract_targets(platforms_dir: str) -> list[tuple[str, str]]: diff --git a/scripts/release_notes.py b/scripts/release_notes.py index 995d18f6..9a40d660 100644 --- a/scripts/release_notes.py +++ b/scripts/release_notes.py @@ -63,7 +63,8 @@ def _links(tag: str, assets: list[str]) -> str: base = f"https://github.com/{REPO}/releases/download/{tag}" if len(assets) == 1: return f"[{assets[0]}]({base}/{assets[0]})" - return " + ".join(f"[part {n}]({base}/{asset})" for n, asset in enumerate(assets, 1)) + parts = " + ".join(f"[part {n}]({base}/{asset})" for n, asset in enumerate(assets, 1)) + return f"{len(assets)} files, all needed: {parts}" def render(record: dict, platforms_dir: str, whats_new: str) -> str: diff --git a/scripts/release_record.py b/scripts/release_record.py index f8510686..1926398f 100644 --- a/scripts/release_record.py +++ b/scripts/release_record.py @@ -103,8 +103,11 @@ def _read_record(dist: Path, tag: str, owners: dict[str, list[str]]) -> dict: members: dict[str, int] = {} for path in paths: with zipfile.ZipFile(path) as archive: + # A part's marker names its siblings and is not pack content: + # the manifest's pack_files is counted before any split. for info in archive.infolist(): - members[info.filename] = info.file_size + if not split_pack.is_marker(info.filename): + members[info.filename] = info.file_size packs[name] = { "files": len(members), "extracted_size": sum(members.values()), diff --git a/scripts/split_pack.py b/scripts/split_pack.py index af6ec7e9..9e830ff7 100644 --- a/scripts/split_pack.py +++ b/scripts/split_pack.py @@ -6,6 +6,11 @@ as parts, each a complete archive of whole files: any tool opens one, and the parts extracted into the same folder are the pack. Members are copied as stored, never recompressed, so the parts depend on the pack alone. +Each part also carries a short text file, its marker, at the archive root: +the pack it belongs to, its number, and the name and size of every part. +Extracted with the files, the markers let `install.py --check` name a part +that is missing from the folder. + Usage: python scripts/split_pack.py dist/ python scripts/split_pack.py dist/RetroArch_v1.22.2_BIOS_Pack.zip @@ -20,6 +25,7 @@ import re import struct import sys import zipfile +import zlib from pathlib import Path from artifacts import ArtifactLockBusy, artifact_lock @@ -29,6 +35,10 @@ ASSET_LIMIT = 2 * 1024**3 PACK_SUFFIX = "_BIOS_Pack.zip" _PART_NAME = re.compile(r"^(?P.+_BIOS_Pack)\.part(?P\d+)of(?P\d+)\.zip$") +_MARKER_NAME = re.compile(r"^(?P.+_BIOS_Pack)\.part(?P\d+)of(?P\d+)\.txt$") +# Sizes are written at a fixed width, so a marker's length, and with it the +# size of its own part, does not depend on the sizes it records. +_SIZE_WIDTH = 13 _LOCAL = struct.Struct("<4s2B4HL2L2H") _CENTRAL = struct.Struct("<4s4B4HL2L5H2L") @@ -58,6 +68,44 @@ def part_name(pack_name: str, index: int, count: int) -> str: return f"{pack_name[: -len('.zip')]}.part{index}of{count}.zip" +def marker_name(pack_name: str, index: int, count: int) -> str: + return f"{pack_name[: -len('.zip')]}.part{index}of{count}.txt" + + +def is_marker(name: str) -> bool: + """Whether an archive member is the marker a split writes in each part.""" + return _MARKER_NAME.match(name) is not None + + +def marker_text(pack_name: str, index: int, sizes: list[int]) -> bytes: + """The marker of part *index* of a pack published in ``len(sizes)`` parts. + + Plain text a person can read, in a form install.py parses back: the + lines that matter are `pack:`, `part:` and one `file:` per part. + """ + count = len(sizes) + lines = [ + "RetroBIOS split pack", + f"This folder holds part {index} of {count} of {pack_name}.", + "Every part is needed: extract them all into this same folder.", + "The installer's --check names a part missing from the folder.", + "", + f"pack: {pack_name}", + f"part: {index} of {count}", + ] + lines += [ + f"file: {size:>{_SIZE_WIDTH}} {part_name(pack_name, number, count)}" + for number, size in enumerate(sizes, 1) + ] + return ("\n".join(lines) + "\n").encode("utf-8") + + +def _marker_weight(pack_name: str, index: int, count: int) -> int: + name = len(marker_name(pack_name, index, count).encode("utf-8")) + body = len(marker_text(pack_name, index, [0] * count)) + return _LOCAL.size + name + body + _CENTRAL.size + name + + def _name_bytes(info: zipfile.ZipInfo) -> bytes: encoding = "utf-8" if info.flag_bits & _UTF8_NAME else "cp437" return info.orig_filename.encode(encoding) @@ -70,23 +118,24 @@ def _weight(info: zipfile.ZipInfo) -> int: def plan_parts( - members: list[zipfile.ZipInfo], limit: int + members: list[zipfile.ZipInfo], limit: int, reserve: int = 0 ) -> list[list[zipfile.ZipInfo]]: - """Fill each part in archive order, up to the limit.""" + """Fill each part in archive order, up to the limit less *reserve*.""" parts: list[list[zipfile.ZipInfo]] = [[]] - size = _END.size + floor = _END.size + reserve + size = floor for info in members: weight = _weight(info) - if _END.size + weight >= limit: + if floor + weight >= limit: raise ValueError( f"{info.filename} needs {weight:,} bytes, more than a part of " f"{limit:,} can hold" ) if info.compress_size > _MAX_FIELD or info.file_size > _MAX_FIELD: raise ValueError(f"{info.filename} needs ZIP64, which a part does not use") - if size + weight >= limit or len(parts[-1]) >= _MAX_ENTRIES: + if size + weight >= limit or len(parts[-1]) >= _MAX_ENTRIES - 1: parts.append([]) - size = _END.size + size = floor parts[-1].append(info) size += weight return parts @@ -109,12 +158,24 @@ def _data_offset(source: io.BufferedReader, info: zipfile.ZipInfo) -> int: return info.header_offset + _LOCAL.size + header[10] + header[11] +def _marker_info(name: str, body: bytes) -> zipfile.ZipInfo: + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_STORED + info.CRC = zlib.crc32(body) + info.compress_size = info.file_size = len(body) + info.create_system = 3 + info.external_attr = (0o100000 | 0o644) << 16 + return info + + def _write_part( - source: io.BufferedReader, members: list[zipfile.ZipInfo], dest: Path + source: io.BufferedReader, members: list[zipfile.ZipInfo], dest: Path, + marker: tuple[str, bytes], ) -> None: directory = [] + marker_info = _marker_info(*marker) with open(dest, "wb") as out: - for info in members: + for info in [marker_info, *members]: name = _name_bytes(info) flags = info.flag_bits & ~_DATA_DESCRIPTOR dos_time, dos_date = _dos_stamp(info) @@ -127,8 +188,12 @@ def _write_part( ) ) out.write(name) - source.seek(_data_offset(source, info)) - remaining = info.compress_size + if info is marker_info: + out.write(marker[1]) + remaining = 0 + else: + source.seek(_data_offset(source, info)) + remaining = info.compress_size while remaining: chunk = source.read(min(_CHUNK, remaining)) if not chunk: @@ -157,7 +222,33 @@ def _write_part( def _identity(archive: zipfile.ZipFile) -> list[tuple[str, int, int]]: - return [(info.filename, info.CRC, info.file_size) for info in archive.infolist()] + return [ + (info.filename, info.CRC, info.file_size) + for info in archive.infolist() + if not is_marker(info.filename) + ] + + +def marker_weight(pack_name: str, count: int) -> int: + """Bytes the largest marker of a pack in *count* parts adds to its part.""" + return max(_marker_weight(pack_name, index, count) for index in (1, count)) + + +def _plan_with_markers( + pack_name: str, members: list[zipfile.ZipInfo], limit: int +) -> list[list[zipfile.ZipInfo]]: + """Plan the parts with room for the marker each one carries. + + The marker lists every part, so its length grows with their count: the + plan is made again until the count it assumed is the count it gives. + """ + count = 1 + while True: + reserve = marker_weight(pack_name, count) + plan = plan_parts(members, limit, reserve) + if len(plan) <= count: + return plan + count = len(plan) def split_pack(zip_path: Path, limit: int = ASSET_LIMIT) -> list[Path]: @@ -174,15 +265,31 @@ def split_pack(zip_path: Path, limit: int = ASSET_LIMIT) -> list[Path]: with zipfile.ZipFile(zip_path) as archive: members = archive.infolist() expected = _identity(archive) - plan = plan_parts(members, limit) + taken = [info.filename for info in members if is_marker(info.filename)] + if taken: + raise ValueError(f"{zip_path.name} already holds {taken[0]}, a part marker name") + plan = _plan_with_markers(zip_path.name, members, limit) + count = len(plan) parts = [ - zip_path.with_name(part_name(zip_path.name, index, len(plan))) - for index in range(1, len(plan) + 1) + zip_path.with_name(part_name(zip_path.name, index, count)) + for index in range(1, count + 1) + ] + sizes = [ + _END.size + _marker_weight(zip_path.name, index, count) + + sum(_weight(info) for info in chosen) + for index, chosen in enumerate(plan, 1) ] try: with open(zip_path, "rb") as source: - for dest, chosen in zip(parts, plan): - _write_part(source, chosen, dest) + for index, (dest, chosen) in enumerate(zip(parts, plan), 1): + marker = (marker_name(zip_path.name, index, count), + marker_text(zip_path.name, index, sizes)) + _write_part(source, chosen, dest, marker) + if dest.stat().st_size != sizes[index - 1]: + raise ValueError( + f"{dest.name} is {dest.stat().st_size:,} bytes, its marker " + f"says {sizes[index - 1]:,}" + ) found: list[tuple[str, int, int]] = [] for dest in parts: with zipfile.ZipFile(dest) as archive: diff --git a/tests/test_doc_surfaces.py b/tests/test_doc_surfaces.py index f873f593..21cfd770 100644 --- a/tests/test_doc_surfaces.py +++ b/tests/test_doc_surfaces.py @@ -111,6 +111,15 @@ class DownloadLinksNameTheFile(unittest.TestCase): ) self.assertIn("part 1 of 2", cell) + def test_a_split_pack_says_how_many_files_to_download(self): + """One link taken for the pack left half of it out of the folder.""" + self.assertIn( + "Download, 2 files, all needed: ", + download_links("batocera", SPLIT_RECORD, "../..", "Download"), + ) + buttons = download_links("batocera", SPLIT_RECORD, REPO, "Pack", " { .x }") + self.assertTrue(buttons.startswith("2 files to download, all needed: "), buttons) + def test_buttons_carry_their_attributes_on_each_part(self): cell = download_links("batocera", SPLIT_RECORD, REPO, "Pack", " { .x }") self.assertEqual(cell.count("{ .x }"), 2) diff --git a/tests/test_download.py b/tests/test_download.py index 5c5c9927..a3b28627 100644 --- a/tests/test_download.py +++ b/tests/test_download.py @@ -446,8 +446,11 @@ class DownloaderCase(unittest.TestCase): "pack staged in the system temp directory", ) searched = list(self.dest.parent.iterdir()) + list(self.dest.rglob("*")) + # A part's marker is extracted on purpose: install.py --check reads it. leftovers = sorted( - p.name for p in searched if "_BIOS_Pack" in p.name or p.name.startswith(".") + p.name for p in searched + if ("_BIOS_Pack" in p.name or p.name.startswith(".")) + and not split_pack.is_marker(p.name) ) self.assertEqual(leftovers, [], "download staging left behind") @@ -573,6 +576,12 @@ class ZipPartsCase(DownloaderCase): """A pack published as parts that are each an archive.""" zip_parts = True + # Bytes deflate cannot shrink: each member outweighs the marker a part + # carries, so the pack splits in two. + payload = { + "bios/scph5501.bin": b"".join(hashlib.sha256(b"us%d" % n).digest() for n in range(64)), + "bios/dc/dc_boot.bin": b"".join(hashlib.sha256(b"dc%d" % n).digest() for n in range(64)), + } class ZipPartsTests: diff --git a/tests/test_pack_counts.py b/tests/test_pack_counts.py index 43629cef..cc0d4541 100644 --- a/tests/test_pack_counts.py +++ b/tests/test_pack_counts.py @@ -341,6 +341,11 @@ class TheTableDescribesTheRelease(unittest.TestCase): self.assertTrue(f"| RetroArch | {files:,} |" in readme) +def _incompressible(seed: int) -> bytes: + """960 bytes deflate cannot shrink, so a member outweighs a part marker.""" + return b"".join(hashlib.sha256(bytes([seed, n])).digest() for n in range(30)) + + class ReleaseRecord(unittest.TestCase): OWNERS = { "Whole_1.0_BIOS_Pack.zip": ["whole"], @@ -356,7 +361,7 @@ class ReleaseRecord(unittest.TestCase): self._pack("Whole_1.0_BIOS_Pack.zip", {"a.bin": b"a" * 10, "README.txt": b"r"}) big = self._pack( "Demo_2.0_BIOS_Pack.zip", - {f"dir/file{n}.bin": bytes([n]) * 700 + bytes(range(256)) for n in range(6)}, + {f"dir/file{n}.bin": _incompressible(n) for n in range(6)}, ) self.parts = split_pack.split_pack(big, limit=big.stat().st_size - 1) @@ -375,7 +380,7 @@ class ReleaseRecord(unittest.TestCase): self.assertEqual(record["tag"], "v1") demo = record["packs"]["Demo_2.0_BIOS_Pack.zip"] self.assertEqual(demo["files"], 6) - self.assertEqual(demo["extracted_size"], 6 * (700 + 256)) + self.assertEqual(demo["extracted_size"], 6 * 960) self.assertEqual(demo["assets"], [part.name for part in self.parts]) self.assertEqual( demo["download_size"], sum(part.stat().st_size for part in self.parts) @@ -399,7 +404,7 @@ class ReleaseRecord(unittest.TestCase): pack, leaves both: summed, the download size doubled.""" self._pack( "Demo_2.0_BIOS_Pack.zip", - {f"dir/file{n}.bin": bytes([n]) * 700 + bytes(range(256)) for n in range(6)}, + {f"dir/file{n}.bin": _incompressible(n) for n in range(6)}, ) with self.assertRaises(release_record.UnfinishedSplitError): release_record.build_record(self.dist, "v1", owners=self.OWNERS) diff --git a/tests/test_split_pack.py b/tests/test_split_pack.py index 3d0f0aa0..497380d2 100644 --- a/tests/test_split_pack.py +++ b/tests/test_split_pack.py @@ -67,6 +67,8 @@ class SplitFixture(unittest.TestCase): with zipfile.ZipFile(part) as archive: self.assertIsNone(archive.testzip()) for name in archive.namelist(): + if split_pack.is_marker(name): + continue self.assertNotIn(name, found, "a file sits in two parts") found[name] = archive.read(name) return found @@ -137,6 +139,61 @@ class PartsAreArchives(SplitFixture): self.assertTrue(modes["bin/engine"] & stat.S_IXUSR) +class EachPartNamesItsSiblings(SplitFixture): + """A part extracted alone looked like a complete pack. Each part now + carries a marker naming the pack, its number and every part with its + size, so a missing one can be named from the extracted folder.""" + + def _markers(self, parts: list[Path]) -> dict[str, str]: + found = {} + for part in parts: + with zipfile.ZipFile(part) as archive: + names = [n for n in archive.namelist() if split_pack.is_marker(n)] + self.assertEqual(len(names), 1, part.name) + found[part.name] = archive.read(names[0]).decode("utf-8") + return found + + def test_every_part_carries_one_marker(self): + parts = split_pack.split_pack(self.pack, limit=LIMIT) + count = len(parts) + for index, part in enumerate(parts, 1): + with zipfile.ZipFile(part) as archive: + self.assertIn(f"Demo_1.0_BIOS_Pack.part{index}of{count}.txt", + archive.namelist()) + + def test_the_marker_names_the_pack_the_part_and_every_size(self): + parts = split_pack.split_pack(self.pack, limit=LIMIT) + count = len(parts) + for index, (name, text) in enumerate(self._markers(parts).items(), 1): + with self.subTest(part=name): + lines = text.splitlines() + self.assertIn("pack: Demo_1.0_BIOS_Pack.zip", lines) + self.assertIn(f"part: {index} of {count}", lines) + listed = [line.split()[1:] for line in lines if line.startswith("file:")] + self.assertEqual( + listed, + [[str(p.stat().st_size), p.name] for p in parts], + ) + + def test_the_parts_still_fit_with_their_markers(self): + for part in split_pack.split_pack(self.pack, limit=LIMIT): + self.assertLess(part.stat().st_size, LIMIT) + + def test_a_pack_holding_a_marker_name_is_refused(self): + members = dict(self.members) + members["Demo_1.0_BIOS_Pack.part1of2.txt"] = b"x" + self.pack.unlink() + self._build(self.pack, members) + with self.assertRaises(ValueError): + split_pack.split_pack(self.pack, limit=LIMIT) + self.assertTrue(self.pack.is_file()) + + def test_a_marker_is_not_a_part(self): + self.assertTrue(split_pack.is_marker("Demo_BIOS_Pack.part2of3.txt")) + self.assertFalse(split_pack.is_part("Demo_BIOS_Pack.part2of3.txt")) + self.assertFalse(split_pack.is_marker("Demo_BIOS_Pack.part2of3.zip")) + + class SplitHoldsTheDirectory(SplitFixture): """split_pack read and deleted packs in a directory a build was writing: the build could truncate the pack under the split, or the split delete From aaabea8bd59c4d8ad86817a678721dae41d2c7e2 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:21:46 +0200 Subject: [PATCH 2/7] feat: explain check refusals, install offline --check names why each flagged file is refused and where each core opens it, and names a pack part missing from the folder from the markers split_pack writes. --from-pack installs from a downloaded pack or its parts with no network request, --manifest reads a saved file list, both checking every hash. The end of a run names the system or core that pulls most of the download when narrowing saves over half. --- install.ps1 | 2 +- install.py | 866 +++++++++++++++++++++++++++++++++- install.sh | 2 +- tests/test_install_offline.py | 488 +++++++++++++++++++ wiki/installer.md | 98 +++- wiki/testing-guide.md | 2 +- 6 files changed, 1432 insertions(+), 26 deletions(-) create mode 100644 tests/test_install_offline.py diff --git a/install.ps1 b/install.ps1 index fb7d9bda..73473be8 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "ad3b338a8f709be45352a12bd9c6a6250819a8fb9afd19498e24f41e3f062d20" + $defaultInstallSha256 = "996cf4a8e21b8f05a524778046d0d32c807fbe3564f6edefdf5bee1add9c76cd" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index 986d2133..dee4a060 100644 --- a/install.py +++ b/install.py @@ -39,6 +39,7 @@ import urllib.parse import urllib.request import xml.etree.ElementTree as ET import zipfile +import zlib from collections.abc import Callable, Iterator from pathlib import Path, PurePosixPath from typing import BinaryIO, NamedTuple, TextIO @@ -2922,6 +2923,7 @@ def _fetch_json(url: str, limit: int, label: str) -> object: A 4xx answer or an invalid document is final: asking again changes nothing. """ + _network_allowed(label) for attempt in range(1, MAX_RETRIES + 1): try: with urllib.request.urlopen(url, timeout=30) as resp: @@ -2946,7 +2948,16 @@ def _fetch_json(url: str, limit: int, label: str) -> object: def fetch_manifest(plat: str) -> dict: - """Download platform manifest JSON.""" + """Download platform manifest JSON, or read the one this run was given.""" + if LOCAL.manifest is not None: + return read_local_manifest(LOCAL.manifest, plat) + if LOCAL.pack is not None: + try: + return _validate_manifest(LOCAL.pack.manifest(plat), plat) + except ValueError as exc: + print(f" The file list in {LOCAL.pack.name} is not valid: {exc}", + file=sys.stderr) + sys.exit(1) url = MANIFEST_URL.format(platform=plat) try: return _validate_manifest(_fetch_json(url, MAX_MANIFEST_BYTES, "manifest"), plat) @@ -3979,6 +3990,9 @@ def _download_one( return f["dest"], False, NOT_WRITABLE return f["dest"], False, exc.strerror or str(exc) + if LOCAL.pack is not None: + return _extract_from_pack(f, bios_path, dest, progress) + _network_allowed(f["dest"]) if f.get("release_asset"): url = RELEASE_URL.format(asset=urllib.parse.quote(f["release_asset"], safe="")) else: @@ -4380,6 +4394,7 @@ def _folder_writable(path: Path) -> bool: # Options that take a value, as a hint gives them back. _VALUED_OPTIONS = ( "--system", "--core", "--emulator", "--region", "--target", "--cores-dir", "--save-dir", + "--from-pack", "--manifest", ) @@ -4567,6 +4582,7 @@ def _report_outcome( assumed: set[Path] | None = None, omitted_entries: list[dict] | None = None, platform_of: dict[Path, str] | None = None, + hints: list[str] | None = None, ) -> None: """Close the run on what happened, and leave with a status that says so. @@ -4574,7 +4590,8 @@ def _report_outcome( which reads as success to a person and to a script alike. """ if transferred and seconds > 0: - print(f"\n{format_size(transferred)} downloaded in " + verb = "copied from the pack" if LOCAL.pack is not None else "downloaded" + print(f"\n{format_size(transferred)} {verb} in " f"{format_duration(seconds)} ({format_size(int(transferred / seconds))}/s).") reasons = {reason for _dest, reason in failures or []} if failures and len(failures) > 1 and len(reasons) == 1 and reasons != {""}: @@ -4652,6 +4669,8 @@ def _report_outcome( if omitted: for line in _omission_lines(omitted, omitted_entries or []): print(line) + for line in hints or []: + print(line) print("To check this later, run the same command with --check.") @@ -4774,8 +4793,9 @@ def _print_plan( if len(by_system[system]) > 8: print(f" ... and {len(by_system[system]) - 8} more") for entry, needed in trees: + source = LOCAL.pack.name if LOCAL.pack is not None else entry["url"] print(f" {term.bold(entry['ref'] + ' data')} {_sized(list(needed.values()))}, " - f"from {entry['url']}") + f"from {source}") for folder, (count, size) in sorted((kept or {}).items()): print(f"\n Would keep {_plural(count, 'file')} you have ({format_size(size)}), " f"whose contents differ, in {folder}") @@ -4875,6 +4895,7 @@ def download_files( def _fetch_archive(url: str, directory: Path) -> Path: """A data directory's source archive, in *directory*, retried like files.""" + _network_allowed(url) reason: Exception | None = None for attempt in range(1, MAX_RETRIES + 1): handle = tempfile.NamedTemporaryFile(dir=directory, suffix=".archive", delete=False) @@ -4981,8 +5002,13 @@ def install_data_directory(entry: dict, bios_path: Path) -> dict: try: root.mkdir(parents=True, exist_ok=True) with tempfile.TemporaryDirectory(dir=root) as scratch: - archive = _fetch_archive(entry["url"], Path(scratch)) - for rel, declared, opener in _archive_members(archive, entry): + if LOCAL.pack is not None: + # The pack holds the tree where it extracts, already mapped. + members = LOCAL.pack.tree(entry.get("dest", "")) + else: + archive = _fetch_archive(entry["url"], Path(scratch)) + members = _archive_members(archive, entry) + for rel, declared, opener in members: meta = needed.pop(rel, None) if meta is None: continue @@ -5841,6 +5867,729 @@ def _requested_narrowing( return systems, cores, regions +# A pack too large for one release file is published as ZIP parts +# (scripts/split_pack.py), each holding at its root a marker that names the +# pack, the part and every part with its size. Extracted with the files, +# the markers say which part a folder lacks. +_PART_FILE = re.compile(r"^(?P.+_BIOS_Pack)\.part(?P\d+)of(?P\d+)\.zip$") +_PART_MARKER = re.compile(r"^(?P.+_BIOS_Pack)\.part(?P\d+)of(?P\d+)\.txt$") +MAX_MARKER_BYTES = 64 * 1024 +MAX_PARTS = 999 + + +class PackPart(NamedTuple): + """What one part's marker says, and the folder it was found in.""" + + pack: str + index: int + count: int + parts: tuple[tuple[str, int], ...] + folder: Path + + +def read_part_marker(path: Path) -> PackPart | None: + """The marker at path, None when it is not one split_pack wrote.""" + match = _PART_MARKER.match(path.name) + if match is None: + return None + try: + with open(path, "rb") as handle: + text = handle.read(MAX_MARKER_BYTES).decode("utf-8") + except (OSError, UnicodeDecodeError): + return None + pack = "" + index = count = 0 + parts: list[tuple[str, int]] = [] + for line in text.splitlines(): + key, _sep, value = line.partition(":") + value = value.strip() + if key == "pack": + pack = value + elif key == "part": + numbers = value.split(" of ") + if len(numbers) == 2 and all(n.isdigit() for n in numbers): + index, count = int(numbers[0]), int(numbers[1]) + elif key == "file": + size, _space, name = value.partition(" ") + if size.isdigit() and _PART_FILE.match(name.strip()): + parts.append((name.strip(), int(size))) + if ( + not pack.endswith("_BIOS_Pack.zip") + or not 1 <= index <= count <= MAX_PARTS + or len(parts) != count + or (index, count) != (int(match["index"]), int(match["count"])) + ): + return None + return PackPart(pack, index, count, tuple(parts), path.parent) + + +def find_part_markers(root: Path) -> list[PackPart]: + """Markers in root and in the folders right below it. + + A part extracted into a folder named after itself (the default of + several archive tools) leaves its marker one level down. + """ + found: list[PackPart] = [] + try: + entries = sorted(root.iterdir()) + except OSError: + return found + for entry in entries: + if entry.is_file(): + marker = read_part_marker(entry) + if marker: + found.append(marker) + elif entry.is_dir() and not entry.is_symlink() and entry.name != BACKUP_DIR: + try: + below = sorted(entry.iterdir()) + except OSError: + continue + for child in below: + marker = read_part_marker(child) if child.is_file() else None + if marker: + found.append(marker) + return found + + +def split_pack_lines(root: Path) -> list[str]: + """What --check says about the parts of a pack extracted into root: one + missing, or one extracted into a folder of its own.""" + by_pack: dict[str, list[PackPart]] = {} + for marker in find_part_markers(root): + by_pack.setdefault(marker.pack, []).append(marker) + lines: list[str] = [] + for pack, markers in sorted(by_pack.items()): + listed = max(markers, key=lambda m: m.count) + here = {m.index for m in markers if m.folder == root and m.count == listed.count} + missing = [n for n in range(1, listed.count + 1) if n not in here] + deep = sorted({m.folder for m in markers if m.folder != root}) + if not missing and not deep: + continue + lines.append(f"\n {pack} comes in {listed.count} parts, all needed.") + for folder in deep: + names = sorted(m.index for m in markers if m.folder == folder) + which = ", ".join(str(n) for n in names) + lines.append(f" {'Parts' if len(names) > 1 else 'Part'} {which} went into " + f"{folder}, a folder of its own: move what it holds up into {root}.") + absent = [n for n in missing + if not any(m.index == n and m.folder != root for m in markers)] + for number in absent: + name, size = listed.parts[number - 1] + lines.append(f" missing: part {number}, {name} ({format_size(size)})") + if absent: + lines.append(f" Download {'it' if len(absent) == 1 else 'them'} and extract " + f"into {root}, beside the other parts.") + return lines + + +class PackSource: + """A pack already downloaded, whole or in parts, read in place of the network. + + Every file it gives is checked against the hashes the file list records, + as a download is. + """ + + def __init__(self, paths: list[Path]) -> None: + self.files = self._resolve(paths) + first = _PART_FILE.match(self.files[0].name) + self.name = f"{first['stem']}.zip" if first else self.files[0].name + self.archives: list[zipfile.ZipFile] = [] + self.members: dict[str, tuple[zipfile.ZipFile, zipfile.ZipInfo]] = {} + try: + for path in self.files: + archive = zipfile.ZipFile(path) + self.archives.append(archive) + for info in archive.infolist(): + if not info.is_dir(): + self.members.setdefault(info.filename, (archive, info)) + except (OSError, zipfile.BadZipFile) as exc: + self.close() + raise ValueError(f"{path.name} cannot be read as a ZIP ({exc}). A download " + "cut short gives this: download it again.") from exc + self.listed = self._pack_list() + self.by_sha1 = {f["sha1"]: f["path"] for f in self.listed} + + @staticmethod + def _resolve(paths: list[Path]) -> list[Path]: + """The archives of one pack: a whole ZIP, or every one of its parts.""" + given: list[Path] = [] + for path in paths: + if path.is_dir(): + given += sorted(p for p in path.iterdir() + if p.is_file() and p.name.endswith(".zip") + and "_BIOS_Pack" in p.name) + elif path.is_file(): + given.append(path) + else: + raise ValueError(f"{path} does not exist") + if not given: + raise ValueError(f"no pack ZIP in {', '.join(map(str, paths))}") + packs: dict[str, list[Path]] = {} + for path in given: + part = _PART_FILE.match(path.name) + packs.setdefault(f"{part['stem']}.zip" if part else path.name, []).append(path) + if len(packs) > 1: + raise ValueError(f"these are several packs ({', '.join(sorted(packs))}): " + "name the one to install from") + name, chosen = next(iter(packs.items())) + whole = [p for p in chosen if not _PART_FILE.match(p.name)] + if whole: + return whole[:1] + count = int(_PART_FILE.match(chosen[0].name)["count"]) + if count > MAX_PARTS or any( + int(_PART_FILE.match(p.name)["count"]) != count for p in chosen + ): + raise ValueError(f"the parts of {name} disagree on how many there are") + stem = name[: -len(".zip")] + folders = list(dict.fromkeys(p.parent for p in chosen)) + found: list[Path] = [] + missing: list[str] = [] + for number in range(1, count + 1): + part_name = f"{stem}.part{number}of{count}.zip" + place = next((folder / part_name for folder in folders + if (folder / part_name).is_file()), None) + if place is None: + missing.append(part_name) + else: + found.append(place) + if missing: + where = ", ".join(map(str, folders)) + raise ValueError(f"{name} comes in {count} parts and {len(missing)} " + f"{'is' if len(missing) == 1 else 'are'} not in {where}: " + f"{', '.join(missing)}") + return found + + def _pack_list(self) -> list[dict]: + """The file list the pack carries, every file with its SHA-1 and size.""" + held = self.members.get("manifest.json") + if held is None: + raise ValueError(f"{self.name} holds no manifest.json. Name the file list " + "with --manifest.") + archive, info = held + if info.file_size > MAX_MANIFEST_BYTES: + raise ValueError(f"the manifest.json of {self.name} exceeds " + f"{MAX_MANIFEST_BYTES} bytes") + try: + data = json.loads(archive.read(info).decode("utf-8")) + except (OSError, zipfile.BadZipFile, UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"the manifest.json of {self.name} cannot be read: {exc}") from exc + files = data.get("files") if isinstance(data, dict) else None + if not isinstance(files, list): + raise ValueError(f"the manifest.json of {self.name} lists no files") + listed = [] + for item in files: + if ( + isinstance(item, dict) + and isinstance(item.get("path"), str) + and isinstance(item.get("sha1"), str) + and _SHA1_RE.fullmatch(item["sha1"].lower()) + and isinstance(item.get("size"), int) + ): + listed.append({"path": item["path"], "sha1": item["sha1"].lower(), + "size": item["size"]}) + return listed + + def platforms(self) -> list[str]: + """The platforms the pack's name spells, several for a grouped pack.""" + stem = "_" + self.name[: -len("_BIOS_Pack.zip")] + "_" + return [plat for plat in AVAILABLE_PLATFORMS + if f"_{PLATFORM_NAMES[plat].replace(' ', '_')}_" in stem] + + def manifest(self, plat: str) -> dict: + """A file list for plat built from the pack's own: the same files at + the same places, as extracting the pack would leave them.""" + return { + "manifest_version": 2, + "platform": plat, + "files": [ + {"dest": f["path"], "sha1": f["sha1"], "size": f["size"], + "repo_path": f"bios/{f['path']}"} + for f in self.listed + ], + } + + def find(self, entry: dict) -> tuple[zipfile.ZipFile, zipfile.ZipInfo] | None: + """The member holding an entry's bytes: by its SHA-1 in the pack's list, + else at its destination. The size has to agree either way.""" + sha1 = str(entry.get("sha1", "")).lower() + for name in (self.by_sha1.get(sha1), entry.get("dest")): + held = self.members.get(name) if isinstance(name, str) else None + if held is not None and held[1].file_size == entry.get("size"): + return held + return None + + def tree(self, prefix: str) -> Iterator[tuple[str, int, Callable[[], BinaryIO]]]: + """(path in the tree, size, opener) of every member below prefix.""" + lead = f"{prefix.rstrip('/')}/" if prefix else "" + for name, (archive, info) in self.members.items(): + if name.startswith(lead): + yield name[len(lead):], info.file_size, ( + lambda archive=archive, info=info: archive.open(info)) + + def close(self) -> None: + for archive in self.archives: + archive.close() + self.archives = [] + + +class LocalSources: + """Where this run reads instead of the network: a file list, a pack.""" + + def __init__(self) -> None: + self.manifest: Path | None = None + self.pack: PackSource | None = None + + @property + def offline(self) -> bool: + return self.pack is not None + + def reset(self) -> None: + if self.pack is not None: + self.pack.close() + self.manifest = None + self.pack = None + + +LOCAL = LocalSources() +NOT_IN_PACK = "not in the pack" + + +class _Offline(OSError): + """A network request in a run told to make none.""" + + +def _network_allowed(what: str) -> None: + if LOCAL.offline: + raise _Offline(f"{what}: this run reads a pack and makes no network request") + + +def read_local_manifest(path: Path, plat: str) -> dict: + """A file list saved on disk, checked as a downloaded one is.""" + try: + with open(path, "rb") as handle: + raw = handle.read(MAX_MANIFEST_BYTES + 1) + except OSError as exc: + print(f" Cannot read the file list {path}: {exc.strerror or exc}.", file=sys.stderr) + sys.exit(1) + try: + if len(raw) > MAX_MANIFEST_BYTES: + raise ValueError(f"it exceeds {MAX_MANIFEST_BYTES} bytes") + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise ValueError(f"not JSON ({exc})") from exc + return _validate_manifest(data, plat) + except ValueError as exc: + EVENTS.emit("error", stage="manifest", reason="invalid", platform=plat, + message=f"the file list is not valid: {exc}") + print(f" The file list {path} is not valid for {plat}: {exc}", file=sys.stderr) + sys.exit(1) + + +def local_manifest_platform(path: Path) -> str | None: + """The platform a saved file list names, None when it names none.""" + try: + with open(path, "rb") as handle: + data = json.loads(handle.read(MAX_MANIFEST_BYTES).decode("utf-8")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + return None + plat = data.get("platform") if isinstance(data, dict) else None + return plat if plat in AVAILABLE_PLATFORMS else None + + +def _extract_from_pack( + f: dict, bios_path: Path, dest: Path, progress: Progress | None, +) -> tuple[str, bool, str]: + """Copy one file out of the pack, checked like a download.""" + pack = LOCAL.pack + held = pack.find(f) if pack is not None else None + if held is None: + return f["dest"], False, NOT_IN_PACK + archive, info = held + expected = {name: f[name].lower() for name in ("sha256", "sha1") if f.get(name)} + tmp_path: Path | None = None + counted = 0 + try: + with archive.open(info) as src, tempfile.NamedTemporaryFile( + mode="wb", dir=dest.parent, prefix=f".{dest.name}.", suffix=".part", + delete=False, + ) as out: + tmp_path = Path(out.name) + size = 0 + while True: + if _stop.is_set(): + raise _Interrupted() + chunk = src.read(min(1024 * 1024, f["size"] + 1 - size)) + if not chunk: + break + size += len(chunk) + if size > f["size"]: + raise ValueError("the pack holds more bytes than the list records") + out.write(chunk) + if progress is not None: + progress.add(len(chunk)) + counted += len(chunk) + if size != f["size"]: + raise ValueError(f"the pack holds {size} bytes, the list records {f['size']}") + actual = _digest_file(tmp_path, tuple(expected)) + for name, label in (("sha256", "SHA-256"), ("sha1", "SHA-1")): + if name in expected and actual[name] != expected[name]: + raise ValueError(f"{label} mismatch in the pack") + os.chmod(tmp_path, (0o777 if f.get("executable") else 0o666) & ~_UMASK) + _clear_read_only(dest) + KEPT.keep(*_kept_slot(*_entry_place(bios_path, f)), dest, expected, tmp_path) + os.replace(tmp_path, dest) + tmp_path = None + return f["dest"], True, "" + except _Interrupted: + return f["dest"], False, "stopped" + except _NotKept as exc: + return f["dest"], False, f"{NOT_KEPT} ({exc})" + except (OSError, ValueError, zipfile.BadZipFile, zlib.error) as exc: + if progress is not None and counted: + progress.add(-counted) + if isinstance(exc, OSError) and _write_refused(exc): + return f["dest"], False, READ_ONLY if _read_only_file(dest) else NOT_WRITABLE + return f["dest"], False, str(exc) + finally: + if tmp_path is not None and tmp_path.exists(): + tmp_path.unlink() + + +# Files --check reads to tell why a file is refused: past these, the folder +# is not one a person filled by hand, and walking it costs more than it tells. +MAX_SCANNED_FILES = 200_000 +_ARCHIVE_SUFFIXES = (".zip", ".7z", ".rar") +# Cores named on the line that says who opens a file, before "and N more". +SHOWN_CORES = 4 + + +class FolderIndex: + """The files under a BIOS folder, by name and by size, hashed on demand.""" + + def __init__(self, root: Path) -> None: + self.root = root = root.resolve() + self.by_name: dict[str, list[Path]] = {} + self.by_size: dict[int, list[Path]] = {} + self.size: dict[Path, int] = {} + self.archives: list[Path] = [] + self._sha1: dict[Path, str] = {} + self._members: dict[Path, list[str]] = {} + seen = 0 + for folder, dirs, names in os.walk(root): + dirs[:] = sorted(d for d in dirs if d != BACKUP_DIR) + for name in sorted(names): + path = Path(folder) / name + try: + if not path.is_file(): + continue + size = path.stat().st_size + except OSError: + continue + self.by_name.setdefault(name.casefold(), []).append(path) + self.by_size.setdefault(size, []).append(path) + self.size[path] = size + if name.casefold().endswith(_ARCHIVE_SUFFIXES): + self.archives.append(path) + seen += 1 + if seen >= MAX_SCANNED_FILES: + return + + def sha1(self, path: Path) -> str: + if path not in self._sha1: + try: + self._sha1[path] = _sha1_file(path) + except OSError: + self._sha1[path] = "" + return self._sha1[path] + + def holding(self, sizes: set[int], sha1s: set[str]) -> list[Path]: + """Files whose contents are one of sha1s, read only when the size fits.""" + return [path for size in sorted(sizes) for path in self.by_size.get(size, []) + if self.sha1(path) in sha1s] + + def zip_members(self, path: Path) -> list[str]: + if path not in self._members: + try: + with zipfile.ZipFile(path) as archive: + self._members[path] = [n.rsplit("/", 1)[-1].casefold() + for n in archive.namelist()] + except (OSError, zipfile.BadZipFile): + self._members[path] = [] + return self._members[path] + + def relative(self, path: Path) -> str: + try: + return path.relative_to(self.root).as_posix() + except ValueError: + return str(path) + + +def _copies(entry: dict) -> list[dict]: + """Every copy the list knows for an entry: its own, other cores', the + platform's declared one.""" + return [entry, *entry.get("alternatives", []), *entry.get("slot_choices", []), + *entry.get("declared_choices", [])] + + +def readers(entry: dict, manifest: dict) -> list[str]: + """The cores that open an entry: as its copies name them, else every core + the platform runs for its systems.""" + named = sorted({core for copy in _copies(entry) for core in copy.get("cores") or []}) + if named: + return named + system_cores = manifest.get("system_cores") or {} + return sorted({core for system in entry_systems(entry) + for core in system_cores.get(system, [])}) + + +def opened_line(entry: dict, manifest: dict, bios_path: Path) -> str: + """Where the emulators read an entry, and which ones.""" + try: + where = _entry_path(bios_path, entry) + except ValueError: + where = bios_path / str(entry.get("dest")) + cores = readers(entry, manifest) + if not cores: + return f"opened at {where}" + shown = ", ".join(cores[:SHOWN_CORES]) + if len(cores) > SHOWN_CORES: + shown += f" and {len(cores) - SHOWN_CORES} more" + return f"opened at {where} by {shown}" + + +def _known_destinations(files: list[dict]) -> set[str]: + return {str(f.get("dest", "")).casefold() for f in files} + + +def _content_owner(sha1: str, entry: dict, files: list[dict]) -> str: + """Why a file holding sha1 is not the one entry wants, if the list knows.""" + for copy in entry.get("alternatives", []) + entry.get("slot_choices", []): + if copy.get("sha1") == sha1 and copy.get("cores"): + return (f"another revision: the copy {', '.join(copy['cores'])} " + "reads, not the one this list installs here") + for copy in entry.get("declared_choices", []): + if copy.get("sha1") == sha1: + return ("another revision: the one the platform's own list declares, " + "not the one the cores read") + for other in files: + if other is entry or other.get("dest") == entry.get("dest"): + continue + if any(copy.get("sha1") == sha1 for copy in _copies(other)): + regions = [r for r in other.get("regions") or [] if r != WORLD_REGION] + if regions: + return (f"another region: the contents of {other['dest']} " + f"({', '.join(regions)})") + return f"another file: the contents of {other['dest']}" + return "" + + +def _found_elsewhere( + entry: dict, index: FolderIndex, bios_path: Path, files: list[dict], + known: set[str], +) -> tuple[str, str]: + """(reason, folder too deep) for an entry missing at its place.""" + dest = str(entry.get("dest", "")) + name = dest.rsplit("/", 1)[-1] + sizes = {copy.get("size") for copy in _copies(entry) if isinstance(copy.get("size"), int)} + sha1s = {copy.get("sha1") for copy in _copies(entry) if copy.get("sha1")} + try: + expected = _entry_path(bios_path, entry) + except ValueError: + return "", "" + same_name = [p for p in index.by_name.get(name.casefold(), []) if p != expected] + matching = [p for p in same_name if index.size[p] in sizes and index.sha1(p) in sha1s] + for path in matching or same_name: + rel = index.relative(path) + contents = "" if path in matching else ", with other contents" + if rel.endswith("/" + dest) and path.name == name: + extra = rel[: -len(dest) - 1] + top = index.root.name + if extra.casefold() == top.casefold(): + return (f"nested folder: it is at {top}/{rel}{contents}, a " + f"{top} folder inside {top}"), extra + depth = extra.count("/") + 1 + levels = "one folder" if depth == 1 else f"{depth} folders" + return f"{levels} too deep: it is at {rel}{contents}", extra + if path.parent == expected.parent and path.name != name: + return f"case of the name: the folder holds {path.name}{contents}", "" + if path.name.casefold() == name.casefold(): + return f"wrong folder: it is at {rel}{contents}", "" + above = index.root.parent / dest + if index.root.parent != index.root and above.is_file(): + return f"one folder too high: it is at {above}", "" + for path in index.holding(sizes, sha1s): + rel = index.relative(path) + if rel.casefold() in known: + return f"its contents are at {rel}: copy that file to {dest}", "" + return f"a name no core opens: its contents are at {rel}, rename it {name}", "" + if name.casefold().endswith(_ARCHIVE_SUFFIXES): + return "", "" + stem = name.rsplit(".", 1)[0].casefold() + folders = {expected.parent, index.root} + for path in index.archives: + if path.parent not in folders or index.relative(path).casefold() in known: + continue + lowered = path.name.casefold() + if lowered.endswith(".zip") and name.casefold() in index.zip_members(path): + return f"still zipped: it is inside {index.relative(path)}", "" + if lowered.rsplit(".", 1)[0] == stem: + return f"still zipped: {index.relative(path)} needs extracting", "" + return "", "" + + +def _wrong_contents(entry: dict, path: Path, index: FolderIndex, files: list[dict]) -> str: + """Why the file at an entry's place is not the one the list installs.""" + try: + size = path.stat().st_size + with open(path, "rb") as handle: + head = handle.read(4) + except OSError: + return "" + sha1 = index.sha1(path) + owner = _content_owner(sha1, entry, files) + if owner: + return owner + if head == b"PK\x03\x04" and not str(entry.get("dest", "")).casefold().endswith(".zip"): + return "still zipped: this file is a ZIP archive, extract what it holds" + expected = {copy.get("size") for copy in _copies(entry) if isinstance(copy.get("size"), int)} + if size not in expected: + want = " or ".join(f"{s:,}" for s in sorted(expected)) + return f"wrong size: {size:,} bytes, the list expects {want}" + return "same size, other contents: another dump, or a damaged copy" + + +class Diagnosis(NamedTuple): + reasons: dict[int, str] + deep: dict[str, int] + + +def diagnose( + missing: list[dict], mismatched: list[dict], files: list[dict], bios_path: Path, +) -> Diagnosis: + """The reason --check gives for each file it flags, from what the folder + holds and what the list records: id(entry) -> reason, and the folders + holding files one level too deep with how many.""" + reasons: dict[int, str] = {} + deep: dict[str, int] = {} + if not bios_path.is_dir() or not (missing or mismatched): + return Diagnosis(reasons, deep) + index = FolderIndex(bios_path) + known = _known_destinations(files) + for entry in missing: + reason, extra = _found_elsewhere(entry, index, bios_path, files, known) + if reason: + reasons[id(entry)] = reason + if extra: + deep[extra] = deep.get(extra, 0) + 1 + for entry in mismatched: + try: + path = _entry_path(bios_path, entry) + except ValueError: + continue + reason = _wrong_contents(entry, path, index, files) + if reason: + reasons[id(entry)] = reason + return Diagnosis(reasons, deep) + + +def check_lines( + missing: list[dict], mismatched: list[dict], files: list[dict], manifest: dict, + bios_path: Path, every: bool, +) -> list[str]: + """The files --check flags, each with why and where its cores read it. + + "missing: scph5501.bin" left a user with the file one folder too deep, + or still in its ZIP, to guess. A flagged file with a reason is always + listed. Plain missing ones stop at ten unless every is set. + """ + found = diagnose(missing, mismatched, files, bios_path) + lines = [] + for extra, count in sorted(found.deep.items(), key=lambda item: -item[1]): + if count >= 3: + lines.append(f" {_plural(count, 'file')} sit too deep, in " + f"{bios_path / extra}: move what it holds up into {bios_path}.") + explained = [e for e in missing if id(e) in found.reasons] + plain = [e for e in missing if id(e) not in found.reasons] + shown = plain if every else plain[:10] + + def block(label: str, entry: dict) -> list[str]: + game = " (game data)" if entry.get("category") == GAME_DATA else "" + rows = [f" {label}: {entry['dest']}{game}"] + if id(entry) in found.reasons: + rows.append(f" {found.reasons[id(entry)]}") + rows.append(f" {opened_line(entry, manifest, bios_path)}") + return rows + + for entry in mismatched: + lines += block("wrong hash", entry) + for entry in explained: + lines += block("missing", entry) + for entry in shown: + lines += block("missing", entry) + if len(shown) < len(plain): + lines.append(f" ... and {len(plain) - len(shown)} more missing " + "(--verbose lists them all)") + return lines + + +# Below this a download is small enough that narrowing it is not worth a line. +BULK_HINT_FROM = 100 * 1024 * 1024 + + +def _bytes(files: list[dict]) -> int: + return sum(f.get("size", 0) for f in files) + + +def bulk_hint( + to_download: list[dict], manifest: dict, systems_chosen: bool, cores_chosen: bool, +) -> list[str]: + """Name what pulls most of a download, when narrowing saves over half of it. + + `--system 3do` fetched 349 MB for RetroArch, and `--core opera` 12.5 MB: + the arcade cores that also read 3DO pulled the rest, and nothing said so. + """ + total = _bytes(to_download) + if total < BULK_HINT_FROM: + return [] + systems = {s for entry in to_download for s in entry_systems(entry)} + if not cores_chosen and len(systems) == 1: + system = next(iter(systems)) + candidates = (manifest.get("system_cores") or {}).get(system) or [] + by_core: dict[str, int] = {} + for entry in to_download: + for core in entry.get("cores") or []: + by_core[core] = by_core.get(core, 0) + entry.get("size", 0) + kept = {core: _bytes(_filter_by_target(to_download, [core])) for core in candidates} + if by_core and kept: + heavy = max(sorted(by_core), key=lambda core: by_core[core]) + light = min(sorted(kept), key=lambda core: kept[core]) + saved = total - kept[light] + if saved * 2 > total: + return [ + f" {heavy} pulls {format_size(by_core[heavy])} of this " + f"{format_size(total)} download for {system}. --core {light} " + f"keeps {format_size(kept[light])} of it and saves " + f"{format_size(saved)} ({saved * 100 // total}%)." + ] + if not systems_chosen and len(systems) > 1: + alone: dict[str, int] = {} + for entry in to_download: + own = entry_systems(entry) + if len(own) == 1: + alone[own[0]] = alone.get(own[0], 0) + entry.get("size", 0) + if alone: + heavy = max(sorted(alone), key=lambda system: alone[system]) + if alone[heavy] * 2 > total: + return [ + f" {heavy} pulls {format_size(alone[heavy])} of this " + f"{format_size(total)} download. --system naming the systems " + f"you play leaves it out and saves {format_size(alone[heavy])} " + f"({alone[heavy] * 100 // total}%), --list-systems names them." + ] + return [] + + def _help_epilog() -> str: """Examples in the form the run was started with, and where the rest is.""" run = _INVOCATION.launcher() @@ -6030,6 +6779,22 @@ def main() -> None: help=f"put back the files an earlier run replaced and kept in FOLDER " f"(a dated folder inside {BACKUP_DIR})", ) + parser.add_argument( + "--from-pack", + action="append", + type=Path, + metavar="PATH", + help="install from a pack already downloaded: its ZIP, one of its parts " + "(the others are found beside it) or the folder holding them. " + "Makes no network request, and checks every file's hash", + ) + parser.add_argument( + "--manifest", + type=Path, + metavar="FILE", + help="read the file list from FILE (install/.json) instead " + "of downloading it", + ) parser.add_argument( "--version", action="version", @@ -6065,7 +6830,7 @@ def main() -> None: "cores_dir", "save_dir", "system", "core", "region", "no_game_data", "list_systems", "list_cores", "romm_firmware", "list_emulators", "events", "quiet", - "yes", "verbose") + "yes", "verbose", "from_pack", "manifest") given = [name for name in alone if getattr(args, name)] if args.jobs != 8: given.append("jobs") @@ -6075,6 +6840,12 @@ def main() -> None: sys.exit(restore_kept(args.restore, args.dry_run)) if args.every_board and args.target: parser.error("--every-board installs every board; --target names one") + if args.from_pack and args.target: + parser.error("--target reads a core list from the network, which --from-pack " + "does not use") + if (args.from_pack or args.manifest) and (args.list_platforms or args.list_targets): + parser.error("--from-pack and --manifest give a file list, which --list-platforms " + "and --list-targets do not read") if args.no_game_data and (args.list_platforms or args.list_targets): parser.error("--no-game-data narrows a file list, not --list-platforms " "or --list-targets") @@ -6122,6 +6893,44 @@ def _run(args: argparse.Namespace) -> None: sys.stdout = real_stdout +def _open_local_sources(args: argparse.Namespace) -> None: + """Take the file list and the pack this run was given, and the platform + they are for when none is named.""" + LOCAL.reset() + if args.manifest is not None: + LOCAL.manifest = args.manifest + named = local_manifest_platform(args.manifest) + if args.platform is None and named: + args.platform = named + print(f" File list: {args.manifest} ({display_name(named)})") + if not args.from_pack: + return + try: + LOCAL.pack = PackSource(args.from_pack) + except ValueError as exc: + print(f"Error: {exc}", file=sys.stderr) + sys.exit(1) + pack = LOCAL.pack + parts = len(pack.files) + print(f" Pack: {pack.name}" + + (f", {parts} parts" if parts > 1 or _PART_FILE.match(pack.files[0].name) else "")) + spelled = pack.platforms() + if args.platform is not None: + plat = normalize_platform(args.platform) + if spelled and plat not in spelled: + print(f"Error: {pack.name} is a pack for " + f"{' and '.join(display_name(n) for n in spelled)}, not " + f"{display_name(plat)}.", file=sys.stderr) + sys.exit(1) + elif len(spelled) == 1: + args.platform = spelled[0] + else: + among = (" (" + ", ".join(spelled) + ")") if spelled else "" + print(f"Error: name the platform {pack.name} is installed for with " + f"--platform{among}.", file=sys.stderr) + sys.exit(1) + + def _missing_dest(dest: Path, ask: bool) -> None: """Say that a --dest folder does not exist before anything goes there. @@ -6254,6 +7063,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: os_type = detect_os() if os_type == "wsl": _enter_wsl() + _open_local_sources(args) # Early exit for listing if args.list_platforms: @@ -6404,6 +7214,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: files_pending = 0 failures: list[tuple[str, str]] = [] installed_paths: list[Path] = [] + bulk_lines: list[str] = [] # --target is checked for every platform before anything is written, as # --installed-cores is below: EmuDeck installed 3.2 GB and only then was @@ -6454,7 +7265,12 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: for plat_name, bios_path in platforms: if bios_path not in installed_paths: installed_paths.append(bios_path) - print(f"\nFetching file index for {plat_name}...") + if LOCAL.manifest is not None: + print(f"\nReading the file index for {plat_name} from {LOCAL.manifest}...") + elif LOCAL.pack is not None: + print(f"\nReading the file index for {plat_name} from {LOCAL.pack.name}...") + else: + print(f"\nFetching file index for {plat_name}...") manifest = fetch_manifest(plat_name) files = manifest.get("files", []) omitted_files = manifest.get("omitted_files", []) @@ -6510,7 +7326,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # narrows to it: a Batocera RPi4 took 8.6 GiB where its board needs # 5.5. Only on the device itself, where the board file exists. target = args.target - if not target and not args.every_board and args.dest is None: + if not target and not args.every_board and args.dest is None and not LOCAL.offline: board = detect_board(plat_name) if board: known = fetch_targets(plat_name) or {} @@ -6687,6 +7503,12 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # Mismatched files need re-download to_download.extend(mismatched) + hint = [] if LOCAL.pack is not None else bulk_hint( + to_download, manifest, + systems_chosen=bool(selection and selection[0]), + cores_chosen=bool((selection and selection[1]) or args.emulator + or args.installed_cores or target), + ) narrowed = bool( target or args.system or args.core or args.region or args.installed_cores @@ -6712,15 +7534,14 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: # 8000 missing files, it hid behind "... and 8059 more". wrong = {id(entry) for entry in mismatched} missing = [entry for entry in to_download if id(entry) not in wrong] - shown = missing if args.verbose else missing[:10] + # A pack extracted from some of its parts: the part is named + # before the hundreds of files it holds. + for line in split_pack_lines(bios_path): + print(line) print(f"\n {_plural(len(to_download), 'file')} to download:") - for entry in mismatched: - print(f" wrong hash: {entry['dest']}") - for entry in shown: - print(f" missing: {entry['dest']}") - if len(shown) < len(missing): - print(f" ... and {len(missing) - len(shown)} more missing " - "(--verbose lists them all)") + for line in check_lines(missing, mismatched, files, manifest, bios_path, + args.verbose): + print(line) print(" Run the same command without --check to fetch them; " "a file with a wrong hash is replaced, and the one in place " f"kept in {BACKUP_DIR} beside the BIOS folder.") @@ -6752,6 +7573,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: for entry, needed in trees for rel in needed ], bios_path) _print_plan(to_download, bios_path, term, trees, kept) + for line in hint: + print(line) continue if to_download: @@ -6771,7 +7594,11 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: files_pending += len(to_download) continue EVENTS.emit("download", platform=plat_name, files=len(to_download), bytes=dl_size) - print(f"\nDownloading {len(to_download)} files ({format_size(dl_size)})...") + if LOCAL.pack is not None: + print(f"\nCopying {len(to_download)} files ({format_size(dl_size)}) " + f"from {LOCAL.pack.name}...") + else: + print(f"\nDownloading {len(to_download)} files ({format_size(dl_size)})...") print(" Press Ctrl+C to stop. Running the same command again " "carries on from here.") # Checked before the folder is created: a run stopped for space @@ -6797,6 +7624,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: to_download, bios_path, jobs=args.jobs, verbose=args.verbose, term=term, progress=progress, ) + bulk_lines += hint total_downloaded += len(to_download) - len(failed) files_pending += len(failed) if interrupted: @@ -6829,7 +7657,8 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: "the source (the pack has them)") print(line) if counts["error"]: - print(f" {entry['ref']}: could not read {entry['url']}: " + print(f" {entry['ref']}: could not read " + f"{LOCAL.pack.name if LOCAL.pack is not None else entry['url']}: " f"{counts['error']}", file=sys.stderr) failures.append((entry["ref"], counts["error"])) total_errors += 1 @@ -6898,6 +7727,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: assumed=assumed_dests, omitted_entries=omitted_seen, platform_of={path: plat for plat, path in platforms}, + hints=bulk_lines, ) diff --git a/install.sh b/install.sh index 376ff2e1..87f82a40 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="ad3b338a8f709be45352a12bd9c6a6250819a8fb9afd19498e24f41e3f062d20" +DEFAULT_INSTALL_SHA256="996cf4a8e21b8f05a524778046d0d32c807fbe3564f6edefdf5bee1add9c76cd" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_offline.py b/tests/test_install_offline.py new file mode 100644 index 00000000..4f69e3ce --- /dev/null +++ b/tests/test_install_offline.py @@ -0,0 +1,488 @@ +"""What --check says about a refused file, and an install with no network. + +A user with the BIOS one folder too deep, still in its ZIP or under another +name read "missing: scph5501.bin" and nothing else. --check now names the +reason and where each core opens the file. A pack extracted from some of +its parts is named by the part it lacks. And a pack downloaded once +installs again from disk, --from-pack, every file checked by its hash. +""" + +from __future__ import annotations + +import contextlib +import hashlib +import io +import json +import os +import shutil +import socket +import subprocess +import sys +import tempfile +import unittest +import zipfile +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO_ROOT)) +sys.path.insert(0, str(REPO_ROOT / "scripts")) +sys.path.insert(0, str(REPO_ROOT / "tests")) + +import install # noqa: E402 +import split_pack # noqa: E402 +from test_install_selection import FakeRepository # noqa: E402 + +US = b"us bios " * 64 +JP = b"jp bios " * 64 +EU = b"eu bios " * 64 +GBA = b"gba bios" * 32 +EXTRA = b"pcsx extra" + + +def _sha1(data: bytes) -> str: + return hashlib.sha1(data).hexdigest() + + +def _entry(dest: str, data: bytes, systems: list[str], **fields) -> dict: + return { + "dest": dest, "sha1": _sha1(data), "size": len(data), + "repo_path": f"bios/{dest.replace('/', '_')}", "cores": None, + "systems": systems, **fields, + } + + +def manifest() -> dict: + """A small RetroArch list: two regions of one BIOS, a core file with a + second revision, a file in a subfolder.""" + return { + "manifest_version": 2, + "platform": "retroarch", + "files": [ + _entry("scph5501.bin", US, ["sony-playstation"], regions=["north-america"]), + _entry("scph5500.bin", JP, ["sony-playstation"], regions=["japan"]), + _entry("gba_bios.bin", GBA, ["nintendo-gba"]), + _entry("pcsx/extra.bin", EXTRA, ["sony-playstation"], cores=["pcsx_rearmed"], + alternatives=[{"sha1": _sha1(EU), "size": len(EU), + "repo_path": "bios/eu", "cores": ["beetle_psx"]}, + {"sha1": _sha1(EXTRA), "size": len(EXTRA), + "repo_path": "bios/x", "cores": ["pcsx_rearmed"]}]), + ], + "system_cores": {"sony-playstation": ["beetle_psx", "pcsx_rearmed"], + "nintendo-gba": ["mgba"]}, + } + + +PAYLOAD = {"scph5501.bin": US, "scph5500.bin": JP, "gba_bios.bin": GBA, + "pcsx/extra.bin": EXTRA} + + +def _closed_port() -> int: + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + +def _offline_env() -> dict: + """An environment where any request the installer makes is refused.""" + env = dict(os.environ) + env["RETROBIOS_BASE_URL"] = f"http://127.0.0.1:{_closed_port()}" + env["HTTPS_PROXY"] = env["HTTP_PROXY"] = f"http://127.0.0.1:{_closed_port()}" + return env + + +class Scratch(unittest.TestCase): + def setUp(self): + scratch = REPO_ROOT / "tmp" + scratch.mkdir(exist_ok=True) + self.root = Path(tempfile.mkdtemp(dir=scratch)) + self.addCleanup(shutil.rmtree, self.root, True) + self.dest = self.root / "system" + self.list = self.root / "retroarch.json" + self.list.write_text(json.dumps(manifest())) + + def run_installer(self, *options: str) -> subprocess.CompletedProcess: + return subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "retroarch", + "--dest", str(self.dest), *options], + env=_offline_env(), capture_output=True, text=True, timeout=120, + check=False, stdin=subprocess.DEVNULL, + ) + + def place(self, relative: str, data: bytes) -> Path: + path = self.dest / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + return path + + def check(self) -> str: + proc = self.run_installer("--check", "--manifest", str(self.list), "--verbose") + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + return proc.stdout + + def complete(self, *skip: str) -> None: + for name, data in PAYLOAD.items(): + if name not in skip: + self.place(name, data) + + +class CheckNamesWhyAFileIsRefused(Scratch): + def test_another_region_is_named(self): + self.complete("scph5501.bin") + self.place("scph5501.bin", JP) + out = self.check() + self.assertIn("wrong hash: scph5501.bin", out) + self.assertIn("another region: the contents of scph5500.bin (japan)", out) + + def test_another_revision_names_the_core_that_reads_it(self): + self.complete("pcsx/extra.bin") + self.place("pcsx/extra.bin", EU) + out = self.check() + self.assertIn("another revision: the copy beetle_psx reads", out) + + def test_a_wrong_size_is_measured(self): + self.complete("gba_bios.bin") + self.place("gba_bios.bin", b"short") + self.assertIn(f"wrong size: 5 bytes, the list expects {len(GBA):,}", self.check()) + + def test_a_zip_under_the_file_name_is_still_zipped(self): + self.complete("gba_bios.bin") + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("gba_bios.bin", GBA) + self.place("gba_bios.bin", buffer.getvalue()) + self.assertIn("still zipped: this file is a ZIP archive", self.check()) + + def test_a_file_inside_an_archive_is_still_zipped(self): + self.complete("gba_bios.bin") + self.dest.mkdir(exist_ok=True) + with zipfile.ZipFile(self.dest / "GBA BIOS.zip", "w") as archive: + archive.writestr("bios/gba_bios.bin", GBA) + self.assertIn("still zipped: it is inside GBA BIOS.zip", self.check()) + + def test_a_case_difference_is_named(self): + probe = self.root / "CaseProbe" + probe.write_text("x") + if (self.root / "caseprobe").exists(): + self.skipTest("this filesystem does not keep case") + self.complete("gba_bios.bin") + self.place("GBA_BIOS.BIN", GBA) + self.assertIn("case of the name: the folder holds GBA_BIOS.BIN", self.check()) + + def test_a_nested_bios_folder_is_named(self): + for name, data in PAYLOAD.items(): + self.place(f"system/{name}", data) + out = self.check() + self.assertIn("nested folder: it is at system/system/scph5501.bin", out) + self.assertIn(f"4 files sit too deep, in {self.dest.resolve() / 'system'}", out) + + def test_a_wrong_folder_is_named(self): + self.complete("pcsx/extra.bin") + self.place("beetle/extra.bin", EXTRA) + self.assertIn("wrong folder: it is at beetle/extra.bin", self.check()) + + def test_a_folder_too_deep_is_named(self): + self.complete("gba_bios.bin") + self.place("gba/roms/gba_bios.bin", GBA) + self.assertIn("2 folders too deep: it is at gba/roms/gba_bios.bin", self.check()) + + def test_a_folder_too_high_is_named(self): + self.complete("gba_bios.bin") + (self.root / "gba_bios.bin").write_bytes(GBA) + self.assertIn(f"one folder too high: it is at {self.root.resolve() / 'gba_bios.bin'}", + self.check()) + + def test_a_name_no_core_opens_is_named(self): + self.complete("scph5501.bin") + self.place("PSX BIOS (USA).bin", US) + self.assertIn("a name no core opens: its contents are at PSX BIOS (USA).bin, " + "rename it scph5501.bin", self.check()) + + def test_each_flagged_file_says_where_its_cores_open_it(self): + self.complete("scph5501.bin", "pcsx/extra.bin") + out = self.check() + here = self.dest.resolve() + self.assertIn(f"opened at {here / 'scph5501.bin'} by beetle_psx, pcsx_rearmed", out) + self.assertIn(f"opened at {here / 'pcsx' / 'extra.bin'} by beetle_psx, pcsx_rearmed", + out) + + def test_a_complete_folder_says_nothing_more(self): + self.complete() + proc = self.run_installer("--check", "--manifest", str(self.list)) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertNotIn("opened at", proc.stdout) + + +def _pack(directory: Path, name: str, members: dict[str, bytes]) -> Path: + """A pack as generate_pack writes one: the files and a manifest.json.""" + listed = [{"path": path, "sha1": _sha1(data), "md5": hashlib.md5(data).hexdigest(), + "size": len(data), "status": "verified", "name": path.rsplit("/", 1)[-1]} + for path, data in members.items()] + path = directory / name + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as archive: + for member, data in members.items(): + archive.writestr(member, data) + archive.writestr("README.txt", b"read me\n") + archive.writestr("manifest.json", json.dumps({"files": listed})) + return path + + +def _noise(seed: str, size: int) -> bytes: + """Bytes deflate cannot shrink.""" + return b"".join(hashlib.sha256(f"{seed}{n}".encode()).digest() + for n in range(size // 32)) + + +BIG = {f"dir{n}/file{n}.bin": _noise(str(n), 2048) for n in range(4)} + + +class SplitParts(Scratch): + def split(self) -> list[Path]: + downloads = self.root / "downloads" + downloads.mkdir() + pack = _pack(downloads, "RetroArch_v1_BIOS_Pack.zip", BIG) + parts = split_pack.split_pack(pack, limit=pack.stat().st_size // 3) + self.assertGreater(len(parts), 2) + return parts + + def extract(self, parts: list[Path], into: Path | None = None) -> None: + for part in parts: + with zipfile.ZipFile(part) as archive: + archive.extractall(into or self.dest) + + def list_of_pack(self) -> None: + self.list.write_text(json.dumps({ + "manifest_version": 2, "platform": "retroarch", + "files": [_entry(path, data, ["arcade"]) for path, data in BIG.items()], + })) + + def test_check_names_the_part_missing_from_the_folder(self): + parts = self.split() + self.list_of_pack() + self.extract(parts[:1] + parts[2:]) + out = self.check() + count = len(parts) + self.assertIn(f"RetroArch_v1_BIOS_Pack.zip comes in {count} parts, all needed.", out) + self.assertIn(f"missing: part 2, RetroArch_v1_BIOS_Pack.part2of{count}.zip", out) + self.assertNotIn("missing: part 1,", out) + + def test_check_names_a_part_extracted_into_a_folder_of_its_own(self): + parts = self.split() + self.list_of_pack() + self.extract(parts[1:]) + own = self.dest / parts[0].stem + self.extract(parts[:1], own) + out = self.check() + self.assertIn(f"Part 1 went into {own}, a folder of its own", out) + self.assertNotIn("missing: part 1,", out) + + def test_a_complete_set_of_parts_says_nothing(self): + parts = self.split() + self.list_of_pack() + self.extract(parts) + proc = self.run_installer("--check", "--manifest", str(self.list)) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertNotIn("comes in", proc.stdout) + + def test_from_pack_finds_the_other_parts_beside_one(self): + parts = self.split() + proc = self.run_installer("--from-pack", str(parts[1])) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + for path, data in BIG.items(): + self.assertEqual((self.dest / path).read_bytes(), data) + + def test_from_pack_names_a_part_that_is_not_there(self): + parts = self.split() + parts[1].unlink() + proc = self.run_installer("--from-pack", str(parts[0])) + self.assertEqual(proc.returncode, 1) + self.assertIn(f"not in {parts[0].parent}: {parts[1].name}", proc.stderr) + self.assertFalse(self.dest.exists()) + + +class FromPackInstallsOffline(Scratch): + def setUp(self): + super().setUp() + self.downloads = self.root / "downloads" + self.downloads.mkdir() + self.pack = _pack(self.downloads, "RetroArch_v1_BIOS_Pack.zip", PAYLOAD) + + def installed(self) -> dict[str, bytes]: + return {p.relative_to(self.dest).as_posix(): p.read_bytes() + for p in self.dest.rglob("*") if p.is_file()} + + def test_the_pack_installs_with_no_network(self): + proc = self.run_installer("--from-pack", str(self.pack)) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(self.installed(), PAYLOAD) + self.assertIn("Copying 4 files", proc.stdout) + self.assertNotIn("Fetching", proc.stdout) + + def test_the_folder_holding_the_pack_is_enough(self): + proc = self.run_installer("--from-pack", str(self.downloads)) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(self.installed(), PAYLOAD) + + def test_the_platform_is_read_from_the_pack_name(self): + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--dest", str(self.dest), + "--from-pack", str(self.pack), "--no-input"], + env=_offline_env(), capture_output=True, text=True, timeout=120, + check=False, stdin=subprocess.DEVNULL, + ) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(self.installed(), PAYLOAD) + + def test_a_pack_for_another_platform_is_refused(self): + proc = subprocess.run( + [sys.executable, str(REPO_ROOT / "install.py"), "--platform", "batocera", + "--dest", str(self.dest), "--from-pack", str(self.pack)], + env=_offline_env(), capture_output=True, text=True, timeout=120, + check=False, stdin=subprocess.DEVNULL, + ) + self.assertEqual(proc.returncode, 1) + self.assertIn("is a pack for RetroArch, not Batocera", proc.stderr) + + def test_a_file_whose_bytes_differ_from_the_list_is_refused(self): + """The hash is checked against the list the run reads, not trusted + from the archive that carries both.""" + listed = manifest() + listed["files"][0]["sha1"] = _sha1(b"something else") + self.list.write_text(json.dumps(listed)) + proc = self.run_installer("--from-pack", str(self.pack), "--manifest", str(self.list)) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertIn("scph5501.bin", proc.stdout) + self.assertFalse((self.dest / "scph5501.bin").exists()) + self.assertEqual((self.dest / "gba_bios.bin").read_bytes(), GBA) + + def test_with_a_saved_list_the_choice_options_apply(self): + proc = self.run_installer("--from-pack", str(self.pack), "--manifest", + str(self.list), "--system", "nintendo-gba") + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(self.installed(), {"gba_bios.bin": GBA}) + + def test_a_file_the_pack_does_not_hold_fails_by_name(self): + listed = manifest() + listed["files"].append(_entry("absent.bin", b"absent", ["nintendo-gba"])) + self.list.write_text(json.dumps(listed)) + proc = self.run_installer("--from-pack", str(self.pack), "--manifest", str(self.list)) + self.assertEqual(proc.returncode, 1, proc.stdout + proc.stderr) + self.assertIn("absent.bin (not in the pack)", proc.stdout) + + def test_a_data_tree_is_rebuilt_from_the_pack(self): + """The tree's source archive is never fetched: the pack holds it + where it extracts.""" + tree = {"Sys/GC/font.bin": b"font bytes", "Sys/Wii/setting.txt": b"setting"} + self.pack = _pack(self.downloads, "RetroArch_v1_BIOS_Pack.zip", + {**PAYLOAD, **{f"dolphin-emu/{k}": v for k, v in tree.items()}}) + listed = manifest() + listed["data_directories"] = [{ + "ref": "dolphin-sys", "dest": "dolphin-emu", "type": "zip", + "url": "https://example.invalid/dolphin.zip", "systems": ["nintendo-gba"], + "files": {rel: {"sha1": _sha1(data), "size": len(data)} + for rel, data in tree.items()}, + }] + self.list.write_text(json.dumps(listed)) + proc = self.run_installer("--from-pack", str(self.pack), "--manifest", str(self.list)) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertIn("dolphin-sys: 2 installed", proc.stdout) + for rel, data in tree.items(): + self.assertEqual((self.dest / "dolphin-emu" / rel).read_bytes(), data) + + def test_a_cut_download_is_named(self): + self.pack.write_bytes(self.pack.read_bytes()[:100]) + proc = self.run_installer("--from-pack", str(self.pack)) + self.assertEqual(proc.returncode, 1) + self.assertIn("cannot be read as a ZIP", proc.stderr) + + def test_target_is_refused_with_from_pack(self): + proc = self.run_installer("--from-pack", str(self.pack), "--target", "switch") + self.assertEqual(proc.returncode, 2) + self.assertIn("--target", proc.stderr) + + def test_no_request_leaves_the_process(self): + """The guard holds in the code, not only in the test's dead proxy.""" + install.LOCAL.reset() + install.LOCAL.pack = install.PackSource([self.pack]) + self.addCleanup(install.LOCAL.reset) + with self.assertRaises(OSError): + install._fetch_json("http://example.invalid/x.json", 10, "manifest") + + +class ManifestFromDisk(unittest.TestCase): + def test_a_saved_list_installs_without_fetching_the_list(self): + with FakeRepository() as repo: + saved = repo.root / "saved.json" + served = repo.root / "install" / "retroarch.json" + saved.write_bytes(served.read_bytes()) + served.unlink() + proc = repo.run("--manifest", str(saved)) + self.assertEqual(proc.returncode, 0, proc.stdout + proc.stderr) + self.assertEqual(len(repo.installed()), 5) + + def test_a_list_for_another_platform_is_refused(self): + with FakeRepository() as repo: + saved = repo.root / "saved.json" + data = json.loads((repo.root / "install" / "retroarch.json").read_text()) + data["platform"] = "batocera" + saved.write_text(json.dumps(data)) + proc = repo.run("--manifest", str(saved)) + self.assertEqual(proc.returncode, 1) + self.assertIn("not valid for retroarch", proc.stderr) + + +def _sized(dest: str, size: int, systems: list[str], cores: list[str] | None = None) -> dict: + return {"dest": dest, "size": size, "sha1": "0" * 40, "systems": systems, + "cores": cores, "repo_path": f"bios/{dest}"} + + +MB = 1024 * 1024 + + +class TheSummaryNamesTheBulk(unittest.TestCase): + def test_a_core_pulling_most_of_a_system_is_named_with_the_saving(self): + files = [ + _sized("opera.bin", 12 * MB, ["3do"]), + _sized("mame/3do.zip", 300 * MB, ["3do"], ["mame"]), + _sized("mame/3do_m2.zip", 37 * MB, ["3do"], ["mame"]), + ] + lines = install.bulk_hint(files, {"system_cores": {"3do": ["mame", "opera"]}}, + systems_chosen=True, cores_chosen=False) + self.assertEqual(lines, [ + " mame pulls 337.0 MB of this 349.0 MB download for 3do. --core opera " + "keeps 12.0 MB of it and saves 337.0 MB (96%)." + ]) + + def test_a_system_pulling_most_of_the_download_is_named(self): + files = [_sized("a.zip", 400 * MB, ["arcade"]), _sized("b.bin", 50 * MB, ["nes"]), + _sized("c.bin", 10 * MB, ["gba"])] + lines = install.bulk_hint(files, {}, systems_chosen=False, cores_chosen=False) + self.assertEqual(len(lines), 1) + self.assertIn("arcade pulls 400.0 MB of this 460.0 MB download", lines[0]) + self.assertIn("saves 400.0 MB (86%)", lines[0]) + + def test_nothing_is_said_under_half(self): + files = [_sized("a.zip", 100 * MB, ["arcade"]), _sized("b.bin", 100 * MB, ["nes"])] + self.assertEqual(install.bulk_hint(files, {}, False, False), []) + + def test_nothing_is_said_for_a_small_download(self): + files = [_sized("a.zip", 40 * MB, ["arcade"]), _sized("b.bin", 1 * MB, ["nes"])] + self.assertEqual(install.bulk_hint(files, {}, False, False), []) + + def test_a_chosen_core_is_not_second_guessed(self): + files = [_sized("opera.bin", 12 * MB, ["3do"]), + _sized("mame/3do.zip", 300 * MB, ["3do"], ["mame"])] + self.assertEqual(install.bulk_hint( + files, {"system_cores": {"3do": ["mame", "opera"]}}, True, True), []) + + def test_the_end_summary_prints_the_line(self): + out = io.StringIO() + with contextlib.redirect_stdout(out): + install._report_outcome([Path("/x")], 3, 0, 0, 0, False, + hints=[" arcade pulls most of it."]) + text = out.getvalue() + self.assertIn(" arcade pulls most of it.", text) + self.assertLess(text.index("Done."), text.index("arcade pulls")) + + +if __name__ == "__main__": + unittest.main() diff --git a/wiki/installer.md b/wiki/installer.md index 1ea30e3c..1544de5a 100644 --- a/wiki/installer.md +++ b/wiki/installer.md @@ -223,6 +223,16 @@ Location: /userdata/bios To check this later, run the same command with --check. ``` +When most of a download of 100 MB or more comes from one system, or from +one core of a single system, the end of the run says so and what narrowing +would save, once that saving is over half: + +``` + mame pulls 336.5 MB of this 349.0 MB download for 3do. --core opera keeps 12.5 MB of it and saves 336.5 MB (96%). +``` + +`--dry-run` prints the same line under its plan. + The word `Done` and a location are the confirmation. Any other ending (`Stopped`, `Finished with errors`, `Nothing was installed`) means something was left undone, and the run says which case it was. @@ -252,6 +262,52 @@ present with contents the platform does not accept. Running the install again replaces it and keeps the old one, see [Files you already had](#files-you-already-had). +### Why a file is refused { #why-a-file-is-refused } + +Each file `--check` flags comes with the reason it found and the place each +core opens it: + +``` + 3 files to download: + wrong hash: scph5501.bin + another region: the contents of scph5500.bin (japan) + opened at /home/me/.config/retroarch/system/scph5501.bin by beetle_psx, pcsx_rearmed + missing: gba_bios.bin + nested folder: it is at system/system/gba_bios.bin, a system folder inside system + opened at /home/me/.config/retroarch/system/gba_bios.bin by mgba + missing: pcsx/extra.bin + still zipped: it is inside PSX BIOS.zip + opened at /home/me/.config/retroarch/system/pcsx/extra.bin by pcsx_rearmed +``` + +| Reason | What the folder holds | +|--------|-----------------------| +| `nested folder` | The file inside a folder named like the BIOS folder itself (`bios/bios/`, `system/system/`) | +| `one folder too deep`, `2 folders too deep` | The file below its place, as a ZIP extracted into a folder of its own leaves it | +| `one folder too high` | The file in the folder above the BIOS folder | +| `wrong folder` | The file under its name in another folder | +| `case of the name` | A name that differs only by capital letters, on a system where that makes two names | +| `still zipped` | The file inside a ZIP in the folder, a `.7z` or `.rar` of the same name, or a ZIP saved under the file's own name | +| `wrong size` | A file of another size than any copy the list accepts | +| `another region`, `another revision`, `another file` | The contents of another file of the list: another region's BIOS, the copy another core reads, or the one the platform's own list declares | +| `a name no core opens` | The right contents under a name no core of the platform reads | +| `same size, other contents` | Another dump, or a damaged copy | + +The reasons are read from the folder and the file list, nothing else. When +three files or more sit too deep in the same folder, a line before the list +names that folder once. Files with a reason are always listed. The plain +missing ones stop at ten unless `--verbose` is given. + +A pack published in parts leaves a marker beside the files of each part +(`RetroArch_v1.22.2_BIOS_Pack.part1of3.txt`). `--check` reads them and names +a part that was not extracted, or one extracted into a folder of its own: + +``` + RetroArch_v1.22.2_BIOS_Pack.zip comes in 3 parts, all needed. + missing: part 2, RetroArch_v1.22.2_BIOS_Pack.part2of3.zip (1.9 GB) + Download it and extract into /home/me/.config/retroarch/system, beside the other parts. +``` + ## Files you already had A file in the BIOS folder whose contents differ from the expected one (another @@ -448,6 +504,8 @@ what the options name, everything by default. | `--yes`, `-y` | Do not ask before a download of 1 GB or more | | `--quiet`, `-q` | Print only failures and the final summary. Implies `--no-input` | | `--events` | Write the run as JSON events on standard output, one object per line, and the usual text on standard error. Implies `--no-input`. See [Machine-readable events](#machine-readable-events) | +| `--from-pack PATH` | Install from a pack already downloaded: its ZIP, one of its parts (the others are found in the same folder), or the folder holding them. Makes no network request and checks every file's hash. See [Without a network](#without-a-network) | +| `--manifest FILE` | Read the file list from FILE, a copy of `install/.json`, instead of downloading it. See [Without a network](#without-a-network) | | `--restore FOLDER` | Put back the files a run replaced and kept in FOLDER, a dated folder inside `retrobios-backup`. Takes no other option but `--dry-run` and `--no-input`. See [Files you already had](#files-you-already-had) | | `--version` | Print the installer's identity, the first 12 hex digits of the SHA-256 the bootstraps pin | | `--list-platforms` | Print the supported platforms and what was detected here | @@ -459,8 +517,9 @@ what the options name, everything by default. `--check` reads the same file list, hashes the destination and exits, with code 1 when a file is missing or does not match and 0 when every one does. Files -with a wrong hash are listed first and in full, then the first ten missing ones -(`--verbose` lists them all). `--dry-run` counts the data trees in its total. Without +with a wrong hash are listed first and in full, then the missing ones whose +reason it found, then the first ten other missing ones (`--verbose` lists them +all). See [Why a file is refused](#why-a-file-is-refused). `--dry-run` counts the data trees in its total. Without it, a file already present with the expected hash is left untouched, and one whose contents do not match is re-downloaded and replaced by the verified copy, the old one kept beside the BIOS folder. @@ -806,6 +865,34 @@ The launcher (`ports/RetroBIOS/RetroBIOS.sh`) and its screens (`retrobios/screens.txt`) read the installer through [Machine-readable events](#machine-readable-events) only. +## Without a network { #without-a-network } + +A pack downloaded once installs again with no network, on this machine or on +one that has none: + +```bash +python install.py --from-pack ~/Downloads/RetroArch_v1.22.2_BIOS_Pack.zip +python install.py --from-pack ~/Downloads/RetroArch_v1.22.2_BIOS_Pack.part1of3.zip +python install.py --from-pack ~/Downloads --dest /media/card/retroarch/system +``` + +One part is enough to name a split pack: the others are taken from the same +folder, and a part that is not there is named before anything is written. +The platform is read from the pack's name when `--platform` is not given. The +file list is the one the pack carries (`manifest.json`), with each file's +SHA-1 and size, and every file is checked against it before it takes its +place. A data tree (Dolphin's `Sys/`, PPSSPP's assets) comes from the pack +too, not from its source archive. `--target` reads a core list from the +network and is refused with `--from-pack`. A board found on the device is not +applied either: every board's files are installed. + +`--manifest FILE` reads the file list from a copy of `install/.json` +instead of downloading it. Alone, the files still come from the repository. +With `--from-pack`, the run reads that list and takes every file from the +pack: `--system`, `--core`, `--region` and `--no-game-data` then apply as +they do online, and a file of the list the pack does not hold fails as +`not in the pack`. The list's platform has to be the one installed for. + ## Where the files come from Each manifest entry names either `repo_path`, served from the repository at the @@ -911,8 +998,8 @@ fields, the installer refuses to narrow rather than install everything. | Code | Meaning | |------|---------| | `0` | everything asked for is installed or already up to date | -| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, or `--restore` could not put a file back | -| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a `--dest` folder that does not exist was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | +| `1` | a failure: the one-liner could not download or verify `install.py`, or found no Python 3.8 (nothing was run), an unusable base URL, an unknown platform or target, `--installed-cores` with no core found, a manifest that fails validation, a fetch that never succeeded, no platform to install for in a non-interactive run, some files failed while others installed, `--check` found files to fetch, a `--from-pack` pack that cannot be read, lacks a part or is for another platform, or `--restore` could not put a file back | +| `2` | nothing was installed: every file the run tried failed (a BIOS folder that cannot be written, for one), the drive has too little free space for the download (checked before any folder is created), or the question about a `--dest` folder that does not exist was answered no. Also an option the installer refused before any download: an unknown option, `--jobs` outside 1 to 32, `--target` with `--every-board` or `--from-pack`. The message then starts with `install.py: error:`. With `--restore`, a folder that is not a dated folder inside `retrobios-backup` | | `130` | stopped with Ctrl+C, or with Ctrl+C or Ctrl+D at a question. Run the same command again to carry on | | `141` | the output was piped to a reader that closed early (`--list-platforms \| head`) | @@ -1054,7 +1141,8 @@ right pack either way. Parts named `Pack.part1of2.zip` are ordinary ZIPs: unzip each one into the same folder, and check that all of them were downloaded. The name says how many -there are. `scripts/download.sh` and `scripts/download.py` download every part +there are, and each part leaves a marker file naming every part, which +`--check` reads to name a part that is missing. `scripts/download.sh` and `scripts/download.py` download every part and check it against the release's checksums before unzipping. Both stage inside the destination folder, never in `/tmp`, which is a RAM disk on most of these systems. diff --git a/wiki/testing-guide.md b/wiki/testing-guide.md index abdbeb0d..de938be7 100644 --- a/wiki/testing-guide.md +++ b/wiki/testing-guide.md @@ -51,7 +51,7 @@ Every module in `tests/`, by the area it guards. | Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` | | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` | | Hardware targets and systems | synthetic + target files | `test_extract_targets`, `test_lakka_targets`, `test_libretro_slugs`, `test_system_ids`, `test_system_selection`, `test_target_names`, `test_target_scrapers`, `test_target_systems`, `test_target_verify` | -| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_selection` | +| Installer and downloads | loopback server | `test_check_release`, `test_download`, `test_install`, `test_install_cores`, `test_install_experience`, `test_install_offline`, `test_install_selection` | | Handheld port | stubbed PortMaster dialogs, fake installer | `test_port` | | Truth and native export | synthetic + cached originals | `test_export_counts`, `test_export_native_layer`, `test_export_report`, `test_exporters`, `test_truth_data_dir_readers`, `test_truth_diff`, `test_truth_layout` | | Scrapers, upstream and provenance | inline sources, synthetic | `test_check_freshness`, `test_dedup`, `test_elec_scrapers`, `test_fbneo_parser`, `test_find_pinned_revision`, `test_hash_merge`, `test_launchers`, `test_launchers_emudeck`, `test_launchers_recalbox`, `test_launchers_retrobat`, `test_launchers_retrodeck`, `test_launchers_retropie`, `test_launchers_rocknix`, `test_mame_devices`, `test_mame_parser`, `test_mame_ref_audit`, `test_profile_sync`, `test_provenance`, `test_refresh_data_dirs`, `test_refresh_failures`, `test_refresh_stale`, `test_retrodeck_scraper_errors`, `test_romset_recipes`, `test_scraper_contract`, `test_upstream`, `test_upstream_index` | From 167195863c959531b0c89c3d932377d7ade21904 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:22:36 +0200 Subject: [PATCH 3/7] docs: point to check reasons and missing parts --- wiki/troubleshooting.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/wiki/troubleshooting.md b/wiki/troubleshooting.md index 7d14c4e1..11e3ecf4 100644 --- a/wiki/troubleshooting.md +++ b/wiki/troubleshooting.md @@ -28,6 +28,7 @@ Read the count line: - **`wrong hash`**: a file is there with the wrong contents: another region or model, or damaged. - **Fewer present than the total**: some files are missing. +- **A line under a file**, like `still zipped`: it says what to fix. - **`0 wrong hash` and all present**: your files are fine. Go to step 3. ## 2. Fix the files @@ -47,7 +48,7 @@ the wrong ones, keeping the old ones. differ only by capital letters, which some ZIPs leave out. - **A ZIP in parts will not open?** Download every part, like `Pack.part1of2.zip` and `Pack.part2of2.zip`. Unzip each one into the same - folder. + folder. `--check` names a part you missed. ## One console only From 08f174bf32cdd88b50eaa5a40af6ec51326cc913 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:22:46 +0200 Subject: [PATCH 4/7] docs: name the part marker in the release steps --- wiki/release-process.md | 4 +++- wiki/tools.md | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/wiki/release-process.md b/wiki/release-process.md index 7d366126..126467c1 100644 --- a/wiki/release-process.md +++ b/wiki/release-process.md @@ -213,7 +213,9 @@ python scripts/pipeline.py --include-archived # 2. A release file must be under 2 GiB. A larger pack becomes parts that are # each a ZIP of whole files (Pack.part1of2.zip): any tool opens one, and # the parts extracted into one folder are the pack. Each part is read back -# and the set compared with the pack before the pack is removed. +# and the set compared with the pack before the pack is removed. Each part +# also holds a marker (Pack.part1of2.txt) naming every part and its size, +# which install.py --check reads to name a part missing from a folder. python scripts/split_pack.py dist/ # 2b. The handheld port, RetroBIOS.zip, built from ports/RetroBIOS with the diff --git a/wiki/tools.md b/wiki/tools.md index aabf6b41..656ca86c 100644 --- a/wiki/tools.md +++ b/wiki/tools.md @@ -688,7 +688,7 @@ python scripts/refresh_stale.py --only platforms,targets --jobs 6 | `list_platforms.py` | List active platforms (`--include-archived` adds the archived ones, used by CI) | | `download.py` | Download a pack from GitHub releases, every part checked (Python, stdlib only) | | `download.sh` | Same, as a shell one-liner (`curl` + `unzip`) | -| `split_pack.py` | Cut a pack over the release asset limit into parts that are each a ZIP of whole files, read back before the pack is removed | +| `split_pack.py` | Cut a pack over the release asset limit into parts that are each a ZIP of whole files, each with a marker naming every part, read back before the pack is removed | | `provenance_report.py` | Dump-catalog coverage and acquisition targets (see above) | | `find_pinned_revision.py` | Walk the upstream commits that touched a file and return the revision whose bytes match a platform's pinned md5 (`--write` saves them) | | `generate_readme.py` | Generate README.md and CONTRIBUTING.md from database | From 31614fa0e47982d3492354f369731fbcd1dec69e Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:23:36 +0200 Subject: [PATCH 5/7] refactor: reuse verified hashes in check reasons --- install.ps1 | 2 +- install.py | 20 ++++++++++++++++---- install.sh | 2 +- 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/install.ps1 b/install.ps1 index 73473be8..6ac2deb2 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "996cf4a8e21b8f05a524778046d0d32c807fbe3564f6edefdf5bee1add9c76cd" + $defaultInstallSha256 = "b50eda2089a5da5fe4cb18f86a214fe3d8c08f04649f4ba94167fd35ab5942ac" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index dee4a060..e5ed739d 100644 --- a/install.py +++ b/install.py @@ -6295,6 +6295,10 @@ class FolderIndex: if seen >= MAX_SCANNED_FILES: return + def seed(self, path: Path, sha1: str) -> None: + """A SHA-1 already known, so the file is not read again.""" + self._sha1[path] = sha1 + def sha1(self, path: Path) -> str: if path not in self._sha1: try: @@ -6466,15 +6470,23 @@ class Diagnosis(NamedTuple): def diagnose( missing: list[dict], mismatched: list[dict], files: list[dict], bios_path: Path, + verified: list[dict] | None = None, ) -> Diagnosis: """The reason --check gives for each file it flags, from what the folder holds and what the list records: id(entry) -> reason, and the folders - holding files one level too deep with how many.""" + holding files too deep with how many. The files the check verified are + not read again: their SHA-1 is the one the list records.""" reasons: dict[int, str] = {} deep: dict[str, int] = {} if not bios_path.is_dir() or not (missing or mismatched): return Diagnosis(reasons, deep) index = FolderIndex(bios_path) + for entry in verified or []: + if entry.get("sha1"): + try: + index.seed(_entry_path(bios_path, entry), entry["sha1"].lower()) + except ValueError: + continue known = _known_destinations(files) for entry in missing: reason, extra = _found_elsewhere(entry, index, bios_path, files, known) @@ -6495,7 +6507,7 @@ def diagnose( def check_lines( missing: list[dict], mismatched: list[dict], files: list[dict], manifest: dict, - bios_path: Path, every: bool, + bios_path: Path, every: bool, verified: list[dict] | None = None, ) -> list[str]: """The files --check flags, each with why and where its cores read it. @@ -6503,7 +6515,7 @@ def check_lines( or still in its ZIP, to guess. A flagged file with a reason is always listed. Plain missing ones stop at ten unless every is set. """ - found = diagnose(missing, mismatched, files, bios_path) + found = diagnose(missing, mismatched, files, bios_path, verified) lines = [] for extra, count in sorted(found.deep.items(), key=lambda item: -item[1]): if count >= 3: @@ -7540,7 +7552,7 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None: print(line) print(f"\n {_plural(len(to_download), 'file')} to download:") for line in check_lines(missing, mismatched, files, manifest, bios_path, - args.verbose): + args.verbose, up_to_date): print(line) print(" Run the same command without --check to fetch them; " "a file with a wrong hash is replaced, and the one in place " diff --git a/install.sh b/install.sh index 87f82a40..5c9f2546 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="996cf4a8e21b8f05a524778046d0d32c807fbe3564f6edefdf5bee1add9c76cd" +DEFAULT_INSTALL_SHA256="b50eda2089a5da5fe4cb18f86a214fe3d8c08f04649f4ba94167fd35ab5942ac" MAX_INSTALLER_BYTES=2097152 cleanup() { From 81fd98b24c0629b8129a5c5c492215c13b42df40 Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:28:07 +0200 Subject: [PATCH 6/7] fix: name save files left in the bios folder --- install.ps1 | 2 +- install.py | 5 ++++- install.sh | 2 +- tests/test_install_offline.py | 14 ++++++++++++++ 4 files changed, 20 insertions(+), 3 deletions(-) diff --git a/install.ps1 b/install.ps1 index 6ac2deb2..013467a5 100644 --- a/install.ps1 +++ b/install.ps1 @@ -11,7 +11,7 @@ $ErrorActionPreference = "Stop" $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" - $defaultInstallSha256 = "b50eda2089a5da5fe4cb18f86a214fe3d8c08f04649f4ba94167fd35ab5942ac" + $defaultInstallSha256 = "568d9399c5639db607cfc3944fa46a0dfa8d47741291d9346ed336cd2500eb00" $maximumInstallerBytes = 2MB $installerExit = 0 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null } diff --git a/install.py b/install.py index e5ed739d..bc0b9076 100644 --- a/install.py +++ b/install.py @@ -6406,7 +6406,10 @@ def _found_elsewhere( for path in matching or same_name: rel = index.relative(path) contents = "" if path in matching else ", with other contents" - if rel.endswith("/" + dest) and path.name == name: + # A dest below the save directory is not below this folder: a copy + # here under the same tail is in the wrong place, not too deep. + below = entry.get("root") != "save" + if below and rel.endswith("/" + dest) and path.name == name: extra = rel[: -len(dest) - 1] top = index.root.name if extra.casefold() == top.casefold(): diff --git a/install.sh b/install.sh index 5c9f2546..999f63c0 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,7 @@ esac TEMP_INSTALLER="" TEMP_DIRECTORY="" DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py" -DEFAULT_INSTALL_SHA256="b50eda2089a5da5fe4cb18f86a214fe3d8c08f04649f4ba94167fd35ab5942ac" +DEFAULT_INSTALL_SHA256="568d9399c5639db607cfc3944fa46a0dfa8d47741291d9346ed336cd2500eb00" MAX_INSTALLER_BYTES=2097152 cleanup() { diff --git a/tests/test_install_offline.py b/tests/test_install_offline.py index 4f69e3ce..ad555075 100644 --- a/tests/test_install_offline.py +++ b/tests/test_install_offline.py @@ -197,6 +197,20 @@ class CheckNamesWhyAFileIsRefused(Scratch): self.assertIn("a name no core opens: its contents are at PSX BIOS (USA).bin, " "rename it scph5501.bin", self.check()) + def test_a_save_directory_file_left_in_the_bios_folder_is_in_the_wrong_folder(self): + """A pack carries the save directory's files under saves/: extracted + into the BIOS folder they sit beside it, not too deep in it.""" + self.place("saves/azahar/keys.txt", b"keys") + saves = self.root / "saves" + saves.mkdir() + install._SAVE_DIRS[self.dest] = install.SavePlace(saves, False) + self.addCleanup(install._SAVE_DIRS.pop, self.dest, None) + entry = _entry("azahar/keys.txt", b"keys", ["nintendo-3ds"], root="save") + found = install.diagnose([entry], [], [entry], self.dest) + self.assertEqual(found.reasons[id(entry)], + "wrong folder: it is at saves/azahar/keys.txt") + self.assertEqual(found.deep, {}) + def test_each_flagged_file_says_where_its_cores_open_it(self): self.complete("scph5501.bin", "pcsx/extra.bin") out = self.check() From ca21e0066b2757435bf0a86aac12c55ea13681bd Mon Sep 17 00:00:00 2001 From: Abdessamad Derraz <3028866+Abdess@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:40:16 +0200 Subject: [PATCH 7/7] docs: count the files of each split pack --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 898e677c..46fa5918 100644 --- a/README.md +++ b/README.md @@ -48,22 +48,22 @@ The Files column counts what a pack holds once extracted, the figure a file mana | Platform | Files | Extracted size | Extract to | Download | |----------|------:|---------------:|-----------|----------| | AmberELEC | - | - | `/storage/roms/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| Batocera | 12,810 | 7.5 GB | `/userdata/bios/` | Download: [part 1 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part3of3.zip) | +| Batocera | 12,810 | 7.5 GB | `/userdata/bios/` | Download, 3 files, all needed: [part 1 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/Batocera_43.1_BIOS_Pack.part3of3.zip) | | BizHawk | 950 | 2.4 GB | `Firmware/` | [Download](../../releases/download/v2026.10.10/BizHawk_2.11.1_BIOS_Pack.zip) | | EmuDeck | 620 | 3.2 GB | `~/Emulation/bios/` | [Download](../../releases/download/v2026.10.10/EmuDeck_2.7EA_BIOS_Pack.zip) | | EmuELEC | - | - | `/storage/roms/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | | KNULLI | - | - | `/userdata/bios/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| Lakka | 8,504 | 6.0 GB | `/storage/system/` | Download: [part 1 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part2of2.zip) | +| Lakka | 8,504 | 6.0 GB | `/storage/system/` | Download, 2 files, all needed: [part 1 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part2of2.zip) | | MiSTer FPGA | 83 | 29 MB | `/media/fat/games/` | [Download](../../releases/download/v2026.10.10/MiSTer_FPGA_2026-09-29_BIOS_Pack.zip) | | MinUI | - | - | `/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | | MyMinUI | - | - | `/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | | NextUI | - | - | `/` | Not in v2026.10.10: [use the installer](https://abdess.github.io/retrobios/wiki/installer/) | -| ROCKNIX | 5,804 | 6.1 GB | `/storage/roms/bios/` | Download: [part 1 of 2](../../releases/download/v2026.10.10/ROCKNIX_20261001_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/ROCKNIX_20261001_BIOS_Pack.part2of2.zip) | -| Recalbox | 3,222 | 3.9 GB | `/recalbox/share/bios/` | Download: [part 1 of 2](../../releases/download/v2026.10.10/Recalbox_10.1.1_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/Recalbox_10.1.1_BIOS_Pack.part2of2.zip) | -| RetroArch | 8,504 | 6.0 GB | `system/` | Download: [part 1 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part2of2.zip) | -| RetroBat | 12,399 | 7.9 GB | `bios/` | Download: [part 1 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part3of3.zip) | -| RetroDECK | 7,793 | 8.0 GB | `~/retrodeck/` | Download: [part 1 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part3of3.zip) | -| RetroPie * | 12,533 | 8.1 GB | `~/RetroPie/BIOS/` | Download: [part 1 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part3of3.zip) | +| ROCKNIX | 5,804 | 6.1 GB | `/storage/roms/bios/` | Download, 2 files, all needed: [part 1 of 2](../../releases/download/v2026.10.10/ROCKNIX_20261001_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/ROCKNIX_20261001_BIOS_Pack.part2of2.zip) | +| Recalbox | 3,222 | 3.9 GB | `/recalbox/share/bios/` | Download, 2 files, all needed: [part 1 of 2](../../releases/download/v2026.10.10/Recalbox_10.1.1_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/Recalbox_10.1.1_BIOS_Pack.part2of2.zip) | +| RetroArch | 8,504 | 6.0 GB | `system/` | Download, 2 files, all needed: [part 1 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part1of2.zip), [part 2 of 2](../../releases/download/v2026.10.10/RetroArch_Lakka_v1.22.2_BIOS_Pack.part2of2.zip) | +| RetroBat | 12,399 | 7.9 GB | `bios/` | Download, 3 files, all needed: [part 1 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroBat_8.2.1_BIOS_Pack.part3of3.zip) | +| RetroDECK | 7,793 | 8.0 GB | `~/retrodeck/` | Download, 3 files, all needed: [part 1 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroDECK_0.10.10b_BIOS_Pack.part3of3.zip) | +| RetroPie * | 12,533 | 8.1 GB | `~/RetroPie/BIOS/` | Download, 3 files, all needed: [part 1 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part1of3.zip), [part 2 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part2of3.zip), [part 3 of 3](../../releases/download/v2026.10.10/RetroPie_4.8.12_BIOS_Pack.part3of3.zip) | | RomM | 1,087 | 1.5 GB | `library/bios/` | [Download](../../releases/download/v2026.10.10/RomM_5.3.1_BIOS_Pack.zip) | The AmberELEC pack holds its own `bios/` folder. Unzip it into `/storage/roms/`. @@ -209,4 +209,4 @@ The scripts and tooling are released under the [MIT License](LICENSE). The BIOS and firmware files are not covered by that license: they are third-party system software, preserved and provided for personal backup, archival, and interoperability with emulation software. [NOTICE](NOTICE) sets out their status and how to ask for a file to be removed. The reasoning, and where it is weakest, is in [the project's legal position](https://abdess.github.io/retrobios/wiki/legal/). -*Auto-generated on 2026-10-22T20:58:47Z* +*Auto-generated on 2026-10-23T05:39:12Z*