feat: mark each pack part with its siblings

scripts/generate_readme.py

@@ -236,7 +236,9 @@ def download_links(
     """Markdown links to the files of a platform's pack in the recorded release.
 
     A pack published whole gets one link labelled *label*. A pack in parts
-    gets one link per part, since every part is needed. *repo* is the
+    gets one link per part, since every part is needed, after the number of
+    files to download: a reader who took one link for the pack extracted
+    half of it. *repo* is the
     repository URL, absolute or relative, and *attrs* an attribute list
     put after each link, or *button* the function that draws each link as a
     button on the site. Without a recorded release the link is the latest
@@ -263,8 +265,8 @@ def download_links(
         for number, url in enumerate(urls, 1)
     ]
     if styled:
-        return " ".join(parts)
-    return f"{label}: " + ", ".join(parts)
+        return f"{len(urls)} files to download, all needed: " + " ".join(parts)
+    return f"{label}, {len(urls)} files, all needed: " + ", ".join(parts)
 
 
 def extract_targets(platforms_dir: str) -> list[tuple[str, str]]:

scripts/release_notes.py

@@ -63,7 +63,8 @@ def _links(tag: str, assets: list[str]) -> str:
     base = f"https://github.com/{REPO}/releases/download/{tag}"
     if len(assets) == 1:
         return f"[{assets[0]}]({base}/{assets[0]})"
-    return " + ".join(f"[part {n}]({base}/{asset})" for n, asset in enumerate(assets, 1))
+    parts = " + ".join(f"[part {n}]({base}/{asset})" for n, asset in enumerate(assets, 1))
+    return f"{len(assets)} files, all needed: {parts}"
 
 
 def render(record: dict, platforms_dir: str, whats_new: str) -> str:

scripts/release_record.py

@@ -103,8 +103,11 @@ def _read_record(dist: Path, tag: str, owners: dict[str, list[str]]) -> dict:
         members: dict[str, int] = {}
         for path in paths:
             with zipfile.ZipFile(path) as archive:
+                # A part's marker names its siblings and is not pack content:
+                # the manifest's pack_files is counted before any split.
                 for info in archive.infolist():
-                    members[info.filename] = info.file_size
+                    if not split_pack.is_marker(info.filename):
+                        members[info.filename] = info.file_size
         packs[name] = {
             "files": len(members),
             "extracted_size": sum(members.values()),

scripts/split_pack.py

@@ -6,6 +6,11 @@ as parts, each a complete archive of whole files: any tool opens one, and
 the parts extracted into the same folder are the pack. Members are copied
 as stored, never recompressed, so the parts depend on the pack alone.
 
+Each part also carries a short text file, its marker, at the archive root:
+the pack it belongs to, its number, and the name and size of every part.
+Extracted with the files, the markers let `install.py --check` name a part
+that is missing from the folder.
+
 Usage:
     python scripts/split_pack.py dist/
     python scripts/split_pack.py dist/RetroArch_v1.22.2_BIOS_Pack.zip
@@ -20,6 +25,7 @@ import re
 import struct
 import sys
 import zipfile
+import zlib
 from pathlib import Path
 
 from artifacts import ArtifactLockBusy, artifact_lock
@@ -29,6 +35,10 @@ ASSET_LIMIT = 2 * 1024**3
 
 PACK_SUFFIX = "_BIOS_Pack.zip"
 _PART_NAME = re.compile(r"^(?P<stem>.+_BIOS_Pack)\.part(?P<index>\d+)of(?P<count>\d+)\.zip$")
+_MARKER_NAME = re.compile(r"^(?P<stem>.+_BIOS_Pack)\.part(?P<index>\d+)of(?P<count>\d+)\.txt$")
+# Sizes are written at a fixed width, so a marker's length, and with it the
+# size of its own part, does not depend on the sizes it records.
+_SIZE_WIDTH = 13
 
 _LOCAL = struct.Struct("<4s2B4HL2L2H")
 _CENTRAL = struct.Struct("<4s4B4HL2L5H2L")
@@ -58,6 +68,44 @@ def part_name(pack_name: str, index: int, count: int) -> str:
     return f"{pack_name[: -len('.zip')]}.part{index}of{count}.zip"
 
 
+def marker_name(pack_name: str, index: int, count: int) -> str:
+    return f"{pack_name[: -len('.zip')]}.part{index}of{count}.txt"
+
+
+def is_marker(name: str) -> bool:
+    """Whether an archive member is the marker a split writes in each part."""
+    return _MARKER_NAME.match(name) is not None
+
+
+def marker_text(pack_name: str, index: int, sizes: list[int]) -> bytes:
+    """The marker of part *index* of a pack published in ``len(sizes)`` parts.
+
+    Plain text a person can read, in a form install.py parses back: the
+    lines that matter are `pack:`, `part:` and one `file:` per part.
+    """
+    count = len(sizes)
+    lines = [
+        "RetroBIOS split pack",
+        f"This folder holds part {index} of {count} of {pack_name}.",
+        "Every part is needed: extract them all into this same folder.",
+        "The installer's --check names a part missing from the folder.",
+        "",
+        f"pack: {pack_name}",
+        f"part: {index} of {count}",
+    ]
+    lines += [
+        f"file: {size:>{_SIZE_WIDTH}} {part_name(pack_name, number, count)}"
+        for number, size in enumerate(sizes, 1)
+    ]
+    return ("\n".join(lines) + "\n").encode("utf-8")
+
+
+def _marker_weight(pack_name: str, index: int, count: int) -> int:
+    name = len(marker_name(pack_name, index, count).encode("utf-8"))
+    body = len(marker_text(pack_name, index, [0] * count))
+    return _LOCAL.size + name + body + _CENTRAL.size + name
+
+
 def _name_bytes(info: zipfile.ZipInfo) -> bytes:
     encoding = "utf-8" if info.flag_bits & _UTF8_NAME else "cp437"
     return info.orig_filename.encode(encoding)
@@ -70,23 +118,24 @@ def _weight(info: zipfile.ZipInfo) -> int:
 
 
 def plan_parts(
-    members: list[zipfile.ZipInfo], limit: int
+    members: list[zipfile.ZipInfo], limit: int, reserve: int = 0
 ) -> list[list[zipfile.ZipInfo]]:
-    """Fill each part in archive order, up to the limit."""
+    """Fill each part in archive order, up to the limit less *reserve*."""
     parts: list[list[zipfile.ZipInfo]] = [[]]
-    size = _END.size
+    floor = _END.size + reserve
+    size = floor
     for info in members:
         weight = _weight(info)
-        if _END.size + weight >= limit:
+        if floor + weight >= limit:
             raise ValueError(
                 f"{info.filename} needs {weight:,} bytes, more than a part of "
                 f"{limit:,} can hold"
             )
         if info.compress_size > _MAX_FIELD or info.file_size > _MAX_FIELD:
             raise ValueError(f"{info.filename} needs ZIP64, which a part does not use")
-        if size + weight >= limit or len(parts[-1]) >= _MAX_ENTRIES:
+        if size + weight >= limit or len(parts[-1]) >= _MAX_ENTRIES - 1:
             parts.append([])
-            size = _END.size
+            size = floor
         parts[-1].append(info)
         size += weight
     return parts
@@ -109,12 +158,24 @@ def _data_offset(source: io.BufferedReader, info: zipfile.ZipInfo) -> int:
     return info.header_offset + _LOCAL.size + header[10] + header[11]
 
 
+def _marker_info(name: str, body: bytes) -> zipfile.ZipInfo:
+    info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0))
+    info.compress_type = zipfile.ZIP_STORED
+    info.CRC = zlib.crc32(body)
+    info.compress_size = info.file_size = len(body)
+    info.create_system = 3
+    info.external_attr = (0o100000 | 0o644) << 16
+    return info
+
+
 def _write_part(
-    source: io.BufferedReader, members: list[zipfile.ZipInfo], dest: Path
+    source: io.BufferedReader, members: list[zipfile.ZipInfo], dest: Path,
+    marker: tuple[str, bytes],
 ) -> None:
     directory = []
+    marker_info = _marker_info(*marker)
     with open(dest, "wb") as out:
-        for info in members:
+        for info in [marker_info, *members]:
             name = _name_bytes(info)
             flags = info.flag_bits & ~_DATA_DESCRIPTOR
             dos_time, dos_date = _dos_stamp(info)
@@ -127,8 +188,12 @@ def _write_part(
                 )
             )
             out.write(name)
-            source.seek(_data_offset(source, info))
-            remaining = info.compress_size
+            if info is marker_info:
+                out.write(marker[1])
+                remaining = 0
+            else:
+                source.seek(_data_offset(source, info))
+                remaining = info.compress_size
             while remaining:
                 chunk = source.read(min(_CHUNK, remaining))
                 if not chunk:
@@ -157,7 +222,33 @@ def _write_part(
 
 
 def _identity(archive: zipfile.ZipFile) -> list[tuple[str, int, int]]:
-    return [(info.filename, info.CRC, info.file_size) for info in archive.infolist()]
+    return [
+        (info.filename, info.CRC, info.file_size)
+        for info in archive.infolist()
+        if not is_marker(info.filename)
+    ]
+
+
+def marker_weight(pack_name: str, count: int) -> int:
+    """Bytes the largest marker of a pack in *count* parts adds to its part."""
+    return max(_marker_weight(pack_name, index, count) for index in (1, count))
+
+
+def _plan_with_markers(
+    pack_name: str, members: list[zipfile.ZipInfo], limit: int
+) -> list[list[zipfile.ZipInfo]]:
+    """Plan the parts with room for the marker each one carries.
+
+    The marker lists every part, so its length grows with their count: the
+    plan is made again until the count it assumed is the count it gives.
+    """
+    count = 1
+    while True:
+        reserve = marker_weight(pack_name, count)
+        plan = plan_parts(members, limit, reserve)
+        if len(plan) <= count:
+            return plan
+        count = len(plan)
 
 
 def split_pack(zip_path: Path, limit: int = ASSET_LIMIT) -> list[Path]:
@@ -174,15 +265,31 @@ def split_pack(zip_path: Path, limit: int = ASSET_LIMIT) -> list[Path]:
     with zipfile.ZipFile(zip_path) as archive:
         members = archive.infolist()
         expected = _identity(archive)
-    plan = plan_parts(members, limit)
+    taken = [info.filename for info in members if is_marker(info.filename)]
+    if taken:
+        raise ValueError(f"{zip_path.name} already holds {taken[0]}, a part marker name")
+    plan = _plan_with_markers(zip_path.name, members, limit)
+    count = len(plan)
     parts = [
-        zip_path.with_name(part_name(zip_path.name, index, len(plan)))
-        for index in range(1, len(plan) + 1)
+        zip_path.with_name(part_name(zip_path.name, index, count))
+        for index in range(1, count + 1)
+    ]
+    sizes = [
+        _END.size + _marker_weight(zip_path.name, index, count)
+        + sum(_weight(info) for info in chosen)
+        for index, chosen in enumerate(plan, 1)
     ]
     try:
         with open(zip_path, "rb") as source:
-            for dest, chosen in zip(parts, plan):
-                _write_part(source, chosen, dest)
+            for index, (dest, chosen) in enumerate(zip(parts, plan), 1):
+                marker = (marker_name(zip_path.name, index, count),
+                          marker_text(zip_path.name, index, sizes))
+                _write_part(source, chosen, dest, marker)
+                if dest.stat().st_size != sizes[index - 1]:
+                    raise ValueError(
+                        f"{dest.name} is {dest.stat().st_size:,} bytes, its marker "
+                        f"says {sizes[index - 1]:,}"
+                    )
         found: list[tuple[str, int, int]] = []
         for dest in parts:
             with zipfile.ZipFile(dest) as archive: