chore: merge archive recipes

scripts/archive_build.py

@@ -0,0 +1,538 @@
+"""A recipe for every archive the collection holds, and the build that uses it.
+
+The collection must be able to regenerate each archive it serves from the
+ROMs it holds plus a committed recipe, without any copy of that archive
+from elsewhere. ``recipes/archives.json`` is that recipe list: one record
+per archive in ``database.json``, keyed by its path.
+
+A record names the members in archive order, each with its size, crc32 and
+sha1, and the container's sha1, md5 and size. ``rebuild`` says what the
+build reproduces:
+
+``exact``
+    The archive is TorrentZip and ``build_torrentzip()`` over its members
+    gives its bytes. Proven at capture by rebuilding it.
+``canonical``
+    Same members, other container bytes: the archive was written by another
+    tool. The build writes the TorrentZip form and checks members only.
+
+``sources`` lists the recipes that agree on the member list: profile
+``contents:`` blocks and the MAME and FBNeo DAT snapshots under
+``recipes/``, then ``collection``, the archive itself. A DAT is provenance,
+never the only source. ``bytes_pinned`` marks an archive whose container
+hash a platform or a profile pins, with each pin named.
+
+The build reads each member by sha1: a loose file first, then a member of
+another archive, then the archive's own copy, and verifies what it wrote
+before moving it into place.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import os
+import re
+import sys
+import zipfile
+from collections import defaultdict
+from collections.abc import Callable, Iterable
+from concurrent.futures import ProcessPoolExecutor
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import BinaryIO
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+from torrentzip import Source as Readable
+from torrentzip import write_torrentzip, zlib_ng_version  # noqa: E402
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+ARCHIVES_FILE = REPO_ROOT / "recipes" / "archives.json"
+# The writer contract: build_torrentzip's layout over stock zlib. A change to
+# the layout is a new builder name and a new proof of every record.
+BUILDER = "torrentzip-1"
+_ARCHIVE_PATH = re.compile(r"\.zip(\.[0-9a-f]{8})?$", re.IGNORECASE)
+_TORRENTZIP_COMMENT = b"TORRENTZIPPED-"
+
+
+def is_archive_path(path: str) -> bool:
+    """A ZIP by its name: ``x.zip`` or a ``.variants/`` copy ``x.zip.1a2b3c4d``."""
+    return bool(_ARCHIVE_PATH.search(path))
+
+
+def database_archives(db: dict) -> dict[str, dict]:
+    """Path -> database entry, for every archive the database lists."""
+    return {
+        entry["path"]: entry
+        for entry in (db.get("files") or {}).values()
+        if is_archive_path(entry.get("path", ""))
+    }
+
+
+# --- the ledger file --------------------------------------------------------
+
+
+def load_ledger(path: Path = ARCHIVES_FILE) -> dict[str, dict]:
+    """Path -> record, from ``recipes/archives.json``, empty when absent."""
+    if not path.is_file():
+        return {}
+    with path.open(encoding="utf-8") as handle:
+        data = json.load(handle)
+    return {record["path"]: record for record in data.get("archives", [])}
+
+
+def ledger_text(records: dict[str, dict]) -> str:
+    """One record per line, so a diff names the archive that changed."""
+    lines = [
+        json.dumps(records[path], ensure_ascii=False, separators=(",", ":"))
+        for path in sorted(records)
+    ]
+    body = ",\n".join(f"  {line}" for line in lines)
+    return f'{{"builder": "{BUILDER}", "archives": [\n{body}\n]}}\n'
+
+
+def write_ledger(records: dict[str, dict], path: Path = ARCHIVES_FILE) -> bool:
+    from artifacts import write_if_changed  # noqa: PLC0415
+
+    path.parent.mkdir(parents=True, exist_ok=True)
+    return write_if_changed(str(path), ledger_text(records))
+
+
+# --- capture ----------------------------------------------------------------
+
+
+class HashSink:
+    """A write target that hashes what it receives, and keeps it if asked."""
+
+    def __init__(self, handle: BinaryIO | None = None) -> None:
+        self.handle = handle
+        self.sha1 = hashlib.sha1()
+        self.md5 = hashlib.md5()
+        self.size = 0
+
+    def write(self, data: bytes) -> int:
+        self.sha1.update(data)
+        self.md5.update(data)
+        self.size += len(data)
+        if self.handle is not None:
+            self.handle.write(data)
+        return len(data)
+
+
+class _Member(io.RawIOBase):
+    """One member of a ZIP, open for reading, closing the archive with it."""
+
+    def __init__(self, archive: Path, name: str) -> None:
+        super().__init__()
+        self._zip = zipfile.ZipFile(archive)
+        self._stream = self._zip.open(name)
+
+    def readable(self) -> bool:
+        return True
+
+    def read(self, size: int = -1) -> bytes:
+        return self._stream.read(size)
+
+    def close(self) -> None:
+        if not self.closed:
+            self._stream.close()
+            self._zip.close()
+        super().close()
+
+
+def _opener(source: Path, member: str | None) -> Callable[[], Readable]:
+    if member is None:
+        return lambda: source.open("rb")
+    return lambda: _Member(source, member)
+
+
+def _file_infos(path: Path) -> tuple[list[zipfile.ZipInfo], list[str], bool]:
+    with zipfile.ZipFile(path) as archive:
+        infos = archive.infolist()
+        torrentzip = archive.comment.startswith(_TORRENTZIP_COMMENT)
+    files = [info for info in infos if not info.is_dir()]
+    directories = [info.filename for info in infos if info.is_dir()]
+    return files, directories, torrentzip
+
+
+def _member_sha1(path: Path, name: str) -> str:
+    digest = hashlib.sha1()
+    with _Member(path, name) as stream:
+        while chunk := stream.read(16 * 1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def read_archive(path: Path, sha1: str) -> dict:
+    """Members, directories and rebuild class of the archive at *path*.
+
+    A TorrentZip archive is rebuilt from its own members: equal bytes make
+    the recipe ``exact``. A mismatch is rebuilt once more before it is
+    believed, because this host sometimes computes wrongly under load. Any
+    other archive is ``canonical`` and only its members are read.
+    """
+    files, directories, torrentzip = _file_infos(path)
+    rebuild = "canonical"
+    by_name: dict[str, str] = {}
+    if torrentzip:
+        for _attempt in range(2):
+            sink = HashSink()
+            written = write_torrentzip(
+                sink, [(info.filename, _opener(path, info.filename)) for info in files]
+            )
+            by_name = {member.name: member.sha1 for member in written}
+            if sink.sha1.hexdigest() == sha1:
+                rebuild = "exact"
+                break
+    else:
+        by_name = {info.filename: _member_sha1(path, info.filename) for info in files}
+    record: dict = {
+        "rebuild": rebuild,
+        "members": [
+            {
+                "name": info.filename,
+                "size": info.file_size,
+                "crc32": f"{info.CRC:08x}",
+                "sha1": by_name[info.filename],
+            }
+            for info in files
+        ],
+    }
+    if directories:
+        record["directories"] = directories
+    return record
+
+
+def _read_job(job: tuple[str, str]) -> tuple[str, dict | str]:
+    path, sha1 = job
+    try:
+        return path, read_archive(Path(path), sha1)
+    except (OSError, zipfile.BadZipFile, UnicodeEncodeError) as exc:
+        return path, f"{type(exc).__name__}: {exc}"
+
+
+def _hash_values(value: object) -> set[str]:
+    raw = value if isinstance(value, list) else str(value or "").split(",")
+    return {str(item).strip().lower() for item in raw if str(item).strip()}
+
+
+def container_pins(platforms_dir: str, emulators_dir: str) -> dict[str, set[str]]:
+    """Container hash -> who pins it: ``platform:<name>`` or ``profile:<name>``.
+
+    A ``zipped_file`` entry pins a ROM inside the archive, and a Recalbox
+    archive md5 is the composite of its members (``zip_digest: composite``,
+    Bios.cpp:64-68): neither pins the container bytes.
+    """
+    from common import (  # noqa: PLC0415
+        list_registered_platforms,
+        load_emulator_profiles,
+        load_platform_config,
+    )
+
+    pins: dict[str, set[str]] = defaultdict(set)
+    for platform in list_registered_platforms(platforms_dir):
+        config = load_platform_config(platform, platforms_dir)
+        for system in config.get("systems", {}).values():
+            for entry in system.get("files", []) or []:
+                if entry.get("zipped_file") or entry.get("zip_digest") == "composite":
+                    continue
+                for key in ("md5", "sha1", "alt_md5"):
+                    for value in _hash_values(entry.get(key)):
+                        pins[value].add(f"platform:{platform}")
+    for name, profile in load_emulator_profiles(emulators_dir, skip_aliases=False).items():
+        for entry in profile.get("files", []) or []:
+            for key in ("md5", "sha1", "sha256"):
+                for value in _hash_values(entry.get(key)):
+                    pins[value].add(f"profile:{name}")
+    return dict(pins)
+
+
+MemberList = tuple[tuple[str, str], ...]
+
+
+def recipe_index(emulators_dir: str, recipes_dir: str) -> dict[MemberList, set[str]]:
+    """Sorted (name, crc32) -> labels of the recipes with exactly those members.
+
+    Matching on the member list, not the archive name, credits a DAT that
+    knows the set under another name (``aga.zip`` is MAME's ``isa_aga``).
+    """
+    from common import load_emulator_profiles  # noqa: PLC0415
+    from romset_recipes import collect_recipes, load_dat_recipes  # noqa: PLC0415
+
+    index: dict[MemberList, set[str]] = defaultdict(set)
+    profiles = load_emulator_profiles(emulators_dir, skip_aliases=False)
+    for per_label in collect_recipes(profiles).values():
+        for label, recipe in per_label.items():
+            index[tuple(sorted(recipe))].add(f"profile:{label}")
+    for per_label in load_dat_recipes(recipes_dir).values():
+        for label, recipe in per_label.items():
+            index[tuple(sorted(recipe))].add(label)
+    return dict(index)
+
+
+def member_key(members: Iterable[dict]) -> MemberList:
+    return tuple(sorted((m["name"], m["crc32"]) for m in members))
+
+
+def capture(
+    db: dict,
+    previous: dict[str, dict],
+    index: dict[MemberList, set[str]],
+    pins: dict[str, set[str]],
+    workers: int = 4,
+    log: Callable[[str], None] = print,
+) -> tuple[dict[str, dict], dict[str, str]]:
+    """A record for every archive in *db*, and the archives that could not be read.
+
+    A previous record whose sha1 still names the file is kept as read: the
+    container bytes fix the members. Sources and pins are recomputed every
+    time, since profiles, snapshots and platform lists move.
+    """
+    if zlib_ng_version():
+        raise ZlibNgError(zlib_ng_version())
+    archives = database_archives(db)
+    reused: dict[str, dict] = {}
+    jobs: list[tuple[str, str]] = []
+    for path, entry in sorted(archives.items()):
+        old = previous.get(path)
+        if old and old.get("sha1") == entry["sha1"] and old.get("members"):
+            reused[path] = old
+        else:
+            jobs.append((path, entry["sha1"]))
+    read: dict[str, dict] = {}
+    failed: dict[str, str] = {}
+    if jobs:
+        log(f"  reading {len(jobs)} archives ({len(reused)} unchanged)")
+        with ProcessPoolExecutor(max_workers=workers) as pool:
+            for path, result in pool.map(_read_job, jobs, chunksize=4):
+                if isinstance(result, str):
+                    failed[path] = result
+                else:
+                    read[path] = result
+    records: dict[str, dict] = {}
+    for path, entry in sorted(archives.items()):
+        body = reused.get(path) or read.get(path)
+        if body is None:
+            continue
+        record = {
+            "path": path,
+            "sha1": entry["sha1"],
+            "md5": entry["md5"],
+            "size": entry["size"],
+            "rebuild": body["rebuild"],
+            "members": body["members"],
+        }
+        if body.get("directories"):
+            record["directories"] = body["directories"]
+        record["sources"] = sorted(index.get(member_key(body["members"]), set())) + [
+            "collection"
+        ]
+        pinned = set()
+        for value in (entry["md5"], entry["sha1"], entry.get("sha256", "")):
+            pinned |= pins.get(str(value).lower(), set())
+        if pinned:
+            record["bytes_pinned"] = True
+            record["pins"] = sorted(pinned)
+        records[path] = record
+    return records, failed
+
+
+class ZlibNgError(RuntimeError):
+    def __init__(self, version: str | None) -> None:
+        super().__init__(
+            f"this Python deflates with zlib-ng {version}: TorrentZip bytes are "
+            "stock zlib's level-9 output, so an exact rebuild cannot be proven "
+            "or claimed here. Run under a Python linked against stock zlib."
+        )
+
+
+# --- build ------------------------------------------------------------------
+
+
+@dataclass
+class Source:
+    path: Path
+    member: str | None  # None for a loose file
+
+
+@dataclass
+class AtomIndex:
+    """Where each member sha1 can be read in the collection."""
+
+    loose: dict[str, Path] = field(default_factory=dict)
+    inside: dict[str, list[tuple[Path, str]]] = field(default_factory=dict)
+
+    @classmethod
+    def from_collection(cls, ledger: dict[str, dict], db: dict, root: Path) -> AtomIndex:
+        index = cls()
+        for entry in (db.get("files") or {}).values():
+            path = root / entry.get("path", "")
+            if not is_archive_path(entry.get("path", "")) and path.is_file():
+                index.loose.setdefault(entry["sha1"], path)
+        for archive, record in sorted(ledger.items()):
+            path = root / archive
+            if not path.is_file():
+                continue
+            for member in record.get("members", []):
+                index.inside.setdefault(member["sha1"], []).append((path, member["name"]))
+        return index
+
+    def find(self, sha1: str, own: Path) -> Source | None:
+        """A loose copy first, then another archive, then the archive itself."""
+        if sha1 in self.loose:
+            return Source(self.loose[sha1], None)
+        holders = self.inside.get(sha1, [])
+        for path, member in holders:
+            if path != own:
+                return Source(path, member)
+        for path, member in holders:
+            return Source(path, member)
+        return None
+
+    def held(self, sha1: str) -> bool:
+        return sha1 in self.loose or sha1 in self.inside
+
+
+@dataclass
+class BuildResult:
+    path: str
+    status: str  # exact, canonical, impossible, failed
+    output: str = ""
+    detail: str = ""
+    missing: list[dict] = field(default_factory=list)
+    own_members: int = 0
+
+
+def _members_match(built: Path, members: list[dict]) -> str:
+    """Empty when *built* holds exactly *members*, else what differs."""
+    expected = sorted((m["name"], m["size"], m["crc32"], m["sha1"]) for m in members)
+    with zipfile.ZipFile(built) as archive:
+        names = [info.filename for info in archive.infolist() if not info.is_dir()]
+        sizes = {info.filename: info.file_size for info in archive.infolist()}
+        crcs = {info.filename: f"{info.CRC:08x}" for info in archive.infolist()}
+    held = sorted(
+        (name, sizes[name], crcs[name], _member_sha1(built, name)) for name in names
+    )
+    if held == expected:
+        return ""
+    missing = sorted(set(expected) - set(held))
+    extra = sorted(set(held) - set(expected))
+    return f"members differ: expected {missing[:3]}, built {extra[:3]}"
+
+
+def _file_sha1(path: Path) -> str:
+    digest = hashlib.sha1()
+    with path.open("rb") as handle:
+        while chunk := handle.read(16 * 1024 * 1024):
+            digest.update(chunk)
+    return digest.hexdigest()
+
+
+def destination(record_path: str, output_dir: Path) -> Path:
+    """Where a build lands: the archive's path below ``bios/``, under *output_dir*."""
+    parts = Path(record_path).parts
+    relative = Path(*parts[1:]) if parts and parts[0] == "bios" else Path(record_path)
+    return output_dir / relative
+
+
+def build_archive(
+    record: dict,
+    atoms: AtomIndex,
+    output_dir: Path,
+    root: Path = REPO_ROOT,
+    attempts: int = 2,
+) -> BuildResult:
+    """Build *record* from held members into *output_dir*, verified.
+
+    The bytes are written to a scratch file beside the destination and read
+    back from disk: an exact recipe must give its recorded sha1, a canonical
+    one its members. A mismatch is built once more, then refused: nothing
+    wrong is ever moved to the destination. Under zlib-ng an exact recipe is
+    checked by its members only and never reported exact.
+    """
+    path = record["path"]
+    own = root / path
+    sources: list[tuple[str, Source]] = []
+    missing: list[dict] = []
+    for member in record["members"]:
+        source = atoms.find(member["sha1"], own)
+        if source is None:
+            missing.append(member)
+        else:
+            sources.append((member["name"], source))
+    if missing:
+        return BuildResult(path, "impossible", missing=missing)
+    own_members = sum(1 for _name, source in sources if source.path == own)
+    ng = zlib_ng_version()
+    exact = record["rebuild"] == "exact" and not ng
+    target = destination(path, output_dir)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    scratch = target.with_name(f".{target.name}.{os.getpid()}.part")
+    detail = ""
+    try:
+        for _attempt in range(attempts):
+            with scratch.open("wb") as handle:
+                write_torrentzip(
+                    handle,
+                    [(name, _opener(source.path, source.member)) for name, source in sources],
+                )
+            if exact:
+                written = _file_sha1(scratch)
+                detail = "" if written == record["sha1"] else (
+                    f"sha1 {written} is not the recorded {record['sha1']}"
+                )
+            else:
+                detail = _members_match(scratch, record["members"])
+            if not detail:
+                os.replace(scratch, target)
+                status = "exact" if exact else "canonical"
+                note = (
+                    f"zlib-ng {ng}: exact bytes not claimed, members verified"
+                    if ng and record["rebuild"] == "exact" else ""
+                )
+                return BuildResult(path, status, str(target), note, own_members=own_members)
+    finally:
+        if scratch.exists():
+            scratch.unlink()
+    return BuildResult(path, "failed", detail=detail, own_members=own_members)
+
+
+def select(ledger: dict[str, dict], wanted: str, root: Path = REPO_ROOT) -> list[dict]:
+    """Records named by a path (relative or absolute) or by a file name."""
+    candidate = Path(wanted)
+    if candidate.is_absolute():
+        try:
+            candidate = candidate.resolve().relative_to(root.resolve())
+        except ValueError:
+            return []
+    key = candidate.as_posix()
+    if key in ledger:
+        return [ledger[key]]
+    return [record for path, record in sorted(ledger.items()) if Path(path).name == wanted]
+
+
+def missing_members(ledger: dict[str, dict], db: dict) -> list[dict]:
+    """Members held nowhere: neither as a loose file nor inside any archive.
+
+    Read from the committed data alone, so it holds without the files: a
+    member is held when database.json has its sha1 as a file, or when the
+    recipe of an archive database.json lists carries it.
+    """
+    archives = database_archives(db)
+    loose = {
+        sha1 for sha1, entry in (db.get("files") or {}).items()
+        if not is_archive_path(entry.get("path", ""))
+    }
+    inside = {
+        member["sha1"]
+        for path, record in ledger.items() if path in archives
+        for member in record.get("members", [])
+    }
+    return [
+        {"archive": path, "member": member["name"], "sha1": member["sha1"]}
+        for path, record in sorted(ledger.items())
+        for member in record.get("members", [])
+        if member["sha1"] not in loose and member["sha1"] not in inside
+    ]

scripts/generate_site.py

@@ -5985,7 +5985,7 @@ def _romset_line(f: dict) -> str:
     MAME and FBNeo check the members of an archive by CRC32, never its own
     bytes or its member names: a set works with a release when it holds
     every CRC32 the release's recipe lists, extra members or not. The
-    main neogeo.zip, which every MAME from 0.190 to 0.289 loads, read "No
+    main neogeo.zip, which every MAME from 0.177 to 0.289 loads, read "No
     match" when names had to agree and no member could be extra. Empty
     when no recipe names the archive or the file is not on disk.
     """

scripts/pipeline.py

@@ -5,7 +5,8 @@ Steps, numbered as the run prints them:
   0    validate_schemas.py          profiles and platform YAMLs, as PR CI checks them
   1    generate_db.py --force       rebuild database.json from bios/
   1b   provenance_report.py         dump-catalog coverage, derivable.json
-  1c   romset_recipes.py            archive identification, reconstruction targets
+  1c   romset_recipes.py            archive identification, reconstruction targets,
+                                    and --capture: recipes/archives.json
   2    refresh_data_dirs.py         Dolphin Sys, PPSSPP assets and the other trees
   2a   mame_hash_scraper            MAME BIOS hashes, and device sets on a new release
   2a2  fbneo_hash_scraper           FBNeo BIOS hashes
@@ -1216,6 +1217,14 @@ def main() -> None:
         [sys.executable, "scripts/romset_recipes.py"],
         STEP_LABELS["romset_recipes"],
     )
+    # A recipe for every archive database.json lists (recipes/archives.json):
+    # only an archive whose sha1 moved is read again, so the step costs the
+    # archives added since the last run.
+    captured, _ = run(
+        [sys.executable, "scripts/romset_recipes.py", "--capture"],
+        f"{STEP_LABELS['romset_recipes']} (archive recipes)",
+    )
+    ok = ok and captured
     results["romset_recipes"] = ok
     all_ok = all_ok and ok
 

scripts/romset_recipes.py

@@ -20,6 +20,24 @@ Two questions follow, and this module answers both:
 Reconstruction is only possible when the pinned archive is itself TorrentZip.
 An archive whose bytes carry metadata unrelated to its contents cannot be
 derived from ROMs by anyone, and is reported as such rather than guessed at.
+
+Every archive the collection holds also has a recipe of its own in
+``recipes/archives.json`` (scripts/archive_build.py):
+
+``--capture``
+    Write a record for every archive in database.json: members in order with
+    size, crc32 and sha1, container sha1, md5 and size, and whether the
+    TorrentZip build reproduces the bytes (``exact``) or only the members
+    (``canonical``). Archives already recorded under the same sha1 are not
+    read again.
+
+``--build NAME --output-dir DIR``
+    Build an archive from the members the collection holds, loose or inside
+    another archive, and verify it before moving it into place.
+
+``--build-all --output-dir DIR``
+    The same for every recorded archive, with one line per archive: exact,
+    canonical, or impossible with the member nothing holds.
 """
 
 from __future__ import annotations
@@ -43,6 +61,8 @@ from common import (
 )
 from torrentzip import build_torrentzip, is_torrentzip
 
+ARCHIVES_FILE = Path(__file__).resolve().parent.parent / "recipes" / "archives.json"
+
 Recipe = list[tuple[str, str]]
 
 
@@ -290,7 +310,9 @@ def platform_targets(platforms_dir: str) -> dict[str, set[str]]:
     """Container MD5s the platforms pin, per archive name.
 
     ``zipped_file`` entries pin an inner ROM instead of the container, so
-    their MD5 is not an archive hash and is skipped.
+    their MD5 is not an archive hash and is skipped. So is a Recalbox md5,
+    the composite of the members (``zip_digest: composite``, Bios.cpp:64-68):
+    no container build can ever give it.
     """
     targets: dict[str, set[str]] = defaultdict(set)
     for name in list_registered_platforms(platforms_dir):
@@ -298,7 +320,11 @@ def platform_targets(platforms_dir: str) -> dict[str, set[str]]:
         for system in config.get("systems", {}).values():
             for entry in system.get("files", []) or []:
                 archive = entry.get("name", "")
-                if not archive.endswith(".zip") or entry.get("zipped_file"):
+                if (
+                    not archive.endswith(".zip")
+                    or entry.get("zipped_file")
+                    or entry.get("zip_digest") == "composite"
+                ):
                     continue
                 for value in str(entry.get("md5") or "").split(","):
                     value = value.strip().lower()
@@ -453,6 +479,95 @@ def print_missing(gap: dict) -> None:
         print(f"  NO RECIPE: {record['archive']} md5={record['md5'][:8]}")
 
 
+def run_capture(args: argparse.Namespace, db: dict) -> int:
+    """Write recipes/archives.json for every archive the database lists."""
+    import archive_build  # noqa: PLC0415
+
+    try:
+        records, failed = archive_build.capture(
+            db,
+            archive_build.load_ledger(Path(args.archives)),
+            archive_build.recipe_index(args.emulators_dir, args.dat_recipes),
+            archive_build.container_pins(args.platforms_dir, args.emulators_dir),
+        )
+    except archive_build.ZlibNgError as exc:
+        print(f"ERROR: {exc}", file=sys.stderr)
+        return 2
+    for path, reason in sorted(failed.items()):
+        print(f"  UNREADABLE: {path}: {reason}", file=sys.stderr)
+    changed = archive_build.write_ledger(records, Path(args.archives))
+    by_class: dict[str, int] = defaultdict(int)
+    for record in records.values():
+        by_class[record["rebuild"]] += 1
+    agreed = sum(1 for record in records.values() if len(record["sources"]) > 1)
+    pinned = sum(1 for record in records.values() if record.get("bytes_pinned"))
+    print(
+        f"{len(records)} archive recipes: {by_class['exact']} exact, "
+        f"{by_class['canonical']} canonical; {agreed} agree with a profile or DAT "
+        f"recipe, {pinned} pinned as bytes"
+    )
+    print(f"{'Wrote' if changed else 'Unchanged'} {args.archives}")
+    return 1 if failed else 0
+
+
+def run_build(args: argparse.Namespace, db: dict) -> int:
+    """Build one archive, or all of them, from the members the collection holds."""
+    import archive_build  # noqa: PLC0415
+
+    if not args.output_dir:
+        print("ERROR: --build and --build-all need --output-dir", file=sys.stderr)
+        return 2
+    ledger = archive_build.load_ledger(Path(args.archives))
+    if args.build_all:
+        records = [ledger[path] for path in sorted(ledger)]
+    else:
+        records = archive_build.select(ledger, args.build)
+        if not records:
+            print(f"ERROR: no archive recipe named {args.build}", file=sys.stderr)
+            return 2
+    ng = archive_build.zlib_ng_version()
+    if ng:
+        print(
+            f"  zlib-ng {ng} in use: TorrentZip bytes are stock zlib's output, so no "
+            "build is claimed exact here; exact recipes are checked by their members"
+        )
+    root = Path(args.bios_dir).resolve().parent
+    atoms = archive_build.AtomIndex.from_collection(ledger, db, root)
+    counts: dict[str, int] = defaultdict(int)
+    own: list[tuple[str, int]] = []
+    for record in records:
+        result = archive_build.build_archive(record, atoms, Path(args.output_dir), root)
+        counts[result.status] += 1
+        if result.own_members:
+            own.append((result.path, result.own_members))
+        if result.status == "impossible":
+            for member in result.missing:
+                print(
+                    f"  IMPOSSIBLE: {result.path}: missing {member['name']} "
+                    f"({member['sha1']})"
+                )
+        elif result.status == "failed":
+            print(f"  FAILED: {result.path}: {result.detail}")
+        else:
+            note = f" ({result.detail})" if result.detail else ""
+            print(f"  {result.status.upper()}: {result.path}{note}")
+            if args.discard:
+                Path(result.output).unlink()
+    print(
+        f"\n{len(records)} archives: {counts['exact']} rebuilt exact, "
+        f"{counts['canonical']} canonical, {counts['impossible']} impossible, "
+        f"{counts['failed']} failed"
+    )
+    print(
+        f"  {len(own)} of them read {sum(n for _p, n in own)} members from their "
+        "own archive, held nowhere else in the collection"
+    )
+    if counts["failed"]:
+        print("ERROR: a build did not match its recipe and was not kept", file=sys.stderr)
+        return 1
+    return 1 if counts["impossible"] else 0
+
+
 def _resolve_modes(args: argparse.Namespace) -> None:
     """Which reports run: both by default, and --missing whenever --write.
 
@@ -502,11 +617,35 @@ def main() -> int:
         default="recipes",
         help="recipe snapshot file, or a directory of them",
     )
+    parser.add_argument(
+        "--archives", default=str(ARCHIVES_FILE),
+        help="archive recipe file (default: recipes/archives.json)",
+    )
+    parser.add_argument(
+        "--capture", action="store_true",
+        help="write a recipe for every archive in the database",
+    )
+    build = parser.add_mutually_exclusive_group()
+    build.add_argument("--build", metavar="ARCHIVE", help="build one archive, by path or name")
+    build.add_argument("--build-all", action="store_true", help="build every recorded archive")
+    parser.add_argument("--output-dir", help="where --build and --build-all write")
+    parser.add_argument(
+        "--discard", action="store_true",
+        help="delete each build once verified: a full proof without the disk space",
+    )
     args = parser.parse_args()
     require_directory(args.bios_dir)
-    _resolve_modes(args)
+    if (args.output_dir or args.discard) and not (args.build or args.build_all):
+        parser.error("--output-dir and --discard need --build or --build-all")
+    if args.capture and (args.build or args.build_all):
+        parser.error("--capture runs alone")
 
     db = load_database(args.db)
+    if args.capture:
+        return run_capture(args, db)
+    if args.build or args.build_all:
+        return run_build(args, db)
+    _resolve_modes(args)
     profiles = load_emulator_profiles(args.emulators_dir, skip_aliases=False)
     profile_recipes = collect_recipes(profiles)
     dat_recipes = load_dat_recipes(args.dat_recipes)
@@ -560,7 +699,7 @@ def main() -> int:
         )
         print(
             "  The label is the oldest version producing these bytes: a recipe "
-            "unchanged since 0.190 is counted there, not at its own version."
+            "unchanged since 0.177 is counted there, not at its own version."
         )
         by_recipe: dict[str, int] = defaultdict(int)
         for record in ident["identified"]:

scripts/scraper/romset_dat_importer.py

@@ -16,9 +16,12 @@ in its repository, so both are fetched without a browser:
 
 Only sets the collection has a reason to know are kept, so the committed
 snapshot stays small: the archive names that emulator profiles or platform
-BIOS lists reference, and the sets holding a dump-catalog entry the
-collection lacks (provenance_report names the set as where to find it).
-``--all`` keeps everything.
+BIOS lists reference, the archives the collection holds, the sets holding a
+dump-catalog entry the collection lacks (provenance_report names the set as
+where to find it), and every set whose members equal an archive the
+collection keeps under another name. That last entry carries the
+collection's name and keeps the upstream one in ``set``. ``--all`` keeps
+everything.
 
 Members without a CRC32 are undumped. A real romset does not carry them, so
 they are dropped from the recipe rather than disqualifying the whole set.
@@ -80,6 +83,83 @@ def relevant_archives(emulators_dir: str, platforms_dir: str) -> set[str]:
 
 
 Targets = dict[tuple[str, int], set[str]]
+MemberList = tuple[tuple[str, str], ...]
+ByMembers = dict[MemberList, set[str]]
+
+_VARIANT_TAG = re.compile(r"\.[0-9a-f]{8}$")
+
+
+def member_list(members: list[dict]) -> MemberList:
+    """Sorted (name, crc32) of a recipe, the key an archive is matched on."""
+    return tuple(
+        sorted((str(m["name"]), str(m.get("crc32") or "").lower()) for m in members)
+    )
+
+
+def collection_archives(db_path: str) -> tuple[set[str], ByMembers]:
+    """Names and member lists of every archive the collection holds.
+
+    A set the collection keeps under another name is still that set:
+    ``aga.zip`` holds exactly MAME's ``isa_aga`` and ``votrax.zip`` its
+    ``votrsc01a``. Matched by name alone, neither ever entered the snapshot.
+    A ``.variants/`` copy stands for the name its tag is cut from.
+    """
+    with open(db_path, encoding="utf-8") as handle:
+        files = json.load(handle).get("files", {})
+    names: set[str] = set()
+    by_members: ByMembers = {}
+    for entry in files.values():
+        path = Path(entry.get("path", ""))
+        name = _VARIANT_TAG.sub("", path.name)
+        if not name.lower().endswith(".zip") or not path.is_file():
+            continue
+        try:
+            with zipfile.ZipFile(path) as archive:
+                held = [
+                    {"name": info.filename, "crc32": f"{info.CRC:08x}"}
+                    for info in archive.infolist() if not info.is_dir()
+                ]
+        except (OSError, zipfile.BadZipFile) as exc:
+            print(f"  UNREADABLE: {path}: {exc}", file=sys.stderr)
+            continue
+        names.add(name)
+        if held:
+            by_members.setdefault(member_list(held), set()).add(name)
+    return names, by_members
+
+
+def _entry(
+    dat: str, archive: str, set_name: str, description: str, members: list[dict]
+) -> dict:
+    return {
+        "dat": dat,
+        "name": archive,
+        "set": set_name,
+        "description": description,
+        "members": sorted(members, key=lambda member: member["name"]),
+    }
+
+
+def named_entries(
+    dat: str,
+    set_name: str,
+    description: str,
+    members: list[dict],
+    kept: bool,
+    by_members: ByMembers | None,
+) -> list[dict]:
+    """The snapshot entries one upstream set gives.
+
+    Under its own name when kept, and under every collection name whose
+    archive holds exactly its members. ``set`` keeps the upstream name.
+    """
+    archives = {f"{set_name}.zip"} if kept else set()
+    if by_members:
+        archives |= by_members.get(member_list(members), set())
+    return [
+        _entry(dat, archive, set_name, description, members)
+        for archive in sorted(archives)
+    ]
 
 
 def catalog_targets(provenance_dir: str, db_path: str) -> Targets:
@@ -116,8 +196,28 @@ def holds_target(members: list[dict], targets: Targets | None) -> bool:
     return False
 
 
+def one_set_per_archive(entries: list[dict]) -> list[dict]:
+    """One entry per archive and member list, its own set preferred.
+
+    Thirty MAME keyboard sets carry the same ROM, so ``a500kbd_ch.zip``
+    matches all of them. The set named like the archive is the one kept,
+    then the first by name.
+    """
+    chosen: dict[tuple[str, MemberList], dict] = {}
+    for entry in entries:
+        key = (entry["name"], member_list(entry["members"]))
+        current = chosen.get(key)
+        own = entry["name"] == f"{entry['set']}.zip"
+        if current is None or (own and current["name"] != f"{current['set']}.zip"):
+            chosen[key] = entry
+    return sorted(chosen.values(), key=lambda e: (e["name"], e["set"]))
+
+
 def recipe_entries(
-    dat: LogiqxDat, keep: set[str] | None = None, targets: Targets | None = None
+    dat: LogiqxDat,
+    keep: set[str] | None = None,
+    targets: Targets | None = None,
+    by_members: ByMembers | None = None,
 ) -> list[dict]:
     """Group a parsed DAT into one recipe per set."""
     by_set: dict[str, list] = {}
@@ -128,32 +228,22 @@ def recipe_entries(
 
     entries: list[dict] = []
     for set_name, roms in sorted(by_set.items()):
-        archive = f"{set_name}.zip"
-        if keep is not None and archive not in keep and not holds_target(
-            [{"crc32": r.crc32, "size": r.size, "sha1": r.sha1} for r in roms], targets
-        ):
-            continue
-        entries.append(
-            {
-                "dat": dat.name or "unnamed",
-                "name": archive,
-                "set": set_name,
-                "description": roms[0].description,
-                "members": sorted(
-                    (
-                        {
-                            "name": rom.name,
-                            "size": rom.size,
-                            "crc32": rom.crc32,
-                            "sha1": rom.sha1,
-                        }
-                        for rom in roms
-                    ),
-                    key=lambda member: member["name"],
-                ),
-            }
+        members = [
+            {"name": rom.name, "size": rom.size, "crc32": rom.crc32, "sha1": rom.sha1}
+            for rom in roms
+        ]
+        kept = (
+            keep is None
+            or f"{set_name}.zip" in keep
+            or holds_target(members, targets)
         )
-    return entries
+        entries.extend(
+            named_entries(
+                dat.name or "unnamed", set_name, roms[0].description, members,
+                kept, by_members,
+            )
+        )
+    return one_set_per_archive(entries)
 
 
 _MACHINE = re.compile(rb'<machine name="([^"]+)"([^>]*)>')
@@ -240,19 +330,31 @@ def listxml_entries(
     label: str,
     keep: set[str] | None,
     targets: Targets | None = None,
+    by_members: ByMembers | None = None,
 ) -> list[dict]:
     """Recipes from a MAME -listxml document, parents resolved.
 
     A non-merged archive carries its parent set's ROMs as well as its own, so
     a machine with ``romof`` is read as the union, its own members winning a
     name collision. Two passes: the parents a set needs are only known once
-    that set has been seen.
+    that set has been seen. Matching by member list reads every machine in
+    one pass, every parent included.
     """
+    def kept(name: str) -> bool:
+        return keep is None or f"{name}.zip" in keep
+
     with open_stream() as stream:
         wanted = _scan_listxml(
-            stream, lambda name: keep is None or f"{name}.zip" in keep, targets
+            stream, (lambda _name: True) if by_members else kept, targets
         )
-    requested = set(wanted)
+    requested = (
+        {
+            name for name, (_parent, roms) in wanted.items()
+            if kept(name) or holds_target(roms, targets)
+        }
+        if by_members
+        else set(wanted)
+    )
     parents = {
         parent for parent, _roms in wanted.values() if parent and parent not in wanted
     }
@@ -263,23 +365,20 @@ def listxml_entries(
 
     entries: list[dict] = []
     for name, (parent, roms) in sorted(wanted.items()):
-        # A parent read only to complete a child is not itself an entry.
-        if name not in requested:
+        if not by_members and name not in requested:
+            # A parent read only to complete a child is not itself an entry.
             continue
         members = {member["name"]: member for member in wanted.get(parent, (None, []))[1]} if parent else {}
         members.update({member["name"]: member for member in roms})
         if not members:
             continue
-        entries.append(
-            {
-                "dat": label,
-                "name": f"{name}.zip",
-                "set": name,
-                "description": f"parent {parent}" if parent else name,
-                "members": sorted(members.values(), key=lambda m: m["name"]),
-            }
+        entries.extend(
+            named_entries(
+                label, name, f"parent {parent}" if parent else name,
+                list(members.values()), name in requested, by_members,
+            )
         )
-    return entries
+    return one_set_per_archive(entries)
 
 
 def _recipe_key(entry: dict) -> tuple[str, str]:
@@ -609,12 +708,13 @@ def main() -> int:
 
     output = args.output or DEFAULT_OUTPUT.format(source=args.source)
     Path(output).parent.mkdir(parents=True, exist_ok=True)
-    keep = (
-        None
-        if args.all
-        else relevant_archives(args.emulators_dir, args.platforms_dir)
-    )
-    targets = None if args.all else catalog_targets(args.provenance_dir, args.db)
+    keep: set[str] | None = None
+    targets = None
+    by_members: ByMembers | None = None
+    if not args.all:
+        held, by_members = collection_archives(args.db)
+        keep = relevant_archives(args.emulators_dir, args.platforms_dir) | held
+        targets = catalog_targets(args.provenance_dir, args.db)
 
     dats: dict[str, str] = {}
     entries: list[dict] = []
@@ -622,7 +722,9 @@ def main() -> int:
 
     if _is_listxml(pack):
         label = f"MAME {pack.stem.replace('lx', '')}"
-        entries = listxml_entries(_listxml_opener(pack), label, keep, targets)
+        entries = listxml_entries(
+            _listxml_opener(pack), label, keep, targets, by_members
+        )
         dats[label] = pack.stem
         print(f"  {label}: {len(entries)} sets retenus")
         members = sum(len(entry["members"]) for entry in entries)
@@ -645,7 +747,7 @@ def main() -> int:
             continue
         if revision:
             dat.name = f"{dat.name or member} @ {revision}"
-        found = recipe_entries(dat, keep, targets)
+        found = recipe_entries(dat, keep, targets, by_members)
         if not found:
             continue
         dats[dat.name or member] = dat.version

scripts/torrentzip.py

@@ -23,16 +23,140 @@ CLI:
 from __future__ import annotations
 
 import argparse
+import functools
 import hashlib
+import io
 import struct
 import zipfile
 import zlib
+from collections.abc import Callable, Iterable
 from pathlib import Path
+from typing import Any, NamedTuple, Protocol
+
+
+class Source(Protocol):
+    """An open member: read in chunks, closed by its ``with`` block."""
+
+    def read(self, size: int = -1, /) -> bytes: ...
+    def __enter__(self) -> Source: ...
+    def __exit__(self, *exc: Any) -> Any: ...
+
+
+class Sink(Protocol):
+    def write(self, data: bytes, /) -> int: ...
 
 # TorrentZip constants: 1996-12-24 23:32:00, deflate level 9
 _DOS_DATE = ((1996 - 1980) << 9) | (12 << 5) | 24
 _DOS_TIME = (23 << 11) | (32 << 5) | 0
 _COMMENT_PREFIX = "TORRENTZIPPED-"
+# Members are read and deflated this much at a time. Deflate output does not
+# depend on how its input is cut, so a 2 GiB disk image streams through in
+# the same bytes a one-shot compression gives.
+CHUNK = 16 * 1024 * 1024
+# A size or offset that does not fit 32 bits takes a ZIP64 field. No
+# TorrentZip-conformant archive needs one, so their layout is untouched.
+ZIP32_LIMIT = 0xFFFFFFFF
+_ZIP64_MARK = 0xFFFFFFFF
+
+
+class Written(NamedTuple):
+    """A member as the writer read it from its source."""
+
+    name: str
+    size: int
+    crc32: str
+    sha1: str
+
+
+def zlib_ng_version(module: object = zlib) -> str | None:
+    """The zlib-ng version this interpreter deflates with, None for zlib.
+
+    TorrentZip bytes are the output of zlib's level-9 deflate. Stock zlib
+    1.2.13 to 1.3.2 reproduces every TorrentZip archive in the collection.
+    zlib-ng 2.2.4 (python.org 3.14 on Windows) reproduces 479 of 1,561 and
+    2.3.3 545, and the two disagree with each other.
+    """
+    version = getattr(module, "ZLIBNG_VERSION", None)
+    if version:
+        return str(version)
+    runtime = str(getattr(module, "ZLIB_RUNTIME_VERSION", "") or "")
+    return runtime if "zlib-ng" in runtime.lower() else None
+
+
+def _deflate(stream: Source) -> tuple[bytes, Written]:
+    compressor = zlib.compressobj(9, zlib.DEFLATED, -15)
+    blob = bytearray()
+    crc, size, sha1 = 0, 0, hashlib.sha1()
+    while chunk := stream.read(CHUNK):
+        crc = zlib.crc32(chunk, crc)
+        size += len(chunk)
+        sha1.update(chunk)
+        blob += compressor.compress(chunk)
+    blob += compressor.flush()
+    return bytes(blob), Written("", size, f"{crc & 0xFFFFFFFF:08x}", sha1.hexdigest())
+
+
+def write_torrentzip(
+    sink: Sink, members: Iterable[tuple[str, Callable[[], Source]]]
+) -> list[Written]:
+    """Write a TorrentZip archive to *sink*, one member in memory at a time.
+
+    *members* pairs each name with a callable opening its bytes. Members are
+    sorted case-insensitively by name, as the format requires. Returns what
+    was read, in archive order, so a caller can check every member.
+    """
+    ordered = sorted(members, key=lambda m: m[0].lower())
+    central = bytearray()
+    offset = 0
+    written: list[Written] = []
+    for name, opener in ordered:
+        raw = name.encode("ascii")
+        with opener() as stream:
+            blob, read = _deflate(stream)
+        crc = int(read.crc32, 16)
+        written.append(read._replace(name=name))
+        wide = read.size >= ZIP32_LIMIT or len(blob) >= ZIP32_LIMIT
+        far = offset >= ZIP32_LIMIT
+        local_extra = struct.pack("<HHQQ", 1, 16, read.size, len(blob)) if wide else b""
+        central_extra = (
+            struct.pack("<QQ", read.size, len(blob)) if wide else b""
+        ) + (struct.pack("<Q", offset) if far else b"")
+        if central_extra:
+            central_extra = struct.pack("<HH", 1, len(central_extra)) + central_extra
+        sizes = (_ZIP64_MARK, _ZIP64_MARK) if wide else (len(blob), read.size)
+        needed = 45 if wide or far else 20
+        header = struct.pack(
+            "<IHHHHHIIIHH", 0x04034B50, needed, 2, 8, _DOS_TIME, _DOS_DATE,
+            crc, *sizes, len(raw), len(local_extra),
+        ) + raw + local_extra
+        sink.write(header)
+        sink.write(blob)
+        central += struct.pack(
+            "<IHHHHHHIIIHHHHHII", 0x02014B50, 0, needed, 2, 8, _DOS_TIME, _DOS_DATE,
+            crc, *sizes, len(raw), len(central_extra), 0, 0, 0, 0,
+            _ZIP64_MARK if far else offset,
+        ) + raw + central_extra
+        offset += len(header) + len(blob)
+
+    cd_offset = offset
+    sink.write(central)
+    count = len(ordered)
+    if cd_offset >= ZIP32_LIMIT or len(central) >= ZIP32_LIMIT or count >= 0xFFFF:
+        end64 = cd_offset + len(central)
+        sink.write(struct.pack(
+            "<IQHHIIQQQQ", 0x06064B50, 44, 45, 45, 0, 0, count, count,
+            len(central), cd_offset,
+        ))
+        sink.write(struct.pack("<IIQI", 0x07064B50, 0, end64, 1))
+    comment = (
+        f"{_COMMENT_PREFIX}{zlib.crc32(bytes(central)) & 0xFFFFFFFF:08X}"
+    ).encode("ascii")
+    sink.write(struct.pack(
+        "<IHHHHIIH", 0x06054B50, 0, 0, min(count, 0xFFFF), min(count, 0xFFFF),
+        min(len(central), _ZIP64_MARK), min(cd_offset, _ZIP64_MARK), len(comment),
+    ))
+    sink.write(comment)
+    return written
 
 
 def build_torrentzip(members: list[tuple[str, bytes]]) -> bytes:
@@ -41,38 +165,11 @@ def build_torrentzip(members: list[tuple[str, bytes]]) -> bytes:
     Members are sorted case-insensitively by name, as the format requires.
     Returns the complete archive bytes.
     """
-    ordered = sorted(members, key=lambda m: m[0].lower())
-    body = bytearray()
-    central = bytearray()
-
-    for name, data in ordered:
-        raw = name.encode("ascii")
-        compressor = zlib.compressobj(9, zlib.DEFLATED, -15)
-        blob = compressor.compress(data) + compressor.flush()
-        crc = zlib.crc32(data) & 0xFFFFFFFF
-        offset = len(body)
-        body += struct.pack(
-            "<IHHHHHIIIHH", 0x04034B50, 20, 2, 8, _DOS_TIME, _DOS_DATE,
-            crc, len(blob), len(data), len(raw), 0,
-        )
-        body += raw + blob
-        central += struct.pack(
-            "<IHHHHHHIIIHHHHHII", 0x02014B50, 0, 20, 2, 8, _DOS_TIME, _DOS_DATE,
-            crc, len(blob), len(data), len(raw), 0, 0, 0, 0, 0, offset,
-        )
-        central += raw
-
-    cd_offset = len(body)
-    body += central
-    comment = (
-        f"{_COMMENT_PREFIX}{zlib.crc32(bytes(central)) & 0xFFFFFFFF:08X}"
-    ).encode("ascii")
-    body += struct.pack(
-        "<IHHHHIIH", 0x06054B50, 0, 0, len(ordered), len(ordered),
-        len(central), cd_offset, len(comment),
+    sink = io.BytesIO()
+    write_torrentzip(
+        sink, [(name, functools.partial(io.BytesIO, data)) for name, data in members]
     )
-    body += comment
-    return bytes(body)
+    return sink.getvalue()
 
 
 def rebuild_torrentzip(source: str | Path) -> bytes:

tests/test_archive_build.py

@@ -0,0 +1,349 @@
+"""Every archive has a recipe, every member is held, and the build reproduces it.
+
+The collection must regenerate each archive it serves from the ROMs it holds
+plus recipes/archives.json, without a copy of that archive from elsewhere.
+"""
+
+from __future__ import annotations
+
+import tests  # noqa: F401  offline, scratch under tmp/, the suite's own cache
+
+import hashlib
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+from unittest import mock
+
+ROOT = Path(__file__).resolve().parent.parent
+TMP_ROOT = ROOT / "tmp" / "tests"
+TMP_ROOT.mkdir(parents=True, exist_ok=True)
+sys.path.insert(0, str(ROOT / "scripts"))
+
+import archive_build  # noqa: E402
+from torrentzip import build_torrentzip  # noqa: E402
+
+ROM_A = b"first rom payload" * 50
+ROM_B = b"second rom payload" * 40
+ROM_C = b"third rom, held only inside its own archive" * 30
+
+
+def _sha1(data: bytes) -> str:
+    return hashlib.sha1(data).hexdigest()
+
+
+class TheLedgerCoversTheCollection(unittest.TestCase):
+    """Read from the committed files: database.json and recipes/archives.json."""
+
+    @classmethod
+    def setUpClass(cls):
+        with (ROOT / "database.json").open(encoding="utf-8") as handle:
+            cls.db = json.load(handle)
+        cls.ledger = archive_build.load_ledger()
+        cls.archives = archive_build.database_archives(cls.db)
+
+    def test_every_database_archive_has_a_recipe(self):
+        absent = sorted(set(self.archives) - set(self.ledger))
+        self.assertEqual(
+            absent, [],
+            f"{len(absent)} archives without a recipe; run "
+            "`python scripts/romset_recipes.py --capture`",
+        )
+
+    def test_no_recipe_outlives_its_archive(self):
+        self.assertEqual(sorted(set(self.ledger) - set(self.archives)), [])
+
+    def test_each_recipe_names_the_container_the_database_holds(self):
+        wrong = [
+            path for path, record in self.ledger.items()
+            if path in self.archives and any(
+                record[key] != self.archives[path][key] for key in ("sha1", "md5", "size")
+            )
+        ]
+        self.assertEqual(wrong, [], "recaptured archives keep a stale recipe")
+
+    def test_each_recipe_is_complete(self):
+        for path, record in self.ledger.items():
+            self.assertIn(record["rebuild"], ("exact", "canonical"), path)
+            self.assertEqual(record["sources"][-1], "collection", path)
+            self.assertTrue(record["members"], path)
+            for member in record["members"]:
+                self.assertEqual(set(member), {"name", "size", "crc32", "sha1"}, path)
+            if record.get("bytes_pinned"):
+                self.assertTrue(record["pins"], path)
+
+    def test_every_member_is_held_in_the_collection(self):
+        """The list printed here is the acquisition list: a member neither held
+        as a loose file nor inside any archive the database lists."""
+        missing = archive_build.missing_members(self.ledger, self.db)
+        listing = "\n".join(
+            f"  {m['archive']}: {m['member']} ({m['sha1']})" for m in missing
+        )
+        self.assertEqual(missing, [], f"members held nowhere:\n{listing}")
+
+    def test_the_file_is_written_the_one_way(self):
+        text = archive_build.ARCHIVES_FILE.read_text(encoding="utf-8")
+        self.assertEqual(text, archive_build.ledger_text(self.ledger))
+
+    def test_recipes_describe_the_archives_on_disk(self):
+        """Names in order, sizes and crc32 against each central directory."""
+        checked = 0
+        for path, record in sorted(self.ledger.items()):
+            archive = ROOT / path
+            if not archive.is_file():
+                continue  # a release asset not restored in this checkout
+            with zipfile.ZipFile(archive) as handle:
+                held = [
+                    (info.filename, info.file_size, f"{info.CRC:08x}")
+                    for info in handle.infolist() if not info.is_dir()
+                ]
+            recorded = [(m["name"], m["size"], m["crc32"]) for m in record["members"]]
+            self.assertEqual(held, recorded, path)
+            checked += 1
+        if not checked:
+            self.skipTest("bios/ is not present")
+
+
+class Fixture(unittest.TestCase):
+    """A small collection: a loose ROM, a TorrentZip set, a plain ZIP."""
+
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory(dir=TMP_ROOT)
+        self.root = Path(self.temp.name)
+        bios = self.root / "bios"
+        bios.mkdir()
+        (bios / "a.rom").write_bytes(ROM_A)
+        self.exact_path = bios / "set.zip"
+        self.exact_path.write_bytes(
+            build_torrentzip([("a.rom", ROM_A), ("b.rom", ROM_B), ("c.rom", ROM_C)])
+        )
+        self.other_path = bios / "other.zip"
+        with zipfile.ZipFile(self.other_path, "w", zipfile.ZIP_DEFLATED) as archive:
+            archive.writestr("dir/", b"")
+            archive.writestr("second.rom", ROM_B)
+            archive.writestr("first.rom", ROM_A)
+        self.db = {"files": {}}
+        for path in (bios / "a.rom", self.exact_path, self.other_path):
+            data = path.read_bytes()
+            self.db["files"][_sha1(data)] = {
+                "path": path.relative_to(self.root).as_posix(),
+                "name": path.name,
+                "sha1": _sha1(data),
+                "md5": hashlib.md5(data).hexdigest(),
+                "size": len(data),
+            }
+
+    def tearDown(self):
+        self.temp.cleanup()
+
+    def capture(self, **kwargs) -> dict[str, dict]:
+        records, failed = archive_build.capture(
+            self.db, kwargs.get("previous", {}), kwargs.get("index", {}),
+            kwargs.get("pins", {}), workers=1, log=lambda _line: None,
+        )
+        self.assertEqual(failed, {})
+        return records
+
+
+class Capture(Fixture):
+    def setUp(self):
+        super().setUp()
+        self._cwd = Path.cwd()
+        import os
+
+        os.chdir(self.root)
+
+    def tearDown(self):
+        import os
+
+        os.chdir(self._cwd)
+        super().tearDown()
+
+    def test_a_torrentzip_archive_is_exact_and_another_canonical(self):
+        records = self.capture()
+        exact, other = records["bios/set.zip"], records["bios/other.zip"]
+        self.assertEqual(exact["rebuild"], "exact")
+        self.assertEqual(other["rebuild"], "canonical")
+        self.assertEqual(
+            [m["name"] for m in other["members"]], ["second.rom", "first.rom"],
+            "members keep the archive's own order",
+        )
+        self.assertEqual(other["directories"], ["dir/"])
+        self.assertEqual(other["members"][0]["sha1"], _sha1(ROM_B))
+        self.assertEqual(exact["sources"], ["collection"])
+
+    def test_dat_agreement_and_pins_are_recorded(self):
+        key = archive_build.member_key(
+            [{"name": n, "crc32": c} for n, c in (
+                ("a.rom", self._crc(ROM_A)), ("b.rom", self._crc(ROM_B)),
+                ("c.rom", self._crc(ROM_C)),
+            )]
+        )
+        md5 = hashlib.md5(self.exact_path.read_bytes()).hexdigest()
+        records = self.capture(
+            index={key: {"mame:MAME mame0289"}}, pins={md5: {"platform:demo"}}
+        )
+        record = records["bios/set.zip"]
+        self.assertEqual(record["sources"], ["mame:MAME mame0289", "collection"])
+        self.assertEqual((record["bytes_pinned"], record["pins"]), (True, ["platform:demo"]))
+        self.assertNotIn("bytes_pinned", records["bios/other.zip"])
+
+    def test_an_unchanged_archive_is_not_read_again(self):
+        previous = self.capture()
+        previous["bios/set.zip"]["members"][0]["sha1"] = "kept"
+        records = self.capture(previous=previous)
+        self.assertEqual(records["bios/set.zip"]["members"][0]["sha1"], "kept")
+
+    def test_zlib_ng_refuses_to_capture(self):
+        with mock.patch.object(archive_build, "zlib_ng_version", lambda: "2.2.4"):
+            with self.assertRaises(archive_build.ZlibNgError):
+                self.capture()
+
+    def test_composite_and_zipped_pins_name_no_container(self):
+        platforms = self.root / "platforms"
+        platforms.mkdir()
+        (platforms / "_registry.yml").write_text(
+            "platforms:\n"
+            "  plain:\n    config: plain.yml\n"
+            "  composite:\n    config: composite.yml\n    zip_digest: composite\n",
+            encoding="utf-8",
+        )
+        entry = "      - name: set.zip\n        destination: set.zip\n"
+        (platforms / "plain.yml").write_text(
+            "platform: Plain\nsystems:\n  arcade:\n    files:\n"
+            f"{entry}        md5: {'a' * 32}\n"
+            f"{entry}        md5: {'b' * 32}\n        zipped_file: a.rom\n",
+            encoding="utf-8",
+        )
+        (platforms / "composite.yml").write_text(
+            "platform: Composite\nsystems:\n  arcade:\n    files:\n"
+            f"{entry}        md5: {'c' * 32}\n",
+            encoding="utf-8",
+        )
+        emulators = self.root / "emulators"
+        emulators.mkdir()
+        (emulators / "core.yml").write_text(
+            "emulator: Core\ntype: libretro\nfiles:\n"
+            f"  - name: set.zip\n    sha1: {'d' * 40}\n",
+            encoding="utf-8",
+        )
+        pins = archive_build.container_pins(str(platforms), str(emulators))
+        self.assertEqual(pins, {"a" * 32: {"platform:plain"}, "d" * 40: {"profile:core"}})
+
+    @staticmethod
+    def _crc(data: bytes) -> str:
+        import zlib
+
+        return f"{zlib.crc32(data) & 0xFFFFFFFF:08x}"
+
+
+class Build(Fixture):
+    def setUp(self):
+        super().setUp()
+        import os
+
+        self._cwd = Path.cwd()
+        os.chdir(self.root)
+        self.ledger = self.capture()
+        os.chdir(self._cwd)
+        self.atoms = archive_build.AtomIndex.from_collection(self.ledger, self.db, self.root)
+        self.out = self.root / "out"
+
+    def build(self, path: str):
+        return archive_build.build_archive(self.ledger[path], self.atoms, self.out, self.root)
+
+    def test_an_exact_recipe_rebuilds_the_bytes(self):
+        result = self.build("bios/set.zip")
+        self.assertEqual(result.status, "exact")
+        built = self.out / "set.zip"
+        self.assertEqual(built.read_bytes(), self.exact_path.read_bytes())
+        self.assertEqual(result.own_members, 1, "c.rom is held only in set.zip")
+
+    def test_members_come_from_a_loose_file_then_another_archive(self):
+        own = self.root / "bios" / "set.zip"
+        self.assertEqual(self.atoms.find(_sha1(ROM_A), own).member, None)
+        self.assertEqual(self.atoms.find(_sha1(ROM_B), own).path.name, "other.zip")
+        self.assertEqual(self.atoms.find(_sha1(ROM_C), own).path, own)
+
+    def test_a_canonical_recipe_is_checked_by_its_members(self):
+        result = self.build("bios/other.zip")
+        self.assertEqual(result.status, "canonical")
+        with zipfile.ZipFile(self.out / "other.zip") as archive:
+            self.assertEqual(sorted(archive.namelist()), ["first.rom", "second.rom"])
+            self.assertTrue(archive.comment.startswith(b"TORRENTZIPPED-"))
+
+    def test_a_member_held_nowhere_makes_it_impossible(self):
+        record = dict(self.ledger["bios/set.zip"])
+        record["members"] = record["members"] + [
+            {"name": "gone.rom", "size": 1, "crc32": "00000000", "sha1": "f" * 40}
+        ]
+        result = archive_build.build_archive(record, self.atoms, self.out, self.root)
+        self.assertEqual(result.status, "impossible")
+        self.assertEqual([m["name"] for m in result.missing], ["gone.rom"])
+        self.assertFalse(self.out.exists())
+
+    def test_a_mismatch_is_never_promoted(self):
+        record = dict(self.ledger["bios/set.zip"], sha1="0" * 40)
+        result = archive_build.build_archive(record, self.atoms, self.out, self.root)
+        self.assertEqual(result.status, "failed")
+        self.assertIn("is not the recorded", result.detail)
+        self.assertEqual(list(self.out.rglob("*")), [], "no file, no scratch left")
+
+    def test_a_canonical_mismatch_is_never_promoted(self):
+        record = dict(self.ledger["bios/other.zip"])
+        record["members"] = [dict(m) for m in record["members"]]
+        record["members"][0]["crc32"] = "00000000"
+        result = archive_build.build_archive(record, self.atoms, self.out, self.root)
+        self.assertEqual(result.status, "failed")
+        self.assertEqual(list(self.out.rglob("*")), [])
+
+    def test_zlib_ng_never_claims_exact(self):
+        with mock.patch.object(archive_build, "zlib_ng_version", lambda: "2.2.4"):
+            result = self.build("bios/set.zip")
+        self.assertEqual(result.status, "canonical")
+        self.assertIn("zlib-ng 2.2.4", result.detail)
+
+    def test_an_archive_is_named_by_path_or_by_name(self):
+        self.assertEqual(
+            [r["path"] for r in archive_build.select(self.ledger, "set.zip", self.root)],
+            ["bios/set.zip"],
+        )
+        self.assertEqual(
+            len(archive_build.select(self.ledger, str(self.exact_path), self.root)), 1
+        )
+        self.assertEqual(archive_build.select(self.ledger, "none.zip", self.root), [])
+
+    def test_missing_members_reads_the_committed_data(self):
+        self.assertEqual(archive_build.missing_members(self.ledger, self.db), [])
+        del self.db["files"][_sha1(self.exact_path.read_bytes())]
+        missing = archive_build.missing_members(self.ledger, self.db)
+        self.assertEqual(
+            [(m["archive"], m["member"]) for m in missing], [("bios/set.zip", "c.rom")]
+        )
+
+    def test_the_command_reports_each_archive(self):
+        archives = self.root / "archives.json"
+        archives.write_text(archive_build.ledger_text(self.ledger), encoding="utf-8")
+        db = self.root / "database.json"
+        db.write_text(json.dumps(self.db), encoding="utf-8")
+        run = subprocess.run(
+            [
+                sys.executable, str(ROOT / "scripts" / "romset_recipes.py"),
+                "--build-all", "--output-dir", str(self.out), "--discard",
+                "--bios-dir", str(self.root / "bios"), "--db", str(db),
+                "--archives", str(archives),
+            ],
+            capture_output=True, text=True, check=False, timeout=120,
+        )
+        self.assertEqual(run.returncode, 0, run.stdout + run.stderr)
+        self.assertIn("EXACT: bios/set.zip", run.stdout)
+        self.assertIn("CANONICAL: bios/other.zip", run.stdout)
+        self.assertIn("2 archives: 1 rebuilt exact, 1 canonical, 0 impossible", run.stdout)
+        self.assertEqual(list(self.out.rglob("*.zip")), [], "--discard keeps nothing")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_archive_recipes.py

@@ -66,6 +66,13 @@ KEPT = {
     "bios/Mattel/Aquarius/.variants/aquarius_ar.zip.73026a45": (
         "a platform pack copy; its TMCTV.png is held only here"
     ),
+    # Each carries slot ROMs beside its own set. Its own members, TorrentZipped,
+    # are byte for byte the archive under the newer MAME name in Arcade/MAME,
+    # so trimming it makes a duplicate: whether the old name stays is the
+    # owner's storage decision (2026-11-06).
+    "bios/Arcade/Arcade/fccpu20.zip": "MAME 0.196 to 0.258 fccpu20, now sys68k_cpu20.zip",
+    "bios/Arcade/Arcade/fcisio1.zip": "MAME 0.196 to 0.258 fcisio1, now sys68k_isio1.zip",
+    "bios/Arcade/Arcade/interpod.zip": "MAME 0.177 to 0.215 interpod, now cbm_interpod.zip",
     "bios/Sega/Dreamcast/dc.zip": (
         "RetroBat's dc/dc.zip holds dc_boot.bin and dc_flash.bin, the names the "
         "Dreamcast BIOS goes by outside MAME, not MAME's dc set"

tests/test_romset_recipes.py

@@ -246,6 +246,25 @@ class ReportsOverAFixture(unittest.TestCase):
         self.assertEqual(len(report["unreproducible"]), 1)
         self.assertEqual(report["unreproducible"][0]["recipes_tried"], ["v1"])
 
+    def test_a_composite_md5_is_no_container_target(self):
+        """Recalbox hashes a ZIP by its members: its md5 names no archive bytes."""
+        platforms = self.root / "platforms"
+        platforms.mkdir()
+        (platforms / "_registry.yml").write_text(
+            "platforms:\n  demo:\n    config: demo.yml\n    zip_digest: composite\n",
+            encoding="utf-8",
+        )
+        (platforms / "demo.yml").write_text(
+            "platform: Demo\nsystems:\n  arcade:\n    files:\n"
+            "      - name: set.zip\n        destination: set.zip\n"
+            f"        md5: {'f' * 32}\n",
+            encoding="utf-8",
+        )
+        self.db["indexes"]["by_md5"] = {}
+
+        report = missing_report(self.recipes, self.pool, self.db, str(platforms))
+        self.assertEqual(report, {"constructible": [], "unreproducible": [], "no_recipe": []})
+
 
 DAT = """<?xml version="1.0"?>
 <datafile>
@@ -410,6 +429,87 @@ class ListXmlRecipes(unittest.TestCase):
             [m["name"] for m in entries[0]["members"]], ["own.rom", "shared.rom"]
         )
 
+    def test_a_set_kept_under_another_name_is_matched_by_members(self):
+        """aga.zip holds exactly MAME's isa_aga: the entry carries the
+        collection's name and keeps the upstream one in ``set``."""
+        from scripts.scraper.romset_dat_importer import member_list
+
+        child = [
+            {"name": "own.rom", "crc32": _crc(ROM_B)},
+            {"name": "shared.rom", "crc32": _crc(ROM_A)},
+        ]
+        entries = listxml_entries(
+            self._opener(self._document()), "MAME 0.289", {"other.zip"},
+            by_members={member_list(child): {"mine.zip"}},
+        )
+        self.assertEqual(
+            [(entry["name"], entry["set"]) for entry in entries],
+            [("mine.zip", "childset")],
+        )
+        self.assertEqual(
+            [m["name"] for m in entries[0]["members"]], ["own.rom", "shared.rom"]
+        )
+
+    def test_matching_by_members_keeps_the_named_sets(self):
+        from scripts.scraper.romset_dat_importer import member_list
+
+        entries = listxml_entries(
+            self._opener(self._document()), "MAME 0.289", {"parentset.zip"},
+            by_members={member_list([{"name": "nothing", "crc32": "0"}]): {"x.zip"}},
+        )
+        self.assertEqual([entry["name"] for entry in entries], ["parentset.zip"])
+
+    def test_one_entry_per_archive_its_own_set_first(self):
+        """Thirty keyboard sets share one ROM: the set named like the archive wins."""
+        from scripts.scraper.romset_dat_importer import one_set_per_archive
+
+        members = [{"name": "kbd.rom", "crc32": "12345678"}]
+        entries = one_set_per_archive([
+            {"name": "a500kbd.zip", "set": "a2000kbd", "members": members},
+            {"name": "a500kbd.zip", "set": "a500kbd", "members": members},
+            {"name": "a500kbd.zip", "set": "a600kbd", "members": members},
+        ])
+        self.assertEqual([entry["set"] for entry in entries], ["a500kbd"])
+
+    def test_a_logiqx_set_is_matched_by_members_too(self):
+        from scripts.scraper.romset_dat_importer import member_list
+
+        members = [
+            {"name": "01", "crc32": _crc(ROM_A)},
+            {"name": "23", "crc32": _crc(ROM_B)},
+        ]
+        entries = recipe_entries(
+            parse_logiqx(DAT.format(crc_a=_crc(ROM_A), crc_b=_crc(ROM_B))),
+            keep=set(), by_members={member_list(members): {"salora.zip"}},
+        )
+        self.assertEqual(
+            [(entry["name"], entry["set"]) for entry in entries],
+            [("salora.zip", "manager")],
+        )
+
+    def test_the_collection_names_and_member_lists(self):
+        """A .variants/ copy stands for the name its tag is cut from."""
+        from scripts.scraper.romset_dat_importer import collection_archives, member_list
+
+        with tempfile.TemporaryDirectory(dir=TMP_ROOT) as temp:
+            root = Path(temp)
+            plain, variant = root / "set.zip", root / "other.zip.1a2b3c4d"
+            for path in (plain, variant):
+                with zipfile.ZipFile(path, "w") as archive:
+                    archive.writestr("a.rom", ROM_A)
+            loose = root / "loose.bin"
+            loose.write_bytes(ROM_B)
+            db = root / "db.json"
+            db.write_text(json.dumps({"files": {
+                str(i): {"path": str(path)} for i, path in enumerate((plain, variant, loose))
+            }}), encoding="utf-8")
+            names, by_members = collection_archives(str(db))
+        self.assertEqual(names, {"set.zip", "other.zip"})
+        self.assertEqual(
+            by_members,
+            {member_list([{"name": "a.rom", "crc32": _crc(ROM_A)}]): {"set.zip", "other.zip"}},
+        )
+
     def test_a_set_holding_a_missing_catalog_dump_is_kept(self):
         """fm7740sx carries three No-Intro dumps the collection lacks. No
         profile names it, so the missing report named no place to find them."""

tests/test_site_systems.py

@@ -752,7 +752,7 @@ class TheRomsetLineMatchesByCrc(unittest.TestCase):
         if main is None or not (REPO_ROOT / main["path"]).is_file():
             self.skipTest("the main neogeo.zip is not in this checkout")
         line = gs._romset_line(main)
-        self.assertIn("Holds every ROM of MAME 0.190 to 0.289", line)
+        self.assertIn("Holds every ROM of MAME 0.177 to 0.289", line)
         self.assertNotIn("No match", line)
 
 

tests/test_torrentzip.py

@@ -155,6 +155,72 @@ def _as_file(test: unittest.TestCase, data: bytes) -> str:
     return path
 
 
+class StreamingWriter(unittest.TestCase):
+    """Members stream through in chunks, one in memory at a time."""
+
+    def _members(self) -> list[tuple[str, bytes]]:
+        import random
+
+        noise = random.Random(7).randbytes(70_000)
+        return [("b.rom", noise + bytes(50_000) + noise[:9_000]), ("A.rom", b"x" * 1_000)]
+
+    def test_chunked_input_gives_the_one_shot_bytes(self):
+        """Deflate output does not depend on how its input is cut."""
+        import io
+        import torrentzip
+
+        members = self._members()
+        whole = torrentzip.build_torrentzip(members)
+        saved = torrentzip.CHUNK
+        try:
+            for chunk in (1, 333, 4096):
+                torrentzip.CHUNK = chunk
+                self.assertEqual(torrentzip.build_torrentzip(members), whole, chunk)
+        finally:
+            torrentzip.CHUNK = saved
+        sink = io.BytesIO()
+        written = torrentzip.write_torrentzip(
+            sink, [(name, lambda d=data: io.BytesIO(d)) for name, data in members]
+        )
+        self.assertEqual(sink.getvalue(), whole)
+        self.assertEqual(
+            [(w.name, w.size, w.sha1) for w in written],
+            [(name, len(data), hashlib.sha1(data).hexdigest())
+             for name, data in sorted(members, key=lambda m: m[0].lower())],
+        )
+
+    def test_a_member_past_32_bits_takes_a_zip64_field(self):
+        """cspect-next-8gb.zip holds an 8 GiB image: its size needs ZIP64."""
+        import torrentzip
+
+        members = self._members()
+        saved = torrentzip.ZIP32_LIMIT
+        torrentzip.ZIP32_LIMIT = 60_000
+        try:
+            data = torrentzip.build_torrentzip(members)
+        finally:
+            torrentzip.ZIP32_LIMIT = saved
+        with zipfile.ZipFile(_as_file(self, data)) as zf:
+            self.assertIsNone(zf.testzip())
+            self.assertEqual(
+                {info.filename: zf.read(info) for info in zf.infolist()}, dict(members)
+            )
+        self.assertNotEqual(data, build_torrentzip(members))
+
+    def test_zlib_ng_is_recognised(self):
+        """python.org 3.14 on Windows links zlib-ng, which gives other bytes."""
+        from types import SimpleNamespace
+
+        from torrentzip import zlib_ng_version
+
+        stock = SimpleNamespace(ZLIB_RUNTIME_VERSION="1.3.1")
+        windows = SimpleNamespace(ZLIB_RUNTIME_VERSION="1.3.1.zlib-ng")
+        binding = SimpleNamespace(ZLIB_RUNTIME_VERSION="1.3.1", ZLIBNG_VERSION="2.2.4")
+        self.assertIsNone(zlib_ng_version(stock))
+        self.assertEqual(zlib_ng_version(windows), "1.3.1.zlib-ng")
+        self.assertEqual(zlib_ng_version(binding), "2.2.4")
+
+
 class CheckModeExitCode(unittest.TestCase):
     """--check printed `conform: False` and exited 0, so no script could use it."""
 

wiki/architecture.md

@@ -30,7 +30,8 @@ platforms/               one YAML config per platform (scraped from upstream)
   _data_dirs.yml         data directory definitions (Dolphin Sys, PPSSPP...)
   targets/               hardware target configs + _overrides.yml
 provenance/              dump-catalog snapshots (redump, no-intro, tosec)
-recipes/                 romset recipes read from MAME and FBNeo DATs
+recipes/                 romset recipes read from MAME and FBNeo DATs, and
+                         archives.json, the recipe of every archive held
 exports/                 each platform's own file, updated, with a report of the changes
 measurements/            records of real runs, one JSONL file per emulator
 scripts/                 all tooling (Python and pyyaml, jsonschema for schema checks)
@@ -318,6 +319,11 @@ How files enter `bios/`, and why.
   An upstream romset whose MD5 a platform pins stays verbatim in `.variants/`.
   A member renamed by hand in an existing archive gives bytes that match no
   distributed romset and that nobody can regenerate.
+- **Every archive has a recipe.** `recipes/archives.json` records the members
+  of each archive the collection holds, by sha1, and whether the TorrentZip
+  build gives its bytes (`exact`) or only its members (`canonical`), so the
+  collection can rebuild it from the ROMs it holds
+  ([Archive recipes](tools.md#archive-recipes)).
 - **A pack carries what the user must obtain.** A file the emulator installs
   itself (`bundled: true`) is not kept once nothing else needs it, and step 4d
   of the pipeline fails on such a copy. Its sha1 and size are written into the

wiki/testing-guide.md

@@ -54,7 +54,7 @@ Every module in `tests/`, by the area it guards.
 | Area | Fixtures | Modules |
 |------|----------|---------|
 | Core regression suites | synthetic | `test_e2e`, `test_audit_regressions`, `test_tool_messages` |
-| File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_choice`, `test_archive_recipes`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_maker_folder_names`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_np2_fonts`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declaration_status`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` |
+| File resolution and identity | synthetic + collection | `test_adler32_resolution`, `test_agnostic_scan`, `test_agnostic_scan_mode`, `test_agnostic_scope`, `test_agnostic_substitute`, `test_archive_build`, `test_archive_choice`, `test_archive_recipes`, `test_bios_companions`, `test_bundled_identity`, `test_casefold_cache`, `test_db_aliases`, `test_declared_names`, `test_derivations`, `test_directory_entries`, `test_extra_identity`, `test_foreign_tree`, `test_foreign_tree_names`, `test_frontend_digest`, `test_game_data_homonyms`, `test_hashing`, `test_identity`, `test_maker_folder_names`, `test_md5_pack_context`, `test_name_tiebreak`, `test_nec_filing`, `test_np2_fonts`, `test_owner_resolution`, `test_path_suffix`, `test_resolution_evidence`, `test_sega_filing`, `test_shipped_content`, `test_twin_declaration_status`, `test_twin_declarations`, `test_validated_choice`, `test_variant_choice`, `test_variant_identity`, `test_zipped_member` |
 | Verification, cross-reference and profile checks | synthetic + profiles | `test_bios_layout`, `test_cache_independent_counts`, `test_case_insensitive_platforms`, `test_checks_have_values`, `test_core_file_fills_declaration`, `test_core_size_refusal`, `test_cross_reference_sources`, `test_crypto_aes`, `test_crypto_unique_data`, `test_crypto_verify`, `test_entry_categories`, `test_file_consumers`, `test_fileless_audit`, `test_flycast_layout`, `test_gap_settlement`, `test_hidden_extras`, `test_launcher_aliases`, `test_launcher_layer`, `test_native_mode`, `test_no_case_collisions`, `test_path_conflict_status`, `test_path_null_copies`, `test_profile_checks`, `test_profile_names`, `test_profile_paths`, `test_profile_repositories`, `test_romm_hash_rule`, `test_shared_groups`, `test_standalone_links`, `test_standalone_mode`, `test_undeclared_dedup`, `test_unplaced_extras`, `test_validation_scope`, `test_verify_all_dir`, `test_verify_data_dir_notice`, `test_verify_emulator_evidence`, `test_verify_listing`, `test_verify_pack_choice`, `test_verify_release_asset` |
 | Packs and manifests | synthetic + real packs | `test_artifact_lock`, `test_atomic_artifacts`, `test_batocera_cores`, `test_core_data_trees`, `test_deterministic_zip`, `test_emulator_pack_counts`, `test_finalize_pack_names`, `test_large_file_cache`, `test_libretro_layout`, `test_manifest_entry`, `test_minui_platform`, `test_no_game_data`, `test_pack_conformance_dirs`, `test_pack_core_extras`, `test_pack_counts`, `test_pack_destinations`, `test_pack_failures`, `test_pack_file_names`, `test_pack_grouping`, `test_pack_integrity`, `test_pack_member_identity`, `test_pack_readme`, `test_platform_inheritance`, `test_platform_keys`, `test_release_packs`, `test_required_only_archives`, `test_required_only_twins`, `test_save_root`, `test_split_extras`, `test_split_names`, `test_split_pack`, `test_system_pack_extras`, `test_system_pack_parity`, `test_torrentzip`, `test_which_pack` |
 | Slots and regions | synthetic + profiles | `test_region`, `test_region_agnostic_scan`, `test_region_groups`, `test_region_held`, `test_region_platform_index`, `test_region_required_only`, `test_slot`, `test_slots`, `test_slots_shipped_file` |
@@ -190,9 +190,21 @@ module level, so nothing leaves the machine.
 
 ### Tests that read the working tree
 
-Three modules assert on real repository data instead of fixtures. Each skips
+Four modules assert on real repository data instead of fixtures. Each skips
 when its input is missing, so a partial checkout still runs green.
 
+**test_archive_build.** Reads `database.json` and `recipes/archives.json` and
+fails when an archive has no recipe, when a recipe outlives its archive or
+names another container, or when a recipe member is held nowhere in the
+collection, neither as a loose file nor inside any archive. That list is the
+acquisition list, printed by the failure, and it is empty today. With `bios/`
+present it also holds each recipe against the archive's central directory:
+names in order, sizes and crc32. Its fixture classes build an exact and a
+canonical archive, refuse a mismatch without leaving a file, report a member
+held nowhere as impossible, and never claim an exact build under zlib-ng.
+After adding or changing an archive, `python scripts/romset_recipes.py
+--capture` (pipeline step 1c) brings the recipes back in line.
+
 **test_torrentzip.** Rebuilds real MAME romsets through the TorrentZip builder
 and asserts the output is byte-identical, which is what keeps arcade ZIP hashes
 stable across rebuilds.

wiki/tools.md

@@ -41,7 +41,7 @@ Pipeline steps:
 | 0 | Validate profiles and platform YAMLs against the schemas | - (no `jsonschema` installed) |
 | 1 | Generate database | - |
 | 1b | Dump-catalog coverage report | - (offline, reads `provenance/`) |
-| 1c | Romset recipes: identify archives, list rebuild targets | - (offline, reads `recipes/`) |
+| 1c | Romset recipes: identify archives, list rebuild targets, then `--capture` writes a recipe for every archive (`recipes/archives.json`) | - (offline, reads `recipes/`) |
 | 2 | Refresh data directories | `--offline` |
 | 2a | Refresh MAME BIOS hashes | `--offline` |
 | 2a2 | Refresh FBNeo BIOS hashes | `--offline` |
@@ -1375,10 +1375,11 @@ before a release (wiki/release-process.md). Its lines stay in
 | `site_insights.py` | What readers look for and do not find, from totals only: searches that show the site but are rarely clicked, file names searched and whether the collection holds them, site searches that found nothing, broken addresses, "Did this page help?" answers, GitHub traffic and release downloads (`--days`, `--skip gsc,goatcounter,github`, `--output`). Reads Search Console, GoatCounter and GitHub with credentials kept outside the repository. [What the site counts](privacy.md) |
 | `make_demo.py` | Record the installer demo of the README and the site into `docs_assets/demo/`: an ArkOS card with six games built under `tmp/demo/` and mounted in a mount namespace of the user, this checkout served on loopback, `docs_assets/demo/install.tape` run by VHS (needs `vhs`, `ttyd`, `ffmpeg`, `unshare`), then the GIF, WebM, MP4 and poster encoded and held to their size targets. `demo.json` records what the recording shows, for the alt text. Never run by the build |
 | `validate_site.py` | Validate rendered metadata, heading order, image alternatives, JSON-LD, local resources, links and fragments, and the page size and search index budgets |
-| `romset_recipes.py` | Identify which emulator version an arcade archive matches, and rebuild a pinned archive from ROMs already held |
+| `romset_recipes.py` | Identify which emulator version an arcade archive matches, rebuild a pinned archive from ROMs already held, record a recipe for every archive the collection holds and build any of them from the members it holds (see [Archive recipes](#archive-recipes)) |
+| `archive_build.py` | The archive recipes behind `romset_recipes.py --capture`, `--build` and `--build-all`: capture, member lookup by sha1, verified build |
 | `scraper/romset_dat_importer.py` | Fetch and import per-set recipes from MAME `-listxml` or FBNeo DATs into `recipes/`. `--fetch` alone takes the newest MAME release, and FBNeo DATs are refreshed by git blob sha. `--fetch <revision>` reads FBNeo at that tag, branch or commit |
 | `deterministic_zip.py` | The fixed date pack members carry. Its archive rebuild is no longer called by the pack, which copies each ZIP byte for byte |
-| `torrentzip.py` | Build TorrentZip archives for MAME/FBNeo ROM sets (archive bytes depend only on contents) |
+| `torrentzip.py` | Build TorrentZip archives for MAME/FBNeo ROM sets (archive bytes depend only on contents, with stock zlib), streaming one member at a time |
 | `crypto_verify.py` | 3DS RSA signature and AES crypto verification |
 | `sect233r1.py` | Pure Python ECDSA verification on sect233r1 curve (3DS OTP cert) |
 | `check_buildbot_system.py` | Detect stale data directories by comparing with buildbot |
@@ -1413,7 +1414,7 @@ Further options, by script:
 | `release_record.py` | `--install-dir` (manifests the archives are held against), `--platforms-dir` (registry the full pack names are rebuilt from: any other pack in the directory is refused), `--output` |
 | `release_notes.py` | `--whats-new FILE` (required, appended as written), `--record`, `--platforms-dir`, `--output` |
 | `restore_large_files.py` | `--cache` (downloaded assets), `--gitignore`, `--db` |
-| `romset_recipes.py` | `--dat-recipes PATH`, a snapshot or a directory of them (default `recipes/`) |
+| `romset_recipes.py` | `--dat-recipes PATH`, a snapshot or a directory of them (default `recipes/`). `--archives FILE` (default `recipes/archives.json`), `--output-dir DIR` for `--build` and `--build-all`, `--discard` deletes each build once verified |
 | `split_pack.py` | `--max-size 1900M`, the ceiling per part (K, M, G, default 2G) |
 | `torrentzip.py` | `--check ZIP` reports conformance, `--rebuild ZIP -o OUT` normalizes an archive |
 | `validate_pr.py` | `--changed` takes the BIOS files the branch changes, a folder named checks every file under it, `--place` copies each recognised file from outside `bios/` to its folder and declared name (or its `.variants/` slot), `--markdown`, `--json` |
@@ -1504,15 +1505,25 @@ check compares with.
 Snapshots land in `recipes/`, not `provenance/`: the latter holds dump catalogues
 and a recipe is not one. Versions accumulate rather than replace each other,
 because a platform pins the archive of whichever version its list was built
-against. Only sets a profile or platform references are kept, plus the sets
-holding a dump-catalog entry the collection lacks (crc32 and size, and sha1
-when both sides carry one), so `provenance_report.py --missing` can name the
-set that ships it: MAME's `fm7740sx` carries the FM77AV40SX `DICROM.rom`,
-`EXTSUB.rom` and `KANJI2.rom` that No-Intro lists. A `romof` parent's
-members are merged into the child, and members with no CRC32 are dropped: those
-are undumped and a real romset does not carry them.
+against. Every MAME release with a `-listxml` asset is imported, 0.177 to
+0.289, not one in five. Kept sets:
 
-The store is compacted: 22 MAME versions produce 31,141 entries but only 2,399
+- the archive names a profile or platform references;
+- the archive names the collection holds;
+- the sets holding a dump-catalog entry the collection lacks (crc32 and size,
+  and sha1 when both sides carry one), so `provenance_report.py --missing` can
+  name the set that ships it: MAME's `fm7740sx` carries the FM77AV40SX
+  `DICROM.rom`, `EXTSUB.rom` and `KANJI2.rom` that No-Intro lists;
+- every set whose member list equals an archive the collection keeps under
+  another name. `aga.zip` holds exactly MAME's `isa_aga`: the entry carries
+  the collection's name and keeps the upstream one in `set`. When several sets
+  share one member list (thirty keyboard sets carry the same ROM), the set
+  named like the archive is kept, then the first by name.
+
+A `romof` parent's members are merged into the child, and members with no
+CRC32 are dropped: those are undumped and a real romset does not carry them.
+
+The store is compacted: 113 MAME versions produce 173,985 entries but only 2,936
 distinct recipes, since most sets do not change between releases. Each is kept
 once, `dats` listing every version that agrees and `dat` naming the earliest, so
 an identification reads as "unchanged since that version".
@@ -1527,6 +1538,61 @@ archive whose bytes carry metadata unrelated to its contents cannot be derived
 from ROMs by anyone, so those are reported as unreproducible rather than guessed
 at. Writing is explicit: the pipeline only ever reports.
 
+### Archive recipes
+
+The collection must regenerate every archive it serves from the ROMs it holds
+plus a committed recipe, without a copy of that archive from elsewhere.
+`recipes/archives.json` holds one record per archive in `database.json`,
+`.variants/` copies included, keyed by path:
+
+- the members in archive order, each with its size, crc32 and sha1;
+- the container's sha1, md5 and size;
+- `rebuild`: `exact` when `build_torrentzip()` over the members gives the
+  container byte for byte (proven at capture by rebuilding it), `canonical`
+  when the archive was written by another tool, so the build gives the same
+  members in other container bytes. Directory entries are listed under
+  `directories` and not rebuilt;
+- `sources`: the profile `contents:` blocks and the MAME and FBNeo DAT
+  recipes whose member list equals the archive's, matched by members rather
+  than by name, then `collection`, the archive itself. A DAT is provenance,
+  never the only source;
+- `bytes_pinned` and `pins` when a platform or a profile pins the container
+  hash. A `zipped_file` md5 and a Recalbox composite md5 name no container
+  bytes and are not pins.
+
+```bash
+python scripts/romset_recipes.py --capture                 # pipeline step 1c
+python scripts/romset_recipes.py --build neogeo.zip --output-dir tmp/built
+python scripts/romset_recipes.py --build "bios/SNK/Neo Geo CD/neocdz.zip" --output-dir tmp/built
+python scripts/romset_recipes.py --build-all --output-dir tmp/built --discard
+```
+
+`--capture` reads only the archives whose sha1 changed since the last run,
+and recomputes sources and pins every time. `--build` takes a path or a file
+name (every archive of that name is built, each at its path below `bios/`
+inside the output directory). Each member is read by its sha1: a loose file
+first, then a member of another archive, then the archive's own copy. The
+result is written beside its destination, read back from disk and checked
+against the recipe: the recorded sha1 for an exact recipe, the members for a
+canonical one. A mismatch is built once more, then refused, and nothing is
+moved into place. `--build-all` prints one line per archive (exact, canonical,
+or impossible with the member nothing holds) and exits 1 when an archive
+failed or was impossible. `--discard` deletes each build once verified, which
+proves the whole collection without the disk space.
+
+The summary also counts the archives that read a member from their own copy:
+that member is held nowhere else, and would have to become a loose file before
+the archive bytes could be dropped from `bios/`.
+
+**Deflate implementation.** TorrentZip bytes are the output of zlib's level-9
+deflate (`compressobj(9, DEFLATED, -15)`). Stock zlib 1.2.13 to 1.3.2
+reproduced all 1,561 TorrentZip archives tested in November 2026, on Linux and
+on Windows. zlib-ng does not: python.org CPython 3.14 and 3.15 on Windows
+(zlib-ng 2.2.4) reproduced 479, and zlib-ng 2.3.3 545, the two versions
+disagreeing with each other. `--capture` refuses to run under zlib-ng, and
+`--build` says so and checks every recipe by its members, never claiming an
+exact rebuild. `torrentzip.zlib_ng_version()` is the test.
+
 ## Large files
 
 Files too large for git are stored as assets on the `large-files` GitHub