feat: install the data trees the pack carries

Manifests list each data directory's source archive and the hash of
every file; the installer rebuilds the tree from the archive and keeps
only the files whose hash matches.

install.ps1

@@ -8,7 +8,7 @@ param(
 
 $ErrorActionPreference = "Stop"
 $defaultInstallUrl = "https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-$defaultInstallSha256 = "e57aa03a8f581b6bf02cb1d3161af6da19719425383790da0ed5f761afaf436c"
+$defaultInstallSha256 = "8e0c7bb63c34f4264ae040f8f322e1a3ff47db3393979466daba1238de9dd942"
 $maximumInstallerBytes = 2MB
 $installerExit = 0
 $installer = if ($PSScriptRoot) { Join-Path $PSScriptRoot "install.py" } else { $null }

install.py

@@ -27,6 +27,7 @@ import re
 import shutil
 import stat
 import sys
+import tarfile
 import tempfile
 import threading
 import time
@@ -34,7 +35,8 @@ import urllib.error
 import urllib.parse
 import urllib.request
 import xml.etree.ElementTree as ET
-from collections.abc import Callable
+import zipfile
+from collections.abc import Callable, Iterator
 from pathlib import Path, PurePosixPath
 from typing import BinaryIO, TextIO
 
@@ -88,6 +90,8 @@ MAX_MANIFEST_BYTES = 16 * 1024 * 1024
 MAX_TARGETS_BYTES = 4 * 1024 * 1024
 MAX_MANIFEST_FILES = 100_000
 MAX_DOWNLOAD_SIZE = 1024 * 1024 * 1024
+# A data directory's source archive: PCSX2's source tarball is the largest.
+MAX_DATA_ARCHIVE_SIZE = 4 * 1024 * 1024 * 1024
 MAX_TOTAL_DOWNLOAD_SIZE = 64 * 1024 * 1024 * 1024
 # Why a manifest leaves a file out, as the installer words it. The manifest
 # schema and the generator are held to this list by the tests.
@@ -1507,6 +1511,52 @@ def _check_entry(
     return size, largest
 
 
+_SHA1_RE = re.compile(r"^[0-9a-f]{40}$")
+
+
+def _check_data_directory(entry: object, where: str) -> None:
+    """A data directory: its source archive, its mapping, its file list."""
+    if not isinstance(entry, dict):
+        raise ValueError(f"{where} must be an object")
+    if not isinstance(entry.get("ref"), str) or not entry["ref"]:
+        raise ValueError(f"{where}.ref is missing")
+    if entry.get("dest", "") != "":
+        _safe_relative_path(entry.get("dest"), f"{where}.dest")
+    url = entry.get("url")
+    parsed = urllib.parse.urlparse(url if isinstance(url, str) else "")
+    if not (parsed.scheme == "https" or (
+        parsed.scheme == "http" and (parsed.hostname or "") in _LOOPBACK_HOSTS
+    )):
+        raise ValueError(f"{where}.url must use HTTPS")
+    kind = entry.get("type")
+    if kind not in ("zip", "tarball"):
+        raise ValueError(f"{where}.type must be zip or tarball")
+    strip = entry.get("strip_components", 0)
+    if not isinstance(strip, int) or isinstance(strip, bool) or not 0 <= strip <= 16:
+        raise ValueError(f"{where}.strip_components is invalid")
+    if kind == "tarball" and (
+        not isinstance(entry.get("source_path"), str) or not entry["source_path"]
+    ):
+        raise ValueError(f"{where}.source_path is missing")
+    for field in ("exclude", "systems"):
+        values = entry.get(field, [])
+        if not isinstance(values, list) or not all(isinstance(v, str) for v in values):
+            raise ValueError(f"{where}.{field} is invalid")
+    files = entry.get("files")
+    if not isinstance(files, dict) or not files or len(files) > MAX_MANIFEST_FILES:
+        raise ValueError(f"{where}.files is invalid")
+    for rel, meta in files.items():
+        _safe_relative_path(rel, f"{where}.files")
+        if (
+            not isinstance(meta, dict)
+            or not isinstance(meta.get("sha1"), str)
+            or not _SHA1_RE.match(meta["sha1"])
+            or not isinstance(meta.get("size"), int)
+            or not 0 <= meta["size"] <= MAX_DOWNLOAD_SIZE
+        ):
+            raise ValueError(f"{where}.files[{rel!r}] is invalid")
+
+
 def _validate_manifest(data: object, plat: str) -> dict:
     """Validate the untrusted install-manifest boundary using stdlib only."""
     if not isinstance(data, dict):
@@ -1538,6 +1588,11 @@ def _validate_manifest(data: object, plat: str) -> dict:
         largest_total += largest
         if largest_total > MAX_TOTAL_DOWNLOAD_SIZE:
             raise ValueError("manifest total size exceeds safety limit")
+    data_dirs = data.get("data_directories", [])
+    if not isinstance(data_dirs, list) or len(data_dirs) > MAX_SELECTION_TAGS:
+        raise ValueError("invalid data_directories list")
+    for index, entry in enumerate(data_dirs):
+        _check_data_directory(entry, f"data_directories[{index}]")
 
     declared_total_files = data.get("total_files")
     if declared_total_files is not None and declared_total_files != len(files):
@@ -2670,6 +2725,150 @@ def download_files(
     return failed, interrupted
 
 
+def _fetch_archive(url: str, directory: Path) -> Path:
+    """A data directory's source archive, in *directory*, retried like files."""
+    reason: Exception | None = None
+    for attempt in range(1, MAX_RETRIES + 1):
+        handle = tempfile.NamedTemporaryFile(dir=directory, suffix=".archive", delete=False)
+        try:
+            with handle, urllib.request.urlopen(url, timeout=60) as resp:
+                received = 0
+                while True:
+                    if _stop.is_set():
+                        raise _Interrupted()
+                    chunk = resp.read(1024 * 1024)
+                    if not chunk:
+                        break
+                    received += len(chunk)
+                    if received > MAX_DATA_ARCHIVE_SIZE:
+                        raise ValueError("archive exceeds the size limit")
+                    handle.write(chunk)
+            return Path(handle.name)
+        except urllib.error.HTTPError as exc:
+            Path(handle.name).unlink(missing_ok=True)
+            if exc.code < 500 and exc.code != 429:
+                raise
+            reason = exc
+        except (urllib.error.URLError, OSError) as exc:
+            Path(handle.name).unlink(missing_ok=True)
+            reason = exc
+        except BaseException:
+            Path(handle.name).unlink(missing_ok=True)
+            raise
+        if attempt < MAX_RETRIES:
+            time.sleep(_retry_delay(attempt, None))
+    raise reason if reason else OSError("download failed")
+
+
+def _archive_members(
+    archive: Path, entry: dict
+) -> Iterator[tuple[str, Callable[[], BinaryIO]]]:
+    """(path in the tree, opener) for each file the archive maps into it.
+
+    The rules refresh_data_dirs applies to build the cache: a ZIP drops
+    strip_components leading parts, a tarball keeps what sits under
+    source_path, and a first part named in exclude is left out.
+    """
+    exclude = set(entry.get("exclude") or [])
+    if entry["type"] == "zip":
+        strip = entry.get("strip_components", 0)
+        with zipfile.ZipFile(archive) as zf:
+            for info in zf.infolist():
+                if info.is_dir():
+                    continue
+                parts = info.filename.split("/")
+                if ".." in parts or info.filename.startswith("/"):
+                    continue
+                if strip:
+                    if len(parts) <= strip:
+                        continue
+                    parts = parts[strip:]
+                if parts[0] in exclude:
+                    continue
+                yield "/".join(parts), (lambda info=info: zf.open(info))
+        return
+    prefix = entry["source_path"].rstrip("/") + "/"
+    with tarfile.open(archive, "r:gz") as tf:
+        for member in tf:
+            if not member.isfile() or not member.name.startswith(prefix):
+                continue
+            rel = member.name[len(prefix):]
+            if not rel or ".." in rel.split("/") or rel.split("/")[0] in exclude:
+                continue
+            yield rel, (lambda member=member: tf.extractfile(member))
+
+
+def data_directory_state(entry: dict, bios_path: Path) -> tuple[Path, dict[str, dict], int]:
+    """(tree root, files missing or wrong, files already right)."""
+    root = _destination_path(bios_path, entry["dest"]) if entry.get("dest") else bios_path.resolve()
+    needed: dict[str, dict] = {}
+    present = 0
+    for rel, meta in entry["files"].items():
+        target = _destination_path(root, rel)
+        if (
+            target.is_file()
+            and target.stat().st_size == meta["size"]
+            and _sha1_file(target) == meta["sha1"]
+        ):
+            present += 1
+        else:
+            needed[rel] = meta
+    return root, needed, present
+
+
+def install_data_directory(entry: dict, bios_path: Path) -> dict:
+    """Rebuild one data tree from its source archive, file by file verified.
+
+    Returns counts: present, installed, changed (the source no longer holds
+    the recorded bytes), absent (not in the archive), and error, the reason
+    the archive could not be read.
+    """
+    root, needed, present = data_directory_state(entry, bios_path)
+    counts = {"present": present, "installed": 0, "changed": 0, "absent": 0, "error": ""}
+    if not needed:
+        return counts
+    try:
+        root.mkdir(parents=True, exist_ok=True)
+        with tempfile.TemporaryDirectory(dir=root) as scratch:
+            archive = _fetch_archive(entry["url"], Path(scratch))
+            for rel, opener in _archive_members(archive, entry):
+                meta = needed.pop(rel, None)
+                if meta is None:
+                    continue
+                target = _destination_path(root, rel)
+                target.parent.mkdir(parents=True, exist_ok=True)
+                digest = hashlib.sha1()
+                size = 0
+                with opener() as src, tempfile.NamedTemporaryFile(
+                    dir=target.parent, prefix=f".{target.name}.", suffix=".part",
+                    delete=False,
+                ) as out:
+                    for chunk in iter(lambda: src.read(1024 * 1024), b""):
+                        digest.update(chunk)
+                        size += len(chunk)
+                        out.write(chunk)
+                if digest.hexdigest() == meta["sha1"] and size == meta["size"]:
+                    os.replace(out.name, target)
+                    counts["installed"] += 1
+                else:
+                    Path(out.name).unlink(missing_ok=True)
+                    counts["changed"] += 1
+    except (urllib.error.URLError, OSError, ValueError, zipfile.BadZipFile,
+            tarfile.TarError) as exc:
+        counts["error"] = str(getattr(exc, "reason", None) or exc)
+    counts["absent"] = len(needed)
+    return counts
+
+
+def select_data_directories(manifest: dict, files: list[dict], narrowed: bool) -> list[dict]:
+    """The data trees this install keeps: all, or those of a kept system."""
+    entries = manifest.get("data_directories") or []
+    if not narrowed:
+        return list(entries)
+    kept = {system for f in files for system in entry_systems(f)}
+    return [e for e in entries if kept & set(e.get("systems") or [])]
+
+
 def do_standalone_copies(
     manifest: dict, bios_path: Path, os_type: str,
     extra_dirs: dict[str, Path] | None = None,
@@ -3516,16 +3715,33 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
         # Mismatched files need re-download
         to_download.extend(mismatched)
 
+        narrowed = bool(
+            target or args.system or args.core or args.region or args.installed_cores
+            or len(files) != len(manifest.get("files", [])) + len(
+                variants if case_kept else []
+            )
+        )
+        data_dirs = select_data_directories(manifest, files, narrowed)
+
         if args.check:
             if to_download:
                 print(f"\n  {len(to_download)} files need downloading.")
                 check_pending += len(to_download)
             else:
                 print("\n  All files up to date.")
+            for entry in data_dirs:
+                _root, needed, present = data_directory_state(entry, bios_path)
+                if needed:
+                    print(f"  {entry['ref']}: {present}/{len(entry['files'])} data files "
+                          f"present, {len(needed)} to fetch")
+                    check_pending += len(needed)
             continue
 
         if args.dry_run:
             _print_plan(to_download, bios_path, term)
+            for entry in data_dirs:
+                print(f"  and the {entry['ref']} tree ({len(entry['files'])} files) "
+                      f"from {entry['url']}")
             continue
 
         if to_download:
@@ -3577,6 +3793,27 @@ def _install(args: argparse.Namespace, real_stdout: TextIO) -> None:
 
         total_up_to_date += len(up_to_date)
 
+        # Data trees the pack carries beside the files: Dolphin's Sys,
+        # PPSSPP's assets. Fetched from their source archive, each file
+        # checked against the hash the list records.
+        if data_dirs and not total_interrupted:
+            print("\nData directories:")
+            for entry in data_dirs:
+                counts = install_data_directory(entry, bios_path)
+                total_downloaded += counts["installed"]
+                total_up_to_date += counts["present"]
+                line = f"  {entry['ref']}: {counts['installed']} installed, " \
+                       f"{counts['present']} already up to date"
+                if counts["changed"] or counts["absent"]:
+                    line += (f", {counts['changed'] + counts['absent']} no longer in "
+                             "the source (the pack has them)")
+                print(line)
+                if counts["error"]:
+                    print(f"  {entry['ref']}: could not read {entry['url']}: "
+                          f"{counts['error']}", file=sys.stderr)
+                    failures.append((entry["ref"], counts["error"]))
+                    total_errors += 1
+
         # Standalone copies
         if (
             manifest.get("standalone_copies")

install.sh

@@ -18,7 +18,7 @@ esac
 TEMP_INSTALLER=""
 TEMP_DIRECTORY=""
 DEFAULT_INSTALL_URL="https://raw.githubusercontent.com/Abdess/retrobios/main/install.py"
-DEFAULT_INSTALL_SHA256="e57aa03a8f581b6bf02cb1d3161af6da19719425383790da0ed5f761afaf436c"
+DEFAULT_INSTALL_SHA256="8e0c7bb63c34f4264ae040f8f322e1a3ff47db3393979466daba1238de9dd942"
 MAX_INSTALLER_BYTES=2097152
 
 cleanup() {

schemas/install-manifest.schema.json

@@ -303,6 +303,81 @@
       "items": {
         "$ref": "#/$defs/file"
       }
+    },
+    "data_directories": {
+      "description": "Data trees the pack carries, rebuilt by the installer from their source archive; each file is kept only when its sha1 matches.",
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": [
+          "ref",
+          "dest",
+          "url",
+          "type",
+          "systems",
+          "files"
+        ],
+        "properties": {
+          "ref": {
+            "type": "string"
+          },
+          "dest": {
+            "type": "string",
+            "pattern": "^(?!/)(?!.*(?:^|/)\\.\\.(?:/|$)).*$"
+          },
+          "url": {
+            "type": "string",
+            "pattern": "^https://"
+          },
+          "type": {
+            "enum": [
+              "zip",
+              "tarball"
+            ]
+          },
+          "strip_components": {
+            "type": "integer",
+            "minimum": 0
+          },
+          "source_path": {
+            "type": "string"
+          },
+          "exclude": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "systems": {
+            "type": "array",
+            "items": {
+              "type": "string"
+            }
+          },
+          "files": {
+            "type": "object",
+            "additionalProperties": {
+              "type": "object",
+              "additionalProperties": false,
+              "required": [
+                "sha1",
+                "size"
+              ],
+              "properties": {
+                "sha1": {
+                  "type": "string",
+                  "pattern": "^[a-f0-9]{40}$"
+                },
+                "size": {
+                  "type": "integer",
+                  "minimum": 0
+                }
+              }
+            }
+          }
+        }
+      }
     }
   },
   "$defs": {

scripts/generate_pack.py

@@ -373,10 +373,14 @@ def _data_directory_members(
     seen_destinations: set,
     seen_lower: set,
     seen_parents: set,
+    refs_out: dict | None = None,
 ) -> Iterator[tuple[str, str]]:
     """Yield (source, destination) for the cached data directories a
     platform's systems declare.
 
+    ``refs_out``, when given, maps each yielded destination to
+    (registry key, system id, directory destination), for the manifest.
+
     These are not BIOS and carry no hash of their own: they are whole
     trees an emulator reads, refreshed from upstream, so what ships is
     whatever the cache holds. The pack and the install manifest both read
@@ -424,6 +428,8 @@ def _data_directory_members(
                         _register_path(full, seen_destinations, seen_parents)
                         if case_insensitive:
                             seen_lower.add(full.lower())
+                        if refs_out is not None:
+                            refs_out[full] = (ref_key, sys_id, dd_prefix)
                         yield src, full
 
 
@@ -4130,16 +4136,19 @@ def generate_manifest(
         ):
             omission["declared_in"] = sorted(declared)
 
-    # Phase 3: data directories. The installer does not fetch them, so they
-    # stay out of the file list; the pack carries them, so they count toward
-    # what an extraction shows.
-    data_sizes = pack_only_sizes + [
-        os.path.getsize(src)
-        for src, _dest in _data_directory_members(
-            pack_systems, data_registry, platform_name, base_dest,
-            case_insensitive, seen_destinations, seen_lower, seen_parents,
-        )
-    ]
+    # Phase 3: data directories. They are not files of the collection: the
+    # installer fetches each from its source archive and keeps the members
+    # whose hash the list below records, the same tree the pack carries.
+    data_refs: dict[str, tuple[str, str, str]] = {}
+    data_members = list(_data_directory_members(
+        pack_systems, data_registry, platform_name, base_dest,
+        case_insensitive, seen_destinations, seen_lower, seen_parents,
+        data_refs,
+    ))
+    data_sizes = pack_only_sizes + [os.path.getsize(src) for src, _dest in data_members]
+    data_directories = _manifest_data_directories(
+        data_members, data_refs, data_registry or {}, base_dest
+    )
 
     now = _build_timestamp(db)
     core_names, system_cores = _manifest_core_index(
@@ -4179,6 +4188,8 @@ def generate_manifest(
         "system_cores": system_cores,
         "files": manifest_files,
     }
+    if data_directories:
+        result["data_directories"] = data_directories
     if case_insensitive and fold_case is None and any(
         d.get("os") not in CASE_FOLDING_OS for d in detect
     ):
@@ -4204,6 +4215,53 @@ def generate_manifest(
     return result
 
 
+def _manifest_data_directories(
+    members: list[tuple[str, str]],
+    refs: dict[str, tuple[str, str, str]],
+    registry: dict,
+    base_dest: str,
+) -> list[dict]:
+    """Each data directory the pack carries, as the installer rebuilds it.
+
+    The source archive and the rule that maps its members onto the tree
+    (strip_components for a ZIP, source_path for a tarball, exclude) come
+    from _data_dirs.yml, the same refresh_data_dirs applies; every file
+    carries the sha1 and size the cache holds, so a member the source
+    changed since is refused rather than installed.
+    """
+    grouped: dict[tuple[str, str], dict] = {}
+    for src, full in members:
+        ref, system, prefix = refs[full]
+        entry = registry.get(ref, {})
+        key = (ref, prefix)
+        group = grouped.get(key)
+        if group is None:
+            version = str(entry.get("version", "master"))
+            dest = prefix
+            if base_dest and (dest == base_dest or dest.startswith(base_dest + "/")):
+                dest = dest[len(base_dest):].lstrip("/")
+            group = grouped[key] = {
+                "ref": ref,
+                "dest": dest,
+                "url": str(entry.get("source_url", "")).format(version=version),
+                "type": "zip" if entry.get("source_type") == "zip" else "tarball",
+                "systems": [],
+                "files": {},
+            }
+            if group["type"] == "zip":
+                group["strip_components"] = int(entry.get("strip_components", 0))
+            else:
+                group["source_path"] = str(entry.get("source_path", "")).format(version=version)
+            if entry.get("exclude"):
+                group["exclude"] = [str(e) for e in entry["exclude"]]
+        if system not in group["systems"]:
+            group["systems"].append(system)
+        rel = full[len(prefix):].lstrip("/") if prefix else full
+        hashes = compute_hashes(src, frozenset({"sha1"}))
+        group["files"][rel] = {"sha1": hashes["sha1"], "size": os.path.getsize(src)}
+    return [grouped[key] for key in sorted(grouped)]
+
+
 def _manifest_core_index(
     config: dict,
     profiles: dict[str, dict],

tests/test_install.py

@@ -2475,5 +2475,88 @@ class ACutFileListIsRetried(unittest.TestCase):
             install._read_limited_json(self._Response(b'{"a"', 10), 1000, "manifest")
 
 
+
+class DataDirectoriesAreInstalled(unittest.TestCase):
+    """The README said the installer does what the pack does; for RetroArch
+    it left out 2,640 data files (Dolphin's Sys, PPSSPP's assets) and
+    --check still said "All files up to date"."""
+
+    def _zip(self, root: Path) -> Path:
+        import zipfile
+
+        archive = root / "Dolphin.zip"
+        with zipfile.ZipFile(archive, "w") as zf:
+            zf.writestr("Dolphin/Sys/GC/font.bin", b"font")
+            zf.writestr("Dolphin/Sys/GC/dsp.bin", b"new bytes upstream")
+            zf.writestr("Dolphin/Themes/x.png", b"theme")
+        return archive
+
+    def _entry(self) -> dict:
+        return {
+            "ref": "dolphin-sys", "dest": "dolphin-emu", "url": "https://example.invalid/D.zip",
+            "type": "zip", "strip_components": 1, "exclude": ["Themes"],
+            "systems": ["nintendo-gamecube"],
+            "files": {
+                "Sys/GC/font.bin": {"sha1": hashlib.sha1(b"font").hexdigest(), "size": 4},
+                "Sys/GC/dsp.bin": {"sha1": hashlib.sha1(b"old").hexdigest(), "size": 3},
+                "Sys/GC/gone.bin": {"sha1": "0" * 40, "size": 1},
+            },
+        }
+
+    def test_verified_members_land_the_rest_are_said(self):
+        work = Path(_tempdir(self))
+        archive = self._zip(work)
+        bios = work / "system"
+        bios.mkdir()
+
+        def fetch(url, directory):
+            copy = Path(directory) / "a.zip"
+            shutil.copy(archive, copy)
+            return copy
+
+        with unittest.mock.patch.object(install, "_fetch_archive", fetch):
+            counts = install.install_data_directory(self._entry(), bios)
+            again = install.install_data_directory(self._entry(), bios)
+        self.assertEqual((counts["installed"], counts["changed"], counts["absent"]), (1, 1, 1))
+        self.assertEqual((bios / "dolphin-emu" / "Sys" / "GC" / "font.bin").read_bytes(), b"font")
+        self.assertFalse((bios / "dolphin-emu" / "Sys" / "GC" / "dsp.bin").exists())
+        self.assertFalse((bios / "dolphin-emu" / "Themes").exists())
+        self.assertEqual(again["present"], 1)
+        self.assertEqual(sorted(p.name for p in (bios / "dolphin-emu").iterdir()), ["Sys"])
+
+    def test_a_tarball_maps_through_source_path(self):
+        import io as _io
+        import tarfile
+
+        work = Path(_tempdir(self))
+        archive = work / "src.tar.gz"
+        with tarfile.open(archive, "w:gz") as tf:
+            data = b"index"
+            info = tarfile.TarInfo("pcsx2-v1/bin/resources/GameIndex.yaml")
+            info.size = len(data)
+            tf.addfile(info, _io.BytesIO(data))
+        entry = {"type": "tarball", "source_path": "pcsx2-v1/bin/resources"}
+        names = [rel for rel, _open in install._archive_members(archive, entry)]
+        self.assertEqual(names, ["GameIndex.yaml"])
+
+    def test_the_trust_boundary(self):
+        entry = self._entry()
+        install._check_data_directory(entry, "d")
+        for bad in ({"url": "http://example.com/x.zip"}, {"dest": "../up"},
+                    {"files": {"../x": {"sha1": "0" * 40, "size": 1}}},
+                    {"type": "rar"}):
+            with self.subTest(bad=bad), self.assertRaises(ValueError):
+                install._check_data_directory({**entry, **bad}, "d")
+
+    def test_the_published_retroarch_list_carries_them(self):
+        manifest = json.loads((REPO_ROOT / "install" / "retroarch.json").read_text())
+        dirs = manifest.get("data_directories") or []
+        files = sum(len(d["files"]) for d in dirs)
+        self.assertEqual(
+            manifest["pack_files"] - manifest["total_files"] - files,
+            2, "the pack's two documents are the only files the installer lacks",
+        )
+
+
 if __name__ == "__main__":
     unittest.main()

wiki/installer.md

@@ -420,6 +420,18 @@ Entries the collection cannot satisfy are not silently dropped: they travel in
 anything. `not_found` means no file in the collection matches the entry, which
 is what the [gap analysis](../gaps.md) tracks.
 
+Some cores read a whole tree of data rather than a few files: Dolphin's `Sys/`,
+PPSSPP's assets, blueMSX's machines. These are not files of the collection, so
+`data_directories` names each tree's source archive (the libretro buildbot, an
+upstream release) and the sha1 and size of every file the pack carries. The
+installer downloads the archive once, maps its members the way the pack was
+built (`strip_components` for a ZIP, `source_path` for a tarball, `exclude`),
+and writes a file only when its hash matches the list. A file the source has
+changed since is left out and counted, and the pack still holds it. `--check`
+counts the data files missing, `--dry-run` names each tree it would fetch, and
+narrowing with `--system`, `--core` or `--target` keeps only the trees of the
+systems that stay.
+
 ## Trust boundary
 
 The manifest is fetched over the network and treated as untrusted input. Every