feat: state how many files each pack holds

schemas/install-manifest.schema.json

@@ -82,6 +82,16 @@
       "type": "integer",
       "minimum": 0
     },
+    "pack_files": {
+      "description": "Files the platform's pack holds once extracted: this list, the data directories and the pack's own documents.",
+      "type": "integer",
+      "minimum": 0
+    },
+    "pack_size": {
+      "description": "Bytes those files occupy once extracted.",
+      "type": "integer",
+      "minimum": 0
+    },
     "total_omitted": {
       "type": "integer",
       "minimum": 0

scripts/generate_pack.py

@@ -288,25 +288,28 @@ def download_external(file_entry: dict, dest_path: str) -> bool:
 
 
 
-def _pack_data_directories(
-    zf,
+# What the builder writes into a platform pack beside the files it carries.
+PACK_DOCUMENTS = ("README.txt", "manifest.json")
+
+
+def _data_directory_members(
     pack_systems: dict,
     data_registry: dict | None,
     platform_name: str,
     base_dest: str,
-    flatten: bool,
     case_insensitive: bool,
     seen_destinations: set,
     seen_lower: set,
     seen_parents: set,
-) -> int:
-    """Add the cached data directories the platform's systems declare.
+):
+    """Yield (source, destination) for the cached data directories a
+    platform's systems declare.
 
     These are not BIOS and carry no hash of their own: they are whole
     trees an emulator reads, refreshed from upstream, so what ships is
-    whatever the cache holds. Returns how many files were added.
+    whatever the cache holds. The pack and the install manifest both read
+    this, the first to write the files and the second to count them.
     """
-    added = 0
     # Data directories from _data_dirs.yml
     for sys_id, system in sorted(pack_systems.items()):
         for dd in system.get("data_directories", []):
@@ -345,8 +348,29 @@ def _pack_data_directories(
                     _register_path(full, seen_destinations, seen_parents)
                     if case_insensitive:
                         seen_lower.add(full.lower())
-                    _add_pack_member(zf, src, _flat(full, base_dest, flatten))
-                    added += 1
+                    yield src, full
+
+
+def _pack_data_directories(
+    zf,
+    pack_systems: dict,
+    data_registry: dict | None,
+    platform_name: str,
+    base_dest: str,
+    flatten: bool,
+    case_insensitive: bool,
+    seen_destinations: set,
+    seen_lower: set,
+    seen_parents: set,
+) -> int:
+    """Add the data directories to the pack. Returns how many files went in."""
+    added = 0
+    for src, full in _data_directory_members(
+        pack_systems, data_registry, platform_name, base_dest,
+        case_insensitive, seen_destinations, seen_lower, seen_parents,
+    ):
+        _add_pack_member(zf, src, _flat(full, base_dest, flatten))
+        added += 1
     return added
 
 
@@ -2880,6 +2904,7 @@ def generate_manifest(
     target_name: str | None = None,
     offline: bool | None = None,
     required_only: bool = False,
+    data_registry: dict | None = None,
 ) -> dict:
     """Generate a JSON manifest for a platform (same resolution as generate_pack).
 
@@ -3100,7 +3125,18 @@ def generate_manifest(
         omitted_by_destination, record_omission, required_only,
     )
 
-    # No phase 3 (data directories) -skipped for manifest
+    # Phase 3: data directories. The installer does not fetch them, so they
+    # stay out of the file list; the pack carries them, so they count toward
+    # what an extraction shows.
+    if data_registry is None:
+        data_registry = load_data_dir_registry(platforms_dir)
+    data_sizes = [
+        os.path.getsize(src)
+        for src, _dest in _data_directory_members(
+            pack_systems, data_registry, platform_name, base_dest,
+            case_insensitive, seen_destinations, seen_lower, seen_parents,
+        )
+    ]
 
     now = _build_timestamp(db)
 
@@ -3122,6 +3158,8 @@ def generate_manifest(
         "standalone_copies": standalone_copies,
         "total_files": len(manifest_files),
         "total_size": total_size,
+        "pack_files": len(manifest_files) + len(data_sizes) + len(PACK_DOCUMENTS),
+        "pack_size": total_size + sum(data_sizes),
         "total_omitted": len(omitted_by_destination),
         "omitted_files": sorted(
             omitted_by_destination.values(), key=lambda entry: entry["dest"]

scripts/generate_readme.py

@@ -97,10 +97,11 @@ def compute_coverage(
 def manifest_totals(
     platform_name: str, install_dir: str = "install"
 ) -> tuple[int | None, int | None]:
-    """Files and bytes a platform's pack ships, from its install manifest.
+    """Files and bytes a platform's pack holds once extracted.
 
-    The manifest is written when packs are built, so it reflects the real
-    pack contents (platform list, core complement and data directories).
+    The install manifest lists what the installer fetches, and states apart
+    what the pack adds to that: the data directories and its own documents.
+    The pack figures are the ones a reader can check against an extraction.
     Returns (None, None) when no manifest exists yet.
     """
     path = os.path.join(install_dir, f"{platform_name}.json")
@@ -111,7 +112,53 @@ def manifest_totals(
             manifest = json.load(f)
     except (json.JSONDecodeError, OSError):
         return None, None
-    return manifest.get("total_files"), manifest.get("total_size")
+    return (
+        manifest.get("pack_files", manifest.get("total_files")),
+        manifest.get("pack_size", manifest.get("total_size")),
+    )
+
+
+def download_table(
+    coverages: dict,
+    archived: set[str],
+    extract_paths: dict[str, str],
+    install_dir: str = "install",
+) -> list[str]:
+    """One row per platform: what its pack holds and where it extracts."""
+    lines = [
+        "| Platform | Files | Extracted size | Extract to | Download |",
+        "|----------|------:|---------------:|-----------|----------|",
+    ]
+    for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]):
+        display = cov["platform"]
+        if name in archived:
+            display = f"{display} *"
+        files, size = manifest_totals(name, install_dir)
+        lines.append(
+            f"| {display} | {f'{files:,}' if files else '-'} |"
+            f" {format_size(size) if size else '-'} |"
+            f" {extract_paths.get(cov['platform'], '')} |"
+            f" [Download]({RELEASE_URL}) |"
+        )
+    return lines
+
+
+def collection_line(total_files: int, comp: dict) -> str:
+    """The size of the whole collection, said as such.
+
+    Printed bare beside the download links it read as the content of one
+    pack, and a complete extraction looked like a broken one.
+    """
+    return (
+        f"- **{total_files:,} files in the collection**, each with its SHA1,"
+        " MD5, SHA256, CRC32 and Adler-32 fingerprints:"
+        f" {comp['systems']['files']:,} system files,"
+        f" {comp['arcade']['files']:,} arcade ROM sets,"
+        f" {comp['game_data']['files']:,} game and engine data files."
+        " That is every platform and emulator together, so no pack holds"
+        " them all: each carries what its own emulators load, counted in"
+        " the download table"
+    )
 
 
 def format_size(size: int) -> str:
@@ -288,7 +335,11 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
         "## Download BIOS packs",
         "",
         "One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file, and `--target` narrows it to one machine; for a region or a bare minimum, build your own pack below.",
-        "The size is what the files occupy once extracted; the ZIP itself"
+        "Files is how many the pack holds once extracted, the figure a file"
+        " manager shows for the folder. It differs from one platform to the"
+        " next because a pack carries only what that platform's emulators"
+        " load."
+        " The size is what the files occupy once extracted; the ZIP itself"
         " downloads smaller, and anything over 2 GB arrives split into"
         " `.zip.001`, `.zip.002` volumes. Open the `.001` with 7-Zip or"
         " PeaZip, or join them first"
@@ -301,8 +352,6 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
         "(https://abdess.github.io/retrobios/wiki/release-process/"
         "#verifying-a-release).",
         "",
-        "| Platform | Extracted size | Extract to | Download |",
-        "|----------|---------------:|-----------|----------|",
     ]
 
     # Where the pack itself is extracted, which is not always the BIOS folder:
@@ -328,16 +377,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
         if entry.get("status") == "archived"
     }
 
-    for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]):
-        display = cov["platform"]
-        if name in archived:
-            display = f"{display} *"
-        path = extract_paths.get(cov["platform"], "")
-        _, size = manifest_totals(name)
-        size_cell = format_size(size) if size else "-"
-        lines.append(
-            f"| {display} | {size_cell} | {path} | [Download]({RELEASE_URL}) |"
-        )
+    lines.extend(download_table(coverages, archived, extract_paths))
 
     if archived:
         lines.extend(
@@ -384,10 +424,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
             f"- **{len(coverages)} platforms** supported with platform-specific verification",
             f"- **{emulator_count} emulators** profiled from source (RetroArch cores + standalone)",
             f"- **{len(system_ids)} systems** handled by those emulators (NES, SNES, PlayStation, Saturn, Dreamcast, ...)",
-            f"- **{total_files:,} files**, each with its SHA1, MD5, SHA256, CRC32 and Adler-32 fingerprints:"
-            f" {comp['systems']['files']:,} system files,"
-            f" {comp['arcade']['files']:,} arcade ROM sets,"
-            f" {comp['game_data']['files']:,} game and engine data files",
+            collection_line(total_files, comp),
             *_catalog_matched_line(db),
             f"- **{size_mb:.0f} MB** total collection size",
             "",

scripts/generate_site.py

@@ -235,7 +235,7 @@ def generate_home(
         "",
         '<div class="rb-stat" markdown>',
         f'<span class="rb-stat-value">{total_files:,}</span>',
-        '<span class="rb-stat-label">Files</span>',
+        f'<span class="rb-stat-label">{COLLECTION_FILES_LABEL}</span>',
         "</div>",
         "",
         '<div class="rb-stat" markdown>',
@@ -264,8 +264,8 @@ def generate_home(
         [
             "## Platforms",
             "",
-            "| Icon | Platform | Files | Checked by | Download |",
-            "|---|----------|-------|-----------|----------|",
+            "| Icon | Platform | Files in the pack | Checked by | Download |",
+            "|---|----------|------:|-----------|----------|",
         ]
     )
 
@@ -281,7 +281,7 @@ def generate_home(
 
         lines.append(
             f"| {logo_md} | [{display}](platforms/{name}.md) | "
-            f"{cov['present']:,} | {mode_label} | "
+            f"{manifest_totals(name)[0] or cov['present']:,} | {mode_label} | "
             f"[Pack]({RELEASE_URL}){{ .md-button .md-button--primary }} |"
         )
 
@@ -410,13 +410,19 @@ def compute_stats(db: dict, coverages: dict, profiles: dict) -> dict:
     }
 
 
+# The home tile counts the whole collection. Labelled "Files" above a table
+# of pack buttons, it read as what one pack contains.
+COLLECTION_FILES_LABEL = "Files collected"
+
+
 def composition_sentence(db: dict) -> str:
     comp = compute_composition(db)
     return (
-        f"Of these files, {comp['systems']['files']:,} are console and "
+        f"The collection holds {comp['systems']['files']:,} console and "
         f"computer system files, {comp['arcade']['files']:,} arcade ROM sets "
         f"(`Arcade/`), and {comp['game_data']['files']:,} game and engine "
-        "data (the `RPG Maker/` and `ScummVM/` trees)."
+        "data files (the `RPG Maker/` and `ScummVM/` trees). No pack holds "
+        "them all: each carries what its platform's emulators load."
     )
 
 
@@ -1148,8 +1154,8 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st
         f"{len(coverages)} supported platforms with "
         f"{total_present:,} verified files.",
         "",
-        "| Platform | Files | Checked by | Status | Download |",
-        "|----------|-------|-----------|--------|----------|",
+        "| Platform | On its BIOS list | Files in the pack | Checked by | Status | Download |",
+        "|----------|------:|------:|-----------|--------|----------|",
     ]
 
     mode_labels = {
@@ -1175,13 +1181,17 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st
 
         lines.append(
             f"| [{display}]({name}.md) | "
-            f"{cov['present']:,} | {mode_html} | {status_html} | "
+            f"{cov['present']:,} | {manifest_totals(name)[0] or cov['present']:,} | "
+            f"{mode_html} | {status_html} | "
             f"[Pack]({RELEASE_URL}){{ .md-button .md-button--primary }} |"
         )
 
     lines.extend(
         [
             "",
+            "The BIOS list is what the platform publishes. The pack adds the "
+            "files its emulators load and the data directories they read, "
+            "which is the count an extraction shows. "
             "Checked by is the test each platform runs on its own, replicated "
             "from its source code. "
             "[How each mode works](../wiki/verification-modes.md).",
@@ -2232,8 +2242,9 @@ def _availability_check(db: dict, data_names):
             for md5_val in parse_md5_list(md5_raw):
                 if by_md5.get(md5_val):
                     return True
-        sha1 = f.get("sha1", "")
-        if sha1 and sha1 in db_files:
+        # A profile lists several sha1 when the code accepts several dumps.
+        sha1 = f.get("sha1") or []
+        if any(value in db_files for value in ([sha1] if isinstance(sha1, str) else sha1)):
             return True
         return False
 

tests/test_pack_counts.py

@@ -0,0 +1,217 @@
+"""How many files a pack holds, said where the pack is downloaded.
+
+The release notes opened on the size of the whole collection, 10 330 files,
+right after "one pack per platform", and nothing anywhere gave the count of
+one pack. A reader who extracted the RetroArch pack and found 4 525 files
+concluded that more than half was missing, when the pack was complete. The
+install manifest counted 1 872 for the same pack, because it lists what the
+installer fetches and leaves out the data directories a pack also carries.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import sys
+import tempfile
+import unittest
+import zipfile
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+sys.path.insert(0, str(REPO_ROOT / "scripts"))
+
+import common  # noqa: E402
+import generate_db  # noqa: E402
+import generate_pack as builder  # noqa: E402
+import generate_readme  # noqa: E402
+import generate_site  # noqa: E402
+
+PROFILE = """\
+emulator: Demo
+type: libretro
+display_name: Demo
+systems: [demo-system]
+cores: [demo]
+files:
+  - name: "extra.bin"
+    required: false
+"""
+
+PLATFORM = {
+    "platform": "Demo",
+    "verification_mode": "existence",
+    "base_destination": "",
+    "cores": ["demo"],
+    "systems": {
+        "demo-system": {
+            "files": [{"name": "boot.bin", "destination": "boot.bin"}],
+            "data_directories": [{"ref": "demo-data", "destination": "demo"}],
+        },
+    },
+}
+
+
+class PackCountFixture(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.root = Path(self._tmp.name)
+        self.bios = self.root / "bios"
+        self.emulators = self.root / "emulators"
+        self.platforms = self.root / "platforms"
+        self.data = self.root / "data" / "demo-data"
+        for directory in (self.bios, self.emulators, self.platforms, self.data):
+            directory.mkdir(parents=True)
+        files: dict[str, dict] = {}
+        for relative, payload in (
+            ("SystemA/boot.bin", b"boot"),
+            ("SystemA/extra.bin", b"extra file"),
+        ):
+            path = self.bios / relative
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_bytes(payload)
+            sha1 = hashlib.sha1(payload).hexdigest()
+            files[sha1] = {
+                "path": str(path),
+                "name": path.name,
+                "size": len(payload),
+                "sha1": sha1,
+                "md5": hashlib.md5(payload).hexdigest(),
+                "sha256": hashlib.sha256(payload).hexdigest(),
+                "crc32": f"{len(payload):08x}",
+            }
+        self.db = {"files": files, "indexes": generate_db.build_indexes(files, {})}
+        for name, payload in (("a.txt", b"one"), ("sub/b.txt", b"three"), ("c.txt", b"")):
+            (self.data / name).parent.mkdir(parents=True, exist_ok=True)
+            (self.data / name).write_bytes(payload)
+        self.registry = {"demo-data": {"local_cache": str(self.data)}}
+        (self.platforms / "demo.yml").write_text(yaml.dump(PLATFORM))
+        (self.platforms / "_registry.yml").write_text(
+            yaml.dump({"platforms": {"demo": {"status": "active"}}})
+        )
+        (self.emulators / "demo.yml").write_text(PROFILE)
+        common._emulator_profiles_cache.clear()
+        self.profiles = common.load_emulator_profiles(str(self.emulators))
+
+    def tearDown(self):
+        common._emulator_profiles_cache.clear()
+        self._tmp.cleanup()
+
+    def _manifest(self) -> dict:
+        return builder.generate_manifest(
+            "demo", str(self.platforms), self.db, str(self.bios),
+            str(self.platforms / "_registry.yml"),
+            emulators_dir=str(self.emulators), emu_profiles=self.profiles,
+            offline=True, data_registry=self.registry,
+        )
+
+
+class ManifestStatesWhatThePackHolds(PackCountFixture):
+    def test_the_count_is_the_one_an_extraction_shows(self):
+        out = self.root / "dist"
+        out.mkdir()
+        zip_path = builder.generate_pack(
+            "demo", str(self.platforms), self.db, str(self.bios), str(out),
+            include_extras=True, emulators_dir=str(self.emulators),
+            emu_profiles=self.profiles, data_registry=self.registry,
+            offline=True,
+        )
+        with zipfile.ZipFile(zip_path) as archive:
+            members = archive.infolist()
+        # manifest.json joins the archive when the pack is finalized.
+        names = {member.filename for member in members} | {"manifest.json"}
+        manifest = self._manifest()
+        self.assertEqual(manifest["pack_files"], len(names))
+        self.assertEqual(
+            manifest["pack_size"],
+            sum(
+                member.file_size
+                for member in members
+                if member.filename not in builder.PACK_DOCUMENTS
+            ),
+        )
+
+    def test_the_installer_count_is_left_alone(self):
+        """install.py refuses a manifest whose total is not its list."""
+        manifest = self._manifest()
+        self.assertEqual(manifest["total_files"], len(manifest["files"]))
+        self.assertEqual(manifest["total_files"], 2)
+        self.assertEqual(manifest["pack_files"], 2 + 3 + len(builder.PACK_DOCUMENTS))
+
+    def test_the_manifest_schema_accepts_the_two_figures(self):
+        from jsonschema import Draft202012Validator
+
+        schema = json.loads(
+            (REPO_ROOT / "schemas" / "install-manifest.schema.json").read_text()
+        )
+        # The fixture stores its files under a temporary directory, which
+        # no entry of a real manifest does: only the document itself is judged.
+        errors = [
+            error.message
+            for error in Draft202012Validator(schema).iter_errors(self._manifest())
+            if not error.absolute_path
+        ]
+        self.assertEqual(errors, [])
+
+
+class ReadmeTotals(unittest.TestCase):
+    def setUp(self):
+        self._tmp = tempfile.TemporaryDirectory()
+        self.install = Path(self._tmp.name)
+
+    def tearDown(self):
+        self._tmp.cleanup()
+
+    def _write(self, **fields) -> None:
+        (self.install / "demo.json").write_text(json.dumps(fields))
+
+    def test_the_pack_figures_win(self):
+        self._write(total_files=1872, total_size=10, pack_files=4525, pack_size=20)
+        self.assertEqual(
+            generate_readme.manifest_totals("demo", str(self.install)), (4525, 20)
+        )
+
+    def test_a_manifest_written_before_them_still_answers(self):
+        self._write(total_files=1872, total_size=10)
+        self.assertEqual(
+            generate_readme.manifest_totals("demo", str(self.install)), (1872, 10)
+        )
+
+    def test_the_download_table_gives_the_file_count_of_each_pack(self):
+        self._write(total_files=1872, total_size=10, pack_files=4525,
+                    pack_size=5882727352)
+        rows = generate_readme.download_table(
+            {"demo": {"platform": "Demo"}}, set(), {"Demo": "`system/`"},
+            str(self.install),
+        )
+        self.assertIn("| Platform | Files | Extracted size |", rows[0])
+        self.assertIn("| Demo | 4,525 | 5.5 GB | `system/` |", rows[2])
+
+
+class TheCollectionTotalNamesTheCollection(unittest.TestCase):
+    COMPOSITION = {
+        "systems": {"files": 5030},
+        "arcade": {"files": 2806},
+        "game_data": {"files": 2539},
+    }
+
+    def test_in_the_readme(self):
+        line = generate_readme.collection_line(10375, self.COMPOSITION)
+        self.assertIn("**10,375 files in the collection**", line)
+        self.assertIn("no pack holds them all", line)
+
+    def test_on_the_site(self):
+        db = {
+            "total_files": 10375,
+            "files": {},
+            "indexes": {},
+        }
+        sentence = generate_site.composition_sentence(db)
+        self.assertTrue(sentence.startswith("The collection"), sentence)
+        self.assertEqual(generate_site.COLLECTION_FILES_LABEL, "Files collected")
+
+
+if __name__ == "__main__":
+    unittest.main()

wiki/release-process.md

@@ -174,8 +174,9 @@ for f in dist/*.zip; do
   split --bytes=1900M --numeric-suffixes=1 --suffix-length=3 "$f" "$f." && rm "$f"
 done
 
-# 5. The two sizes a pack has. Extracted runs well above downloaded, so the
-#    notes table names the one it carries.
+# 5. The two sizes a pack has, and its file count. Extracted runs well above
+#    downloaded, so the notes table names the one it carries. The count is
+#    what a file manager shows once the pack is extracted.
 python3 - <<'PY'
 import json, pathlib, sys
 sys.path.insert(0, "scripts")
@@ -195,7 +196,7 @@ for manifest in sorted(pathlib.Path("install").glob("*.json")):
     data = json.loads(manifest.read_text())
     download = sum(p.stat().st_size for p in parts[base])
     print(f"{base:<42} download {size(download):>8}"
-          f"   extracted {size(data['total_size']):>8}   {data['total_files']} files")
+          f"   extracted {size(data['pack_size']):>8}   {data['pack_files']:,} files")
 PY
 
 # 6. Create the release as a DRAFT, upload every asset, and only then publish it.
@@ -262,3 +263,10 @@ one the table carries, the header names it. Someone sizing a USB drive is
 reading that column. The README table is the extracted size, from the install
 manifests. `SHA256SUMS.txt` lists the checksums of the full ZIPs before
 splitting.
+
+The table carries a Files column, the `pack_files` step 5 prints, and the
+notes never open on the size of the collection. That total covers every
+platform and emulator together and no pack holds it: v2026.09.04 led with
+"10,330 files" right after "one pack per platform", and someone who extracted
+the complete RetroArch pack and counted 4,525 reported half of it missing.
+The collection total belongs under "What's new", worded as the collection.