feat: state how many files each pack holds

scripts/generate_pack.py

@@ -288,25 +288,28 @@ def download_external(file_entry: dict, dest_path: str) -> bool:
 
 
 
-def _pack_data_directories(
-    zf,
+# What the builder writes into a platform pack beside the files it carries.
+PACK_DOCUMENTS = ("README.txt", "manifest.json")
+
+
+def _data_directory_members(
     pack_systems: dict,
     data_registry: dict | None,
     platform_name: str,
     base_dest: str,
-    flatten: bool,
     case_insensitive: bool,
     seen_destinations: set,
     seen_lower: set,
     seen_parents: set,
-) -> int:
-    """Add the cached data directories the platform's systems declare.
+):
+    """Yield (source, destination) for the cached data directories a
+    platform's systems declare.
 
     These are not BIOS and carry no hash of their own: they are whole
     trees an emulator reads, refreshed from upstream, so what ships is
-    whatever the cache holds. Returns how many files were added.
+    whatever the cache holds. The pack and the install manifest both read
+    this, the first to write the files and the second to count them.
     """
-    added = 0
     # Data directories from _data_dirs.yml
     for sys_id, system in sorted(pack_systems.items()):
         for dd in system.get("data_directories", []):
@@ -345,8 +348,29 @@ def _pack_data_directories(
                     _register_path(full, seen_destinations, seen_parents)
                     if case_insensitive:
                         seen_lower.add(full.lower())
-                    _add_pack_member(zf, src, _flat(full, base_dest, flatten))
-                    added += 1
+                    yield src, full
+
+
+def _pack_data_directories(
+    zf,
+    pack_systems: dict,
+    data_registry: dict | None,
+    platform_name: str,
+    base_dest: str,
+    flatten: bool,
+    case_insensitive: bool,
+    seen_destinations: set,
+    seen_lower: set,
+    seen_parents: set,
+) -> int:
+    """Add the data directories to the pack. Returns how many files went in."""
+    added = 0
+    for src, full in _data_directory_members(
+        pack_systems, data_registry, platform_name, base_dest,
+        case_insensitive, seen_destinations, seen_lower, seen_parents,
+    ):
+        _add_pack_member(zf, src, _flat(full, base_dest, flatten))
+        added += 1
     return added
 
 
@@ -2880,6 +2904,7 @@ def generate_manifest(
     target_name: str | None = None,
     offline: bool | None = None,
     required_only: bool = False,
+    data_registry: dict | None = None,
 ) -> dict:
     """Generate a JSON manifest for a platform (same resolution as generate_pack).
 
@@ -3100,7 +3125,18 @@ def generate_manifest(
         omitted_by_destination, record_omission, required_only,
     )
 
-    # No phase 3 (data directories) -skipped for manifest
+    # Phase 3: data directories. The installer does not fetch them, so they
+    # stay out of the file list; the pack carries them, so they count toward
+    # what an extraction shows.
+    if data_registry is None:
+        data_registry = load_data_dir_registry(platforms_dir)
+    data_sizes = [
+        os.path.getsize(src)
+        for src, _dest in _data_directory_members(
+            pack_systems, data_registry, platform_name, base_dest,
+            case_insensitive, seen_destinations, seen_lower, seen_parents,
+        )
+    ]
 
     now = _build_timestamp(db)
 
@@ -3122,6 +3158,8 @@ def generate_manifest(
         "standalone_copies": standalone_copies,
         "total_files": len(manifest_files),
         "total_size": total_size,
+        "pack_files": len(manifest_files) + len(data_sizes) + len(PACK_DOCUMENTS),
+        "pack_size": total_size + sum(data_sizes),
         "total_omitted": len(omitted_by_destination),
         "omitted_files": sorted(
             omitted_by_destination.values(), key=lambda entry: entry["dest"]

scripts/generate_readme.py

@@ -97,10 +97,11 @@ def compute_coverage(
 def manifest_totals(
     platform_name: str, install_dir: str = "install"
 ) -> tuple[int | None, int | None]:
-    """Files and bytes a platform's pack ships, from its install manifest.
+    """Files and bytes a platform's pack holds once extracted.
 
-    The manifest is written when packs are built, so it reflects the real
-    pack contents (platform list, core complement and data directories).
+    The install manifest lists what the installer fetches, and states apart
+    what the pack adds to that: the data directories and its own documents.
+    The pack figures are the ones a reader can check against an extraction.
     Returns (None, None) when no manifest exists yet.
     """
     path = os.path.join(install_dir, f"{platform_name}.json")
@@ -111,7 +112,53 @@ def manifest_totals(
             manifest = json.load(f)
     except (json.JSONDecodeError, OSError):
         return None, None
-    return manifest.get("total_files"), manifest.get("total_size")
+    return (
+        manifest.get("pack_files", manifest.get("total_files")),
+        manifest.get("pack_size", manifest.get("total_size")),
+    )
+
+
+def download_table(
+    coverages: dict,
+    archived: set[str],
+    extract_paths: dict[str, str],
+    install_dir: str = "install",
+) -> list[str]:
+    """One row per platform: what its pack holds and where it extracts."""
+    lines = [
+        "| Platform | Files | Extracted size | Extract to | Download |",
+        "|----------|------:|---------------:|-----------|----------|",
+    ]
+    for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]):
+        display = cov["platform"]
+        if name in archived:
+            display = f"{display} *"
+        files, size = manifest_totals(name, install_dir)
+        lines.append(
+            f"| {display} | {f'{files:,}' if files else '-'} |"
+            f" {format_size(size) if size else '-'} |"
+            f" {extract_paths.get(cov['platform'], '')} |"
+            f" [Download]({RELEASE_URL}) |"
+        )
+    return lines
+
+
+def collection_line(total_files: int, comp: dict) -> str:
+    """The size of the whole collection, said as such.
+
+    Printed bare beside the download links it read as the content of one
+    pack, and a complete extraction looked like a broken one.
+    """
+    return (
+        f"- **{total_files:,} files in the collection**, each with its SHA1,"
+        " MD5, SHA256, CRC32 and Adler-32 fingerprints:"
+        f" {comp['systems']['files']:,} system files,"
+        f" {comp['arcade']['files']:,} arcade ROM sets,"
+        f" {comp['game_data']['files']:,} game and engine data files."
+        " That is every platform and emulator together, so no pack holds"
+        " them all: each carries what its own emulators load, counted in"
+        " the download table"
+    )
 
 
 def format_size(size: int) -> str:
@@ -288,7 +335,11 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
         "## Download BIOS packs",
         "",
         "One pack per platform, and it holds everything the platform runs: its own BIOS list plus every file its emulator cores load. Pick your platform, download the ZIP, extract to the BIOS path. The installer above does the same file by file, and `--target` narrows it to one machine; for a region or a bare minimum, build your own pack below.",
-        "The size is what the files occupy once extracted; the ZIP itself"
+        "Files is how many the pack holds once extracted, the figure a file"
+        " manager shows for the folder. It differs from one platform to the"
+        " next because a pack carries only what that platform's emulators"
+        " load."
+        " The size is what the files occupy once extracted; the ZIP itself"
         " downloads smaller, and anything over 2 GB arrives split into"
         " `.zip.001`, `.zip.002` volumes. Open the `.001` with 7-Zip or"
         " PeaZip, or join them first"
@@ -301,8 +352,6 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
         "(https://abdess.github.io/retrobios/wiki/release-process/"
         "#verifying-a-release).",
         "",
-        "| Platform | Extracted size | Extract to | Download |",
-        "|----------|---------------:|-----------|----------|",
     ]
 
     # Where the pack itself is extracted, which is not always the BIOS folder:
@@ -328,16 +377,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
         if entry.get("status") == "archived"
     }
 
-    for name, cov in sorted(coverages.items(), key=lambda x: x[1]["platform"]):
-        display = cov["platform"]
-        if name in archived:
-            display = f"{display} *"
-        path = extract_paths.get(cov["platform"], "")
-        _, size = manifest_totals(name)
-        size_cell = format_size(size) if size else "-"
-        lines.append(
-            f"| {display} | {size_cell} | {path} | [Download]({RELEASE_URL}) |"
-        )
+    lines.extend(download_table(coverages, archived, extract_paths))
 
     if archived:
         lines.extend(
@@ -384,10 +424,7 @@ def generate_readme(db: dict, platforms_dir: str) -> str:
             f"- **{len(coverages)} platforms** supported with platform-specific verification",
             f"- **{emulator_count} emulators** profiled from source (RetroArch cores + standalone)",
             f"- **{len(system_ids)} systems** handled by those emulators (NES, SNES, PlayStation, Saturn, Dreamcast, ...)",
-            f"- **{total_files:,} files**, each with its SHA1, MD5, SHA256, CRC32 and Adler-32 fingerprints:"
-            f" {comp['systems']['files']:,} system files,"
-            f" {comp['arcade']['files']:,} arcade ROM sets,"
-            f" {comp['game_data']['files']:,} game and engine data files",
+            collection_line(total_files, comp),
             *_catalog_matched_line(db),
             f"- **{size_mb:.0f} MB** total collection size",
             "",

scripts/generate_site.py

@@ -235,7 +235,7 @@ def generate_home(
         "",
         '<div class="rb-stat" markdown>',
         f'<span class="rb-stat-value">{total_files:,}</span>',
-        '<span class="rb-stat-label">Files</span>',
+        f'<span class="rb-stat-label">{COLLECTION_FILES_LABEL}</span>',
         "</div>",
         "",
         '<div class="rb-stat" markdown>',
@@ -264,8 +264,8 @@ def generate_home(
         [
             "## Platforms",
             "",
-            "| Icon | Platform | Files | Checked by | Download |",
-            "|---|----------|-------|-----------|----------|",
+            "| Icon | Platform | Files in the pack | Checked by | Download |",
+            "|---|----------|------:|-----------|----------|",
         ]
     )
 
@@ -281,7 +281,7 @@ def generate_home(
 
         lines.append(
             f"| {logo_md} | [{display}](platforms/{name}.md) | "
-            f"{cov['present']:,} | {mode_label} | "
+            f"{manifest_totals(name)[0] or cov['present']:,} | {mode_label} | "
             f"[Pack]({RELEASE_URL}){{ .md-button .md-button--primary }} |"
         )
 
@@ -410,13 +410,19 @@ def compute_stats(db: dict, coverages: dict, profiles: dict) -> dict:
     }
 
 
+# The home tile counts the whole collection. Labelled "Files" above a table
+# of pack buttons, it read as what one pack contains.
+COLLECTION_FILES_LABEL = "Files collected"
+
+
 def composition_sentence(db: dict) -> str:
     comp = compute_composition(db)
     return (
-        f"Of these files, {comp['systems']['files']:,} are console and "
+        f"The collection holds {comp['systems']['files']:,} console and "
         f"computer system files, {comp['arcade']['files']:,} arcade ROM sets "
         f"(`Arcade/`), and {comp['game_data']['files']:,} game and engine "
-        "data (the `RPG Maker/` and `ScummVM/` trees)."
+        "data files (the `RPG Maker/` and `ScummVM/` trees). No pack holds "
+        "them all: each carries what its platform's emulators load."
     )
 
 
@@ -1148,8 +1154,8 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st
         f"{len(coverages)} supported platforms with "
         f"{total_present:,} verified files.",
         "",
-        "| Platform | Files | Checked by | Status | Download |",
-        "|----------|-------|-----------|--------|----------|",
+        "| Platform | On its BIOS list | Files in the pack | Checked by | Status | Download |",
+        "|----------|------:|------:|-----------|--------|----------|",
     ]
 
     mode_labels = {
@@ -1175,13 +1181,17 @@ def generate_platform_index(coverages: dict, registry: dict | None = None) -> st
 
         lines.append(
             f"| [{display}]({name}.md) | "
-            f"{cov['present']:,} | {mode_html} | {status_html} | "
+            f"{cov['present']:,} | {manifest_totals(name)[0] or cov['present']:,} | "
+            f"{mode_html} | {status_html} | "
             f"[Pack]({RELEASE_URL}){{ .md-button .md-button--primary }} |"
         )
 
     lines.extend(
         [
             "",
+            "The BIOS list is what the platform publishes. The pack adds the "
+            "files its emulators load and the data directories they read, "
+            "which is the count an extraction shows. "
             "Checked by is the test each platform runs on its own, replicated "
             "from its source code. "
             "[How each mode works](../wiki/verification-modes.md).",
@@ -2232,8 +2242,9 @@ def _availability_check(db: dict, data_names):
             for md5_val in parse_md5_list(md5_raw):
                 if by_md5.get(md5_val):
                     return True
-        sha1 = f.get("sha1", "")
-        if sha1 and sha1 in db_files:
+        # A profile lists several sha1 when the code accepts several dumps.
+        sha1 = f.get("sha1") or []
+        if any(value in db_files for value in ([sha1] if isinstance(sha1, str) else sha1)):
             return True
         return False