chore: merge narrowed pack comparison

scripts/generate_pack.py

@@ -139,14 +139,17 @@ class PackRecord:
     """A pack build kept in memory: the paths it writes, no archive.
 
     ``members`` is every path written, ``files`` the ones the build counts as
-    the profile's files (archives and loose files, not trees or notes). The
-    pipeline builds every emulator pack this way and compares ``files`` with
-    what verify --emulator credits, without reading or compressing a byte.
+    the profile's files (archives and loose files, not trees or notes), and
+    ``trees`` the members a platform pack takes from its data directories.
+    The pipeline builds every emulator pack and every narrowed platform pack
+    this way and compares them with what verify credits, without reading or
+    compressing a byte.
     """
 
     def __init__(self) -> None:
         self.members: list[str] = []
         self.files: list[str] = []
+        self.trees: list[str] = []
 
 
 def _add_pack_member(zf: zipfile.ZipFile, source_path: str, arcname: str) -> None:
@@ -586,6 +589,8 @@ def _pack_data_directories(
         required_only=required_only, core_trees=core_trees,
     ):
         _add_pack_member(zf, src, _flat(full, base_dest, flatten))
+        if isinstance(zf, PackRecord):
+            zf.trees.append(_flat(full, base_dest, flatten))
         added += 1
     return added
 
@@ -790,6 +795,7 @@ def generate_pack(
     offline: bool | None = None,
     pack_name: str | None = None,
     no_game_data: bool = False,
+    record: PackRecord | None = None,
 ) -> str | None:
     """Generate a ZIP pack for a platform.
 
@@ -800,6 +806,9 @@ def generate_pack(
     its group, and a name built from every system of a hundred-system group
     is longer than a file name can be.
 
+    ``record`` builds the pack in memory (PackRecord): no archive is written,
+    no external file fetched, and the paths are kept on it.
+
     Returns the path to the generated ZIP, or None on failure.
     """
     config = load_platform_config(platform_name, platforms_dir)
@@ -832,7 +841,8 @@ def generate_pack(
             platform_name, platforms_dir, narrowings, system_filter, pack_name
         ),
     )
-    os.makedirs(output_dir, exist_ok=True)
+    if record is None:
+        os.makedirs(output_dir, exist_ok=True)
 
     # Case-insensitive dedup only for platforms targeting Windows/macOS.
     # Linux-only platforms (Batocera, Recalbox, RetroDECK, Lakka, RomM)
@@ -919,7 +929,10 @@ def generate_pack(
         undeclared=undeclared,
     )
 
-    with _writing_pack(zip_path) as zf:
+    with (
+        contextlib.nullcontext(record) if record is not None
+        else _writing_pack(zip_path)
+    ) as zf:
       if source != "truth":
         for sys_id, system in sorted(pack_systems.items()):
             for file_entry in system.get("files", []):
@@ -1040,7 +1053,7 @@ def generate_pack(
                     if case_insensitive:
                         seen_lower.add(dedup_key.lower())
                     try:
-                        if download_external(file_entry, tmp_path):
+                        if record is None and download_external(file_entry, tmp_path):
                             extract = file_entry.get("extract", False)
                             if extract and tmp_path.endswith(".zip"):
                                 _extract_zip_to_archive(tmp_path, _flat(full_dest, base_dest, flatten), zf)
@@ -1371,6 +1384,13 @@ def _extract_zip_to_archive(
 ) -> None:
     """Extract contents of a source ZIP into target ZIP under dest_prefix."""
     with zipfile.ZipFile(source_zip, "r") as src:
+        if isinstance(target_zf, PackRecord):
+            target_zf.members.extend(
+                f"{dest_prefix}/{name}" if dest_prefix else name
+                for name in map(sanitize_pack_path, src.namelist())
+                if name and not name.endswith("/")
+            )
+            return
         for info in src.infolist():
             if info.is_dir():
                 continue

scripts/pipeline.py

@@ -22,7 +22,8 @@ Steps, numbered as the run prints them:
   4c   --manifest-targets           per-target core lists
   4d   bundled_only.py --strict     copies only an emulator install provides
   4e   generate_games_index.py      install/games/: what the games-on-card mode reads
-  5    consistency check            verify == pack: counts, paths, emulator and system packs
+  5    consistency check            verify == pack: counts, paths, emulator and system packs,
+                                    narrowed platform packs (see check_narrowed_packs)
   6    --verify-packs               pack integrity, extraction and hashes
   7    generate_readme.py           README.md and CONTRIBUTING.md
        generate_installer_doc.py    run samples and tables of wiki/installer.md
@@ -616,6 +617,165 @@ def check_emulator_packs(
     return all_ok and systems_ok
 
 
+# How each _narrowings() dimension is compared by check_narrowed_packs, or why
+# it is not. A test reads the builder's signature against these two tables, so
+# a new narrowing fails it until it is compared or a reason is written here.
+NARROWINGS_COMPARED = frozenset({"regions", "target_name", "required_only", "no_game_data"})
+NARROWINGS_NOT_COMPARED = {
+    "standalone": "an emulator pack mode, compared by check_emulator_packs",
+    "source": "verify has no platform-only or truth-only report to compare with",
+    "one_per_slot": "verify has no slot choice to compare with",
+}
+
+
+def _matrix_coverage(full: bool, count: int) -> range | list[int]:
+    """Which of *count* variants of one dimension a run compares.
+
+    The full matrix takes every one. The default takes the first, the last
+    and one in between, which reaches each code path the dimension takes
+    (an empty selection, a single system, a large one) without its cost.
+    """
+    if full or count <= 3:
+        return range(count)
+    return sorted({0, count // 2, count - 1})
+
+
+def narrowed_variants(
+    platform: str, config: dict, profiles: dict, platforms_dir: str, full: bool,
+) -> list[tuple[str, dict]]:
+    """(label, builder arguments) of each narrowed pack compared for a platform.
+
+    Every region a file of the platform's cores declares, every hardware
+    target, every system, --no-game-data and --required-only. Without
+    *full*, regions, targets and systems are sampled (_matrix_coverage).
+    """
+    import region as region_mod  # noqa: PLC0415
+    from common import list_available_targets  # noqa: PLC0415
+
+    regions = sorted({
+        r for entry in region_mod.platform_region_index(config, profiles).values()
+        for r in entry["regions"]
+    })
+    targets = [t["name"] for t in list_available_targets(platform, platforms_dir)]
+    systems = sorted(config.get("systems") or {})
+    variants: list[tuple[str, dict]] = [
+        ("--required-only", {"required_only": True}),
+        ("--no-game-data", {"no_game_data": True}),
+    ]
+    variants += [
+        (f"--region {regions[i]}", {"regions": [regions[i]]})
+        for i in _matrix_coverage(full, len(regions))
+    ]
+    variants += [
+        (f"--target {targets[i]}", {"target_name": targets[i]})
+        for i in _matrix_coverage(full, len(targets))
+    ]
+    variants += [
+        (f"--system {systems[i]}", {"system_filter": [systems[i]]})
+        for i in _matrix_coverage(full, len(systems))
+    ]
+    return variants
+
+
+def check_narrowed_packs(
+    emulators_dir: str = "emulators", platforms_dir: str = "platforms",
+    db_path: str = "database.json", include_archived: bool = False,
+    full: bool = False,
+) -> bool:
+    """Build each narrowed platform pack in memory and compare it with verify.
+
+    The paths the pack writes against the ones verify credits for the same
+    narrowing, as check_pack_destinations compares the full packs. Both sides
+    run offline: a release asset the cache lacks is missing on each.
+
+    Coverage: --required-only and --no-game-data for every platform, and
+    --region, --target and --system. Measured on 2026-11-05, the full matrix
+    is 1,150 packs and took 25 minutes, one to two seconds a pack, most
+    of it platform-wide work (slot overrides, the gap report) that each build
+    redoes. A run therefore takes the first, middle and last region, target
+    and system of each platform, 155 packs in about 3 minutes: each value of
+    a dimension runs the same code, so three reach every path a narrowing
+    takes (the empty end, one system, a large one). *full* (--full-narrowings)
+    compares every value. NARROWINGS_NOT_COMPARED says why --source and
+    --one-per-slot are not compared.
+    """
+    import generate_pack  # noqa: PLC0415
+    import packresolve  # noqa: PLC0415
+    import verify  # noqa: PLC0415
+    from common import (  # noqa: PLC0415
+        build_zip_contents_index,
+        group_identical_platforms,
+        list_registered_platforms,
+        load_database,
+        load_emulator_profiles,
+        load_platform_config,
+        load_target_config,
+    )
+
+    print(f"\n--- {step_title(STEP_LABELS['consistency'])}, narrowed packs ---")
+    db = load_database(db_path)
+    zip_contents = build_zip_contents_index(db)
+    profiles = load_emulator_profiles(emulators_dir)
+    registry = load_data_dir_registry(platforms_dir)
+    groups = group_identical_platforms(
+        list_registered_platforms(platforms_dir, include_archived=include_archived),
+        platforms_dir,
+    )
+    was_offline = packresolve._OFFLINE
+    packresolve.set_offline(True)
+    all_ok = True
+    compared = 0
+    try:
+        for _members, platform in groups:
+            config = load_platform_config(platform, platforms_dir)
+            for label, narrowing in narrowed_variants(
+                platform, config, profiles, platforms_dir, full,
+            ):
+                target = narrowing.get("target_name")
+                target_cores = (
+                    load_target_config(platform, target, platforms_dir) if target else None
+                )
+                record = generate_pack.PackRecord()
+                with contextlib.redirect_stdout(io.StringIO()), \
+                        contextlib.redirect_stderr(io.StringIO()):
+                    built = generate_pack.generate_pack(
+                        platform, platforms_dir, db, "bios", "",
+                        emulators_dir=emulators_dir, zip_contents=zip_contents,
+                        data_registry=registry, emu_profiles=profiles,
+                        target_cores=target_cores, offline=True, record=record,
+                        **narrowing,
+                    )
+                    report = verify.verify_platform(
+                        load_platform_config(platform, platforms_dir), db,
+                        emulators_dir, profiles, target_cores=target_cores,
+                        data_dir_registry=registry,
+                        **{k: v for k, v in narrowing.items() if k != "target_name"},
+                    ) if built else None
+                compared += 1
+                name = f"{platform} {label}"
+                if report is None:
+                    # The builder declines a system no core of the platform runs.
+                    print(f"  {name}: the builder wrote no pack")
+                    all_ok = False
+                    continue
+                credited = set(report["pack_destinations"])
+                trees = set(record.trees) - credited
+                written = {
+                    m for m in record.members
+                    if not _PACK_NOTES.search(m) and m not in trees
+                }
+                lines = _set_difference_lines(name, credited, written)
+                if lines:
+                    print("\n".join(lines))
+                    all_ok = False
+    finally:
+        packresolve.set_offline(was_offline)
+    scope = "full matrix" if full else "sampled"
+    print(f"  {compared} narrowed platform packs compared ({scope}), "
+          f"{'all equal' if all_ok else 'differences above'}")
+    return all_ok
+
+
 class _Skipped:
     """A step that did not run. Truthy, so it never fails the pipeline, and
     distinct, so the summary does not report it as work that was done."""
@@ -636,7 +796,8 @@ def _refuse_unapplied_flags(
     """A flag the run would drop is refused before any step.
 
     --all-variants builds every source itself, so a --source beside it is
-    dropped; --skip-packs builds none, so both are.
+    dropped; --skip-packs builds none, so both are, and step 5 does not run
+    for --full-narrowings to widen.
     """
     # --source defaults to None so a typed `--source full` is seen: compared
     # with its default value, it went through beside the modes that refuse it.
@@ -645,7 +806,8 @@ def _refuse_unapplied_flags(
         parser.error("--source is incompatible with --all-variants")
     if args.skip_packs:
         for flag, on in (("--source", source_given),
-                         ("--all-variants", args.all_variants)):
+                         ("--all-variants", args.all_variants),
+                         ("--full-narrowings", args.full_narrowings)):
             if on:
                 parser.error(f"{flag} is incompatible with --skip-packs")
     if args.source is None:
@@ -775,6 +937,11 @@ def main() -> None:
         help="Build the six source x required packs of each platform; "
         "not with --source or --skip-packs",
     )
+    parser.add_argument(
+        "--full-narrowings", action="store_true",
+        help="Step 5 compares every region, target and system of each "
+        "platform, not a sample of three (about 25 minutes)",
+    )
     parser.add_argument(
         "--check-buildbot",
         action="store_true",
@@ -1143,6 +1310,9 @@ def main() -> None:
         # Counts can agree while the paths differ, both ways at once.
         ok = check_pack_destinations(destinations_file, pack_output) and ok
         ok = check_emulator_packs() and ok
+        ok = check_narrowed_packs(
+            include_archived=args.include_archived, full=args.full_narrowings,
+        ) and ok
         print(
             f"--- {step_title(STEP_LABELS['consistency'])}, counts and paths: "
             f"{'OK' if ok else 'FAILED'} ---"

scripts/verify.py

@@ -948,6 +948,7 @@ def _mark_hidden(
     resolver: object | None = None,
     placed_kind: dict[str, str] | None = None,
     system_ids: list[str] | None = None,
+    required_only: bool = False,
 ) -> set[str]:
     """What the builder holds but cannot place beside the files it packed.
 
@@ -963,7 +964,7 @@ def _mark_hidden(
         emu_profiles if emu_profiles is not None
         else load_emulator_profiles(emulators_dir),
         shipped, target_cores, region_drops, placed, report, resolver, placed_kind,
-        system_ids=system_ids,
+        system_ids=system_ids, required_only=required_only,
     )
     for u in undeclared:
         reason = hidden.get(
@@ -1146,6 +1147,7 @@ def verify_platform(
     regions: list[str] | None = None,
     system_filter: list[str] | None = None,
     no_game_data: bool = False,
+    required_only: bool = False,
 ) -> dict:
     """Verify all BIOS files for a platform, including cross-reference gaps.
 
@@ -1156,6 +1158,9 @@ def verify_platform(
     same list would carry, using the same selection function as the builder.
     ``system_filter`` does the same for a pack built with --system: the
     systems it keeps and the core extras those systems own.
+    ``required_only`` reports on the pack built with --required-only: the
+    declarations, trees and core extras marked ``required: false`` are left
+    out before any of them takes a path, as the builder leaves them out.
     """
     left_out: list[str] = []
     if no_game_data:
@@ -1221,6 +1226,16 @@ def verify_platform(
     validation_owners = destination_owners(platform_profiles)
 
     verify_systems = platform_scope_systems(config, profiles, target_cores, system_filter)
+    if required_only:
+        verify_systems = {
+            sys_id: {
+                **system,
+                "files": [
+                    fe for fe in system.get("files", []) if fe.get("required") is not False
+                ],
+            }
+            for sys_id, system in verify_systems.items()
+        }
 
     details = []
     file_status: dict[str, str] = {}
@@ -1252,6 +1267,7 @@ def verify_platform(
             config.get("base_destination", ""),
             profiles,
             target_cores=target_cores,
+            required_only=required_only,
             held_out=held,
         )
         region_index = region_mod.platform_region_index(config, profiles, target_cores)
@@ -1445,6 +1461,8 @@ def verify_platform(
         from packextras import report_owned_by_systems
 
         undeclared = report_owned_by_systems(undeclared, system_filter)
+    if required_only:
+        undeclared = [u for u in undeclared if u.get("required") is not False]
     if region_drops:
         undeclared = [
             u
@@ -1467,6 +1485,7 @@ def verify_platform(
         ),
         placed_kind,
         system_filter,
+        required_only,
     )
     # What the builder places from the cores, one per pack path, by category:
     # several entries claim one path (rom.key from three Amiga cores) and the